From 26e24a69495a164089eab7eb3e0b63cd712f4d3b Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Mon, 7 Apr 2025 11:45:30 +0200 Subject: [PATCH 01/34] Remove old OCF module Change-Id: Icf9704d338fe8fe5d3f6ef8eb5f9255af4a649d2 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 1794 ------------------------------- module/bdev/ocf/vbdev_ocf.h | 206 ---- module/bdev/ocf/vbdev_ocf_rpc.c | 631 ----------- 3 files changed, 2631 deletions(-) delete mode 100644 module/bdev/ocf/vbdev_ocf.c delete mode 100644 module/bdev/ocf/vbdev_ocf.h delete mode 100644 module/bdev/ocf/vbdev_ocf_rpc.c diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c deleted file mode 100644 index 08e586f2061..00000000000 --- a/module/bdev/ocf/vbdev_ocf.c +++ /dev/null @@ -1,1794 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (C) 2018 Intel Corporation. - * All rights reserved. - */ - -#include -#include -#include - -#include "ctx.h" -#include "data.h" -#include "volume.h" -#include "utils.h" -#include "vbdev_ocf.h" - -#include "spdk/bdev_module.h" -#include "spdk/thread.h" -#include "spdk/string.h" -#include "spdk/log.h" -#include "spdk/cpuset.h" - -/* This namespace UUID was generated using uuid_generate() method. */ -#define BDEV_OCF_NAMESPACE_UUID "f92b7f49-f6c0-44c8-bd23-3205e8c3b6ad" - -static struct spdk_bdev_module ocf_if; - -static TAILQ_HEAD(, vbdev_ocf) g_ocf_vbdev_head - = TAILQ_HEAD_INITIALIZER(g_ocf_vbdev_head); - -static TAILQ_HEAD(, examining_bdev) g_ocf_examining_bdevs_head - = TAILQ_HEAD_INITIALIZER(g_ocf_examining_bdevs_head); - -static bool g_fini_started = false; - -/* Structure for keeping list of bdevs that are claimed but not used yet */ -struct examining_bdev { - struct spdk_bdev *bdev; - TAILQ_ENTRY(examining_bdev) tailq; -}; - -/* Add bdev to list of claimed */ -static void -examine_start(struct spdk_bdev *bdev) -{ - struct examining_bdev *entry = malloc(sizeof(*entry)); - - assert(entry); - entry->bdev = bdev; - TAILQ_INSERT_TAIL(&g_ocf_examining_bdevs_head, entry, tailq); -} - -/* Find bdev on list of claimed bdevs, then remove it, - * if it was the last one on list then report examine done */ -static void -examine_done(int status, struct vbdev_ocf *vbdev, void *cb_arg) -{ - struct spdk_bdev *bdev = cb_arg; - struct examining_bdev *entry, *safe, *found = NULL; - - TAILQ_FOREACH_SAFE(entry, &g_ocf_examining_bdevs_head, tailq, safe) { - if (entry->bdev == bdev) { - if (found) { - goto remove; - } else { - found = entry; - } - } - } - - assert(found); - spdk_bdev_module_examine_done(&ocf_if); - -remove: - TAILQ_REMOVE(&g_ocf_examining_bdevs_head, found, tailq); - free(found); -} - -/* Free allocated strings and structure itself - * Used at shutdown only */ -static void -free_vbdev(struct vbdev_ocf *vbdev) -{ - if (!vbdev) { - return; - } - - free(vbdev->name); - free(vbdev->cache.name); - free(vbdev->core.name); - free(vbdev); -} - -/* Get existing cache base - * that is attached to other vbdev */ -static struct vbdev_ocf_base * -get_other_cache_base(struct vbdev_ocf_base *base) -{ - struct vbdev_ocf *vbdev; - - TAILQ_FOREACH(vbdev, &g_ocf_vbdev_head, tailq) { - if (&vbdev->cache == base || !vbdev->cache.attached) { - continue; - } - if (!strcmp(vbdev->cache.name, base->name)) { - return &vbdev->cache; - } - } - - return NULL; -} - -static bool -is_ocf_cache_running(struct vbdev_ocf *vbdev) -{ - if (vbdev->cache.attached && vbdev->ocf_cache) { - return ocf_cache_is_running(vbdev->ocf_cache); - } - return false; -} - -static bool -is_ocf_cache_detached(struct vbdev_ocf *vbdev) -{ - if (vbdev->cache.attached && vbdev->ocf_cache) { - return ocf_cache_is_detached(vbdev->ocf_cache); - } - return false; -} - -/* Get existing OCF cache instance - * that is started by other vbdev */ -static ocf_cache_t -get_other_cache_instance(struct vbdev_ocf *vbdev) -{ - struct vbdev_ocf *cmp; - - TAILQ_FOREACH(cmp, &g_ocf_vbdev_head, tailq) { - if (cmp->state.doing_finish || cmp == vbdev) { - continue; - } - if (strcmp(cmp->cache.name, vbdev->cache.name)) { - continue; - } - if (is_ocf_cache_running(cmp) || is_ocf_cache_detached(cmp)) { - return cmp->ocf_cache; - } - } - - return NULL; -} - -static void -_remove_base_bdev(void *ctx) -{ - struct spdk_bdev_desc *desc = ctx; - - spdk_bdev_close(desc); -} - -/* Close and unclaim base bdev */ -static void -remove_base_bdev(struct vbdev_ocf_base *base) -{ - if (base->attached) { - if (base->management_channel) { - spdk_put_io_channel(base->management_channel); - } - - spdk_bdev_module_release_bdev(base->bdev); - /* Close the underlying bdev on its same opened thread. */ - if (base->thread && base->thread != spdk_get_thread()) { - spdk_thread_send_msg(base->thread, _remove_base_bdev, base->desc); - } else { - spdk_bdev_close(base->desc); - } - base->attached = false; - } -} - -/* Finish unregister operation */ -static void -unregister_finish(struct vbdev_ocf *vbdev) -{ - spdk_bdev_destruct_done(&vbdev->exp_bdev, vbdev->state.stop_status); - - if (vbdev->ocf_cache) { - ocf_mngt_cache_put(vbdev->ocf_cache); - } - - if (vbdev->cache_ctx) { - vbdev_ocf_cache_ctx_put(vbdev->cache_ctx); - } - vbdev_ocf_mngt_continue(vbdev, 0); -} - -static void -close_core_bdev(struct vbdev_ocf *vbdev) -{ - remove_base_bdev(&vbdev->core); - vbdev_ocf_mngt_continue(vbdev, 0); -} - -static void -remove_core_cmpl(void *priv, int error) -{ - struct vbdev_ocf *vbdev = priv; - - ocf_mngt_cache_unlock(vbdev->ocf_cache); - vbdev_ocf_mngt_continue(vbdev, error); -} - -/* Try to lock cache, then remove core */ -static void -remove_core_cache_lock_cmpl(ocf_cache_t cache, void *priv, int error) -{ - struct vbdev_ocf *vbdev = (struct vbdev_ocf *)priv; - - if (error) { - SPDK_ERRLOG("Error %d, can not lock cache instance %s\n", - error, vbdev->name); - vbdev_ocf_mngt_continue(vbdev, error); - return; - } - - ocf_mngt_cache_remove_core(vbdev->ocf_core, remove_core_cmpl, vbdev); -} - -/* Detach core base */ -static void -detach_core(struct vbdev_ocf *vbdev) -{ - if (is_ocf_cache_running(vbdev)) { - ocf_mngt_cache_lock(vbdev->ocf_cache, remove_core_cache_lock_cmpl, vbdev); - } else { - vbdev_ocf_mngt_continue(vbdev, 0); - } -} - -static void -close_cache_bdev(struct vbdev_ocf *vbdev) -{ - remove_base_bdev(&vbdev->cache); - vbdev_ocf_mngt_continue(vbdev, 0); -} - -/* Detach cache base */ -static void -detach_cache(struct vbdev_ocf *vbdev) -{ - vbdev->state.stop_status = vbdev->mngt_ctx.status; - - /* If some other vbdev references this cache bdev, - * we detach this only by changing the flag, without actual close */ - if (get_other_cache_base(&vbdev->cache)) { - vbdev->cache.attached = false; - } - - vbdev_ocf_mngt_continue(vbdev, 0); -} - -static void -stop_vbdev_cmpl(ocf_cache_t cache, void *priv, int error) -{ - struct vbdev_ocf *vbdev = priv; - - vbdev_ocf_queue_put(vbdev->cache_ctx->mngt_queue); - ocf_mngt_cache_unlock(cache); - - vbdev_ocf_mngt_continue(vbdev, error); -} - -/* Try to lock cache, then stop it */ -static void -stop_vbdev_cache_lock_cmpl(ocf_cache_t cache, void *priv, int error) -{ - struct vbdev_ocf *vbdev = (struct vbdev_ocf *)priv; - - if (error) { - SPDK_ERRLOG("Error %d, can not lock cache instance %s\n", - error, vbdev->name); - vbdev_ocf_mngt_continue(vbdev, error); - return; - } - - ocf_mngt_cache_stop(vbdev->ocf_cache, stop_vbdev_cmpl, vbdev); -} - -/* Stop OCF cache object - * vbdev_ocf is not operational after this */ -static void -stop_vbdev(struct vbdev_ocf *vbdev) -{ - if (!is_ocf_cache_running(vbdev)) { - vbdev_ocf_mngt_continue(vbdev, 0); - return; - } - - if (!g_fini_started && get_other_cache_instance(vbdev)) { - SPDK_NOTICELOG("Not stopping cache instance '%s'" - " because it is referenced by other OCF bdev\n", - vbdev->cache.name); - vbdev_ocf_mngt_continue(vbdev, 0); - return; - } - - ocf_mngt_cache_lock(vbdev->ocf_cache, stop_vbdev_cache_lock_cmpl, vbdev); -} - -static void -flush_vbdev_cmpl(ocf_cache_t cache, void *priv, int error) -{ - struct vbdev_ocf *vbdev = priv; - - ocf_mngt_cache_unlock(cache); - vbdev_ocf_mngt_continue(vbdev, error); -} - -static void -flush_vbdev_cache_lock_cmpl(ocf_cache_t cache, void *priv, int error) -{ - struct vbdev_ocf *vbdev = (struct vbdev_ocf *)priv; - - if (error) { - SPDK_ERRLOG("Error %d, can not lock cache instance %s\n", - error, vbdev->name); - vbdev_ocf_mngt_continue(vbdev, error); - return; - } - - ocf_mngt_cache_flush(vbdev->ocf_cache, flush_vbdev_cmpl, vbdev); -} - -static void -flush_vbdev(struct vbdev_ocf *vbdev) -{ - if (!is_ocf_cache_running(vbdev)) { - vbdev_ocf_mngt_continue(vbdev, -EINVAL); - return; - } - - ocf_mngt_cache_lock(vbdev->ocf_cache, flush_vbdev_cache_lock_cmpl, vbdev); -} - -/* Procedures called during dirty unregister */ -vbdev_ocf_mngt_fn unregister_path_dirty[] = { - flush_vbdev, - stop_vbdev, - detach_cache, - close_cache_bdev, - detach_core, - close_core_bdev, - unregister_finish, - NULL -}; - -/* Procedures called during clean unregister */ -vbdev_ocf_mngt_fn unregister_path_clean[] = { - flush_vbdev, - detach_core, - close_core_bdev, - stop_vbdev, - detach_cache, - close_cache_bdev, - unregister_finish, - NULL -}; - -/* Start asynchronous management operation using unregister_path */ -static void -unregister_cb(void *opaque) -{ - struct vbdev_ocf *vbdev = opaque; - vbdev_ocf_mngt_fn *unregister_path; - int rc; - - unregister_path = vbdev->state.doing_clean_delete ? - unregister_path_clean : unregister_path_dirty; - - rc = vbdev_ocf_mngt_start(vbdev, unregister_path, NULL, NULL); - if (rc) { - SPDK_ERRLOG("Unable to unregister OCF bdev: %d\n", rc); - spdk_bdev_destruct_done(&vbdev->exp_bdev, rc); - } -} - -/* Clean remove case - remove core and then cache, this order - * will remove instance permanently */ -static void -_vbdev_ocf_destruct_clean(struct vbdev_ocf *vbdev) -{ - if (vbdev->core.attached) { - detach_core(vbdev); - close_core_bdev(vbdev); - } - - if (vbdev->cache.attached) { - detach_cache(vbdev); - close_cache_bdev(vbdev); - } -} - -/* Dirty shutdown/hot remove case - remove cache and then core, this order - * will allow us to recover this instance in the future */ -static void -_vbdev_ocf_destruct_dirty(struct vbdev_ocf *vbdev) -{ - if (vbdev->cache.attached) { - detach_cache(vbdev); - close_cache_bdev(vbdev); - } - - if (vbdev->core.attached) { - detach_core(vbdev); - close_core_bdev(vbdev); - } -} - -/* Unregister io device with callback to unregister_cb - * This function is called during spdk_bdev_unregister */ -static int -vbdev_ocf_destruct(void *opaque) -{ - struct vbdev_ocf *vbdev = opaque; - - if (vbdev->state.doing_finish) { - return -EALREADY; - } - - if (vbdev->state.starting && !vbdev->state.started) { - /* Prevent before detach cache/core during register path of - this bdev */ - return -EBUSY; - } - - vbdev->state.doing_finish = true; - - if (vbdev->state.started) { - spdk_io_device_unregister(vbdev, unregister_cb); - /* Return 1 because unregister is delayed */ - return 1; - } - - if (vbdev->state.doing_clean_delete) { - _vbdev_ocf_destruct_clean(vbdev); - } else { - _vbdev_ocf_destruct_dirty(vbdev); - } - - return 0; -} - -/* Stop OCF cache and unregister SPDK bdev */ -int -vbdev_ocf_delete(struct vbdev_ocf *vbdev, void (*cb)(void *, int), void *cb_arg) -{ - int rc = 0; - - if (vbdev->state.started) { - spdk_bdev_unregister(&vbdev->exp_bdev, cb, cb_arg); - } else { - rc = vbdev_ocf_destruct(vbdev); - if (rc == 0 && cb) { - cb(cb_arg, 0); - } - } - - return rc; -} - -/* Remove cores permanently and then stop OCF cache and unregister SPDK bdev */ -int -vbdev_ocf_delete_clean(struct vbdev_ocf *vbdev, void (*cb)(void *, int), - void *cb_arg) -{ - vbdev->state.doing_clean_delete = true; - - return vbdev_ocf_delete(vbdev, cb, cb_arg); -} - - -/* If vbdev is online, return its object */ -struct vbdev_ocf * -vbdev_ocf_get_by_name(const char *name) -{ - struct vbdev_ocf *vbdev; - - if (name == NULL) { - assert(false); - return NULL; - } - - TAILQ_FOREACH(vbdev, &g_ocf_vbdev_head, tailq) { - if (vbdev->name == NULL || vbdev->state.doing_finish) { - continue; - } - if (strcmp(vbdev->name, name) == 0) { - return vbdev; - } - } - return NULL; -} - -/* Return matching base if parent vbdev is online */ -struct vbdev_ocf_base * -vbdev_ocf_get_base_by_name(const char *name) -{ - struct vbdev_ocf *vbdev; - - if (name == NULL) { - assert(false); - return NULL; - } - - TAILQ_FOREACH(vbdev, &g_ocf_vbdev_head, tailq) { - if (vbdev->state.doing_finish) { - continue; - } - - if (vbdev->cache.name && strcmp(vbdev->cache.name, name) == 0) { - return &vbdev->cache; - } - if (vbdev->core.name && strcmp(vbdev->core.name, name) == 0) { - return &vbdev->core; - } - } - return NULL; -} - -/* Execute fn for each OCF device that is online or waits for base devices */ -void -vbdev_ocf_foreach(vbdev_ocf_foreach_fn fn, void *ctx) -{ - struct vbdev_ocf *vbdev; - - assert(fn != NULL); - - TAILQ_FOREACH(vbdev, &g_ocf_vbdev_head, tailq) { - if (!vbdev->state.doing_finish) { - fn(vbdev, ctx); - } - } -} - -/* Called from OCF when SPDK_IO is completed */ -static void -vbdev_ocf_io_submit_cb(ocf_io_t io, void *priv1, void *priv2, int error) -{ - struct spdk_bdev_io *bdev_io = priv1; - - if (error == 0) { - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); - } else if (error == -OCF_ERR_NO_MEM) { - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); - } else { - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); - } - - ocf_io_put(io); -} - -/* Configure io parameters and send it to OCF */ -static int -io_submit_to_ocf(struct spdk_bdev_io *bdev_io, ocf_io_t io) -{ - switch (bdev_io->type) { - case SPDK_BDEV_IO_TYPE_WRITE: - case SPDK_BDEV_IO_TYPE_READ: - ocf_core_submit_io(io); - return 0; - case SPDK_BDEV_IO_TYPE_FLUSH: - ocf_core_submit_flush(io); - return 0; - case SPDK_BDEV_IO_TYPE_UNMAP: - ocf_core_submit_discard(io); - return 0; - case SPDK_BDEV_IO_TYPE_RESET: - case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: - default: - SPDK_ERRLOG("Unsupported IO type: %d\n", bdev_io->type); - return -EINVAL; - } -} - -/* Submit SPDK-IO to OCF */ -static void -io_handle(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) -{ - struct vbdev_ocf *vbdev = bdev_io->bdev->ctxt; - ocf_io_t io = NULL; - struct bdev_ocf_data *data = NULL; - struct vbdev_ocf_qctx *qctx = spdk_io_channel_get_ctx(ch); - uint64_t len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen; - uint64_t offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen; - int dir, flags = 0; - int err; - - switch (bdev_io->type) { - case SPDK_BDEV_IO_TYPE_READ: - dir = OCF_READ; - break; - case SPDK_BDEV_IO_TYPE_WRITE: - dir = OCF_WRITE; - break; - case SPDK_BDEV_IO_TYPE_FLUSH: - dir = OCF_WRITE; - break; - case SPDK_BDEV_IO_TYPE_UNMAP: - dir = OCF_WRITE; - break; - default: - err = -EINVAL; - goto fail; - } - - if (bdev_io->type == SPDK_BDEV_IO_TYPE_FLUSH) { - flags = OCF_WRITE_FLUSH; - } - - io = ocf_volume_new_io(ocf_core_get_front_volume(vbdev->ocf_core), qctx->queue, offset, len, dir, 0, - flags); - if (!io) { - err = -ENOMEM; - goto fail; - } - - data = vbdev_ocf_data_from_spdk_io(bdev_io); - if (!data) { - err = -ENOMEM; - goto fail; - } - - err = ocf_io_set_data(io, data, 0); - if (err) { - goto fail; - } - - ocf_io_set_cmpl(io, bdev_io, NULL, vbdev_ocf_io_submit_cb); - - err = io_submit_to_ocf(bdev_io, io); - if (err) { - goto fail; - } - - return; - -fail: - if (io) { - ocf_io_put(io); - } - - if (err == -ENOMEM) { - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); - } else { - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); - } -} - -static void -vbdev_ocf_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, - bool success) -{ - if (!success) { - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); - return; - } - - io_handle(ch, bdev_io); -} - -/* Called from bdev layer when an io to Cache vbdev is submitted */ -static void -vbdev_ocf_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) -{ - switch (bdev_io->type) { - case SPDK_BDEV_IO_TYPE_READ: - /* User does not have to allocate io vectors for the request, - * so in case they are not allocated, we allocate them here */ - spdk_bdev_io_get_buf(bdev_io, vbdev_ocf_get_buf_cb, - bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); - break; - case SPDK_BDEV_IO_TYPE_WRITE: - case SPDK_BDEV_IO_TYPE_FLUSH: - case SPDK_BDEV_IO_TYPE_UNMAP: - io_handle(ch, bdev_io); - break; - case SPDK_BDEV_IO_TYPE_RESET: - case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: - default: - SPDK_ERRLOG("Unknown I/O type %d\n", bdev_io->type); - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); - break; - } -} - -/* Called from bdev layer */ -static bool -vbdev_ocf_io_type_supported(void *opaque, enum spdk_bdev_io_type io_type) -{ - struct vbdev_ocf *vbdev = opaque; - - switch (io_type) { - case SPDK_BDEV_IO_TYPE_READ: - case SPDK_BDEV_IO_TYPE_WRITE: - case SPDK_BDEV_IO_TYPE_FLUSH: - case SPDK_BDEV_IO_TYPE_UNMAP: - return spdk_bdev_io_type_supported(vbdev->core.bdev, io_type); - case SPDK_BDEV_IO_TYPE_RESET: - case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: - default: - return false; - } -} - -/* Called from bdev layer */ -static struct spdk_io_channel * -vbdev_ocf_get_io_channel(void *opaque) -{ - struct vbdev_ocf *bdev = opaque; - - return spdk_get_io_channel(bdev); -} - -static int -vbdev_ocf_dump_info_json(void *opaque, struct spdk_json_write_ctx *w) -{ - struct vbdev_ocf *vbdev = opaque; - - spdk_json_write_named_string(w, "cache_device", vbdev->cache.name); - spdk_json_write_named_string(w, "core_device", vbdev->core.name); - - spdk_json_write_named_string(w, "mode", - ocf_get_cache_modename(ocf_cache_get_mode(vbdev->ocf_cache))); - spdk_json_write_named_uint32(w, "cache_line_size", - ocf_get_cache_line_size(vbdev->ocf_cache)); - spdk_json_write_named_bool(w, "metadata_volatile", - vbdev->cfg.cache.metadata_volatile); - - return 0; -} - -static void -vbdev_ocf_write_json_config(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) -{ - struct vbdev_ocf *vbdev = bdev->ctxt; - - spdk_json_write_object_begin(w); - - spdk_json_write_named_string(w, "method", "bdev_ocf_create"); - - spdk_json_write_named_object_begin(w, "params"); - spdk_json_write_named_string(w, "name", vbdev->name); - spdk_json_write_named_string(w, "mode", - ocf_get_cache_modename(ocf_cache_get_mode(vbdev->ocf_cache))); - spdk_json_write_named_uint32(w, "cache_line_size", - ocf_get_cache_line_size(vbdev->ocf_cache)); - spdk_json_write_named_string(w, "cache_bdev_name", vbdev->cache.name); - spdk_json_write_named_string(w, "core_bdev_name", vbdev->core.name); - spdk_json_write_object_end(w); - - spdk_json_write_object_end(w); -} - -/* Cache vbdev function table - * Used by bdev layer */ -static struct spdk_bdev_fn_table cache_dev_fn_table = { - .destruct = vbdev_ocf_destruct, - .io_type_supported = vbdev_ocf_io_type_supported, - .submit_request = vbdev_ocf_submit_request, - .get_io_channel = vbdev_ocf_get_io_channel, - .write_config_json = vbdev_ocf_write_json_config, - .dump_info_json = vbdev_ocf_dump_info_json, -}; - -/* Poller function for the OCF queue - * We execute OCF requests here synchronously */ -static int -queue_poll(void *opaque) -{ - struct vbdev_ocf_qctx *qctx = opaque; - uint32_t iono = ocf_queue_pending_io(qctx->queue); - int i, max = spdk_min(32, iono); - - for (i = 0; i < max; i++) { - ocf_queue_run_single(qctx->queue); - } - - if (iono > 0) { - return SPDK_POLLER_BUSY; - } else { - return SPDK_POLLER_IDLE; - } -} - -/* Called during ocf_submit_io, ocf_purge* - * and any other requests that need to submit io */ -static void -vbdev_ocf_ctx_queue_kick(ocf_queue_t q) -{ -} - -/* OCF queue deinitialization - * Called at ocf_cache_stop */ -static void -vbdev_ocf_ctx_queue_stop(ocf_queue_t q) -{ - struct vbdev_ocf_qctx *qctx = ocf_queue_get_priv(q); - - if (qctx) { - spdk_put_io_channel(qctx->cache_ch); - spdk_put_io_channel(qctx->core_ch); - spdk_poller_unregister(&qctx->poller); - if (qctx->allocated) { - free(qctx); - } - } -} - -/* Queue ops is an interface for running queue thread - * stop() operation in called just before queue gets destroyed */ -const struct ocf_queue_ops queue_ops = { - .kick_sync = vbdev_ocf_ctx_queue_kick, - .kick = vbdev_ocf_ctx_queue_kick, - .stop = vbdev_ocf_ctx_queue_stop, -}; - -/* Called on cache vbdev creation at every thread - * We allocate OCF queues here and SPDK poller for it */ -static int -io_device_create_cb(void *io_device, void *ctx_buf) -{ - struct vbdev_ocf *vbdev = io_device; - struct vbdev_ocf_qctx *qctx = ctx_buf; - int rc; - - rc = vbdev_ocf_queue_create(vbdev->ocf_cache, &qctx->queue, &queue_ops); - if (rc) { - return rc; - } - - ocf_queue_set_priv(qctx->queue, qctx); - - qctx->vbdev = vbdev; - qctx->cache_ch = spdk_bdev_get_io_channel(vbdev->cache.desc); - qctx->core_ch = spdk_bdev_get_io_channel(vbdev->core.desc); - qctx->poller = SPDK_POLLER_REGISTER(queue_poll, qctx, 0); - - return rc; -} - -/* Called per thread - * Put OCF queue and relaunch poller with new context to finish pending requests */ -static void -io_device_destroy_cb(void *io_device, void *ctx_buf) -{ - /* Making a copy of context to use it after io channel will be destroyed */ - struct vbdev_ocf_qctx *copy = malloc(sizeof(*copy)); - struct vbdev_ocf_qctx *qctx = ctx_buf; - - if (copy) { - ocf_queue_set_priv(qctx->queue, copy); - memcpy(copy, qctx, sizeof(*copy)); - spdk_poller_unregister(&qctx->poller); - copy->poller = SPDK_POLLER_REGISTER(queue_poll, copy, 0); - copy->allocated = true; - } else { - SPDK_ERRLOG("Unable to stop OCF queue properly: %s\n", - spdk_strerror(ENOMEM)); - } - - vbdev_ocf_queue_put(qctx->queue); -} - -/* OCF management queue deinitialization */ -static void -vbdev_ocf_ctx_mngt_queue_stop(ocf_queue_t q) -{ - struct spdk_poller *poller = ocf_queue_get_priv(q); - - if (poller) { - spdk_poller_unregister(&poller); - } -} - -static int -mngt_queue_poll(void *opaque) -{ - ocf_queue_t q = opaque; - uint32_t iono = ocf_queue_pending_io(q); - int i, max = spdk_min(32, iono); - - for (i = 0; i < max; i++) { - ocf_queue_run_single(q); - } - - if (iono > 0) { - return SPDK_POLLER_BUSY; - } else { - return SPDK_POLLER_IDLE; - } -} - -static void -vbdev_ocf_ctx_mngt_queue_kick(ocf_queue_t q) -{ -} - -/* Queue ops is an interface for running queue thread - * stop() operation in called just before queue gets destroyed */ -const struct ocf_queue_ops mngt_queue_ops = { - .kick_sync = NULL, - .kick = vbdev_ocf_ctx_mngt_queue_kick, - .stop = vbdev_ocf_ctx_mngt_queue_stop, -}; - -static void -vbdev_ocf_mngt_exit(struct vbdev_ocf *vbdev, vbdev_ocf_mngt_fn *rollback_path, int rc) -{ - vbdev->state.starting = false; - vbdev_ocf_mngt_stop(vbdev, rollback_path, rc); -} - -/* Create exported spdk object */ -static void -finish_register(struct vbdev_ocf *vbdev) -{ - struct spdk_uuid ns_uuid; - int result; - - /* Copy properties of the base bdev */ - vbdev->exp_bdev.blocklen = vbdev->core.bdev->blocklen; - vbdev->exp_bdev.write_cache = vbdev->core.bdev->write_cache; - vbdev->exp_bdev.required_alignment = vbdev->core.bdev->required_alignment; - - vbdev->exp_bdev.name = vbdev->name; - vbdev->exp_bdev.product_name = "SPDK OCF"; - - vbdev->exp_bdev.blockcnt = vbdev->core.bdev->blockcnt; - vbdev->exp_bdev.ctxt = vbdev; - vbdev->exp_bdev.fn_table = &cache_dev_fn_table; - vbdev->exp_bdev.module = &ocf_if; - - /* Generate UUID based on namespace UUID + base bdev UUID. */ - spdk_uuid_parse(&ns_uuid, BDEV_OCF_NAMESPACE_UUID); - result = spdk_uuid_generate_sha1(&vbdev->exp_bdev.uuid, &ns_uuid, - (const char *)&vbdev->core.bdev->uuid, sizeof(struct spdk_uuid)); - if (result) { - SPDK_ERRLOG("Unable to generate new UUID for ocf bdev\n"); - vbdev_ocf_mngt_exit(vbdev, unregister_path_dirty, result); - return; - } - - /* Finally register vbdev in SPDK */ - spdk_io_device_register(vbdev, io_device_create_cb, io_device_destroy_cb, - sizeof(struct vbdev_ocf_qctx), vbdev->name); - result = spdk_bdev_register(&vbdev->exp_bdev); - if (result) { - SPDK_ERRLOG("Could not register exposed bdev %s\n", - vbdev->name); - vbdev_ocf_mngt_exit(vbdev, unregister_path_dirty, result); - return; - } else { - vbdev->state.started = true; - } - - vbdev_ocf_mngt_continue(vbdev, result); -} - -static void -add_core_cmpl(ocf_cache_t cache, ocf_core_t core, void *priv, int error) -{ - struct vbdev_ocf *vbdev = priv; - - ocf_mngt_cache_unlock(cache); - - if (error) { - SPDK_ERRLOG("Error %d, failed to add core device to cache instance %s," - "starting rollback\n", error, vbdev->name); - vbdev_ocf_mngt_exit(vbdev, unregister_path_dirty, error); - return; - } else { - vbdev->ocf_core = core; - } - - vbdev_ocf_mngt_continue(vbdev, error); -} - -/* Try to lock cache, then add core */ -static void -add_core_cache_lock_cmpl(ocf_cache_t cache, void *priv, int error) -{ - struct vbdev_ocf *vbdev = (struct vbdev_ocf *)priv; - - if (error) { - SPDK_ERRLOG("Error %d, can not lock cache instance %s," - "starting rollback\n", error, vbdev->name); - vbdev_ocf_mngt_exit(vbdev, unregister_path_dirty, error); - } - ocf_mngt_cache_add_core(vbdev->ocf_cache, &vbdev->cfg.core, add_core_cmpl, vbdev); -} - -/* Add core for existing OCF cache instance */ -static void -add_core(struct vbdev_ocf *vbdev) -{ - ocf_mngt_cache_lock(vbdev->ocf_cache, add_core_cache_lock_cmpl, vbdev); -} - -static void -start_cache_cmpl(ocf_cache_t cache, void *priv, int error) -{ - struct vbdev_ocf *vbdev = priv; - uint64_t volume_size; - uint64_t mem_needed; - - ocf_mngt_cache_unlock(cache); - - if (error) { - SPDK_ERRLOG("Error %d during start cache %s, starting rollback\n", - error, vbdev->name); - - if (error == -OCF_ERR_NO_MEM) { - volume_size = vbdev->cache.bdev->blockcnt * vbdev->cache.bdev->blocklen; - mem_needed = ocf_mngt_get_ram_needed(cache, volume_size); - - SPDK_NOTICELOG("Try to increase hugepage memory size or cache line size. " - "For your configuration:\nDevice size: %"PRIu64" bytes\n" - "Cache line size: %"PRIu64" bytes\nFree memory needed to start " - "cache: %"PRIu64" bytes\n", - volume_size, vbdev->cfg.cache.cache_line_size, mem_needed); - } - - vbdev_ocf_mngt_exit(vbdev, unregister_path_dirty, error); - return; - } - - vbdev_ocf_mngt_continue(vbdev, error); -} - -static int -create_management_queue(struct vbdev_ocf *vbdev) -{ - struct spdk_poller *mngt_poller; - int rc; - - rc = vbdev_ocf_queue_create_mngt(vbdev->ocf_cache, - &vbdev->cache_ctx->mngt_queue, &mngt_queue_ops); - if (rc) { - SPDK_ERRLOG("Unable to create mngt_queue: %d\n", rc); - return rc; - } - - mngt_poller = SPDK_POLLER_REGISTER(mngt_queue_poll, vbdev->cache_ctx->mngt_queue, 100); - if (mngt_poller == NULL) { - SPDK_ERRLOG("Unable to initiate mngt request: %s", spdk_strerror(ENOMEM)); - return -ENOMEM; - } - - ocf_queue_set_priv(vbdev->cache_ctx->mngt_queue, mngt_poller); - - return 0; -} - -/* Start OCF cache, attach caching device */ -static void -start_cache(struct vbdev_ocf *vbdev) -{ - ocf_cache_t existing; - uint32_t cache_block_size = vbdev->cache.bdev->blocklen; - uint32_t core_block_size = vbdev->core.bdev->blocklen; - int rc; - - if (is_ocf_cache_running(vbdev)) { - vbdev_ocf_mngt_stop(vbdev, NULL, -EALREADY); - return; - } - - if (cache_block_size > core_block_size) { - SPDK_ERRLOG("Cache bdev block size (%d) is bigger then core bdev block size (%d)\n", - cache_block_size, core_block_size); - vbdev_ocf_mngt_exit(vbdev, unregister_path_dirty, -EINVAL); - return; - } - - existing = get_other_cache_instance(vbdev); - if (existing) { - SPDK_NOTICELOG("OCF bdev %s connects to existing cache device %s\n", - vbdev->name, vbdev->cache.name); - vbdev->ocf_cache = existing; - ocf_mngt_cache_get(vbdev->ocf_cache); - vbdev->cache_ctx = ocf_cache_get_priv(existing); - vbdev_ocf_cache_ctx_get(vbdev->cache_ctx); - vbdev_ocf_mngt_continue(vbdev, 0); - return; - } - - vbdev->cache_ctx = calloc(1, sizeof(struct vbdev_ocf_cache_ctx)); - if (vbdev->cache_ctx == NULL) { - vbdev_ocf_mngt_exit(vbdev, unregister_path_dirty, -ENOMEM); - return; - } - - vbdev_ocf_cache_ctx_get(vbdev->cache_ctx); - - rc = ocf_mngt_cache_start(vbdev_ocf_ctx, &vbdev->ocf_cache, &vbdev->cfg.cache, NULL); - if (rc) { - SPDK_ERRLOG("Could not start cache %s: %d\n", vbdev->name, rc); - vbdev_ocf_mngt_exit(vbdev, unregister_path_dirty, rc); - return; - } - ocf_mngt_cache_get(vbdev->ocf_cache); - - ocf_cache_set_priv(vbdev->ocf_cache, vbdev->cache_ctx); - - rc = create_management_queue(vbdev); - if (rc) { - SPDK_ERRLOG("Unable to create mngt_queue: %d\n", rc); - vbdev_ocf_mngt_exit(vbdev, unregister_path_dirty, rc); - return; - } - - if (vbdev->cfg.loadq) { - ocf_mngt_cache_load(vbdev->ocf_cache, &vbdev->cfg.attach, start_cache_cmpl, vbdev); - } else { - ocf_mngt_cache_attach(vbdev->ocf_cache, &vbdev->cfg.attach, start_cache_cmpl, vbdev); - } -} - -/* Procedures called during register operation */ -vbdev_ocf_mngt_fn register_path[] = { - start_cache, - add_core, - finish_register, - NULL -}; - -/* Start cache instance and register OCF bdev */ -static void -register_vbdev(struct vbdev_ocf *vbdev, vbdev_ocf_mngt_callback cb, void *cb_arg) -{ - int rc; - - if (!(vbdev->core.attached && vbdev->cache.attached) || vbdev->state.started) { - cb(-EPERM, vbdev, cb_arg); - return; - } - - vbdev->state.starting = true; - rc = vbdev_ocf_mngt_start(vbdev, register_path, cb, cb_arg); - if (rc) { - cb(rc, vbdev, cb_arg); - } -} - -/* Init OCF configuration options - * for core and cache devices */ -static int -init_vbdev_config(struct vbdev_ocf *vbdev) -{ - struct vbdev_ocf_config *cfg = &vbdev->cfg; - struct ocf_volume_uuid uuid; - ocf_volume_type_t type; - int ret; - - - /* Initialize OCF defaults first */ - ocf_mngt_cache_attach_config_set_default(&cfg->attach); - ocf_mngt_cache_config_set_default(&cfg->cache); - ocf_mngt_core_config_set_default(&cfg->core); - - ret = snprintf(cfg->cache.name, sizeof(cfg->cache.name), "%s", vbdev->name); - if (ret < 0 || (size_t) ret >= sizeof(cfg->cache.name)) { - return -EINVAL; - } - ret = snprintf(cfg->core.name, sizeof(cfg->core.name), "%s", vbdev->core.name); - if (ret < 0 || (size_t) ret >= sizeof(cfg->core.name)) { - return -EINVAL; - } - - cfg->attach.open_cores = false; - cfg->attach.device.perform_test = false; - cfg->attach.discard_on_start = false; - - vbdev->cfg.cache.locked = true; - - cfg->core.volume_type = SPDK_OBJECT; - - if (vbdev->cfg.loadq) { - /* When doing cache_load(), we need to set try_add to true, - * otherwise OCF will interpret this core as new - * instead of the inactive one */ - vbdev->cfg.core.try_add = true; - } else { - /* When cache is initialized as new, set force flag to true, - * to ignore warnings about existing metadata */ - cfg->attach.force = true; - } - - /* Serialize bdev names in OCF UUID to interpret on future loads - * Core UUID is a triple of (core name, vbdev name, cache name) - * Cache UUID is cache bdev name */ - type = ocf_ctx_get_volume_type(vbdev_ocf_ctx, SPDK_OBJECT); - if (!type) { - SPDK_ERRLOG("Fail to get volume type\n"); - return -EINVAL; - } - uuid.size = strlen(vbdev->cache.name) + 1; - uuid.data = vbdev->cache.name; - ret = ocf_volume_create(&cfg->attach.device.volume, type, &uuid); - if (ret) { - SPDK_ERRLOG("Fail to create volume\n"); - return -EINVAL; - } - - snprintf(vbdev->uuid, VBDEV_OCF_MD_MAX_LEN, "%s %s %s", - vbdev->core.name, vbdev->name, vbdev->cache.name); - cfg->core.uuid.size = strlen(vbdev->uuid) + 1; - cfg->core.uuid.data = vbdev->uuid; - vbdev->uuid[strlen(vbdev->core.name)] = 0; - vbdev->uuid[strlen(vbdev->core.name) + 1 + strlen(vbdev->name)] = 0; - - return 0; -} - -/* Allocate vbdev structure object and add it to the global list */ -static int -init_vbdev(const char *vbdev_name, - const char *cache_mode_name, - const uint64_t cache_line_size, - const char *cache_name, - const char *core_name, - bool loadq) -{ - struct vbdev_ocf *vbdev; - int rc = 0; - - if (spdk_bdev_get_by_name(vbdev_name) || vbdev_ocf_get_by_name(vbdev_name)) { - SPDK_ERRLOG("Device with name '%s' already exists\n", vbdev_name); - return -EPERM; - } - - vbdev = calloc(1, sizeof(*vbdev)); - if (!vbdev) { - goto error_mem; - } - - vbdev->name = strdup(vbdev_name); - if (!vbdev->name) { - goto error_mem; - } - - vbdev->cache.name = strdup(cache_name); - if (!vbdev->cache.name) { - goto error_mem; - } - - vbdev->core.name = strdup(core_name); - if (!vbdev->core.name) { - goto error_mem; - } - - vbdev->cache.parent = vbdev; - vbdev->core.parent = vbdev; - vbdev->cache.is_cache = true; - vbdev->core.is_cache = false; - vbdev->cfg.loadq = loadq; - - rc = init_vbdev_config(vbdev); - if (rc) { - SPDK_ERRLOG("Fail to init vbdev config\n"); - goto error_free; - } - - - if (cache_mode_name) { - vbdev->cfg.cache.cache_mode - = ocf_get_cache_mode(cache_mode_name); - } else if (!loadq) { /* In load path it is OK to pass NULL as cache mode */ - SPDK_ERRLOG("No cache mode specified\n"); - rc = -EINVAL; - goto error_free; - } - if (vbdev->cfg.cache.cache_mode < 0) { - SPDK_ERRLOG("Incorrect cache mode '%s'\n", cache_mode_name); - rc = -EINVAL; - goto error_free; - } - - ocf_cache_line_size_t set_cache_line_size = cache_line_size ? - (ocf_cache_line_size_t)cache_line_size * KiB : - ocf_cache_line_size_default; - if (set_cache_line_size == 0) { - SPDK_ERRLOG("Cache line size should be non-zero.\n"); - rc = -EINVAL; - goto error_free; - } - vbdev->cfg.attach.cache_line_size = set_cache_line_size; - vbdev->cfg.cache.cache_line_size = set_cache_line_size; - - TAILQ_INSERT_TAIL(&g_ocf_vbdev_head, vbdev, tailq); - return rc; - -error_mem: - rc = -ENOMEM; -error_free: - free_vbdev(vbdev); - return rc; -} - -/* Read configuration file at the start of SPDK application - * This adds vbdevs to global list if some mentioned in config */ -static int -vbdev_ocf_init(void) -{ - int status; - - status = vbdev_ocf_ctx_init(); - if (status) { - SPDK_ERRLOG("OCF ctx initialization failed with=%d\n", status); - return status; - } - - status = vbdev_ocf_volume_init(); - if (status) { - vbdev_ocf_ctx_cleanup(); - SPDK_ERRLOG("OCF volume initialization failed with=%d\n", status); - return status; - } - - return status; -} - -/* Called after application shutdown started - * Release memory of allocated structures here */ -static void -vbdev_ocf_module_fini(void) -{ - struct vbdev_ocf *vbdev; - - while ((vbdev = TAILQ_FIRST(&g_ocf_vbdev_head))) { - TAILQ_REMOVE(&g_ocf_vbdev_head, vbdev, tailq); - free_vbdev(vbdev); - } - - vbdev_ocf_volume_cleanup(); - vbdev_ocf_ctx_cleanup(); -} - -/* When base device gets unplugged this is called - * We will unregister cache vbdev here - * When cache device is removed, we delete every OCF bdev that used it */ -static void -hotremove_cb(struct vbdev_ocf_base *base) -{ - struct vbdev_ocf *vbdev; - - if (!base->is_cache) { - if (base->parent->state.doing_finish) { - return; - } - - SPDK_NOTICELOG("Deinitializing '%s' because its core device '%s' was removed\n", - base->parent->name, base->name); - vbdev_ocf_delete(base->parent, NULL, NULL); - return; - } - - TAILQ_FOREACH(vbdev, &g_ocf_vbdev_head, tailq) { - if (vbdev->state.doing_finish) { - continue; - } - if (strcmp(base->name, vbdev->cache.name) == 0) { - SPDK_NOTICELOG("Deinitializing '%s' because" - " its cache device '%s' was removed\n", - vbdev->name, base->name); - vbdev_ocf_delete(vbdev, NULL, NULL); - } - } -} - -static void -base_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, - void *event_ctx) -{ - switch (type) { - case SPDK_BDEV_EVENT_REMOVE: - if (event_ctx) { - hotremove_cb(event_ctx); - } - break; - default: - SPDK_NOTICELOG("Unsupported bdev event: type %d\n", type); - break; - } -} - -/* Open base SPDK bdev and claim it */ -static int -attach_base(struct vbdev_ocf_base *base) -{ - int status; - - if (base->attached) { - return -EALREADY; - } - - /* If base cache bdev was already opened by other vbdev, - * we just copy its descriptor here */ - if (base->is_cache) { - struct vbdev_ocf_base *existing = get_other_cache_base(base); - if (existing) { - base->desc = existing->desc; - base->management_channel = existing->management_channel; - base->attached = true; - return 0; - } - } - - status = spdk_bdev_open_ext(base->name, true, base_bdev_event_cb, base, &base->desc); - if (status) { - SPDK_ERRLOG("Unable to open device '%s' for writing\n", base->name); - return status; - } - - status = spdk_bdev_module_claim_bdev(base->bdev, base->desc, - &ocf_if); - if (status) { - SPDK_ERRLOG("Unable to claim device '%s'\n", base->name); - spdk_bdev_close(base->desc); - return status; - } - - base->management_channel = spdk_bdev_get_io_channel(base->desc); - if (!base->management_channel) { - SPDK_ERRLOG("Unable to get io channel '%s'\n", base->name); - spdk_bdev_module_release_bdev(base->bdev); - spdk_bdev_close(base->desc); - return -ENOMEM; - } - - /* Save the thread where the base device is opened */ - base->thread = spdk_get_thread(); - - base->attached = true; - return status; -} - -/* Attach base bdevs */ -static int -attach_base_bdevs(struct vbdev_ocf *vbdev, - struct spdk_bdev *cache_bdev, - struct spdk_bdev *core_bdev) -{ - int rc = 0; - - if (cache_bdev) { - vbdev->cache.bdev = cache_bdev; - rc |= attach_base(&vbdev->cache); - } - - if (core_bdev) { - vbdev->core.bdev = core_bdev; - rc |= attach_base(&vbdev->core); - } - - return rc; -} - -/* Init and then start vbdev if all base devices are present */ -void -vbdev_ocf_construct(const char *vbdev_name, - const char *cache_mode_name, - const uint64_t cache_line_size, - const char *cache_name, - const char *core_name, - bool loadq, - void (*cb)(int, struct vbdev_ocf *, void *), - void *cb_arg) -{ - int rc; - struct spdk_bdev *cache_bdev = spdk_bdev_get_by_name(cache_name); - struct spdk_bdev *core_bdev = spdk_bdev_get_by_name(core_name); - struct vbdev_ocf *vbdev; - - rc = init_vbdev(vbdev_name, cache_mode_name, cache_line_size, cache_name, core_name, loadq); - if (rc) { - cb(rc, NULL, cb_arg); - return; - } - - vbdev = vbdev_ocf_get_by_name(vbdev_name); - if (vbdev == NULL) { - cb(-ENODEV, NULL, cb_arg); - return; - } - - if (cache_bdev == NULL) { - SPDK_NOTICELOG("OCF bdev '%s' is waiting for cache device '%s' to connect\n", - vbdev->name, cache_name); - } - if (core_bdev == NULL) { - SPDK_NOTICELOG("OCF bdev '%s' is waiting for core device '%s' to connect\n", - vbdev->name, core_name); - } - - rc = attach_base_bdevs(vbdev, cache_bdev, core_bdev); - if (rc) { - cb(rc, vbdev, cb_arg); - return; - } - - if (core_bdev && cache_bdev) { - register_vbdev(vbdev, cb, cb_arg); - } else { - cb(0, vbdev, cb_arg); - } -} - -/* Set new cache mode on OCF cache */ -void -vbdev_ocf_set_cache_mode(struct vbdev_ocf *vbdev, - const char *cache_mode_name, - void (*cb)(int, struct vbdev_ocf *, void *), - void *cb_arg) -{ - ocf_cache_t cache; - ocf_cache_mode_t cache_mode; - int rc; - - cache = vbdev->ocf_cache; - cache_mode = ocf_get_cache_mode(cache_mode_name); - - rc = ocf_mngt_cache_trylock(cache); - if (rc) { - cb(rc, vbdev, cb_arg); - return; - } - - rc = ocf_mngt_cache_set_mode(cache, cache_mode); - ocf_mngt_cache_unlock(cache); - cb(rc, vbdev, cb_arg); -} - -/* Set sequential cutoff parameters on OCF cache */ -void -vbdev_ocf_set_seqcutoff(struct vbdev_ocf *vbdev, const char *policy_name, uint32_t threshold, - uint32_t promotion_count, void (*cb)(int, void *), void *cb_arg) -{ - ocf_cache_t cache; - ocf_seq_cutoff_policy policy; - int rc; - - cache = vbdev->ocf_cache; - - policy = ocf_get_seqcutoff_policy(policy_name); - if (policy == ocf_seq_cutoff_policy_max) { - cb(OCF_ERR_INVAL, cb_arg); - return; - } - - rc = ocf_mngt_cache_trylock(cache); - if (rc) { - cb(rc, cb_arg); - return; - } - - rc = ocf_mngt_core_set_seq_cutoff_policy_all(cache, policy); - if (rc) { - goto end; - } - - if (threshold) { - threshold = threshold * KiB; - - rc = ocf_mngt_core_set_seq_cutoff_threshold_all(cache, threshold); - if (rc) { - goto end; - } - } - - if (promotion_count) { - rc = ocf_mngt_core_set_seq_cutoff_promotion_count_all(cache, promotion_count); - } - -end: - ocf_mngt_cache_unlock(cache); - cb(rc, cb_arg); -} - -/* This called if new device is created in SPDK application - * If that device named as one of base bdevs of OCF vbdev, - * claim and open them */ -static void -vbdev_ocf_examine(struct spdk_bdev *bdev) -{ - const char *bdev_name = spdk_bdev_get_name(bdev); - struct vbdev_ocf *vbdev; - - TAILQ_FOREACH(vbdev, &g_ocf_vbdev_head, tailq) { - if (vbdev->state.doing_finish) { - continue; - } - - if (!strcmp(bdev_name, vbdev->cache.name)) { - attach_base_bdevs(vbdev, bdev, NULL); - continue; - } - if (!strcmp(bdev_name, vbdev->core.name)) { - attach_base_bdevs(vbdev, NULL, bdev); - break; - } - } - spdk_bdev_module_examine_done(&ocf_if); -} - -struct metadata_probe_ctx { - struct vbdev_ocf_base base; - ocf_volume_t volume; - - struct ocf_volume_uuid *core_uuids; - unsigned int uuid_count; - - int result; - int refcnt; -}; - -static void -_examine_ctx_put(void *ctx) -{ - struct spdk_bdev_desc *desc = ctx; - - spdk_bdev_close(desc); -} - -static void -examine_ctx_put(struct metadata_probe_ctx *ctx) -{ - unsigned int i; - - ctx->refcnt--; - if (ctx->refcnt > 0) { - return; - } - - if (ctx->result) { - SPDK_ERRLOG("OCF metadata probe for bdev '%s' failed with %d\n", - spdk_bdev_get_name(ctx->base.bdev), ctx->result); - } - - if (ctx->base.desc) { - /* Close the underlying bdev on its same opened thread. */ - if (ctx->base.thread && ctx->base.thread != spdk_get_thread()) { - spdk_thread_send_msg(ctx->base.thread, _examine_ctx_put, ctx->base.desc); - } else { - spdk_bdev_close(ctx->base.desc); - } - } - - if (ctx->volume) { - ocf_volume_destroy(ctx->volume); - } - - if (ctx->core_uuids) { - for (i = 0; i < ctx->uuid_count; i++) { - free(ctx->core_uuids[i].data); - } - } - free(ctx->core_uuids); - - examine_done(ctx->result, NULL, ctx->base.bdev); - free(ctx); -} - -static void -metadata_probe_cb(void *priv, int rc, - struct ocf_metadata_probe_status *status) -{ - struct metadata_probe_ctx *ctx = priv; - - if (rc) { - /* -ENODATA means device does not have cache metadata on it */ - if (rc != -OCF_ERR_NO_METADATA) { - ctx->result = rc; - } - } - - examine_ctx_put(ctx); -} - -/* This is called after vbdev_ocf_examine - * It allows to delay application initialization - * until all OCF bdevs get registered - * If vbdev has all of its base devices it starts asynchronously here - * We first check if bdev appears in configuration, - * if not we do metadata_probe() to create its configuration from bdev metadata */ -static void -vbdev_ocf_examine_disk(struct spdk_bdev *bdev) -{ - const char *bdev_name = spdk_bdev_get_name(bdev); - struct vbdev_ocf *vbdev; - struct metadata_probe_ctx *ctx; - bool created_from_config = false; - int rc; - - examine_start(bdev); - - TAILQ_FOREACH(vbdev, &g_ocf_vbdev_head, tailq) { - if (vbdev->state.doing_finish || vbdev->state.started) { - continue; - } - - if (!strcmp(bdev_name, vbdev->cache.name)) { - examine_start(bdev); - register_vbdev(vbdev, examine_done, bdev); - created_from_config = true; - continue; - } - if (!strcmp(bdev_name, vbdev->core.name)) { - examine_start(bdev); - register_vbdev(vbdev, examine_done, bdev); - examine_done(0, NULL, bdev); - return; - } - } - - /* If devices is discovered during config we do not check for metadata */ - if (created_from_config) { - examine_done(0, NULL, bdev); - return; - } - - /* Metadata probe path - * We create temporary OCF volume and a temporary base structure - * to use them for ocf_metadata_probe() and for bottom adapter IOs - * Then we get UUIDs of core devices an create configurations based on them */ - ctx = calloc(1, sizeof(*ctx)); - if (!ctx) { - examine_done(-ENOMEM, NULL, bdev); - return; - } - - ctx->base.bdev = bdev; - ctx->refcnt = 1; - - rc = spdk_bdev_open_ext(bdev_name, true, base_bdev_event_cb, NULL, &ctx->base.desc); - if (rc) { - ctx->result = rc; - examine_ctx_put(ctx); - return; - } - - rc = ocf_ctx_volume_create(vbdev_ocf_ctx, &ctx->volume, NULL, SPDK_OBJECT); - if (rc) { - ctx->result = rc; - examine_ctx_put(ctx); - return; - } - - rc = ocf_volume_open(ctx->volume, &ctx->base); - if (rc) { - ctx->result = rc; - examine_ctx_put(ctx); - return; - } - - /* Save the thread where the base device is opened */ - ctx->base.thread = spdk_get_thread(); - - ocf_metadata_probe(vbdev_ocf_ctx, ctx->volume, metadata_probe_cb, ctx); -} - -static int -vbdev_ocf_get_ctx_size(void) -{ - return sizeof(struct bdev_ocf_data); -} - -static void -fini_start(void) -{ - g_fini_started = true; -} - -/* Module-global function table - * Does not relate to vbdev instances */ -static struct spdk_bdev_module ocf_if = { - .name = "ocf", - .module_init = vbdev_ocf_init, - .fini_start = fini_start, - .module_fini = vbdev_ocf_module_fini, - .get_ctx_size = vbdev_ocf_get_ctx_size, - .examine_config = vbdev_ocf_examine, - .examine_disk = vbdev_ocf_examine_disk, -}; -SPDK_BDEV_MODULE_REGISTER(ocf, &ocf_if); diff --git a/module/bdev/ocf/vbdev_ocf.h b/module/bdev/ocf/vbdev_ocf.h deleted file mode 100644 index b901b3b8753..00000000000 --- a/module/bdev/ocf/vbdev_ocf.h +++ /dev/null @@ -1,206 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (C) 2018 Intel Corporation. - * All rights reserved. - */ - -#ifndef SPDK_VBDEV_OCF_H -#define SPDK_VBDEV_OCF_H - -#include - -#include "spdk/bdev.h" -#include "spdk/bdev_module.h" - -#define VBDEV_OCF_MD_MAX_LEN 4096 - -struct vbdev_ocf; - -/* Context for OCF queue poller - * Used for mapping SPDK threads to OCF queues */ -struct vbdev_ocf_qctx { - /* OCF queue. Contains OCF requests */ - struct ocf_queue *queue; - /* Poller for OCF queue. Runs OCF requests */ - struct spdk_poller *poller; - /* Reference to parent vbdev */ - struct vbdev_ocf *vbdev; - /* Base devices channels */ - struct spdk_io_channel *cache_ch; - struct spdk_io_channel *core_ch; - /* If true, we have to free this context on queue stop */ - bool allocated; - /* Link to per-bdev list of queue contexts */ - TAILQ_ENTRY(vbdev_ocf_qctx) tailq; -}; - -/* Important states */ -struct vbdev_ocf_state { - /* From the moment when clean delete started */ - bool doing_clean_delete; - /* From the moment when finish started */ - bool doing_finish; - /* From the moment when reset IO received, until it is completed */ - bool doing_reset; - /* From the moment when exp_bdev is registered */ - bool started; - /* From the moment when register path started */ - bool starting; - /* Status of last attempt for stopping this device */ - int stop_status; -}; - -/* - * OCF cache configuration options - */ -struct vbdev_ocf_config { - /* Initial cache configuration */ - struct ocf_mngt_cache_config cache; - - /* Cache device config */ - struct ocf_mngt_cache_attach_config attach; - - /* Core initial config */ - struct ocf_mngt_core_config core; - - /* Load flag, if set to true, then we will try load cache instance from disk, - * otherwise we will create new cache on that disk */ - bool loadq; -}; - -/* Types for management operations */ -typedef void (*vbdev_ocf_mngt_fn)(struct vbdev_ocf *); -typedef void (*vbdev_ocf_mngt_callback)(int, struct vbdev_ocf *, void *); - -/* Context for asynchronous management operations - * Single management operation usually contains a list of sub procedures, - * this structure handles sharing between those sub procedures */ -struct vbdev_ocf_mngt_ctx { - /* Pointer to function that is currently being executed - * It gets incremented on each step until it dereferences to NULL */ - vbdev_ocf_mngt_fn *current_step; - - /* Function that gets invoked by poller on each iteration */ - vbdev_ocf_mngt_fn poller_fn; - /* Poller timeout time stamp - when the poller should stop with error */ - uint64_t timeout_ts; - - /* Status of management operation */ - int status; - - /* External callback and its argument */ - vbdev_ocf_mngt_callback cb; - void *cb_arg; -}; - -/* Base device info */ -struct vbdev_ocf_base { - /* OCF internal name */ - char *name; - - /* True if this is a caching device */ - bool is_cache; - - /* Connected SPDK block device */ - struct spdk_bdev *bdev; - - /* SPDK device io handle */ - struct spdk_bdev_desc *desc; - - /* True if SPDK bdev has been claimed and opened for writing */ - bool attached; - - /* Channel for cleaner operations */ - struct spdk_io_channel *management_channel; - - /* Reference to main vbdev */ - struct vbdev_ocf *parent; - - /* thread where base device is opened */ - struct spdk_thread *thread; -}; - -/* - * The main information provider - * It's also registered as io_device - */ -struct vbdev_ocf { - /* Exposed unique name */ - char *name; - - /* Base bdevs */ - struct vbdev_ocf_base cache; - struct vbdev_ocf_base core; - - /* Base bdevs OCF objects */ - ocf_cache_t ocf_cache; - ocf_core_t ocf_core; - - /* Parameters */ - struct vbdev_ocf_config cfg; - struct vbdev_ocf_state state; - - /* Management context */ - struct vbdev_ocf_mngt_ctx mngt_ctx; - - /* Cache context */ - struct vbdev_ocf_cache_ctx *cache_ctx; - - /* Status of flushing operation */ - struct { - bool in_progress; - int status; - } flush; - - /* Exposed SPDK bdev. Registered in bdev layer */ - struct spdk_bdev exp_bdev; - - /* OCF uuid for core device of this vbdev */ - char uuid[VBDEV_OCF_MD_MAX_LEN]; - - /* Link to global list of this type structures */ - TAILQ_ENTRY(vbdev_ocf) tailq; -}; - -void vbdev_ocf_construct( - const char *vbdev_name, - const char *cache_mode_name, - const uint64_t cache_line_size, - const char *cache_name, - const char *core_name, - bool loadq, - void (*cb)(int, struct vbdev_ocf *, void *), - void *cb_arg); - -/* If vbdev is online, return its object */ -struct vbdev_ocf *vbdev_ocf_get_by_name(const char *name); - -/* Return matching base if parent vbdev is online */ -struct vbdev_ocf_base *vbdev_ocf_get_base_by_name(const char *name); - -/* Stop OCF cache and unregister SPDK bdev */ -int vbdev_ocf_delete(struct vbdev_ocf *vbdev, void (*cb)(void *, int), void *cb_arg); - -int vbdev_ocf_delete_clean(struct vbdev_ocf *vbdev, void (*cb)(void *, int), void *cb_arg); - -/* Set new cache mode on OCF cache */ -void vbdev_ocf_set_cache_mode( - struct vbdev_ocf *vbdev, - const char *cache_mode_name, - void (*cb)(int, struct vbdev_ocf *, void *), - void *cb_arg); - -/* Set sequential cutoff parameters on OCF cache */ -void vbdev_ocf_set_seqcutoff( - struct vbdev_ocf *vbdev, - const char *policy_name, - uint32_t threshold, - uint32_t promotion_count, - void (*cb)(int, void *), - void *cb_arg); - -typedef void (*vbdev_ocf_foreach_fn)(struct vbdev_ocf *, void *); - -/* Execute fn for each OCF device that is online or waits for base devices */ -void vbdev_ocf_foreach(vbdev_ocf_foreach_fn fn, void *ctx); - -#endif diff --git a/module/bdev/ocf/vbdev_ocf_rpc.c b/module/bdev/ocf/vbdev_ocf_rpc.c deleted file mode 100644 index 1396bb0ea47..00000000000 --- a/module/bdev/ocf/vbdev_ocf_rpc.c +++ /dev/null @@ -1,631 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (C) 2018 Intel Corporation. - * All rights reserved. - */ - -#include "vbdev_ocf.h" -#include "stats.h" -#include "utils.h" -#include "spdk/log.h" -#include "spdk/rpc.h" -#include "spdk/string.h" - -/* Common structure to hold the name parameter for RPC methods using bdev name only. */ -struct rpc_bdev_ocf_name { - char *name; /* main vbdev name */ -}; - -/* Common free function for RPC methods using bdev name only. */ -static void -free_rpc_bdev_ocf_name(struct rpc_bdev_ocf_name *r) -{ - free(r->name); -} - -/* Common function to decode the name input parameter for RPC methods using bdev name only. */ -static const struct spdk_json_object_decoder rpc_bdev_ocf_name_decoders[] = { - {"name", offsetof(struct rpc_bdev_ocf_name, name), spdk_json_decode_string}, -}; - - -/* Structure to hold the parameters for this RPC method. */ -struct rpc_bdev_ocf_create { - char *name; /* main vbdev */ - char *mode; /* OCF mode (choose one) */ - uint64_t cache_line_size; /* OCF cache line size */ - char *cache_bdev_name; /* sub bdev */ - char *core_bdev_name; /* sub bdev */ -}; - -static void -free_rpc_bdev_ocf_create(struct rpc_bdev_ocf_create *r) -{ - free(r->name); - free(r->core_bdev_name); - free(r->cache_bdev_name); - free(r->mode); -} - -/* Structure to decode the input parameters for this RPC method. */ -static const struct spdk_json_object_decoder rpc_bdev_ocf_create_decoders[] = { - {"name", offsetof(struct rpc_bdev_ocf_create, name), spdk_json_decode_string}, - {"mode", offsetof(struct rpc_bdev_ocf_create, mode), spdk_json_decode_string}, - {"cache_line_size", offsetof(struct rpc_bdev_ocf_create, cache_line_size), spdk_json_decode_uint64, true}, - {"cache_bdev_name", offsetof(struct rpc_bdev_ocf_create, cache_bdev_name), spdk_json_decode_string}, - {"core_bdev_name", offsetof(struct rpc_bdev_ocf_create, core_bdev_name), spdk_json_decode_string}, -}; - -static void -construct_cb(int status, struct vbdev_ocf *vbdev, void *cb_arg) -{ - struct spdk_jsonrpc_request *request = cb_arg; - struct spdk_json_write_ctx *w; - - if (status) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Could not create OCF vbdev: %d", - status); - } else { - w = spdk_jsonrpc_begin_result(request); - spdk_json_write_string(w, vbdev->name); - spdk_jsonrpc_end_result(request, w); - } -} - -static void -rpc_bdev_ocf_create(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) -{ - struct rpc_bdev_ocf_create req = {NULL}; - int ret; - - ret = spdk_json_decode_object(params, rpc_bdev_ocf_create_decoders, - SPDK_COUNTOF(rpc_bdev_ocf_create_decoders), - &req); - if (ret) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parameters"); - free_rpc_bdev_ocf_create(&req); - return; - } - - vbdev_ocf_construct(req.name, req.mode, req.cache_line_size, req.cache_bdev_name, - req.core_bdev_name, false, construct_cb, request); - free_rpc_bdev_ocf_create(&req); -} -SPDK_RPC_REGISTER("bdev_ocf_create", rpc_bdev_ocf_create, SPDK_RPC_RUNTIME) - -static void -delete_cb(void *cb_arg, int status) -{ - struct spdk_jsonrpc_request *request = cb_arg; - - if (status) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Could not delete OCF vbdev: %d", - status); - } else { - spdk_jsonrpc_send_bool_response(request, true); - } -} - -static void -rpc_bdev_ocf_delete(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) -{ - struct rpc_bdev_ocf_name req = {NULL}; - struct vbdev_ocf *vbdev; - int status; - - status = spdk_json_decode_object(params, rpc_bdev_ocf_name_decoders, - SPDK_COUNTOF(rpc_bdev_ocf_name_decoders), - &req); - if (status) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parameters"); - goto end; - } - - vbdev = vbdev_ocf_get_by_name(req.name); - if (vbdev == NULL) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - spdk_strerror(ENODEV)); - goto end; - } - - status = vbdev_ocf_delete_clean(vbdev, delete_cb, request); - if (status) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Could not delete OCF vbdev: %s", - spdk_strerror(-status)); - goto end; - } - -end: - free_rpc_bdev_ocf_name(&req); -} -SPDK_RPC_REGISTER("bdev_ocf_delete", rpc_bdev_ocf_delete, SPDK_RPC_RUNTIME) - -struct get_ocf_stats_ctx { - struct spdk_jsonrpc_request *request; - char *core_name; -}; - -static void -rpc_bdev_ocf_get_stats_cmpl(ocf_cache_t cache, void *priv, int error) -{ - struct get_ocf_stats_ctx *ctx = (struct get_ocf_stats_ctx *) priv; - struct spdk_json_write_ctx *w; - struct vbdev_ocf_stats stats; - - if (error) { - goto end; - } - - error = vbdev_ocf_stats_get(cache, ctx->core_name, &stats); - - ocf_mngt_cache_read_unlock(cache); - - if (error) { - goto end; - } - - w = spdk_jsonrpc_begin_result(ctx->request); - vbdev_ocf_stats_write_json(w, &stats); - spdk_jsonrpc_end_result(ctx->request, w); - -end: - if (error) { - spdk_jsonrpc_send_error_response_fmt(ctx->request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Could not get stats: %s", - spdk_strerror(-error)); - } - free(ctx); -} - -static void -rpc_bdev_ocf_get_stats(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) -{ - struct rpc_bdev_ocf_name req = {NULL}; - struct vbdev_ocf *vbdev; - struct get_ocf_stats_ctx *ctx; - - ctx = calloc(1, sizeof(*ctx)); - if (!ctx) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Not enough memory to process request"); - goto end; - } - - if (spdk_json_decode_object(params, rpc_bdev_ocf_name_decoders, - SPDK_COUNTOF(rpc_bdev_ocf_name_decoders), - &req)) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parameters"); - free(ctx); - goto end; - } - - vbdev = vbdev_ocf_get_by_name(req.name); - if (vbdev == NULL) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - spdk_strerror(ENODEV)); - free(ctx); - goto end; - } - - ctx->core_name = vbdev->core.name; - ctx->request = request; - ocf_mngt_cache_read_lock(vbdev->ocf_cache, rpc_bdev_ocf_get_stats_cmpl, ctx); - -end: - free_rpc_bdev_ocf_name(&req); -} -SPDK_RPC_REGISTER("bdev_ocf_get_stats", rpc_bdev_ocf_get_stats, SPDK_RPC_RUNTIME) - -static void -rpc_bdev_ocf_reset_stats_cmpl(ocf_cache_t cache, void *priv, int error) -{ - struct get_ocf_stats_ctx *ctx = (struct get_ocf_stats_ctx *) priv; - - if (error) { - goto end; - } - - error = vbdev_ocf_stats_reset(cache, ctx->core_name); - - ocf_mngt_cache_read_unlock(cache); - -end: - if (error) { - spdk_jsonrpc_send_error_response_fmt(ctx->request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Could not reset stats: %s", - spdk_strerror(-error)); - } else { - spdk_jsonrpc_send_bool_response(ctx->request, true); - } - free(ctx); -} - -static void -rpc_bdev_ocf_reset_stats(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) -{ - struct rpc_bdev_ocf_name req = {NULL}; - struct vbdev_ocf *vbdev; - struct get_ocf_stats_ctx *ctx; - - ctx = calloc(1, sizeof(*ctx)); - if (!ctx) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Not enough memory to process request"); - goto end; - } - - if (spdk_json_decode_object(params, rpc_bdev_ocf_name_decoders, - SPDK_COUNTOF(rpc_bdev_ocf_name_decoders), - &req)) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parameters"); - free(ctx); - goto end; - } - - vbdev = vbdev_ocf_get_by_name(req.name); - if (vbdev == NULL) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - spdk_strerror(ENODEV)); - free(ctx); - goto end; - } - - ctx->core_name = vbdev->core.name; - ctx->request = request; - ocf_mngt_cache_read_lock(vbdev->ocf_cache, rpc_bdev_ocf_reset_stats_cmpl, ctx); - -end: - free_rpc_bdev_ocf_name(&req); -} -SPDK_RPC_REGISTER("bdev_ocf_reset_stats", rpc_bdev_ocf_reset_stats, SPDK_RPC_RUNTIME) - -/* Structure to decode the input parameters for this RPC method. */ -static const struct spdk_json_object_decoder rpc_bdev_ocf_get_bdevs_decoders[] = { - {"name", offsetof(struct rpc_bdev_ocf_name, name), spdk_json_decode_string, true}, -}; - -struct bdev_get_bdevs_ctx { - char *name; - struct spdk_json_write_ctx *w; -}; - -static void -bdev_get_bdevs_fn(struct vbdev_ocf *vbdev, void *ctx) -{ - struct bdev_get_bdevs_ctx *cctx = ctx; - struct spdk_json_write_ctx *w = cctx->w; - - if (cctx->name != NULL && - strcmp(vbdev->name, cctx->name) && - strcmp(vbdev->cache.name, cctx->name) && - strcmp(vbdev->core.name, cctx->name)) { - return; - } - - spdk_json_write_object_begin(w); - spdk_json_write_named_string(w, "name", vbdev->name); - spdk_json_write_named_bool(w, "started", vbdev->state.started); - - spdk_json_write_named_object_begin(w, "cache"); - spdk_json_write_named_string(w, "name", vbdev->cache.name); - spdk_json_write_named_bool(w, "attached", vbdev->cache.attached); - spdk_json_write_object_end(w); - - spdk_json_write_named_object_begin(w, "core"); - spdk_json_write_named_string(w, "name", vbdev->core.name); - spdk_json_write_named_bool(w, "attached", vbdev->core.attached); - spdk_json_write_object_end(w); - - spdk_json_write_object_end(w); -} - -static void -rpc_bdev_ocf_get_bdevs(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) -{ - struct spdk_json_write_ctx *w; - struct rpc_bdev_ocf_name req = {NULL}; - struct bdev_get_bdevs_ctx cctx; - - if (params && spdk_json_decode_object(params, rpc_bdev_ocf_get_bdevs_decoders, - SPDK_COUNTOF(rpc_bdev_ocf_get_bdevs_decoders), - &req)) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parameters"); - goto end; - } - - if (req.name) { - if (!(vbdev_ocf_get_by_name(req.name) || vbdev_ocf_get_base_by_name(req.name))) { - spdk_jsonrpc_send_error_response(request, - SPDK_JSONRPC_ERROR_INVALID_PARAMS, - spdk_strerror(ENODEV)); - goto end; - } - } - - w = spdk_jsonrpc_begin_result(request); - - cctx.name = req.name; - cctx.w = w; - - spdk_json_write_array_begin(w); - vbdev_ocf_foreach(bdev_get_bdevs_fn, &cctx); - spdk_json_write_array_end(w); - spdk_jsonrpc_end_result(request, w); - -end: - free_rpc_bdev_ocf_name(&req); -} -SPDK_RPC_REGISTER("bdev_ocf_get_bdevs", rpc_bdev_ocf_get_bdevs, SPDK_RPC_RUNTIME) - -/* Structure to hold the parameters for this RPC method. */ -struct rpc_bdev_ocf_set_cache_mode { - char *name; /* main vbdev name */ - char *mode; /* OCF cache mode to switch to */ -}; - -static void -free_rpc_bdev_ocf_set_cache_mode(struct rpc_bdev_ocf_set_cache_mode *r) -{ - free(r->name); - free(r->mode); -} - -/* Structure to decode the input parameters for this RPC method. */ -static const struct spdk_json_object_decoder rpc_bdev_ocf_set_cache_mode_decoders[] = { - {"name", offsetof(struct rpc_bdev_ocf_set_cache_mode, name), spdk_json_decode_string}, - {"mode", offsetof(struct rpc_bdev_ocf_set_cache_mode, mode), spdk_json_decode_string}, -}; - -static void -cache_mode_cb(int status, struct vbdev_ocf *vbdev, void *cb_arg) -{ - struct spdk_jsonrpc_request *request = cb_arg; - struct spdk_json_write_ctx *w; - - if (status) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Could not change OCF vbdev cache mode: %d", - status); - } else { - w = spdk_jsonrpc_begin_result(request); - spdk_json_write_string(w, ocf_get_cache_modename( - ocf_cache_get_mode(vbdev->ocf_cache))); - spdk_jsonrpc_end_result(request, w); - } -} - -static void -rpc_bdev_ocf_set_cache_mode(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) -{ - struct rpc_bdev_ocf_set_cache_mode req = {NULL}; - struct vbdev_ocf *vbdev; - int status; - - status = spdk_json_decode_object(params, rpc_bdev_ocf_set_cache_mode_decoders, - SPDK_COUNTOF(rpc_bdev_ocf_set_cache_mode_decoders), - &req); - if (status) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parameters"); - goto end; - } - - vbdev = vbdev_ocf_get_by_name(req.name); - if (vbdev == NULL) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - spdk_strerror(ENODEV)); - goto end; - } - - vbdev_ocf_set_cache_mode(vbdev, req.mode, cache_mode_cb, request); - -end: - free_rpc_bdev_ocf_set_cache_mode(&req); -} -SPDK_RPC_REGISTER("bdev_ocf_set_cache_mode", rpc_bdev_ocf_set_cache_mode, SPDK_RPC_RUNTIME) - -static void -seqcutoff_cb(int status, void *cb_arg) -{ - struct spdk_jsonrpc_request *request = cb_arg; - - if (status) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "OCF could not set sequential cutoff parameters: %d", status); - } else { - spdk_jsonrpc_send_bool_response(request, true); - } -} - -/* Structure to hold the parameters for this RPC method. */ -struct rpc_bdev_ocf_set_seqcutoff { - char *name; /* main vbdev name */ - char *policy; - uint32_t threshold; - uint32_t promotion_count; -}; - -static void -free_rpc_bdev_ocf_set_seqcutoff(struct rpc_bdev_ocf_set_seqcutoff *r) -{ - free(r->name); - free(r->policy); -} - -/* Structure to decode the input parameters for this RPC method. */ -static const struct spdk_json_object_decoder rpc_bdev_ocf_set_seqcutoff_decoders[] = { - {"name", offsetof(struct rpc_bdev_ocf_set_seqcutoff, name), spdk_json_decode_string}, - {"policy", offsetof(struct rpc_bdev_ocf_set_seqcutoff, policy), spdk_json_decode_string}, - {"threshold", offsetof(struct rpc_bdev_ocf_set_seqcutoff, threshold), spdk_json_decode_uint32, true}, - {"promotion_count", offsetof(struct rpc_bdev_ocf_set_seqcutoff, promotion_count), spdk_json_decode_uint32, true}, -}; - -static void -rpc_bdev_ocf_set_seqcutoff(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) -{ - struct rpc_bdev_ocf_set_seqcutoff req = {NULL}; - struct vbdev_ocf *vbdev; - int ret; - - ret = spdk_json_decode_object(params, rpc_bdev_ocf_set_seqcutoff_decoders, - SPDK_COUNTOF(rpc_bdev_ocf_set_seqcutoff_decoders), &req); - if (ret) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parameters"); - goto end; - } - - vbdev = vbdev_ocf_get_by_name(req.name); - if (vbdev == NULL) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - spdk_strerror(ENODEV)); - goto end; - } - - vbdev_ocf_set_seqcutoff(vbdev, req.policy, req.threshold, req.promotion_count, seqcutoff_cb, - request); - -end: - free_rpc_bdev_ocf_set_seqcutoff(&req); -} -SPDK_RPC_REGISTER("bdev_ocf_set_seqcutoff", rpc_bdev_ocf_set_seqcutoff, SPDK_RPC_RUNTIME) - -struct get_ocf_flush_start_ctx { - struct spdk_jsonrpc_request *request; - struct vbdev_ocf *vbdev; -}; - -static void -rpc_bdev_ocf_flush_start_cmpl(ocf_cache_t cache, void *priv, int error) -{ - struct get_ocf_flush_start_ctx *ctx = priv; - - ctx->vbdev->flush.in_progress = false; - ctx->vbdev->flush.status = error; - - ocf_mngt_cache_read_unlock(cache); - - free(ctx); -} - -static void -rpc_bdev_ocf_flush_start_lock_cmpl(ocf_cache_t cache, void *priv, int error) -{ - struct get_ocf_flush_start_ctx *ctx = priv; - - if (error) { - spdk_jsonrpc_send_error_response_fmt(ctx->request, - SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Could not lock cache: %d", error); - free(ctx); - return; - } - - ctx->vbdev->flush.in_progress = true; - ocf_mngt_cache_flush(cache, rpc_bdev_ocf_flush_start_cmpl, ctx); - - spdk_jsonrpc_send_bool_response(ctx->request, true); -} - -static void -rpc_bdev_ocf_flush_start(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) -{ - struct rpc_bdev_ocf_name req = {NULL}; - struct get_ocf_flush_start_ctx *ctx; - int status; - - ctx = calloc(1, sizeof(*ctx)); - if (!ctx) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Not enough memory to process request"); - goto end; - } - - status = spdk_json_decode_object(params, rpc_bdev_ocf_name_decoders, - SPDK_COUNTOF(rpc_bdev_ocf_name_decoders), - &req); - if (status) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parameters"); - free(ctx); - goto end; - } - - ctx->vbdev = vbdev_ocf_get_by_name(req.name); - if (ctx->vbdev == NULL) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - spdk_strerror(ENODEV)); - free(ctx); - goto end; - } - - if (!ctx->vbdev->ocf_cache) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Couldn't flush cache: device not attached"); - free(ctx); - goto end; - } - - ctx->request = request; - ocf_mngt_cache_read_lock(ctx->vbdev->ocf_cache, rpc_bdev_ocf_flush_start_lock_cmpl, ctx); - -end: - free_rpc_bdev_ocf_name(&req); -} -SPDK_RPC_REGISTER("bdev_ocf_flush_start", rpc_bdev_ocf_flush_start, SPDK_RPC_RUNTIME) - -static void -rpc_bdev_ocf_flush_status(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) -{ - struct rpc_bdev_ocf_name req = {NULL}; - struct spdk_json_write_ctx *w; - struct vbdev_ocf *vbdev; - int status; - - status = spdk_json_decode_object(params, rpc_bdev_ocf_name_decoders, - SPDK_COUNTOF(rpc_bdev_ocf_name_decoders), - &req); - if (status) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parameters"); - goto end; - } - - vbdev = vbdev_ocf_get_by_name(req.name); - if (vbdev == NULL) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - spdk_strerror(ENODEV)); - goto end; - } - - w = spdk_jsonrpc_begin_result(request); - - spdk_json_write_object_begin(w); - spdk_json_write_named_bool(w, "in_progress", vbdev->flush.in_progress); - if (!vbdev->flush.in_progress) { - spdk_json_write_named_int32(w, "status", vbdev->flush.status); - } - spdk_json_write_object_end(w); - - spdk_jsonrpc_end_result(request, w); - -end: - free_rpc_bdev_ocf_name(&req); -} -SPDK_RPC_REGISTER("bdev_ocf_flush_status", rpc_bdev_ocf_flush_status, SPDK_RPC_RUNTIME) From 42ff735168cb564dba173ef6f3729f43bb52612c Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Sun, 6 Apr 2025 17:25:59 +0200 Subject: [PATCH 02/34] OCF module v2 Change-Id: I58f0fa973cc78b1c7bfefd9ffd783a6f752baba3 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/ctx.c | 47 +- module/bdev/ocf/ctx.h | 14 +- module/bdev/ocf/data.c | 39 +- module/bdev/ocf/data.h | 13 +- module/bdev/ocf/utils.c | 68 +- module/bdev/ocf/utils.h | 27 +- module/bdev/ocf/vbdev_ocf.c | 1120 +++++++++++++++++++++++++++++ module/bdev/ocf/vbdev_ocf.h | 75 ++ module/bdev/ocf/vbdev_ocf_cache.c | 308 ++++++++ module/bdev/ocf/vbdev_ocf_cache.h | 66 ++ module/bdev/ocf/vbdev_ocf_core.c | 462 ++++++++++++ module/bdev/ocf/vbdev_ocf_core.h | 81 +++ module/bdev/ocf/vbdev_ocf_rpc.c | 313 ++++++++ module/bdev/ocf/volume.c | 59 +- module/bdev/ocf/volume.h | 18 +- python/spdk/cli/bdev.py | 180 +++-- 16 files changed, 2636 insertions(+), 254 deletions(-) create mode 100644 module/bdev/ocf/vbdev_ocf.c create mode 100644 module/bdev/ocf/vbdev_ocf.h create mode 100644 module/bdev/ocf/vbdev_ocf_cache.c create mode 100644 module/bdev/ocf/vbdev_ocf_cache.h create mode 100644 module/bdev/ocf/vbdev_ocf_core.c create mode 100644 module/bdev/ocf/vbdev_ocf_core.h create mode 100644 module/bdev/ocf/vbdev_ocf_rpc.c diff --git a/module/bdev/ocf/ctx.c b/module/bdev/ocf/ctx.c index 72bb0cc51c4..cc90daa7a92 100644 --- a/module/bdev/ocf/ctx.c +++ b/module/bdev/ocf/ctx.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2018 Intel Corporation. + * Copyright (C) 2025 Huawei Technologies * All rights reserved. */ @@ -13,13 +14,14 @@ #include "ctx.h" #include "data.h" +#include "vbdev_ocf_cache.h" ocf_ctx_t vbdev_ocf_ctx; static ctx_data_t * vbdev_ocf_ctx_data_alloc(uint32_t pages) { - struct bdev_ocf_data *data; + struct vbdev_ocf_data *data; void *buf; uint32_t sz; @@ -46,7 +48,7 @@ vbdev_ocf_ctx_data_alloc(uint32_t pages) static void vbdev_ocf_ctx_data_free(ctx_data_t *ctx_data) { - struct bdev_ocf_data *data = ctx_data; + struct vbdev_ocf_data *data = ctx_data; int i; if (!data) { @@ -105,7 +107,7 @@ iovec_flatten(struct iovec *iov, size_t iovcnt, void *buf, size_t size, size_t o static uint32_t vbdev_ocf_ctx_data_rd(void *dst, ctx_data_t *src, uint32_t size) { - struct bdev_ocf_data *s = src; + struct vbdev_ocf_data *s = src; uint32_t size_local; size_local = iovec_flatten(s->iovs, s->iovcnt, dst, size, s->seek); @@ -146,7 +148,7 @@ buf_to_iovec(const void *buf, size_t size, struct iovec *iov, size_t iovcnt, siz static uint32_t vbdev_ocf_ctx_data_wr(ctx_data_t *dst, const void *src, uint32_t size) { - struct bdev_ocf_data *d = dst; + struct vbdev_ocf_data *d = dst; uint32_t size_local; size_local = buf_to_iovec(src, size, d->iovs, d->iovcnt, d->seek); @@ -186,7 +188,7 @@ iovset(struct iovec *iov, size_t iovcnt, int byte, size_t size, size_t offset) static uint32_t vbdev_ocf_ctx_data_zero(ctx_data_t *dst, uint32_t size) { - struct bdev_ocf_data *d = dst; + struct vbdev_ocf_data *d = dst; uint32_t size_local; size_local = iovset(d->iovs, d->iovcnt, 0, size, d->seek); @@ -198,7 +200,7 @@ vbdev_ocf_ctx_data_zero(ctx_data_t *dst, uint32_t size) static uint32_t vbdev_ocf_ctx_data_seek(ctx_data_t *dst, ctx_data_seek_t seek, uint32_t offset) { - struct bdev_ocf_data *d = dst; + struct vbdev_ocf_data *d = dst; uint32_t off = 0; switch (seek) { @@ -219,8 +221,8 @@ static uint64_t vbdev_ocf_ctx_data_cpy(ctx_data_t *dst, ctx_data_t *src, uint64_t to, uint64_t from, uint64_t bytes) { - struct bdev_ocf_data *s = src; - struct bdev_ocf_data *d = dst; + struct vbdev_ocf_data *s = src; + struct vbdev_ocf_data *d = dst; uint32_t it_iov = 0; uint32_t it_off = 0; uint32_t n, sz; @@ -255,7 +257,7 @@ vbdev_ocf_ctx_data_cpy(ctx_data_t *dst, ctx_data_t *src, uint64_t to, static void vbdev_ocf_ctx_data_secure_erase(ctx_data_t *ctx_data) { - struct bdev_ocf_data *data = ctx_data; + struct vbdev_ocf_data *data = ctx_data; struct iovec *iovs = data->iovs; int i; @@ -285,18 +287,19 @@ vbdev_ocf_queue_put(ocf_queue_t queue) ocf_queue_put(queue); } -void -vbdev_ocf_cache_ctx_put(struct vbdev_ocf_cache_ctx *ctx) +int +vbdev_ocf_queue_poller(void *ctx) { - if (env_atomic_dec_return(&ctx->refcnt) == 0) { - free(ctx); + ocf_queue_t queue = ctx; + int i, queue_runs; + + queue_runs = spdk_min(ocf_queue_pending_io(queue), VBDEV_OCF_QUEUE_RUN_MAX); + + for (i = 0; i < queue_runs; i++) { + ocf_queue_run_single(queue); } -} -void -vbdev_ocf_cache_ctx_get(struct vbdev_ocf_cache_ctx *ctx) -{ - env_atomic_inc(&ctx->refcnt); + return queue_runs ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; } struct cleaner_priv { @@ -332,13 +335,13 @@ vbdev_ocf_ctx_cleaner_init(ocf_cleaner_t c) { struct cleaner_priv *priv = calloc(1, sizeof(*priv)); ocf_cache_t cache = ocf_cleaner_get_cache(c); - struct vbdev_ocf_cache_ctx *cctx = ocf_cache_get_priv(cache); + struct vbdev_ocf_cache *vbdev_ocf_cache = ocf_cache_get_priv(cache); if (priv == NULL) { return -ENOMEM; } - priv->mngt_queue = cctx->mngt_queue; + priv->mngt_queue = vbdev_ocf_cache->ocf_cache_mngt_q; ocf_cleaner_set_cmpl(c, cleaner_cmpl); ocf_cleaner_set_priv(c, priv); @@ -353,10 +356,12 @@ vbdev_ocf_ctx_cleaner_stop(ocf_cleaner_t c) if (priv) { spdk_poller_unregister(&priv->poller); + vbdev_ocf_queue_put(priv->mngt_queue); free(priv); } } +// this one should only set next_run to "now" and poller registration should be move to _init with saved thread static void vbdev_ocf_ctx_cleaner_kick(ocf_cleaner_t cleaner) { @@ -408,7 +413,7 @@ vbdev_ocf_ctx_log_printf(ocf_logger_t logger, ocf_logger_lvl_t lvl, } static const struct ocf_ctx_config vbdev_ocf_ctx_cfg = { - .name = "OCF SPDK", + .name = "SPDK_OCF", .ops = { .data = { diff --git a/module/bdev/ocf/ctx.h b/module/bdev/ocf/ctx.h index 3d3e5bbcc58..fd9a175f4ff 100644 --- a/module/bdev/ocf/ctx.h +++ b/module/bdev/ocf/ctx.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2018 Intel Corporation. + * Copyright (C) 2025 Huawei Technologies * All rights reserved. */ @@ -7,8 +8,6 @@ #define VBDEV_OCF_CTX_H #include -#include "ocf_env.h" -#include "spdk/thread.h" extern ocf_ctx_t vbdev_ocf_ctx; @@ -16,14 +15,8 @@ extern ocf_ctx_t vbdev_ocf_ctx; #define SPDK_OBJECT 1 -/* Context of cache instance */ -struct vbdev_ocf_cache_ctx { - ocf_queue_t mngt_queue; - env_atomic refcnt; -}; - -void vbdev_ocf_cache_ctx_put(struct vbdev_ocf_cache_ctx *ctx); -void vbdev_ocf_cache_ctx_get(struct vbdev_ocf_cache_ctx *ctx); +// why? +#define VBDEV_OCF_QUEUE_RUN_MAX 32 int vbdev_ocf_ctx_init(void); void vbdev_ocf_ctx_cleanup(void); @@ -34,5 +27,6 @@ int vbdev_ocf_queue_create(ocf_cache_t cache, ocf_queue_t *queue, const struct o int vbdev_ocf_queue_create_mngt(ocf_cache_t cache, ocf_queue_t *queue, const struct ocf_queue_ops *ops); void vbdev_ocf_queue_put(ocf_queue_t queue); +int vbdev_ocf_queue_poller(void *ctx); #endif diff --git a/module/bdev/ocf/data.c b/module/bdev/ocf/data.c index ff3f347ed60..46c3cc55086 100644 --- a/module/bdev/ocf/data.c +++ b/module/bdev/ocf/data.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2018 Intel Corporation. + * Copyright (C) 2025 Huawei Technologies * All rights reserved. */ @@ -7,10 +8,10 @@ #include "spdk/bdev.h" #include "data.h" -struct bdev_ocf_data * +struct vbdev_ocf_data * vbdev_ocf_data_alloc(uint32_t iovcnt) { - struct bdev_ocf_data *data; + struct vbdev_ocf_data *data; data = env_malloc(sizeof(*data), ENV_MEM_NOIO); if (!data) { @@ -34,7 +35,7 @@ vbdev_ocf_data_alloc(uint32_t iovcnt) } void -vbdev_ocf_data_free(struct bdev_ocf_data *data) +vbdev_ocf_data_free(struct vbdev_ocf_data *data) { if (!data) { return; @@ -48,7 +49,7 @@ vbdev_ocf_data_free(struct bdev_ocf_data *data) } void -vbdev_ocf_iovs_add(struct bdev_ocf_data *data, void *base, size_t len) +vbdev_ocf_iovs_add(struct vbdev_ocf_data *data, void *base, size_t len) { assert(NULL != data); assert(data->iovalloc != -1); @@ -62,33 +63,3 @@ vbdev_ocf_iovs_add(struct bdev_ocf_data *data, void *base, size_t len) data->iovs[data->iovcnt].iov_len = len; data->iovcnt++; } - -struct bdev_ocf_data * -vbdev_ocf_data_from_spdk_io(struct spdk_bdev_io *bdev_io) -{ - struct bdev_ocf_data *data; - - if (bdev_io == NULL) { - return NULL; - } - - switch (bdev_io->type) { - case SPDK_BDEV_IO_TYPE_WRITE: - case SPDK_BDEV_IO_TYPE_READ: - assert(bdev_io->u.bdev.iovs); - break; - case SPDK_BDEV_IO_TYPE_FLUSH: - case SPDK_BDEV_IO_TYPE_UNMAP: - break; - default: - SPDK_ERRLOG("Unsupported IO type %d\n", bdev_io->type); - return NULL; - } - - data = (struct bdev_ocf_data *)bdev_io->driver_ctx; - data->iovs = bdev_io->u.bdev.iovs; - data->iovcnt = bdev_io->u.bdev.iovcnt; - data->size = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen; - - return data; -} diff --git a/module/bdev/ocf/data.h b/module/bdev/ocf/data.h index e1b571ddfca..6514f00dabd 100644 --- a/module/bdev/ocf/data.h +++ b/module/bdev/ocf/data.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2018 Intel Corporation. + * Copyright (C) 2025 Huawei Technologies * All rights reserved. */ @@ -9,7 +10,7 @@ #include "ocf_env.h" #include "spdk/bdev_module.h" -struct bdev_ocf_data { +struct vbdev_ocf_data { struct iovec *iovs; int iovcnt; int iovalloc; @@ -17,14 +18,12 @@ struct bdev_ocf_data { uint32_t seek; }; -struct bdev_ocf_data *vbdev_ocf_data_from_spdk_io(struct spdk_bdev_io *bdev_io); +struct vbdev_ocf_data *vbdev_ocf_data_alloc(uint32_t nvecs); -struct bdev_ocf_data *vbdev_ocf_data_alloc(uint32_t nvecs); +void vbdev_ocf_data_free(struct vbdev_ocf_data *data); -void vbdev_ocf_data_free(struct bdev_ocf_data *data); +struct vbdev_ocf_data *vbdev_ocf_data_from_iov(struct iovec *iovs); -struct bdev_ocf_data *vbdev_ocf_data_from_iov(struct iovec *iovs); - -void vbdev_ocf_iovs_add(struct bdev_ocf_data *data, void *base, size_t len); +void vbdev_ocf_iovs_add(struct vbdev_ocf_data *data, void *base, size_t len); #endif diff --git a/module/bdev/ocf/utils.c b/module/bdev/ocf/utils.c index 4c70c70fadd..f3ce5ace56e 100644 --- a/module/bdev/ocf/utils.c +++ b/module/bdev/ocf/utils.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2018 Intel Corporation. + * Copyright (C) 2025 Huawei Technologies * All rights reserved. */ @@ -7,7 +8,6 @@ #include "spdk/log.h" #include "utils.h" -#include "vbdev_ocf.h" static char *cache_modes[ocf_cache_mode_max] = { [ocf_cache_mode_wt] = "wt", @@ -66,69 +66,3 @@ ocf_get_seqcutoff_policy(const char *policy_name) return ocf_seq_cutoff_policy_max; } - -int -vbdev_ocf_mngt_start(struct vbdev_ocf *vbdev, vbdev_ocf_mngt_fn *path, - vbdev_ocf_mngt_callback cb, void *cb_arg) -{ - if (vbdev->mngt_ctx.current_step) { - return -EBUSY; - } - - memset(&vbdev->mngt_ctx, 0, sizeof(vbdev->mngt_ctx)); - - vbdev->mngt_ctx.current_step = path; - vbdev->mngt_ctx.cb = cb; - vbdev->mngt_ctx.cb_arg = cb_arg; - - (*vbdev->mngt_ctx.current_step)(vbdev); - - return 0; -} - -void -vbdev_ocf_mngt_stop(struct vbdev_ocf *vbdev, vbdev_ocf_mngt_fn *rollback_path, int status) -{ - if (status) { - vbdev->mngt_ctx.status = status; - } - - if (vbdev->mngt_ctx.status && rollback_path) { - vbdev->mngt_ctx.poller_fn = NULL; - vbdev->mngt_ctx.current_step = rollback_path; - (*vbdev->mngt_ctx.current_step)(vbdev); - return; - } - - if (vbdev->mngt_ctx.cb) { - vbdev->mngt_ctx.cb(vbdev->mngt_ctx.status, vbdev, vbdev->mngt_ctx.cb_arg); - } - - memset(&vbdev->mngt_ctx, 0, sizeof(vbdev->mngt_ctx)); -} - -void -vbdev_ocf_mngt_continue(struct vbdev_ocf *vbdev, int status) -{ - if (vbdev->mngt_ctx.current_step == NULL) { - return; - } - - assert((*vbdev->mngt_ctx.current_step) != NULL); - - vbdev->mngt_ctx.status = status; - - vbdev->mngt_ctx.current_step++; - if (*vbdev->mngt_ctx.current_step) { - (*vbdev->mngt_ctx.current_step)(vbdev); - return; - } - - vbdev_ocf_mngt_stop(vbdev, NULL, 0); -} - -int -vbdev_ocf_mngt_get_status(struct vbdev_ocf *vbdev) -{ - return vbdev->mngt_ctx.status; -} diff --git a/module/bdev/ocf/utils.h b/module/bdev/ocf/utils.h index 8f1688c6303..7a5003030fd 100644 --- a/module/bdev/ocf/utils.h +++ b/module/bdev/ocf/utils.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2018 Intel Corporation. + * Copyright (C) 2025 Huawei Technologies * All rights reserved. */ @@ -7,7 +8,6 @@ #define VBDEV_OCF_UTILS_H #include -#include "vbdev_ocf.h" ocf_cache_mode_t ocf_get_cache_mode(const char *cache_mode); const char *ocf_get_cache_modename(ocf_cache_mode_t mode); @@ -17,29 +17,4 @@ int ocf_get_cache_line_size(ocf_cache_t cache); /* Get sequential cutoff policy by name */ ocf_seq_cutoff_policy ocf_get_seqcutoff_policy(const char *policy_name); - -/* Initiate management operation - * Receives NULL terminated array of functions (path) - * and callback (cb) - * and callback argument (cb_arg) - * This function may fail with ENOMEM or EBUSY */ -int vbdev_ocf_mngt_start(struct vbdev_ocf *vbdev, vbdev_ocf_mngt_fn *path, - vbdev_ocf_mngt_callback cb, void *cb_arg); - -/* Continue execution with polling operation (fn) - * fn must invoke vbdev_ocf_mngt_continue() to stop polling - * Poller has default timeout of 5 seconds */ -void vbdev_ocf_mngt_poll(struct vbdev_ocf *vbdev, vbdev_ocf_mngt_fn fn); - -/* Continue execution with next function that is on path - * If next function is NULL, finish management operation and invoke callback */ -void vbdev_ocf_mngt_continue(struct vbdev_ocf *vbdev, int status); - -/* Stop the execution, if status is non zero set it, - * if rollback function is not null invoke rollback - * else invoke callback with last status returned */ -void vbdev_ocf_mngt_stop(struct vbdev_ocf *vbdev, vbdev_ocf_mngt_fn *rollback_path, int status); - -/* Get status */ -int vbdev_ocf_mngt_get_status(struct vbdev_ocf *vbdev); #endif diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c new file mode 100644 index 00000000000..925b6e3291e --- /dev/null +++ b/module/bdev/ocf/vbdev_ocf.c @@ -0,0 +1,1120 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (C) 2025 Huawei Technologies + * All rights reserved. + */ + +#include + +#include "spdk/bdev_module.h" +#include "spdk/string.h" // rm ? + +#include "vbdev_ocf.h" +#include "vbdev_ocf_cache.h" +#include "vbdev_ocf_core.h" +#include "ctx.h" +#include "data.h" +#include "volume.h" + +/* This namespace UUID was generated using uuid_generate() method. */ +#define BDEV_OCF_NAMESPACE_UUID "f92b7f49-f6c0-44c8-bd23-3205e8c3b6ad" + +static int vbdev_ocf_module_init(void); +static void vbdev_ocf_module_fini_start(void); +static void vbdev_ocf_module_fini(void); +static int vbdev_ocf_module_get_ctx_size(void); + +struct spdk_bdev_module ocf_if = { + .name = "OCF", + .module_init = vbdev_ocf_module_init, + .fini_start = vbdev_ocf_module_fini_start, + .module_fini = vbdev_ocf_module_fini, + .get_ctx_size = vbdev_ocf_module_get_ctx_size, + .examine_config = NULL, // todo + .examine_disk = NULL, // todo ? + .async_fini_start = true, +}; + +SPDK_BDEV_MODULE_REGISTER(ocf, &ocf_if) + +static int vbdev_ocf_fn_destruct(void *ctx); +static void vbdev_ocf_fn_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io); +static bool vbdev_ocf_fn_io_type_supported(void *ctx, enum spdk_bdev_io_type); +static struct spdk_io_channel *vbdev_ocf_fn_get_io_channel(void *ctx); +static int vbdev_ocf_fn_dump_info_json(void *ctx, struct spdk_json_write_ctx *w); +static void vbdev_ocf_fn_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w); + +struct spdk_bdev_fn_table vbdev_ocf_fn_table = { + .destruct = vbdev_ocf_fn_destruct, + .submit_request = vbdev_ocf_fn_submit_request, + .io_type_supported = vbdev_ocf_fn_io_type_supported, + .get_io_channel = vbdev_ocf_fn_get_io_channel, + .dump_info_json = vbdev_ocf_fn_dump_info_json, + .write_config_json = vbdev_ocf_fn_write_config_json, + .dump_device_stat_json = NULL, // todo ? + .reset_device_stat = NULL, // todo ? +}; + +// rm +#define __bdev_to_io_dev(bdev) (((char *)bdev) + 1) + +static bool +vbdev_ocf_device_exists(const char *name) { + struct vbdev_ocf_cache *cache; + struct vbdev_ocf_core *core; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF: looking for '%s' in existing device names\n", name); + + vbdev_ocf_foreach_core_incomplete(core) { + if (!strcmp(name, core->name)) { + return true; + } + } + vbdev_ocf_foreach_cache(cache) { + if (!strcmp(name, cache->name)) { + return true; + } + vbdev_ocf_foreach_core_in_cache(core, cache) { + if (!strcmp(name, core->name)) { + return true; + } + } + } + + return false; +} + +static int +vbdev_ocf_module_init(void) +{ + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF: starting module\n"); + + if ((rc = vbdev_ocf_ctx_init())) { + SPDK_ERRLOG("OCF: failed to initialize context: %d\n", rc); + return rc; + } + + if ((rc = vbdev_ocf_volume_init())) { + vbdev_ocf_ctx_cleanup(); + SPDK_ERRLOG("OCF: failed to register volume: %d\n", rc); + return rc; + } + + return rc; +} + +static void +_cache_stop_module_fini_stop_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +{ + struct vbdev_ocf_cache *cache = cb_arg; + struct vbdev_ocf_core *core; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", cache->name); + + ocf_mngt_cache_unlock(ocf_cache); + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to stop OCF cache (OCF error: %d)\n", + cache->name, error); + return; + } + + vbdev_ocf_foreach_core_in_cache(core, cache) { + /* It's important to set ocf_core back to NULL before unregistering. + * This is an indicator for destruct that OCF cache isn't there any more. */ + core->ocf_core = NULL; + } + + vbdev_ocf_cache_base_detach(cache); + + if (cache == STAILQ_LAST(&g_vbdev_ocf_caches, vbdev_ocf_cache, link)) { + spdk_bdev_module_fini_start_done(); + } +} + +static void +_cache_stop_module_fini_flush_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +{ + struct vbdev_ocf_cache *cache = cb_arg; + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to flush OCF cache (OCF error: %d)\n", + cache->name, error); + ocf_mngt_cache_unlock(ocf_cache); + return; + } + + ocf_mngt_cache_stop(ocf_cache, _cache_stop_module_fini_stop_cb, cache); +} + +static void +_cache_stop_module_fini_lock_cb(ocf_cache_t ocf_cache, void *lock_arg, int lock_err) +{ + struct vbdev_ocf_cache *cache = lock_arg; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop of OCF cache\n", cache->name); + + if (lock_err) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + cache->name, lock_err); + return; + } + + if (ocf_mngt_cache_is_dirty(ocf_cache)) { + ocf_mngt_cache_flush(ocf_cache, _cache_stop_module_fini_flush_cb, cache); + } else { + ocf_mngt_cache_stop(ocf_cache, _cache_stop_module_fini_stop_cb, cache); + } +} + +static void +vbdev_ocf_module_fini_start(void) +{ + struct vbdev_ocf_cache *cache; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF: initiating module stop\n"); + + if (STAILQ_EMPTY(&g_vbdev_ocf_caches)) { + spdk_bdev_module_fini_start_done(); + return; + } + + /* Stop all OCF caches before unregistering all bdevs. */ + vbdev_ocf_foreach_cache(cache) { + ocf_mngt_cache_lock(cache->ocf_cache, _cache_stop_module_fini_lock_cb, cache); + } +} + +static void +vbdev_ocf_module_fini(void) +{ + struct vbdev_ocf_cache *cache; + struct vbdev_ocf_core *core; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF: finishing module stop\n"); + + while ((core = STAILQ_FIRST(&g_vbdev_ocf_incomplete_cores))) { + vbdev_ocf_core_remove_incomplete(core); + vbdev_ocf_core_destroy(core); + } + while ((cache = STAILQ_FIRST(&g_vbdev_ocf_caches))) { + if (vbdev_ocf_cache_is_incomplete(cache)) { + /* If cache is incomplete it won't be started yet and won't have + * any cores attached, thus just removing init params here. */ + vbdev_ocf_cache_remove_incomplete(cache); + } else { + while ((core = STAILQ_FIRST(&cache->cores))) { + vbdev_ocf_core_remove_from_cache(core); + vbdev_ocf_core_destroy(core); + } + } + vbdev_ocf_cache_destroy(cache); + } + + vbdev_ocf_volume_cleanup(); + vbdev_ocf_ctx_cleanup(); +} + +static int +vbdev_ocf_module_get_ctx_size(void) +{ + return sizeof(struct vbdev_ocf_data); +} + +static void +_io_device_unregister_cb(void *io_device) +{ + struct vbdev_ocf_core *core = io_device; + + vbdev_ocf_core_base_detach(core); + + /* This one finally calls the callback from spdk_bdev_unregister_by_name(). */ + spdk_bdev_destruct_done(&core->ocf_vbdev, 0); +} + +// remove this wrapper ? +static void +vbdev_ocf_destruct_done(struct vbdev_ocf_core *core) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': finishing destruct\n", core->ocf_vbdev.name); + + spdk_io_device_unregister(core, _io_device_unregister_cb); +} + +static void +_core_remove_destruct_remove_cb(void *cb_arg, int error) +{ + struct vbdev_ocf_core *core = cb_arg; + struct vbdev_ocf_cache *cache = vbdev_ocf_core_get_cache(core); + + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': finishing remove of OCF core\n", + core->ocf_vbdev.name); + + ocf_mngt_cache_unlock(cache->ocf_cache); + + if (error) { + SPDK_ERRLOG("OCF vbdev '%s': failed to remove OCF core device (OCF error: %d)\n", + core->ocf_vbdev.name, error); + spdk_bdev_destruct_done(&core->ocf_vbdev, error); + return; + } + + vbdev_ocf_core_remove_from_cache(core); + + vbdev_ocf_destruct_done(core); +} + +static void +_core_remove_destruct_flush_cb(ocf_core_t ocf_core, void *cb_arg, int error) +{ + struct vbdev_ocf_core *core = cb_arg; + + assert(ocf_core == core->ocf_core); + + if (error) { // WARN only ? (may be hotremoved) + SPDK_ERRLOG("OCF vbdev '%s': failed to flush OCF core device (OCF error: %d)\n", + core->ocf_vbdev.name, error); + ocf_mngt_cache_unlock(vbdev_ocf_core_get_cache(core)->ocf_cache); + spdk_bdev_destruct_done(&core->ocf_vbdev, error); + return; + } + + ocf_mngt_cache_remove_core(ocf_core, _core_remove_destruct_remove_cb, core); +} + +static void +_core_remove_destruct_lock_cb(ocf_cache_t ocf_cache, void *lock_arg, int lock_err) +{ + struct vbdev_ocf_core *core = lock_arg; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': initiating remove of OCF core\n", + core->ocf_vbdev.name); + + if (lock_err) { + SPDK_ERRLOG("OCF vbdev '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + core->ocf_vbdev.name, lock_err); + spdk_bdev_destruct_done(&core->ocf_vbdev, lock_err); + return; + } + + if (ocf_mngt_core_is_dirty(core->ocf_core)) { + ocf_mngt_core_flush(core->ocf_core, _core_remove_destruct_flush_cb, core); + } else { + ocf_mngt_cache_remove_core(core->ocf_core, _core_remove_destruct_remove_cb, core); + } +} + +/* This is called internally by SPDK during vbdev_ocf_core_unregister(). */ +static int +vbdev_ocf_fn_destruct(void *ctx) +{ + struct vbdev_ocf_core *core = ctx; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': initiating destruct\n", core->ocf_vbdev.name); + + if (vbdev_ocf_core_cache_is_started(core)) { + ocf_mngt_cache_lock(vbdev_ocf_core_get_cache(core)->ocf_cache, + _core_remove_destruct_lock_cb, core); + } else { + vbdev_ocf_destruct_done(core); + } + + /* Return one to indicate async destruct. */ + return 1; +} + +static void +_vbdev_ocf_submit_io_cb(ocf_io_t io, void *priv1, void *priv2, int error) +{ + struct spdk_bdev_io *bdev_io = priv1; + + // rm ? + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': finishing submit of IO request\n", + bdev_io->bdev->name); + + ocf_io_put(io); + + if (error == -OCF_ERR_NO_MEM) { + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); + } else if (error) { + SPDK_ERRLOG("OCF vbdev '%s': failed to complete OCF IO\n", bdev_io->bdev->name); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } else { + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); + } +} + +typedef void (*submit_io_to_ocf_fn)(ocf_io_t io); + +static void +vbdev_ocf_submit_io(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, uint64_t offset, + uint32_t len, uint32_t dir, uint64_t flags, submit_io_to_ocf_fn submit_io_fn) +{ + struct vbdev_ocf_core *core = bdev_io->bdev->ctxt; + struct vbdev_ocf_data *data = (struct vbdev_ocf_data *)bdev_io->driver_ctx; + struct vbdev_ocf_core_io_channel_ctx *ch_ctx = spdk_io_channel_get_ctx(ch); + ocf_io_t io = NULL; + + // impossible to be true ? + if (!core->ocf_core) { + SPDK_ERRLOG("OCF vbdev '%s': failed to submit IO - no OCF core device\n", + bdev_io->bdev->name); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + return; + } + + io = ocf_volume_new_io(ocf_core_get_front_volume(core->ocf_core), ch_ctx->queue, + offset, len, dir, 0, flags); + if (!io) { + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); + return; + } + + data->iovs = bdev_io->u.bdev.iovs; + data->iovcnt = bdev_io->u.bdev.iovcnt; + data->size = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen; + + ocf_io_set_data(io, data, 0); + ocf_io_set_cmpl(io, bdev_io, NULL, _vbdev_ocf_submit_io_cb); + submit_io_fn(io); +} + +static void +_io_read_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success) +{ + uint64_t offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen; + uint32_t len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen; + + if (!success) { + SPDK_ERRLOG("OCF vbdev '%s': failed to allocate IO buffer - size of the " + "buffer to allocate might be greater than the permitted maximum\n", + bdev_io->bdev->name); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + return; + } + + vbdev_ocf_submit_io(ch, bdev_io, offset, len, OCF_READ, 0, ocf_core_submit_io); +} + +static void +vbdev_ocf_fn_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) +{ + uint64_t offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen; + uint32_t len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen; + + // rm ? + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': initiating submit of IO request\n", + bdev_io->bdev->name); + + switch (bdev_io->type) { + case SPDK_BDEV_IO_TYPE_READ: + // align buffer for write as well ? (comment in old vbdev_ocf.c) + // from doc: This function *must* be called from the thread issuing bdev_io. + spdk_bdev_io_get_buf(bdev_io, _io_read_get_buf_cb, len); + break; + case SPDK_BDEV_IO_TYPE_WRITE: + vbdev_ocf_submit_io(ch, bdev_io, offset, len, OCF_WRITE, 0, ocf_core_submit_io); + break; + case SPDK_BDEV_IO_TYPE_UNMAP: + vbdev_ocf_submit_io(ch, bdev_io, offset, len, OCF_WRITE, 0, ocf_core_submit_discard); + break; + case SPDK_BDEV_IO_TYPE_FLUSH: + vbdev_ocf_submit_io(ch, bdev_io, 0, 0, OCF_WRITE, OCF_WRITE_FLUSH, ocf_core_submit_flush); + break; + default: + SPDK_ERRLOG("OCF vbdev '%s': unsupported IO type: %s\n", bdev_io->bdev->name, + spdk_bdev_get_io_type_name(bdev_io->type)); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } +} + +static bool +vbdev_ocf_fn_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) +{ + struct vbdev_ocf_core *core = ctx; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': checking if IO type '%s' is supported\n", + core->ocf_vbdev.name, spdk_bdev_get_io_type_name(io_type)); + + switch (io_type) { + case SPDK_BDEV_IO_TYPE_READ: + case SPDK_BDEV_IO_TYPE_WRITE: + case SPDK_BDEV_IO_TYPE_UNMAP: + case SPDK_BDEV_IO_TYPE_FLUSH: + return spdk_bdev_io_type_supported(core->base.bdev, io_type); + default: + return false; + } +} + +static struct spdk_io_channel * +vbdev_ocf_fn_get_io_channel(void *ctx) // ctx == ocf_vbdev.ctxt +{ + struct vbdev_ocf_core *core = ctx; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': got request for IO channel\n", core->ocf_vbdev.name); + + return spdk_get_io_channel(core); +} + +static int +vbdev_ocf_fn_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) +{ + return 0; +} + +static void +vbdev_ocf_fn_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) +{ +} + +static void +_cache_stop_err_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +{ + struct vbdev_ocf_cache *cache = cb_arg; + + ocf_mngt_cache_unlock(ocf_cache); + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to stop OCF cache (OCF error: %d)\n", + cache->name, error); + return; + } +} + +static void +vbdev_ocf_cache_start_rollback(struct vbdev_ocf_cache *cache) +{ + vbdev_ocf_queue_put(cache->ocf_cache_mngt_q); + ocf_mngt_cache_stop(cache->ocf_cache, _cache_stop_err_cb, cache); + vbdev_ocf_cache_base_detach(cache); + vbdev_ocf_cache_destroy(cache); +} + +static void +_cache_start_rpc_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +{ + struct vbdev_ocf_cache_start_ctx *cache_start_ctx = cb_arg; + struct vbdev_ocf_cache *cache = cache_start_ctx->cache; + int rc = 0; // rm ? + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing start\n", cache->name); + + ocf_volume_destroy(cache->ocf_cache_att_cfg.device.volume); + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to attach OCF cache device\n", + cache->name); + // here or in ocf_mngt_cache_start() ? + if (error == -OCF_ERR_NO_MEM) { + uint64_t mem_needed, volume_size; + + volume_size = cache->base.bdev->blockcnt * cache->base.bdev->blocklen; + mem_needed = ocf_mngt_get_ram_needed(ocf_cache, volume_size); + SPDK_ERRLOG("Not enough memory. Try to increase hugepage memory size or cache line size.\n"); + SPDK_NOTICELOG("Needed memory to start cache in this configuration " + "(device size: %"PRIu64", cache line size: %"PRIu64"): %"PRIu64"\n", + volume_size, cache->ocf_cache_cfg.cache_line_size, mem_needed); + } + vbdev_ocf_cache_start_rollback(cache); + cache_start_ctx->rpc_cb_fn(NULL, cache_start_ctx->rpc_cb_arg, error); + free(cache_start_ctx); + return; + } + + SPDK_NOTICELOG("OCF cache '%s': started\n", cache->name); + + // check for cores in g_vbdev_ocf_incomplete_cores + // check if (cache_block_size > core_block_size) + + ocf_mngt_cache_unlock(ocf_cache); + cache_start_ctx->rpc_cb_fn(cache, cache_start_ctx->rpc_cb_arg, rc); + free(cache_start_ctx); +} + +/* RPC entry point. */ +void +vbdev_ocf_cache_start(const char *cache_name, const char *bdev_name, const char *cache_mode, + const uint8_t cache_line_size, vbdev_ocf_cache_start_cb rpc_cb_fn, void *rpc_cb_arg) +{ + struct vbdev_ocf_cache *cache; + struct vbdev_ocf_cache_start_ctx *cache_start_ctx; + struct ocf_volume_uuid volume_uuid; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating start\n", cache_name); + + if (vbdev_ocf_device_exists(cache_name)) { + SPDK_ERRLOG("OCF: device '%s' already exists\n", cache_name); + rc = -EEXIST; + goto err_create; + } + + /* Allocate memory for cache struct and put it on the global cache list. */ + if ((rc = vbdev_ocf_cache_create(cache_name, &cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to create cache\n", cache_name); + goto err_create; + } + + /* Create OCF configs for cache and attach. */ + if ((rc = vbdev_ocf_cache_set_config(cache, cache_mode, cache_line_size))) { + SPDK_ERRLOG("OCF cache '%s': failed to create OCF config\n", cache_name); + goto err_base; + } + + /* Check if base device for this cache is already present. */ + if ((rc = vbdev_ocf_cache_base_attach(cache, bdev_name))) { + if (rc == -ENODEV) { + SPDK_NOTICELOG("OCF cache '%s': start deferred - waiting for base bdev '%s'\n", + cache_name, bdev_name); + + /* If not, just save cache init params for use in examine and exit. */ + if ((rc = vbdev_ocf_cache_add_incomplete(cache, bdev_name))) { + SPDK_ERRLOG("OCF cache '%s': failed to save init params - removing cache\n", + cache_name); + goto err_base; + } + rpc_cb_fn(cache, rpc_cb_arg, -ENODEV); + return; + } + SPDK_ERRLOG("OCF cache '%s': failed to open base bdev '%s'\n", cache_name, bdev_name); + goto err_base; + } + + /* Start OCF cache. */ + + if ((rc = ocf_uuid_set_str(&volume_uuid, cache->name))) { // set spdk uuid ? + SPDK_ERRLOG("OCF cache '%s': failed to set OCF volume uuid\n", cache_name); + goto err_volume; + } + + // move elsewhere + cache->ocf_cache_att_cfg.device.volume_params = &cache->base; // for ocf_volume_open() in ocf_mngt_cache_attach() + + // do it right after set_config() and then ocf_volume_destroy() in all callbacks ? + if ((rc = ocf_ctx_volume_create(vbdev_ocf_ctx, &cache->ocf_cache_att_cfg.device.volume, + &volume_uuid, SPDK_OBJECT))) { + SPDK_ERRLOG("OCF cache '%s': failed to create OCF volume\n", cache_name); + goto err_volume; + } + + // maybe this as well for incomplete cache ? + // but then how to check for incomplition? is_cache_detached() ? + // in examine only attach (both in start_rpc and attach_rpc) ? + if ((rc = ocf_mngt_cache_start(vbdev_ocf_ctx, &cache->ocf_cache, &cache->ocf_cache_cfg, cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to start OCF cache\n", cache_name); + goto err_start; + } + + if ((rc = vbdev_ocf_cache_mngt_queue_create(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to create management queue\n", cache_name); + goto err_queue; + } + + cache_start_ctx = calloc(1, sizeof(struct vbdev_ocf_cache_start_ctx)); + if (!cache_start_ctx) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache start context\n", + cache_name); + rc = -ENOMEM; + goto err_alloc; + } + cache_start_ctx->cache = cache; + cache_start_ctx->rpc_cb_fn = rpc_cb_fn; + cache_start_ctx->rpc_cb_arg = rpc_cb_arg; + + ocf_mngt_cache_attach(cache->ocf_cache, &cache->ocf_cache_att_cfg, + _cache_start_rpc_cb, cache_start_ctx); + + return; + +err_alloc: + vbdev_ocf_queue_put(cache->ocf_cache_mngt_q); +err_queue: + ocf_mngt_cache_stop(cache->ocf_cache, _cache_stop_err_cb, cache); +err_start: + ocf_volume_destroy(cache->ocf_cache_att_cfg.device.volume); +err_volume: + vbdev_ocf_cache_base_detach(cache); +err_base: + vbdev_ocf_cache_destroy(cache); +err_create: + rpc_cb_fn(NULL, rpc_cb_arg, rc); +} + +static void +_core_unregister_cache_stop_cb(void *cb_arg, int error) +{ + struct vbdev_ocf_core *core = cb_arg; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", + core->name); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during cache stop: %s\n", + core->name, spdk_strerror(-error)); + } + + // destroy core despite the error ? + vbdev_ocf_core_destroy(core); +} + +static void +_cache_stop_rpc_stop_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +{ + struct vbdev_ocf_cache_stop_ctx *cache_stop_ctx = cb_arg; + struct vbdev_ocf_cache *cache = cache_stop_ctx->cache; + struct vbdev_ocf_core *core; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", cache->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop\n", cache->name); + + ocf_mngt_cache_unlock(ocf_cache); + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to stop OCF cache\n", + cache->name); + cache_stop_ctx->rpc_cb_fn(cache_stop_ctx->rpc_cb_arg, error); + free(cache_stop_ctx); + return; + } + + SPDK_NOTICELOG("OCF cache '%s': stopped\n", cache->name); + + vbdev_ocf_foreach_core_in_cache(core, cache) { + /* It's important to set ocf_core back to NULL before unregistering. + * This is an indicator for destruct that OCF cache isn't there any more. */ + core->ocf_core = NULL; + + if ((rc = vbdev_ocf_core_unregister(core, _core_unregister_cache_stop_cb, core))) { + SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during cache stop: %s\n", + core->name, spdk_strerror(-rc)); + // destroy core despite the error ? + vbdev_ocf_core_destroy(core); + } + } + + vbdev_ocf_cache_base_detach(cache); + vbdev_ocf_cache_destroy(cache); + + cache_stop_ctx->rpc_cb_fn(cache_stop_ctx->rpc_cb_arg, rc); + free(cache_stop_ctx); +} + +static void +_cache_stop_rpc_flush_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +{ + struct vbdev_ocf_cache_stop_ctx *cache_stop_ctx = cb_arg; + struct vbdev_ocf_cache *cache = cache_stop_ctx->cache; + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to flush OCF cache\n", + cache->name); + ocf_mngt_cache_unlock(ocf_cache); + cache_stop_ctx->rpc_cb_fn(cache_stop_ctx->rpc_cb_arg, error); + free(cache_stop_ctx); + return; + } + + ocf_mngt_cache_stop(ocf_cache, _cache_stop_rpc_stop_cb, cache_stop_ctx); +} + +static void +_cache_stop_rpc_lock_cb(ocf_cache_t ocf_cache, void *lock_arg, int lock_err) +{ + struct vbdev_ocf_cache_stop_ctx *cache_stop_ctx = lock_arg; + struct vbdev_ocf_cache *cache = cache_stop_ctx->cache; + + assert(ocf_cache == cache->ocf_cache); + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop of OCF cache\n", cache->name); + + if (lock_err) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock\n", + cache->name); + cache_stop_ctx->rpc_cb_fn(cache_stop_ctx->rpc_cb_arg, lock_err); + free(cache_stop_ctx); + return; + } + + if (ocf_mngt_cache_is_dirty(ocf_cache)) { + ocf_mngt_cache_flush(ocf_cache, _cache_stop_rpc_flush_cb, cache_stop_ctx); + } else { + ocf_mngt_cache_stop(ocf_cache, _cache_stop_rpc_stop_cb, cache_stop_ctx); + } +} + +/* RPC entry point. */ +void +vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_cache_stop_cb rpc_cb_fn, void *rpc_cb_arg) +{ + struct vbdev_ocf_cache *cache; + struct vbdev_ocf_cache_stop_ctx *cache_stop_ctx; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop\n", cache_name); + + cache = vbdev_ocf_cache_get_by_name(cache_name); + if (!cache) { + SPDK_ERRLOG("OCF cache '%s': device not found\n", cache_name); + rpc_cb_fn(rpc_cb_arg, -ENODEV); + return; + } + + /* If cache was not started yet due to lack of base device, just free its structs and exit. */ + if (vbdev_ocf_cache_is_incomplete(cache)) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': removing as incomplete\n", cache->name); + assert(!cache->base.attached); + vbdev_ocf_cache_remove_incomplete(cache); + vbdev_ocf_cache_destroy(cache); + rpc_cb_fn(rpc_cb_arg, 0); + return; + } + + // TODO: send hotremove to each core opener first - check how spdk_bdev_unregister does that + // take from bdev_unregister_unsafe() + // or stop cache after unregister ? + + cache_stop_ctx = calloc(1, sizeof(struct vbdev_ocf_cache_stop_ctx)); + if (!cache_stop_ctx) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache stop context\n", + cache_name); + rpc_cb_fn(rpc_cb_arg, -ENOMEM); + return; + } + cache_stop_ctx->cache = cache; + cache_stop_ctx->rpc_cb_fn = rpc_cb_fn; + cache_stop_ctx->rpc_cb_arg = rpc_cb_arg; + + ocf_mngt_cache_lock(cache->ocf_cache, _cache_stop_rpc_lock_cb, cache_stop_ctx); +} + +static void +_core_remove_err_cb(void *cb_arg, int error) +{ + struct vbdev_ocf_core *core = cb_arg; + struct vbdev_ocf_cache *cache = vbdev_ocf_core_get_cache(core); + + ocf_mngt_cache_unlock(cache->ocf_cache); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to remove OCF core device (OCF error: %d)\n", + core->name, error); + return; + } + + vbdev_ocf_core_remove_from_cache(core); +} + +static void +vbdev_ocf_core_add_rollback(struct vbdev_ocf_core *core) +{ + ocf_mngt_cache_remove_core(core->ocf_core, _core_remove_err_cb, core); + vbdev_ocf_core_base_detach(core); + vbdev_ocf_core_destroy(core); +} + +static void +_core_add_rpc_cb(ocf_cache_t ocf_cache, ocf_core_t ocf_core, void *cb_arg, int error) +{ + struct vbdev_ocf_core_add_ctx *core_add_ctx = cb_arg; + struct vbdev_ocf_cache *cache = core_add_ctx->cache; + struct vbdev_ocf_core *core = core_add_ctx->core; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", core->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add\n", core->name); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s'\n", + core->name, cache->name); + ocf_mngt_cache_unlock(ocf_cache); + vbdev_ocf_core_base_detach(core); + vbdev_ocf_core_destroy(core); + core_add_ctx->rpc_cb_fn(NULL, core_add_ctx->rpc_cb_arg, error); + free(core_add_ctx); + return; + } + + vbdev_ocf_core_add_to_cache(core, cache); + core->ocf_core = ocf_core; + + if ((rc = vbdev_ocf_core_register(core))) { + SPDK_ERRLOG("OCF core '%s': failed to register vbdev\n", core->name); + vbdev_ocf_core_add_rollback(core); + core_add_ctx->rpc_cb_fn(NULL, core_add_ctx->rpc_cb_arg, rc); + free(core_add_ctx); + return; + } + + SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n", core->name, cache->name); + + ocf_mngt_cache_unlock(ocf_cache); + core_add_ctx->rpc_cb_fn(core, core_add_ctx->rpc_cb_arg, rc); + free(core_add_ctx); +} + +static void +_vbdev_ocf_core_add_rpc_lock_cb(ocf_cache_t ocf_cache, void *lock_arg, int lock_err) +{ + struct vbdev_ocf_core_add_ctx *core_add_ctx = lock_arg; + struct vbdev_ocf_cache *cache = core_add_ctx->cache; + struct vbdev_ocf_core *core = core_add_ctx->core; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add of OCF core\n", core->name); + + if (lock_err) { + SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock\n", + core->name); + vbdev_ocf_core_base_detach(core); + vbdev_ocf_core_destroy(core); + core_add_ctx->rpc_cb_fn(NULL, core_add_ctx->rpc_cb_arg, lock_err); + free(core_add_ctx); + return; + } + + ocf_mngt_cache_add_core(cache->ocf_cache, &core->ocf_core_cfg, _core_add_rpc_cb, core_add_ctx); +} + +/* RPC entry point. */ +void +vbdev_ocf_core_add(const char *core_name, const char *bdev_name, const char *cache_name, + vbdev_ocf_core_add_cb rpc_cb_fn, void *rpc_cb_arg) +{ + struct vbdev_ocf_cache *cache; + struct vbdev_ocf_core *core; + struct vbdev_ocf_core_add_ctx *core_add_ctx; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add\n", core_name); + + if (vbdev_ocf_device_exists(core_name)) { + SPDK_ERRLOG("OCF: device '%s' already exists\n", core_name); + rc = -EEXIST; + goto err_create; + } + + /* Allocate memory for core struct. */ + if ((rc = vbdev_ocf_core_create(core_name, &core))) { + SPDK_ERRLOG("OCF core '%s': failed to create core\n", core_name); + goto err_create; + } + + /* Create OCF core config. */ + if ((rc = vbdev_ocf_core_set_config(core))) { + SPDK_ERRLOG("OCF core '%s': failed to create OCF config\n", core_name); + goto err_base; + } + + /* First, check if base device for this core is already present. */ + if ((rc = vbdev_ocf_core_base_attach(core, bdev_name))) { + if (rc == -ENODEV) { + SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for base bdev '%s'\n", + core_name, bdev_name); + /* If not, just save core init params for use in examine, + * put core on the temporary incomplete core list and exit. */ + if ((rc = vbdev_ocf_core_add_incomplete(core, bdev_name, cache_name))) { + SPDK_ERRLOG("OCF core '%s': failed to save init params - removing core\n", + core_name); + goto err_base; + } + rpc_cb_fn(core, rpc_cb_arg, -ENODEV); + return; + } + SPDK_ERRLOG("OCF core '%s': failed to open base bdev '%s'\n", core_name, bdev_name); + goto err_base; + } + + // move elsewhere + core->ocf_core_cfg.volume_params = &core->base; // for ocf_volume_open() in ocf_mngt_cache_add_core() + + /* Second, check if OCF cache for this core is already present and started. */ + cache = vbdev_ocf_cache_get_by_name(cache_name); + if (!cache || vbdev_ocf_cache_is_incomplete(cache)) { + SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache '%s'\n", + core_name, cache_name); + + /* If not, just save core init params for use in examine, + * put core on the temporary incomplete core list and exit. */ + if ((rc = vbdev_ocf_core_add_incomplete(core, bdev_name, cache_name))) { + SPDK_ERRLOG("OCF core '%s': failed to save init params - removing core\n", + core_name); + goto err_alloc; + } + rpc_cb_fn(core, rpc_cb_arg, -ENODEV); + return; + } + + // check if (cache_block_size > core_block_size) + + core_add_ctx = calloc(1, sizeof(struct vbdev_ocf_core_add_ctx)); + if (!core_add_ctx) { + SPDK_ERRLOG("OCF core '%s': failed to allocate memory for vbdev_ocf_core_add_ctx\n", + core_name); + rc = -ENOMEM; + goto err_alloc; + } + core_add_ctx->core = core; + core_add_ctx->cache = cache; + core_add_ctx->rpc_cb_fn = rpc_cb_fn; + core_add_ctx->rpc_cb_arg = rpc_cb_arg; + + ocf_mngt_cache_lock(cache->ocf_cache, _vbdev_ocf_core_add_rpc_lock_cb, core_add_ctx); + + return; + +err_alloc: + vbdev_ocf_core_base_detach(core); +err_base: + vbdev_ocf_core_destroy(core); +err_create: + rpc_cb_fn(NULL, rpc_cb_arg, rc); +} + +static void +_core_unregister_core_rm_cb(void *cb_arg, int error) +{ + struct vbdev_ocf_core_remove_ctx *core_rm_ctx = cb_arg; + struct vbdev_ocf_core *core = core_rm_ctx->core; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", + core->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing removal\n", core->name); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during core removal\n", + core->name); + } else { + SPDK_NOTICELOG("OCF core '%s': removed from cache\n", core->name); + vbdev_ocf_core_destroy(core); + } + + core_rm_ctx->rpc_cb_fn(core_rm_ctx->rpc_cb_arg, error); + free(core_rm_ctx); +} + +/* RPC entry point. */ +void +vbdev_ocf_core_remove(const char *core_name, vbdev_ocf_core_remove_cb rpc_cb_fn, void *rpc_cb_arg) +{ + struct vbdev_ocf_core *core; + struct vbdev_ocf_core_remove_ctx *core_rm_ctx; + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating removal\n", core_name); + + core = vbdev_ocf_core_get_by_name(core_name); + if (!core) { + SPDK_ERRLOG("OCF core '%s': device not found\n", core_name); + rpc_cb_fn(rpc_cb_arg, -ENODEV); + return; + } + + /* If core was not added yet due to lack of base or cache device, + * just free its structs (and detach its base if exists) and exit. */ + if (vbdev_ocf_core_is_incomplete(core)) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': removing as incomplete\n", core->name); + if (core->base.attached) { + vbdev_ocf_core_base_detach(core); + } + vbdev_ocf_core_remove_incomplete(core); + vbdev_ocf_core_destroy(core); + rpc_cb_fn(rpc_cb_arg, 0); + return; + } + + core_rm_ctx = calloc(1, sizeof(struct vbdev_ocf_core_remove_ctx)); + if (!core_rm_ctx) { + SPDK_ERRLOG("OCF core '%s': failed to allocate memory for core remove context\n", core_name); + rpc_cb_fn(rpc_cb_arg, -ENOMEM); + return; + } + core_rm_ctx->core = core; + core_rm_ctx->rpc_cb_fn = rpc_cb_fn; + core_rm_ctx->rpc_cb_arg = rpc_cb_arg; + + if ((rc = vbdev_ocf_core_unregister(core, _core_unregister_core_rm_cb, core_rm_ctx))) { + SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during core removal\n", + core->name); + rpc_cb_fn(rpc_cb_arg, rc); + free(core_rm_ctx); + } +} + +static void +_write_cache_info_begin(struct spdk_json_write_ctx *w, struct vbdev_ocf_cache *cache) +{ + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "type", "OCF_cache"); + spdk_json_write_named_string(w, "name", cache->name); + spdk_json_write_named_string(w, "base_bdev_name", + cache->base.bdev ? spdk_bdev_get_name(cache->base.bdev) : "" ); + spdk_json_write_named_uint16(w, "cores_count", cache->cores_count); +} + +static void +_write_cache_info_end(struct spdk_json_write_ctx *w, struct vbdev_ocf_cache *cache) +{ + spdk_json_write_object_end(w); +} + +static void +_write_core_info(struct spdk_json_write_ctx *w, struct vbdev_ocf_core *core) +{ + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "type", "OCF_core"); + spdk_json_write_named_string(w, "name", core->name); + spdk_json_write_named_string(w, "base_bdev_name", + core->base.bdev ? spdk_bdev_get_name(core->base.bdev) : "" ); + spdk_json_write_named_string(w, "cache_name", vbdev_ocf_core_get_cache(core)->name); + spdk_json_write_object_end(w); +} + +/* RPC entry point. */ +void +vbdev_ocf_get_bdevs(const char *name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2) +{ + struct spdk_json_write_ctx *w = rpc_cb_arg1; + struct vbdev_ocf_cache *cache; + struct vbdev_ocf_core *core; + bool found = false; + + if (name) { + vbdev_ocf_foreach_cache(cache) { + vbdev_ocf_foreach_core_in_cache(core, cache) { + if (strcmp(name, core->name)) { + continue; + } + found = true; + + // dump_info_json() instead? + _write_core_info(w, core); + break; + } + if (found) { + break; + } + if (strcmp(name, cache->name)) { + continue; + } + + _write_cache_info_begin(w, cache); + _write_cache_info_end(w, cache); + break; + } + } else { + vbdev_ocf_foreach_cache(cache) { + _write_cache_info_begin(w, cache); + spdk_json_write_named_array_begin(w, "cores"); + vbdev_ocf_foreach_core_in_cache(core, cache) { + _write_core_info(w, core); + } + spdk_json_write_array_end(w); + _write_cache_info_end(w, cache); + } + } + + rpc_cb_fn(rpc_cb_arg1, rpc_cb_arg2); +} + +SPDK_LOG_REGISTER_COMPONENT(vbdev_ocf) diff --git a/module/bdev/ocf/vbdev_ocf.h b/module/bdev/ocf/vbdev_ocf.h new file mode 100644 index 00000000000..48db95a29e4 --- /dev/null +++ b/module/bdev/ocf/vbdev_ocf.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (C) 2025 Huawei Technologies + * All rights reserved. + */ + +#ifndef SPDK_VBDEV_OCF_H +#define SPDK_VBDEV_OCF_H + +#include "vbdev_ocf_cache.h" +#include "vbdev_ocf_core.h" + +typedef void (*vbdev_ocf_cache_start_cb)(struct vbdev_ocf_cache *vbdev_cache, void *cb_arg, int error); +typedef void (*vbdev_ocf_cache_stop_cb)(void *cb_arg, int error); +typedef void (*vbdev_ocf_core_add_cb)(struct vbdev_ocf_core *vbdev_core, void *cb_arg, int error); +typedef void (*vbdev_ocf_core_remove_cb)(void *cb_arg, int error); +typedef void (*vbdev_ocf_get_bdevs_cb)(void *cb_arg1, void *cb_arg2); + +// merge into one for all start/stop/add/remove ? +struct vbdev_ocf_cache_start_ctx { + struct vbdev_ocf_cache * cache; + vbdev_ocf_cache_start_cb rpc_cb_fn; + void * rpc_cb_arg; +}; + +struct vbdev_ocf_cache_stop_ctx { + struct vbdev_ocf_cache * cache; + vbdev_ocf_cache_stop_cb rpc_cb_fn; + void * rpc_cb_arg; +}; + +struct vbdev_ocf_core_add_ctx { + struct vbdev_ocf_cache * cache; + struct vbdev_ocf_core * core; + vbdev_ocf_core_add_cb rpc_cb_fn; + void * rpc_cb_arg; +}; + +struct vbdev_ocf_core_remove_ctx { + struct vbdev_ocf_core * core; + vbdev_ocf_core_remove_cb rpc_cb_fn; + void * rpc_cb_arg; +}; + +/* RPC entry point. */ +void vbdev_ocf_cache_start(const char *cache_name, + const char *bdev_name, + const char *cache_mode, + const uint8_t cache_line_size, + vbdev_ocf_cache_start_cb cb_fn, + void *cb_arg); + +/* RPC entry point. */ +void vbdev_ocf_cache_stop(const char *cache_name, + vbdev_ocf_cache_stop_cb cb_fn, + void *cb_arg); + +/* RPC entry point. */ +void vbdev_ocf_core_add(const char *core_name, + const char *bdev_name, + const char *cache_name, + vbdev_ocf_core_add_cb cb_fn, + void *cb_arg); + +/* RPC entry point. */ +void vbdev_ocf_core_remove(const char *core_name, + vbdev_ocf_core_remove_cb cb_fn, + void *cb_arg); + +/* RPC entry point. */ +void vbdev_ocf_get_bdevs(const char *name, + vbdev_ocf_get_bdevs_cb cb_fn, + void *cb_arg1, + void *cb_arg2); + +#endif diff --git a/module/bdev/ocf/vbdev_ocf_cache.c b/module/bdev/ocf/vbdev_ocf_cache.c new file mode 100644 index 00000000000..0f21a66be49 --- /dev/null +++ b/module/bdev/ocf/vbdev_ocf_cache.c @@ -0,0 +1,308 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (C) 2025 Huawei Technologies + * All rights reserved. + */ + +#include "vbdev_ocf_cache.h" +#include "ctx.h" +#include "utils.h" + +struct vbdev_ocf_caches_head g_vbdev_ocf_caches = STAILQ_HEAD_INITIALIZER(g_vbdev_ocf_caches); + +int +vbdev_ocf_cache_create(const char *cache_name, struct vbdev_ocf_cache **out) +{ + struct vbdev_ocf_cache *cache; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': allocating vbdev_ocf_cache and adding it to cache list\n", + cache_name); + + cache = calloc(1, sizeof(struct vbdev_ocf_cache)); + if (!cache) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for vbdev_ocf_cache\n", + cache_name); + return -ENOMEM; + } + + cache->name = strdup(cache_name); + if (!cache->name) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for vbdev_ocf_cache name\n", + cache_name); + free(cache); + return -ENOMEM; + } + + STAILQ_INIT(&cache->cores); + STAILQ_INSERT_TAIL(&g_vbdev_ocf_caches, cache, link); + + *out = cache; + + return rc; +} + +void +vbdev_ocf_cache_destroy(struct vbdev_ocf_cache *cache) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': deallocating vbdev_ocf_cache and removing it from cache list\n", + cache->name); + + STAILQ_REMOVE(&g_vbdev_ocf_caches, cache, vbdev_ocf_cache, link); + free(cache->name); + free(cache); +} + +int +vbdev_ocf_cache_set_config(struct vbdev_ocf_cache *cache, const char *cache_mode, + const uint8_t cache_line_size) +{ + struct ocf_mngt_cache_config *ocf_cache_cfg = &cache->ocf_cache_cfg; + struct ocf_mngt_cache_attach_config *ocf_cache_att_cfg = &cache->ocf_cache_att_cfg; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': setting OCF config\n", cache->name); + + ocf_mngt_cache_config_set_default(ocf_cache_cfg); + ocf_mngt_cache_attach_config_set_default(ocf_cache_att_cfg); + + strncpy(ocf_cache_cfg->name, cache->name, OCF_CACHE_NAME_SIZE); + if (cache_mode) { + ocf_cache_cfg->cache_mode = ocf_get_cache_mode(cache_mode); + } + if (cache_line_size) { + ocf_cache_cfg->cache_line_size = cache_line_size * KiB; + ocf_cache_att_cfg->cache_line_size = cache_line_size * KiB; + } + ocf_cache_cfg->locked = true; + ocf_cache_att_cfg->open_cores = false; + ocf_cache_att_cfg->discard_on_start = false; // needed ? + ocf_cache_att_cfg->device.perform_test = false; // needed ? + // TODO: add load option + ocf_cache_att_cfg->force = true; + + return rc; // rm? +} + +// vbdev_ocf_cache_detach() instead of this ? +static void +vbdev_ocf_cache_hotremove(struct spdk_bdev *bdev, void *event_ctx) +{ + struct vbdev_ocf_cache *cache = event_ctx; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': hot removal of base bdev '%s'\n", + cache->name, bdev->name); + + assert(bdev == cache->base.bdev); + + if (vbdev_ocf_cache_is_running(cache)) { + // OCF cache flush + // OCF cache detach + // (to comply with SPDK hotremove support) + } + + vbdev_ocf_cache_base_detach(cache); +} + +static void +_vbdev_ocf_cache_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) +{ + struct vbdev_ocf_cache *cache = event_ctx; + + switch (type) { + case SPDK_BDEV_EVENT_REMOVE: + vbdev_ocf_cache_hotremove(bdev, event_ctx); + break; + default: + SPDK_NOTICELOG("OCF cache '%s': unsupported bdev event type: %d\n", cache->name, type); + } +} + +int +vbdev_ocf_cache_base_attach(struct vbdev_ocf_cache *cache, const char *bdev_name) +{ + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': attaching base bdev '%s'\n", + cache->name, bdev_name); + + if ((rc = spdk_bdev_open_ext(bdev_name, true, _vbdev_ocf_cache_event_cb, cache, &cache->base.desc))) { + return rc; + } + + if ((rc = spdk_bdev_module_claim_bdev_desc(cache->base.desc, + SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, + NULL, &ocf_if))) { + SPDK_ERRLOG("OCF cache '%s': failed to claim base bdev '%s'\n", cache->name, bdev_name); + spdk_bdev_close(cache->base.desc); + return rc; + } + + cache->base.mngt_ch = spdk_bdev_get_io_channel(cache->base.desc); + if (!cache->base.mngt_ch) { + SPDK_ERRLOG("OCF cache '%s': failed to get IO channel for base bdev '%s'\n", + cache->name, bdev_name); + spdk_bdev_close(cache->base.desc); + return -ENOMEM; + } + + cache->base.bdev = spdk_bdev_desc_get_bdev(cache->base.desc); + cache->base.thread = spdk_get_thread(); + cache->base.is_cache = true; + cache->base.attached = true; + + // why not ? what is it then ?!? + //assert(__bdev_to_io_dev(cache->base.bdev) == cache->base.bdev->ctxt); + + return rc; +} + +void +vbdev_ocf_cache_base_detach(struct vbdev_ocf_cache *cache) +{ + struct vbdev_ocf_base *base = &cache->base; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': detaching base bdev '%s'\n", + cache->name, base->bdev->name); + + vbdev_ocf_base_detach(base); +} + +int +vbdev_ocf_cache_add_incomplete(struct vbdev_ocf_cache *cache, const char *bdev_name) +{ + struct vbdev_ocf_cache_init_params *init_params; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': allocating init params\n", cache->name); + + init_params = calloc(1, sizeof(struct vbdev_ocf_cache_init_params)); + if (!init_params) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for init params\n", + cache->name); + return -ENOMEM; + } + + init_params->bdev_name = strdup(bdev_name); + if (!init_params->bdev_name) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for base bdev name\n", + cache->name); + free(init_params); + return -ENOMEM; + } + + cache->init_params = init_params; + + return 0; +} + +void +vbdev_ocf_cache_remove_incomplete(struct vbdev_ocf_cache *cache) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': deallocating init params\n", cache->name); + + free(cache->init_params->bdev_name); + free(cache->init_params); + /* It's important to set init_params back to NULL. + * This is an indicator that cache is not incomplete any more. */ + cache->init_params = NULL; +} + +static void +_cache_mngt_queue_stop(void *ctx) +{ + struct vbdev_ocf_cache_mngt_queue_ctx *mngt_q_ctx = ctx; + + spdk_poller_unregister(&mngt_q_ctx->poller); + free(mngt_q_ctx); +} + +static void +vbdev_ocf_cache_mngt_queue_stop(ocf_queue_t queue) +{ + struct vbdev_ocf_cache_mngt_queue_ctx *mngt_q_ctx = ocf_queue_get_priv(queue); + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': destroying OCF management queue\n", + mngt_q_ctx->cache->name); + + if (mngt_q_ctx->thread && mngt_q_ctx->thread != spdk_get_thread()) { + spdk_thread_send_msg(mngt_q_ctx->thread, _cache_mngt_queue_stop, mngt_q_ctx); + } else { + _cache_mngt_queue_stop(mngt_q_ctx); + } +} + +static void +vbdev_ocf_cache_mngt_queue_kick(ocf_queue_t queue) +{ +} + +const struct ocf_queue_ops cache_mngt_queue_ops = { + .kick_sync = NULL, + .kick = vbdev_ocf_cache_mngt_queue_kick, + .stop = vbdev_ocf_cache_mngt_queue_stop, +}; + +int +vbdev_ocf_cache_mngt_queue_create(struct vbdev_ocf_cache *cache) +{ + struct vbdev_ocf_cache_mngt_queue_ctx *mngt_q_ctx; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': creating OCF management queue\n", cache->name); + + mngt_q_ctx = calloc(1, sizeof(struct vbdev_ocf_cache_mngt_queue_ctx)); + if (!mngt_q_ctx) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for management queue context\n", + cache->name); + return -ENOMEM; + } + + if ((rc = vbdev_ocf_queue_create_mngt(cache->ocf_cache, &cache->ocf_cache_mngt_q, &cache_mngt_queue_ops))) { + SPDK_ERRLOG("OCF cache '%s': failed to create OCF management queue\n", cache->name); + free(mngt_q_ctx); + return rc; + } + ocf_queue_set_priv(cache->ocf_cache_mngt_q, mngt_q_ctx); + + mngt_q_ctx->poller = SPDK_POLLER_REGISTER(vbdev_ocf_queue_poller, cache->ocf_cache_mngt_q, 1000); + if (!mngt_q_ctx->poller) { + SPDK_ERRLOG("OCF cache '%s': failed to create management queue poller\n", cache->name); + vbdev_ocf_queue_put(cache->ocf_cache_mngt_q); + return -ENOMEM; + } + + mngt_q_ctx->cache = cache; // keep? (only for DEBUGLOG) + mngt_q_ctx->thread = spdk_get_thread(); + + return rc; +} + +struct vbdev_ocf_cache * +vbdev_ocf_cache_get_by_name(const char *cache_name) +{ + struct vbdev_ocf_cache *cache; + + vbdev_ocf_foreach_cache(cache) { + if (strcmp(cache_name, cache->name)) { + continue; + } + return cache; + } + return NULL; +} + +bool +vbdev_ocf_cache_is_running(struct vbdev_ocf_cache *cache) +{ + ocf_cache_t ocf_cache = cache->ocf_cache; + + if (ocf_cache && ocf_cache_is_running(ocf_cache)) { + return true; + } + return false; +} + +bool +vbdev_ocf_cache_is_incomplete(struct vbdev_ocf_cache *cache) +{ + return !!cache->init_params; +} diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h new file mode 100644 index 00000000000..8a7afd0408d --- /dev/null +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (C) 2025 Huawei Technologies + * All rights reserved. + */ + +#ifndef SPDK_VBDEV_OCF_CACHE_H +#define SPDK_VBDEV_OCF_CACHE_H + +#include +#include "spdk/bdev_module.h" +#include "volume.h" + +/* Global list of all caches (started and incomplete). */ +extern STAILQ_HEAD(vbdev_ocf_caches_head, vbdev_ocf_cache) g_vbdev_ocf_caches; + +/* OCF module interface. */ +extern struct spdk_bdev_module ocf_if; + +#define vbdev_ocf_foreach_cache(cache) \ + STAILQ_FOREACH(cache, &g_vbdev_ocf_caches, link) + +struct vbdev_ocf_cache_init_params { + char * bdev_name; +}; + +struct vbdev_ocf_cache { + char * name; + struct spdk_uuid uuid; + + struct vbdev_ocf_base base; + + STAILQ_HEAD(, vbdev_ocf_core) cores; + uint16_t cores_count; + + ocf_cache_t ocf_cache; + ocf_queue_t ocf_cache_mngt_q; + struct ocf_mngt_cache_config ocf_cache_cfg; + struct ocf_mngt_cache_attach_config ocf_cache_att_cfg; + + struct vbdev_ocf_cache_init_params * init_params; + + STAILQ_ENTRY(vbdev_ocf_cache) link; +}; + +struct vbdev_ocf_cache_mngt_queue_ctx { + struct spdk_poller * poller; + struct spdk_thread * thread; + /* Currently kept only for its name used in debug log. */ + struct vbdev_ocf_cache * cache; // rm ? +}; + +int vbdev_ocf_cache_create(const char *cache_name, struct vbdev_ocf_cache **out); +void vbdev_ocf_cache_destroy(struct vbdev_ocf_cache *cache); +int vbdev_ocf_cache_set_config(struct vbdev_ocf_cache *cache, const char *cache_mode, + const uint8_t cache_line_size); +int vbdev_ocf_cache_base_attach(struct vbdev_ocf_cache *cache, const char *bdev_name); +void vbdev_ocf_cache_base_detach(struct vbdev_ocf_cache *cache); +int vbdev_ocf_cache_add_incomplete(struct vbdev_ocf_cache *cache, const char *bdev_name); +void vbdev_ocf_cache_remove_incomplete(struct vbdev_ocf_cache *cache); +int vbdev_ocf_cache_mngt_queue_create(struct vbdev_ocf_cache *cache); + +struct vbdev_ocf_cache *vbdev_ocf_cache_get_by_name(const char *cache_name); +bool vbdev_ocf_cache_is_running(struct vbdev_ocf_cache *cache); +bool vbdev_ocf_cache_is_incomplete(struct vbdev_ocf_cache *cache); + +#endif diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c new file mode 100644 index 00000000000..ff4344dcb7e --- /dev/null +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -0,0 +1,462 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (C) 2025 Huawei Technologies + * All rights reserved. + */ + +#include "spdk/string.h" // rm ? + +#include "vbdev_ocf_core.h" +#include "vbdev_ocf_cache.h" +#include "ctx.h" + +struct vbdev_ocf_incomplete_cores_head g_vbdev_ocf_incomplete_cores = + STAILQ_HEAD_INITIALIZER(g_vbdev_ocf_incomplete_cores); + +int +vbdev_ocf_core_create(const char *core_name, struct vbdev_ocf_core **out) +{ + struct vbdev_ocf_core *core; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': allocating vbdev_ocf_core and adding it to core list\n", + core_name); + + core = calloc(1, sizeof(struct vbdev_ocf_core)); + if (!core) { + SPDK_ERRLOG("OCF core '%s': failed to allocate memory for vbdev_ocf_core\n", + core_name); + return -ENOMEM; + } + + core->name = strdup(core_name); + if (!core->name) { + SPDK_ERRLOG("OCF core '%s': failed to allocate memory for vbdev_ocf_core name\n", + core_name); + free(core); + return -ENOMEM; + } + + *out = core; + + return 0; +} + +void +vbdev_ocf_core_destroy(struct vbdev_ocf_core *core) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': deallocating vbdev_ocf_core\n", + core->name); + + free(core->name); + free(core); +} + +int +vbdev_ocf_core_set_config(struct vbdev_ocf_core *core) +{ + struct ocf_mngt_core_config *ocf_core_cfg = &core->ocf_core_cfg; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': setting OCF config\n", core->name); + + ocf_mngt_core_config_set_default(ocf_core_cfg); + + strncpy(ocf_core_cfg->name, core->name, OCF_CORE_NAME_SIZE); + if ((rc = ocf_uuid_set_str(&ocf_core_cfg->uuid, core->name))) { // set spdk uuid ? + return rc; + } + ocf_core_cfg->volume_type = SPDK_OBJECT; + + return rc; +} + +static void +_core_unregister_core_hotrm_cb(void *cb_arg, int error) +{ + struct vbdev_ocf_core *core = cb_arg; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", + core->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing hot removal of base bdev\n", + core->name); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during hot removal: %s\n", + core->name, spdk_strerror(-error)); + } else { + /* Do not destroy core struct in hotremove; it will be needed in examine. + * Just indicate that it's not connected to cache anymore. */ + core->ocf_core = NULL; + } +} + +static void +vbdev_ocf_core_hotremove(struct spdk_bdev *bdev, void *event_ctx) +{ + struct vbdev_ocf_core *core = event_ctx; + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating hot removal of base bdev '%s'\n", + core->name, bdev->name); + + assert(bdev == core->base.bdev); + + if (vbdev_ocf_core_is_incomplete(core)) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': hot removing as incomplete\n", core->name); + vbdev_ocf_core_base_detach(core); + return; + } + + if ((rc = vbdev_ocf_core_unregister(core, _core_unregister_core_hotrm_cb, core))) { + SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during core hot removal: %s\n", + core->name, spdk_strerror(-rc)); + // detach base despite the error ? + vbdev_ocf_core_base_detach(core); + return; + } +} + +static void +_vbdev_ocf_core_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) +{ + struct vbdev_ocf_core *core = event_ctx; + + switch (type) { + case SPDK_BDEV_EVENT_REMOVE: + vbdev_ocf_core_hotremove(bdev, event_ctx); + break; + default: + SPDK_NOTICELOG("OCF core '%s': unsupported bdev event type: %d\n", core->name, type); + } +} + +int +vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core, const char *bdev_name) +{ + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': attaching base bdev '%s'\n", + core->name, bdev_name); + + if ((rc = spdk_bdev_open_ext(bdev_name, true, _vbdev_ocf_core_event_cb, core, &core->base.desc))) { + return rc; + } + + if ((rc = spdk_bdev_module_claim_bdev_desc(core->base.desc, + SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, + NULL, &ocf_if))) { + SPDK_ERRLOG("OCF core '%s': failed to claim base bdev '%s'\n", core->name, bdev_name); + spdk_bdev_close(core->base.desc); + return rc; + } + + core->base.mngt_ch = spdk_bdev_get_io_channel(core->base.desc); + if (!core->base.mngt_ch) { + SPDK_ERRLOG("OCF core '%s': failed to get IO channel for base bdev '%s'\n", + core->name, bdev_name); + spdk_bdev_close(core->base.desc); + return -ENOMEM; + } + + core->base.bdev = spdk_bdev_desc_get_bdev(core->base.desc); + core->base.thread = spdk_get_thread(); + core->base.is_cache = false; + core->base.attached = true; + + // why not ? what is it then ?!? + //assert(__bdev_to_io_dev(core->base.bdev) == core->base.bdev->ctxt); + + return rc; +} + +void +vbdev_ocf_core_base_detach(struct vbdev_ocf_core *core) +{ + struct vbdev_ocf_base *base = &core->base; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': detaching base bdev '%s'\n", + core->name, base->bdev->name); + + vbdev_ocf_base_detach(base); +} + +int +vbdev_ocf_core_add_incomplete(struct vbdev_ocf_core *core, const char *bdev_name, + const char *cache_name) +{ + struct vbdev_ocf_core_init_params *init_params; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': allocating init params and adding core to incomplete list\n", + core->name); + + init_params = calloc(1, sizeof(struct vbdev_ocf_core_init_params)); + if (!init_params) { + SPDK_ERRLOG("OCF core '%s': failed to allocate memory for init params\n", + core->name); + return -ENOMEM; + } + + init_params->bdev_name = strdup(bdev_name); + if (!init_params->bdev_name) { + SPDK_ERRLOG("OCF core '%s': failed to allocate memory for base bdev name\n", + core->name); + free(init_params); + return -ENOMEM; + } + + init_params->cache_name = strdup(cache_name); + if (!init_params->cache_name) { + SPDK_ERRLOG("OCF core '%s': failed to allocate memory for cache bdev name\n", + core->name); + free(init_params->bdev_name); + free(init_params); + return -ENOMEM; + } + + core->init_params = init_params; + STAILQ_INSERT_TAIL(&g_vbdev_ocf_incomplete_cores, core, link); + + return 0; +} + +void +vbdev_ocf_core_remove_incomplete(struct vbdev_ocf_core *core) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': deallocating init params and removing core from incomplete list\n", + core->name); + + free(core->init_params->bdev_name); + free(core->init_params->cache_name); + free(core->init_params); + /* It's important to set init_params back to NULL. + * This is an indicator that core is not incomplete any more. */ + core->init_params = NULL; + STAILQ_REMOVE(&g_vbdev_ocf_incomplete_cores, core, vbdev_ocf_core, link); +} + +void +vbdev_ocf_core_add_to_cache(struct vbdev_ocf_core *core, struct vbdev_ocf_cache *cache) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': adding core to list in cache\n", core->name); + + core->cache = cache; + cache->cores_count++; + STAILQ_INSERT_TAIL(&cache->cores, core, link); +} + +void +vbdev_ocf_core_remove_from_cache(struct vbdev_ocf_core *core) +{ + struct vbdev_ocf_cache *cache; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': removing core from list in cache\n", core->name); + + cache = vbdev_ocf_core_get_cache(core); + cache->cores_count--; + STAILQ_REMOVE(&cache->cores, core, vbdev_ocf_core, link); +} + +static void +_core_io_queue_stop(void *ctx) +{ + struct vbdev_ocf_core_io_channel_ctx *ch_ctx = ctx; + + spdk_poller_unregister(&ch_ctx->poller); + spdk_put_io_channel(ch_ctx->cache_ch); + spdk_put_io_channel(ch_ctx->core_ch); + free(ch_ctx); +} + +static void +vbdev_ocf_core_io_queue_stop(ocf_queue_t queue) +{ + struct vbdev_ocf_core_io_channel_ctx *ch_ctx = ocf_queue_get_priv(queue); + + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': deallocating external IO channel context\n", + ch_ctx->core->ocf_vbdev.name); + + if (ch_ctx->thread && ch_ctx->thread != spdk_get_thread()) { + spdk_thread_send_msg(ch_ctx->thread, _core_io_queue_stop, ch_ctx); + } else { + _core_io_queue_stop(ch_ctx); + } +} + +static void +vbdev_ocf_core_io_queue_kick(ocf_queue_t queue) +{ +} + +const struct ocf_queue_ops core_io_queue_ops = { + .kick_sync = NULL, + .kick = vbdev_ocf_core_io_queue_kick, + .stop = vbdev_ocf_core_io_queue_stop, +}; + +static int +_vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) +{ + struct vbdev_ocf_core *core = io_device; + struct vbdev_ocf_cache *cache = vbdev_ocf_core_get_cache(core); + struct vbdev_ocf_core_io_channel_ctx *ch_destroy_ctx = ctx_buf; + struct vbdev_ocf_core_io_channel_ctx *ch_ctx; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': creating IO channel and allocating external context\n", + core->ocf_vbdev.name); + + /* Do not use provided buffer for IO channel context, as it will be freed + * when this channel is destroyed. Instead allocate our own and keep it + * in queue priv. It will be needed later, after the channel was closed, + * as it may be referenced by backfill. */ + ch_ctx = calloc(1, sizeof(struct vbdev_ocf_core_io_channel_ctx)); + if (!ch_ctx) { + SPDK_ERRLOG("OCF vbdev '%s': failed to allocate memory for IO channel context\n", + core->ocf_vbdev.name); + return -ENOMEM; + } + + if ((rc = vbdev_ocf_queue_create(cache->ocf_cache, &ch_ctx->queue, &core_io_queue_ops))) { + SPDK_ERRLOG("OCF vbdev '%s': failed to create OCF queue\n", core->ocf_vbdev.name); + free(ch_ctx); + return rc; + } + ocf_queue_set_priv(ch_ctx->queue, ch_ctx); + + ch_ctx->cache_ch = spdk_bdev_get_io_channel(cache->base.desc); + if (!ch_ctx->cache_ch) { + SPDK_ERRLOG("OCF vbdev '%s': failed to create IO channel for base bdev '%s'\n", + core->ocf_vbdev.name, cache->base.bdev->name); + vbdev_ocf_queue_put(ch_ctx->queue); + return -ENOMEM; + } + + ch_ctx->core_ch = spdk_bdev_get_io_channel(core->base.desc); + if (!ch_ctx->core_ch) { + SPDK_ERRLOG("OCF vbdev '%s': failed to create IO channel for base bdev '%s'\n", + core->ocf_vbdev.name, core->base.bdev->name); + spdk_put_io_channel(ch_ctx->cache_ch); + vbdev_ocf_queue_put(ch_ctx->queue); + return -ENOMEM; + } + + ch_ctx->poller = SPDK_POLLER_REGISTER(vbdev_ocf_queue_poller, ch_ctx->queue, 0); + if (!ch_ctx->poller) { + SPDK_ERRLOG("OCF vbdev '%s': failed to create IO queue poller\n", core->ocf_vbdev.name); + spdk_put_io_channel(ch_ctx->core_ch); + spdk_put_io_channel(ch_ctx->cache_ch); + vbdev_ocf_queue_put(ch_ctx->queue); + return -ENOMEM; + } + + ch_ctx->core = core; // keep? (only for DEBUGLOG) + ch_ctx->thread = spdk_get_thread(); + + /* Save queue pointer in buffer provided by the IO channel callback. + * Only this will be needed in channel destroy callback to decrement + * the refcount. The rest is freed in queue stop callback. */ + ch_destroy_ctx->queue = ch_ctx->queue; + + return rc; +} + +static void +_vbdev_ocf_ch_destroy_cb(void *io_device, void *ctx_buf) +{ + struct vbdev_ocf_core *core = io_device; + struct vbdev_ocf_core_io_channel_ctx *ch_destroy_ctx = ctx_buf; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': destroying IO channel\n", core->ocf_vbdev.name); + + vbdev_ocf_queue_put(ch_destroy_ctx->queue); +} + +int +vbdev_ocf_core_register(struct vbdev_ocf_core *core) +{ + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': registering OCF vbdev in SPDK bdev layer\n", core->name); + + core->ocf_vbdev.ctxt = core; + core->ocf_vbdev.name = core->name; + core->ocf_vbdev.product_name = "OCF_disk"; + core->ocf_vbdev.write_cache = core->base.bdev->write_cache; + core->ocf_vbdev.blocklen = core->base.bdev->blocklen; + core->ocf_vbdev.blockcnt = core->base.bdev->blockcnt; + // ? + //core->ocf_vbdev.required_alignment = core->base.bdev->required_alignment; + //core->ocf_vbdev.optimal_io_boundary = core->base.bdev->optimal_io_boundary; + // generate UUID based on namespace UUID + base bdev UUID (take from old module?) + core->ocf_vbdev.fn_table = &vbdev_ocf_fn_table; + core->ocf_vbdev.module = &ocf_if; + + spdk_io_device_register(core, _vbdev_ocf_ch_create_cb, _vbdev_ocf_ch_destroy_cb, + sizeof(struct vbdev_ocf_core_io_channel_ctx), core->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': io_device created at 0x%p\n", core->ocf_vbdev.name, core); + + if ((rc = spdk_bdev_register(&core->ocf_vbdev))) { // needs to be called from SPDK app thread + SPDK_ERRLOG("OCF vbdev '%s': failed to register SPDK bdev\n", core->ocf_vbdev.name); + spdk_io_device_unregister(core, NULL); + return rc; + } + + return rc; +} + +int +vbdev_ocf_core_unregister(struct vbdev_ocf_core *core, spdk_bdev_unregister_cb cb_fn, void *cb_arg) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating unregister of OCF vbdev\n", core->name); + + return spdk_bdev_unregister_by_name(core->ocf_vbdev.name, &ocf_if, cb_fn, cb_arg); +} + +struct vbdev_ocf_cache * +vbdev_ocf_core_get_cache(struct vbdev_ocf_core *core) +{ + return core->cache; +} + +struct vbdev_ocf_core * +vbdev_ocf_core_get_by_name(const char *core_name) +{ + struct vbdev_ocf_cache *cache; + struct vbdev_ocf_core *core; + + vbdev_ocf_foreach_core_incomplete(core) { + if (strcmp(core_name, core->name)) { + continue; + } + return core; + } + vbdev_ocf_foreach_cache(cache) { + vbdev_ocf_foreach_core_in_cache(core, cache) { + if (strcmp(core_name, core->name)) { + continue; + } + return core; + } + } + return NULL; +} + +bool +vbdev_ocf_core_cache_is_started(struct vbdev_ocf_core *core) +{ + ocf_core_t ocf_core = core->ocf_core; + ocf_cache_t ocf_cache; + + if (ocf_core) { + ocf_cache = ocf_core_get_cache(ocf_core); + if (ocf_cache_is_running(ocf_cache) || ocf_cache_is_detached(ocf_cache)) { // smth else ? + return true; + } + } + return false; +} + +bool +vbdev_ocf_core_is_incomplete(struct vbdev_ocf_core *core) +{ + return !!core->init_params; +} diff --git a/module/bdev/ocf/vbdev_ocf_core.h b/module/bdev/ocf/vbdev_ocf_core.h new file mode 100644 index 00000000000..a868ad87d9a --- /dev/null +++ b/module/bdev/ocf/vbdev_ocf_core.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (C) 2025 Huawei Technologies + * All rights reserved. + */ + +#ifndef SPDK_VBDEV_OCF_CORE_H +#define SPDK_VBDEV_OCF_CORE_H + +#include +#include "spdk/bdev_module.h" +#include "volume.h" + +/* Global list of all incomplete cores (not added to + * the cache yet due to lack of cache or base device. */ +extern STAILQ_HEAD(vbdev_ocf_incomplete_cores_head, vbdev_ocf_core) g_vbdev_ocf_incomplete_cores; + +/* OCF module interface. */ +extern struct spdk_bdev_module ocf_if; + +/* Function table of exposed OCF vbdev. */ +extern struct spdk_bdev_fn_table vbdev_ocf_fn_table; + +#define vbdev_ocf_foreach_core_incomplete(core) \ + STAILQ_FOREACH(core, &g_vbdev_ocf_incomplete_cores, link) + +#define vbdev_ocf_foreach_core_in_cache(core, cache) \ + STAILQ_FOREACH(core, &cache->cores, link) + +struct vbdev_ocf_core_init_params { + char * bdev_name; + char * cache_name; +}; + +struct vbdev_ocf_core { + char * name; + struct spdk_uuid uuid; + + struct vbdev_ocf_base base; + + struct vbdev_ocf_cache * cache; + + ocf_core_t ocf_core; + struct ocf_mngt_core_config ocf_core_cfg; + + /* Exposed OCF vbdev; the one which is registered in bdev layer for usage */ + struct spdk_bdev ocf_vbdev; + + struct vbdev_ocf_core_init_params * init_params; + + STAILQ_ENTRY(vbdev_ocf_core) link; +}; + +struct vbdev_ocf_core_io_channel_ctx { + ocf_queue_t queue; + struct spdk_io_channel * cache_ch; + struct spdk_io_channel * core_ch; + struct spdk_poller * poller; + struct spdk_thread * thread; + /* Currently kept only for its name used in debug log. */ + struct vbdev_ocf_core * core; // rm ? +}; + +int vbdev_ocf_core_create(const char *core_name, struct vbdev_ocf_core **out); +void vbdev_ocf_core_destroy(struct vbdev_ocf_core *core); +int vbdev_ocf_core_set_config(struct vbdev_ocf_core *core); +int vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core, const char *bdev_name); +void vbdev_ocf_core_base_detach(struct vbdev_ocf_core *core); +int vbdev_ocf_core_add_incomplete(struct vbdev_ocf_core *core, const char *bdev_name, + const char *cache_name); +void vbdev_ocf_core_remove_incomplete(struct vbdev_ocf_core *core); +void vbdev_ocf_core_add_to_cache(struct vbdev_ocf_core *core, struct vbdev_ocf_cache *cache); +void vbdev_ocf_core_remove_from_cache(struct vbdev_ocf_core *core); +int vbdev_ocf_core_register(struct vbdev_ocf_core *core); +int vbdev_ocf_core_unregister(struct vbdev_ocf_core *core, spdk_bdev_unregister_cb cb_fn, void *cb_arg); + +struct vbdev_ocf_cache *vbdev_ocf_core_get_cache(struct vbdev_ocf_core *core); +struct vbdev_ocf_core *vbdev_ocf_core_get_by_name(const char *core_name); +bool vbdev_ocf_core_cache_is_started(struct vbdev_ocf_core *core); +bool vbdev_ocf_core_is_incomplete(struct vbdev_ocf_core *core); + +#endif diff --git a/module/bdev/ocf/vbdev_ocf_rpc.c b/module/bdev/ocf/vbdev_ocf_rpc.c new file mode 100644 index 00000000000..45a2ff68ef5 --- /dev/null +++ b/module/bdev/ocf/vbdev_ocf_rpc.c @@ -0,0 +1,313 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (C) 2025 Huawei Technologies + * All rights reserved. + */ + +#include "vbdev_ocf.h" +#include "spdk/rpc.h" +#include "spdk/string.h" + +struct rpc_bdev_ocf_start_cache { + char *cache_name; + char *bdev_name; + char *cache_mode; + uint8_t cache_line_size; +}; + +static void +free_rpc_bdev_ocf_start_cache(struct rpc_bdev_ocf_start_cache *r) { + free(r->cache_name); + free(r->bdev_name); + free(r->cache_mode); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_start_cache_decoders[] = { + {"cache_name", offsetof(struct rpc_bdev_ocf_start_cache, cache_name), spdk_json_decode_string}, + {"bdev_name", offsetof(struct rpc_bdev_ocf_start_cache, bdev_name), spdk_json_decode_string}, + {"cache_mode", offsetof(struct rpc_bdev_ocf_start_cache, cache_mode), spdk_json_decode_string, true}, + {"cache_line_size", offsetof(struct rpc_bdev_ocf_start_cache, cache_line_size), spdk_json_decode_uint8, true}, +}; + +static void +rpc_bdev_ocf_start_cache_cb(struct vbdev_ocf_cache *vbdev_cache, void *cb_arg, int error) +{ + struct spdk_jsonrpc_request *request = cb_arg; + struct spdk_json_write_ctx *w; + + if (error && error != -ENODEV) { + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Could not start OCF cache: %s", + spdk_strerror(-error)); + return; + } + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_string(w, vbdev_cache->name); + spdk_jsonrpc_end_result(request, w); +} + +static void +rpc_bdev_ocf_start_cache(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_bdev_ocf_start_cache req = {}; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_start_cache_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_start_cache_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parametes"); + goto cleanup; + } + + vbdev_ocf_cache_start(req.cache_name, req.bdev_name, req.cache_mode, req.cache_line_size, + rpc_bdev_ocf_start_cache_cb, request); + +cleanup: + free_rpc_bdev_ocf_start_cache(&req); +} +SPDK_RPC_REGISTER("bdev_ocf_start_cache", rpc_bdev_ocf_start_cache, SPDK_RPC_RUNTIME) + +struct rpc_bdev_ocf_stop_cache { + char *cache_name; +}; + +static void +free_rpc_bdev_ocf_stop_cache(struct rpc_bdev_ocf_stop_cache *r) { + free(r->cache_name); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_stop_cache_decoders[] = { + {"cache_name", offsetof(struct rpc_bdev_ocf_stop_cache, cache_name), spdk_json_decode_string}, +}; + +static void +rpc_bdev_ocf_stop_cache_cb(void *cb_arg, int error) +{ + struct spdk_jsonrpc_request *request = cb_arg; + + if (error) { + spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); +} + +static void +rpc_bdev_ocf_stop_cache(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_bdev_ocf_stop_cache req = {}; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_stop_cache_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_stop_cache_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parametes"); + goto cleanup; + } + + vbdev_ocf_cache_stop(req.cache_name, rpc_bdev_ocf_stop_cache_cb, request); + +cleanup: + free_rpc_bdev_ocf_stop_cache(&req); +} +SPDK_RPC_REGISTER("bdev_ocf_stop_cache", rpc_bdev_ocf_stop_cache, SPDK_RPC_RUNTIME) + +struct rpc_bdev_ocf_add_core { + char *core_name; + char *bdev_name; + char *cache_name; +}; + +static void +free_rpc_bdev_ocf_add_core(struct rpc_bdev_ocf_add_core *r) { + free(r->core_name); + free(r->bdev_name); + free(r->cache_name); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_add_core_decoders[] = { + {"core_name", offsetof(struct rpc_bdev_ocf_add_core, core_name), spdk_json_decode_string}, + {"bdev_name", offsetof(struct rpc_bdev_ocf_add_core, bdev_name), spdk_json_decode_string}, + {"cache_name", offsetof(struct rpc_bdev_ocf_add_core, cache_name), spdk_json_decode_string}, +}; + +static void +rpc_bdev_ocf_add_core_cb(struct vbdev_ocf_core *vbdev_core, void *cb_arg, int error) +{ + struct spdk_jsonrpc_request *request = cb_arg; + struct spdk_json_write_ctx *w; + + if (error && error != -ENODEV) { + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Could not add core to OCF cache: %s", + spdk_strerror(-error)); + return; + } + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_string(w, vbdev_core->name); + spdk_jsonrpc_end_result(request, w); +} + +static void +rpc_bdev_ocf_add_core(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_bdev_ocf_add_core req = {}; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_add_core_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_add_core_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parametes"); + goto cleanup; + } + + vbdev_ocf_core_add(req.core_name, req.bdev_name, req.cache_name, rpc_bdev_ocf_add_core_cb, request); + +cleanup: + free_rpc_bdev_ocf_add_core(&req); +} +SPDK_RPC_REGISTER("bdev_ocf_add_core", rpc_bdev_ocf_add_core, SPDK_RPC_RUNTIME) + +struct rpc_bdev_ocf_remove_core { + char *core_name; +}; + +static void +free_rpc_bdev_ocf_remove_core(struct rpc_bdev_ocf_remove_core *r) { + free(r->core_name); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_remove_core_decoders[] = { + {"core_name", offsetof(struct rpc_bdev_ocf_remove_core, core_name), spdk_json_decode_string}, +}; + +static void +rpc_bdev_ocf_remove_core_cb(void *cb_arg, int error) +{ + struct spdk_jsonrpc_request *request = cb_arg; + + if (error) { + spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); +} + +static void +rpc_bdev_ocf_remove_core(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_bdev_ocf_remove_core req = {}; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_remove_core_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_remove_core_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parametes"); + goto cleanup; + } + + vbdev_ocf_core_remove(req.core_name, rpc_bdev_ocf_remove_core_cb, request); + +cleanup: + free_rpc_bdev_ocf_remove_core(&req); +} +SPDK_RPC_REGISTER("bdev_ocf_remove_core", rpc_bdev_ocf_remove_core, SPDK_RPC_RUNTIME) + +struct rpc_bdev_ocf_get_bdevs { + char *name; +}; + +static void +free_rpc_bdev_ocf_get_bdevs(struct rpc_bdev_ocf_get_bdevs *r) { + free(r->name); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_get_bdevs_decoders[] = { + {"name", offsetof(struct rpc_bdev_ocf_get_bdevs, name), spdk_json_decode_string, true}, +}; + +static void +rpc_bdev_ocf_get_bdevs_cb(void *cb_arg1, void *cb_arg2) +{ + struct spdk_json_write_ctx *w = cb_arg1; + struct spdk_jsonrpc_request *request = cb_arg2; + + spdk_json_write_array_end(w); + spdk_jsonrpc_end_result(request, w); +} + +static void +rpc_bdev_ocf_get_bdevs(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_bdev_ocf_get_bdevs req = {}; + struct spdk_json_write_ctx *w; + + if (params && spdk_json_decode_object(params, rpc_bdev_ocf_get_bdevs_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_get_bdevs_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parametes"); + goto cleanup; + } + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_array_begin(w); + + vbdev_ocf_get_bdevs(params ? req.name : NULL, rpc_bdev_ocf_get_bdevs_cb, w, request); + +cleanup: + free_rpc_bdev_ocf_get_bdevs(&req); +} +SPDK_RPC_REGISTER("bdev_ocf_get_bdevs", rpc_bdev_ocf_get_bdevs, SPDK_RPC_RUNTIME) + +static void +rpc_bdev_ocf_set_cachemode(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ +} +SPDK_RPC_REGISTER("bdev_ocf_set_cachemode", rpc_bdev_ocf_set_cachemode, SPDK_RPC_RUNTIME) + +static void +rpc_bdev_ocf_set_cleaning(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ +} +SPDK_RPC_REGISTER("bdev_ocf_set_cleaning", rpc_bdev_ocf_set_cleaning, SPDK_RPC_RUNTIME) + +static void +rpc_bdev_ocf_set_seqcutoff(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ +} +SPDK_RPC_REGISTER("bdev_ocf_set_seqcutoff", rpc_bdev_ocf_set_seqcutoff, SPDK_RPC_RUNTIME) + +static void +rpc_bdev_ocf_get_stats(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ +} +SPDK_RPC_REGISTER("bdev_ocf_get_stats", rpc_bdev_ocf_get_stats, SPDK_RPC_RUNTIME) + +static void +rpc_bdev_ocf_reset_stats(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ +} +SPDK_RPC_REGISTER("bdev_ocf_reset_stats", rpc_bdev_ocf_reset_stats, SPDK_RPC_RUNTIME) + +static void +rpc_bdev_ocf_flush_start(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ +} +SPDK_RPC_REGISTER("bdev_ocf_flush_start", rpc_bdev_ocf_flush_start, SPDK_RPC_RUNTIME) + +static void +rpc_bdev_ocf_flush_status(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ +} +SPDK_RPC_REGISTER("bdev_ocf_flush_status", rpc_bdev_ocf_flush_status, SPDK_RPC_RUNTIME) + +SPDK_LOG_REGISTER_COMPONENT(vbdev_ocf_rpc) diff --git a/module/bdev/ocf/volume.c b/module/bdev/ocf/volume.c index ea4b54d16f4..075131d73f0 100644 --- a/module/bdev/ocf/volume.c +++ b/module/bdev/ocf/volume.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2019 Intel Corporation. + * Copyright (C) 2025 Huawei Technologies * All rights reserved. */ @@ -10,27 +11,21 @@ #include "spdk/thread.h" #include "spdk/log.h" +#include "vbdev_ocf_core.h" #include "data.h" #include "volume.h" #include "ctx.h" -#include "vbdev_ocf.h" static int vbdev_ocf_volume_open(ocf_volume_t volume, void *opts) { struct vbdev_ocf_base **priv = ocf_volume_get_priv(volume); - struct vbdev_ocf_base *base; - if (opts) { - base = opts; - } else { - base = vbdev_ocf_get_base_by_name(ocf_volume_get_uuid(volume)->data); - if (base == NULL) { - return -ENODEV; - } - } + // refactor (like ocf_core_volume in ocf_core.c ?) - *priv = base; + assert(opts); + + *priv = opts; return 0; } @@ -125,18 +120,18 @@ vbdev_forward_get_channel(ocf_volume_t volume, ocf_forward_token_t token) *((struct vbdev_ocf_base **) ocf_volume_get_priv(volume)); ocf_queue_t queue = ocf_forward_get_io_queue(token); - struct vbdev_ocf_qctx *qctx; + struct vbdev_ocf_core_io_channel_ctx *ch_ctx; if (unlikely(ocf_queue_is_mngt(queue))) { - return base->management_channel; + return base->mngt_ch; } - qctx = ocf_queue_get_priv(queue); - if (unlikely(qctx == NULL)) { + ch_ctx = ocf_queue_get_priv(queue); + if (unlikely(ch_ctx == NULL)) { return NULL; } - return (base->is_cache) ? qctx->cache_ch : qctx->core_ch; + return (base->is_cache) ? ch_ctx->cache_ch : ch_ctx->core_ch; } static void @@ -147,7 +142,7 @@ vbdev_forward_io(ocf_volume_t volume, ocf_forward_token_t token, struct vbdev_ocf_base *base = *((struct vbdev_ocf_base **) ocf_volume_get_priv(volume)); - struct bdev_ocf_data *data = ocf_forward_get_data(token); + struct vbdev_ocf_data *data = ocf_forward_get_data(token); struct spdk_io_channel *ch; spdk_bdev_io_completion_cb cb = vbdev_forward_io_cb; bool iovs_allocated = false; @@ -215,6 +210,14 @@ vbdev_forward_flush(ocf_volume_t volume, ocf_forward_token_t token) uint64_t bytes = base->bdev->blockcnt * base->bdev->blocklen; int status; + // workaround (?); check for better solution + /* If base device doesn't support flush just warn about it and exit. */ + if (unlikely(!spdk_bdev_io_type_supported(base->bdev, SPDK_BDEV_IO_TYPE_FLUSH))) { + SPDK_WARNLOG("Base bdev '%s': attempt to flush device that doesn't support it\n", base->bdev->name); + ocf_forward_end(token, 0); + return; + } + ch = vbdev_forward_get_channel(volume, token); if (unlikely(ch == NULL)) { ocf_forward_end(token, -EFAULT); @@ -282,7 +285,7 @@ vbdev_forward_io_simple(ocf_volume_t volume, ocf_forward_token_t token, struct vbdev_ocf_base *base = *((struct vbdev_ocf_base **) ocf_volume_get_priv(volume)); - struct bdev_ocf_data *data = ocf_forward_get_data(token); + struct vbdev_ocf_data *data = ocf_forward_get_data(token); struct vbdev_forward_io_simple_ctx *ctx; int status = -1; @@ -339,6 +342,26 @@ static struct ocf_volume_properties vbdev_volume_props = { }, }; +static void +_base_detach(void *ctx) +{ + struct vbdev_ocf_base *base = ctx; + + spdk_put_io_channel(base->mngt_ch); + spdk_bdev_close(base->desc); +} + +void +vbdev_ocf_base_detach(struct vbdev_ocf_base *base) +{ + if (base->thread && base->thread != spdk_get_thread()) { + spdk_thread_send_msg(base->thread, _base_detach, base); + } else { + _base_detach(base); + } + base->attached = false; +} + int vbdev_ocf_volume_init(void) { diff --git a/module/bdev/ocf/volume.h b/module/bdev/ocf/volume.h index 7e6629d4715..461e14ca582 100644 --- a/module/bdev/ocf/volume.h +++ b/module/bdev/ocf/volume.h @@ -1,16 +1,22 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2018 Intel Corporation. + * Copyright (C) 2025 Huawei Technologies * All rights reserved. */ -#ifndef VBDEV_OCF_DOBJ_H -#define VBDEV_OCF_DOBJ_H +#ifndef VBDEV_OCF_VOLUME_H +#define VBDEV_OCF_VOLUME_H -#include - -#include "ctx.h" -#include "data.h" +struct vbdev_ocf_base { + bool is_cache; + bool attached; + struct spdk_bdev * bdev; + struct spdk_bdev_desc * desc; + struct spdk_io_channel * mngt_ch; + struct spdk_thread * thread; +}; +void vbdev_ocf_base_detach(struct vbdev_ocf_base *base); int vbdev_ocf_volume_init(void); void vbdev_ocf_volume_cleanup(void); diff --git a/python/spdk/cli/bdev.py b/python/spdk/cli/bdev.py index b52b18917e1..7a3583415c0 100644 --- a/python/spdk/cli/bdev.py +++ b/python/spdk/cli/bdev.py @@ -103,92 +103,142 @@ def bdev_crypto_delete(args): p.add_argument('name', help='crypto bdev name') p.set_defaults(func=bdev_crypto_delete) - def bdev_ocf_create(args): - print_json(args.client.bdev_ocf_create( - name=args.name, - mode=args.mode, - cache_line_size=args.cache_line_size, - cache_bdev_name=args.cache_bdev_name, - core_bdev_name=args.core_bdev_name)) - p = subparsers.add_parser('bdev_ocf_create', help='Add an OCF block device') - p.add_argument('name', help='Name of resulting OCF bdev') - p.add_argument('mode', help='OCF cache mode', choices=['wb', 'wt', 'pt', 'wa', 'wi', 'wo']) - p.add_argument( - '--cache-line-size', - help='OCF cache line size. The unit is KiB', - type=int, - choices=[4, 8, 16, 32, 64] - ) - p.add_argument('cache_bdev_name', help='Name of underlying cache bdev') - p.add_argument('core_bdev_name', help='Name of underlying core bdev') - p.set_defaults(func=bdev_ocf_create) - - def bdev_ocf_delete(args): - args.client.bdev_ocf_delete(name=args.name) - - p = subparsers.add_parser('bdev_ocf_delete', help='Delete an OCF block device') - p.add_argument('name', help='Name of OCF bdev') - p.set_defaults(func=bdev_ocf_delete) - - def bdev_ocf_get_stats(args): - print_dict(args.client.bdev_ocf_get_stats(name=args.name)) - p = subparsers.add_parser('bdev_ocf_get_stats', help='Get statistics of chosen OCF block device') - p.add_argument('name', help='Name of OCF bdev') - p.set_defaults(func=bdev_ocf_get_stats) - - def bdev_ocf_reset_stats(args): - print_dict(args.client.bdev_ocf_reset_stats(name=args.name)) - p = subparsers.add_parser('bdev_ocf_reset_stats', help='Reset statistics of chosen OCF block device') - p.add_argument('name', help='Name of OCF bdev') - p.set_defaults(func=bdev_ocf_reset_stats) + def bdev_ocf_start_cache(args): + print_json(args.client.bdev_ocf_start_cache( + cache_name=args.cache_name, + bdev_name=args.bdev_name, + cache_mode=args.cache_mode, + cache_line_size=args.cache_line_size)) + p = subparsers.add_parser('bdev_ocf_start_cache', help='Start OCF cache instance') + p.add_argument('cache_name', help='name for the new OCF cache vbdev') + p.add_argument('bdev_name', help='name of the bdev to use as cache') + p.add_argument('--cache-mode', + help='choose between {wt|wb|wa|wo|wi|pt}; default wt (Write-Through)', + choices=['wt', 'wb', 'wa', 'wo', 'wi', 'pt']) + p.add_argument('--cache-line-size', + help='choose between {4|8|16|32|64}; default 4 [KiB]', + type=int, + choices=[4, 8, 16, 32, 64]) + p.set_defaults(func=bdev_ocf_start_cache) + + def bdev_ocf_stop_cache(args): + args.client.bdev_ocf_stop_cache(cache_name=args.cache_name) + p = subparsers.add_parser('bdev_ocf_stop_cache', help='Stop OCF cache instance') + p.add_argument('cache_name', help='name of the cache vbdev to stop') + p.set_defaults(func=bdev_ocf_stop_cache) + + def bdev_ocf_add_core(args): + print_json(args.client.bdev_ocf_add_core( + core_name=args.core_name, + bdev_name=args.bdev_name, + cache_name=args.cache_name)) + p = subparsers.add_parser('bdev_ocf_add_core', help='Add core device to OCF cache') + p.add_argument('core_name', help='name for the new OCF core vbdev') + p.add_argument('bdev_name', help='name of the bdev to use as core') + p.add_argument('cache_name', help='name of already started OCF cache vbdev') + p.set_defaults(func=bdev_ocf_add_core) + + def bdev_ocf_remove_core(args): + args.client.bdev_ocf_remove_core(core_name=args.core_name) + p = subparsers.add_parser('bdev_ocf_remove_core', help='Remove core device from OCF cache') + p.add_argument('core_name', help='name of the core vbdev to remove from OCF cache instance') + p.set_defaults(func=bdev_ocf_remove_core) def bdev_ocf_get_bdevs(args): print_dict(args.client.bdev_ocf_get_bdevs(name=args.name)) - p = subparsers.add_parser('bdev_ocf_get_bdevs', help='Get list of OCF devices including unregistered ones') - p.add_argument('name', nargs='?', help='name of OCF vbdev or name of cache device or name of core device (optional)') + p = subparsers.add_parser('bdev_ocf_get_bdevs', help='Get info about OCF devices') + p.add_argument('name', nargs='?', help='optional name of specific OCF vbdev; shows all by default') p.set_defaults(func=bdev_ocf_get_bdevs) - def bdev_ocf_set_cache_mode(args): - print_json(args.client.bdev_ocf_set_cache_mode( - name=args.name, - mode=args.mode)) - p = subparsers.add_parser('bdev_ocf_set_cache_mode', - help='Set cache mode of OCF block device') - p.add_argument('name', help='Name of OCF bdev') - p.add_argument('mode', help='OCF cache mode', choices=['wb', 'wt', 'pt', 'wa', 'wi', 'wo']) - p.set_defaults(func=bdev_ocf_set_cache_mode) + def bdev_ocf_set_cachemode(args): + args.client.bdev_ocf_set_cachemode( + cache_name=args.cache_name, + cache_mode=args.cache_mode) + p = subparsers.add_parser('bdev_ocf_set_cachemode', help='Set cache mode of OCF cache') + p.add_argument('cache_name', help='name of the cache vbdev') + p.add_argument('cache_mode', + help='choose between {wt|wb|wa|wo|wi|pt} (Write-Through, Write-Back, Write-Around, Write-Only, Write-Invalidate, Pass-Through)', + choices=['wt', 'wb', 'wa', 'wo', 'wi', 'pt']) + p.set_defaults(func=bdev_ocf_set_cachemode) + + def bdev_ocf_set_cleaning(args): + args.client.bdev_ocf_set_cleaning( + cache_name=args.cache_name, + policy=args.policy, + acp_wake_up=args.acp_wake_up, + acp_flush_max_buffers=args.acp_flush_max_buffers, + alru_wake_up=args.alru_wake_up, + alru_flush_max_buffers=args.alru_flush_max_buffers, + alru_staleness_time=args.alru_staleness_time, + alru_activity_threshold=args.alru_activity_threshold) + p = subparsers.add_parser('bdev_ocf_set_cleaning', help='Set cleaning parameters for OCF cache device') + p.add_argument('cache_name', help='name of the cache vbdev') + p.add_argument('-p', '--policy', + help='cleaning policy; choose between {acp|alru|nop}', + choices=['acp', 'alru', 'nop']) + p.add_argument('-u', '--acp-wake-up', + help='time between ACP cleaning thread iterations: <0-10000> [ms]; default 10', + type=int) + p.add_argument('-m', '--acp-flush-max-buffers', + help='number of dirty cache lines to be flushed in a single ACP cleaning thread iteration: <1-10000>; default 128', + type=int) + p.add_argument('-v', '--alru-wake-up', + help='cleaning thread sleep time after an idle wake up: <0-3600> [s]; default 20', + type=int) + p.add_argument('-n', '--alru-flush-max-buffers', + help='number of dirty cache lines to be flushed in one cleaning cycle: <1-10000>; default 100', + type=int) + p.add_argument('-s', '--alru-staleness-time', + help='time that has to pass from the last write operation before a dirty cache line can be scheduled to be flushed: <1-3600> [s]; default 120', + type=int) + p.add_argument('-t', '--alru-activity-threshold', + help='cache idle time before flushing thread can start <0-1000000> [ms]; default 10000', + type=int) + p.set_defaults(func=bdev_ocf_set_cleaning) def bdev_ocf_set_seqcutoff(args): args.client.bdev_ocf_set_seqcutoff( - name=args.name, + core_name=args.core_name, policy=args.policy, threshold=args.threshold, promotion_count=args.promotion_count) - p = subparsers.add_parser('bdev_ocf_set_seqcutoff', - help='Set sequential cutoff parameters on all cores for the given OCF cache device') - p.add_argument('name', help='Name of OCF cache bdev') - p.add_argument('-t', '--threshold', type=int, - help='Activation threshold [KiB]') - p.add_argument('-c', '--promotion-count', type=int, - help='Promotion request count') - p.add_argument('-p', '--policy', choices=['always', 'full', 'never'], required=True, - help='Sequential cutoff policy') + p = subparsers.add_parser('bdev_ocf_set_seqcutoff', help='Set sequential cut-off parameters for OCF core device') + p.add_argument('core_name', help='name of the core vbdev') + p.add_argument('-p', '--policy', + help='sequential cut-off policy; choose between {always|full|never}', + choices=['always', 'full', 'never']) + p.add_argument('-t', '--threshold', + help='activation threshold [KiB]', + type=int) + p.add_argument('-c', '--promotion_count', + help='request count threshold for cutting off the sequential stream', + type=int) p.set_defaults(func=bdev_ocf_set_seqcutoff) def bdev_ocf_flush_start(args): args.client.bdev_ocf_flush_start(name=args.name) - p = subparsers.add_parser('bdev_ocf_flush_start', - help='Start flushing OCF cache device') - p.add_argument('name', help='Name of OCF bdev') + p = subparsers.add_parser('bdev_ocf_flush_start', help='Flush all dirty data from the OCF cache device to core devices') + p.add_argument('name', help='name of the OCF vbdev to flush') p.set_defaults(func=bdev_ocf_flush_start) def bdev_ocf_flush_status(args): print_json(args.client.bdev_ocf_flush_status(name=args.name)) - p = subparsers.add_parser('bdev_ocf_flush_status', - help='Get flush status of OCF cache device') - p.add_argument('name', help='Name of OCF bdev') + p = subparsers.add_parser('bdev_ocf_flush_status', help='Get flush status of OCF vbdev') + p.add_argument('name', help='name of the OCF vbdev') p.set_defaults(func=bdev_ocf_flush_status) + def bdev_ocf_get_stats(args): + print_dict(args.client.bdev_ocf_get_stats(name=args.name)) + p = subparsers.add_parser('bdev_ocf_get_stats', help='Get statistics of chosen OCF block device') + p.add_argument('name', help='Name of OCF bdev') + p.set_defaults(func=bdev_ocf_get_stats) + + def bdev_ocf_reset_stats(args): + print_dict(args.client.bdev_ocf_reset_stats(name=args.name)) + p = subparsers.add_parser('bdev_ocf_reset_stats', help='Reset statistics of chosen OCF block device') + p.add_argument('name', help='Name of OCF bdev') + p.set_defaults(func=bdev_ocf_reset_stats) + def bdev_malloc_create(args): num_blocks = (args.total_size * 1024 * 1024) // args.block_size print_json(args.client.bdev_malloc_create( From b54fa70eacfdea131684b13c029fbfc110a47124 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Wed, 7 May 2025 11:21:33 +0200 Subject: [PATCH 03/34] OCF module: move data to OCF structs Change-Id: Ie8d756c4374f5fb75e39a885b203a6b6883d1bf5 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/ctx.c | 2 +- module/bdev/ocf/vbdev_ocf.c | 1422 ++++++++++++++++++----------- module/bdev/ocf/vbdev_ocf.h | 77 +- module/bdev/ocf/vbdev_ocf_cache.c | 289 +++--- module/bdev/ocf/vbdev_ocf_cache.h | 49 +- module/bdev/ocf/vbdev_ocf_core.c | 372 +++----- module/bdev/ocf/vbdev_ocf_core.h | 62 +- module/bdev/ocf/vbdev_ocf_rpc.c | 141 ++- module/bdev/ocf/volume.h | 1 + python/spdk/cli/bdev.py | 36 +- test/ocf/common.sh | 37 +- 11 files changed, 1415 insertions(+), 1073 deletions(-) diff --git a/module/bdev/ocf/ctx.c b/module/bdev/ocf/ctx.c index cc90daa7a92..d5dec5badef 100644 --- a/module/bdev/ocf/ctx.c +++ b/module/bdev/ocf/ctx.c @@ -341,7 +341,7 @@ vbdev_ocf_ctx_cleaner_init(ocf_cleaner_t c) return -ENOMEM; } - priv->mngt_queue = vbdev_ocf_cache->ocf_cache_mngt_q; + priv->mngt_queue = vbdev_ocf_cache->cache_mngt_q; ocf_cleaner_set_cmpl(c, cleaner_cmpl); ocf_cleaner_set_priv(c, priv); diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 925b6e3291e..d61da72e5e1 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -9,8 +9,6 @@ #include "spdk/string.h" // rm ? #include "vbdev_ocf.h" -#include "vbdev_ocf_cache.h" -#include "vbdev_ocf_core.h" #include "ctx.h" #include "data.h" #include "volume.h" @@ -22,6 +20,7 @@ static int vbdev_ocf_module_init(void); static void vbdev_ocf_module_fini_start(void); static void vbdev_ocf_module_fini(void); static int vbdev_ocf_module_get_ctx_size(void); +static void vbdev_ocf_module_examine_config(struct spdk_bdev *bdev); struct spdk_bdev_module ocf_if = { .name = "OCF", @@ -29,7 +28,7 @@ struct spdk_bdev_module ocf_if = { .fini_start = vbdev_ocf_module_fini_start, .module_fini = vbdev_ocf_module_fini, .get_ctx_size = vbdev_ocf_module_get_ctx_size, - .examine_config = NULL, // todo + .examine_config = vbdev_ocf_module_examine_config, .examine_disk = NULL, // todo ? .async_fini_start = true, }; @@ -57,27 +56,53 @@ struct spdk_bdev_fn_table vbdev_ocf_fn_table = { // rm #define __bdev_to_io_dev(bdev) (((char *)bdev) + 1) +static int +_bdev_exists_core_visit(ocf_core_t core, void *cb_arg) +{ + const char *name = cb_arg; + + if (!strcmp(name, ocf_core_get_name(core))) { + return -EEXIST; + } + + return 0; +} + +static int +_bdev_exists_cache_visit(ocf_cache_t cache, void *cb_arg) +{ + char *name = cb_arg; + + if (!strcmp(name, ocf_cache_get_name(cache))) { + return -EEXIST; + } + + return ocf_core_visit(cache, _bdev_exists_core_visit, name, false); +} + static bool -vbdev_ocf_device_exists(const char *name) { - struct vbdev_ocf_cache *cache; - struct vbdev_ocf_core *core; +vbdev_ocf_bdev_exists(const char *name) +{ + struct vbdev_ocf_core *core_ctx; + int rc; - SPDK_DEBUGLOG(vbdev_ocf, "OCF: looking for '%s' in existing device names\n", name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF: looking for '%s' in existing bdev names\n", name); - vbdev_ocf_foreach_core_incomplete(core) { - if (!strcmp(name, core->name)) { + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + if (!strcmp(name, vbdev_ocf_core_get_name(core_ctx))) { return true; } } - vbdev_ocf_foreach_cache(cache) { - if (!strcmp(name, cache->name)) { - return true; - } - vbdev_ocf_foreach_core_in_cache(core, cache) { - if (!strcmp(name, core->name)) { - return true; - } - } + + rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _bdev_exists_cache_visit, (char *)name); + if (rc == -EEXIST) { + return true; + } else if (rc) { + SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + } + + if (spdk_bdev_get_by_name(name)) { + return true; } return false; @@ -105,111 +130,176 @@ vbdev_ocf_module_init(void) } static void -_cache_stop_module_fini_stop_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +_module_fini_cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_cache *cache = cb_arg; - struct vbdev_ocf_core *core; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", cache->name); - - ocf_mngt_cache_unlock(ocf_cache); + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", + ocf_cache_get_name(cache)); if (error) { SPDK_ERRLOG("OCF cache '%s': failed to stop OCF cache (OCF error: %d)\n", - cache->name, error); - return; - } - - vbdev_ocf_foreach_core_in_cache(core, cache) { - /* It's important to set ocf_core back to NULL before unregistering. - * This is an indicator for destruct that OCF cache isn't there any more. */ - core->ocf_core = NULL; + ocf_cache_get_name(cache), error); } vbdev_ocf_cache_base_detach(cache); + vbdev_ocf_cache_destroy(cache); + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); - if (cache == STAILQ_LAST(&g_vbdev_ocf_caches, vbdev_ocf_cache, link)) { + if (!ocf_mngt_cache_get_count(vbdev_ocf_ctx)) { spdk_bdev_module_fini_start_done(); } } static void -_cache_stop_module_fini_flush_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +_module_fini_cache_flush_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_cache *cache = cb_arg; - if (error) { SPDK_ERRLOG("OCF cache '%s': failed to flush OCF cache (OCF error: %d)\n", - cache->name, error); - ocf_mngt_cache_unlock(ocf_cache); - return; + ocf_cache_get_name(cache), error); } - ocf_mngt_cache_stop(ocf_cache, _cache_stop_module_fini_stop_cb, cache); + ocf_mngt_cache_stop(cache, _module_fini_cache_stop_cb, cb_arg); } static void -_cache_stop_module_fini_lock_cb(ocf_cache_t ocf_cache, void *lock_arg, int lock_err) +_module_fini_cache_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_cache *cache = lock_arg; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop of OCF cache\n", cache->name); - - if (lock_err) { + if (error) { SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", - cache->name, lock_err); - return; + ocf_cache_get_name(cache), error); } - if (ocf_mngt_cache_is_dirty(ocf_cache)) { - ocf_mngt_cache_flush(ocf_cache, _cache_stop_module_fini_flush_cb, cache); + if (ocf_mngt_cache_is_dirty(cache)) { + ocf_mngt_cache_flush(cache, _module_fini_cache_flush_cb, cb_arg); } else { - ocf_mngt_cache_stop(ocf_cache, _cache_stop_module_fini_stop_cb, cache); + ocf_mngt_cache_stop(cache, _module_fini_cache_stop_cb, cb_arg); + } +} + +static int +_module_fini_cache_visit_stop(ocf_cache_t cache, void *cb_arg) +{ + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop of OCF cache\n", + ocf_cache_get_name(cache)); + + printf("*** (cache stop) cache refcnt before get: %d\n", ocf_cache_get_refcnt(cache)); + if ((rc = ocf_mngt_cache_get(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to increment ref count: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-rc)); + return rc; + } + printf("*** (cache stop) cache refcnt after get: %d\n", ocf_cache_get_refcnt(cache)); + + ocf_mngt_cache_lock(cache, _module_fini_cache_lock_cb, cb_arg); + + return rc; +} + +static void +_module_fini_core_unregister_cb(void *cb_arg, int error) +{ + ocf_core_t core = cb_arg; + ocf_cache_t cache = ocf_core_get_cache(core); + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", + ocf_core_get_name(core)); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during module stop: %s\n", + ocf_core_get_name(core), spdk_strerror(-error)); + } + + printf("*** (core unregister) cache refcnt before put: %d\n", ocf_cache_get_refcnt(cache)); + ocf_mngt_cache_put(cache); // needed ? + printf("*** (core unregister) cache refcnt after put: %d\n", ocf_cache_get_refcnt(cache)); + + //if (ocf_cache_get_refcnt(cache) < 2) { + // ocf_mngt_cache_lock(cache, _module_fini_cache_lock_cb, cb_arg); + //} +} + +static int +_module_fini_core_visit(ocf_core_t core, void *cb_arg) +{ + ocf_cache_t cache = ocf_core_get_cache(core); + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + int rc = 0; + + printf("*** (core unregister) cache refcnt before get: %d\n", ocf_cache_get_refcnt(cache)); + if ((rc = ocf_mngt_cache_get(cache))) { + SPDK_ERRLOG("OCF core '%s': failed to increment cache '%s' ref count: %s\n", + ocf_core_get_name(core), ocf_cache_get_name(cache), spdk_strerror(-rc)); + return rc; + } + printf("*** (core unregister) cache refcnt after get: %d\n", ocf_cache_get_refcnt(cache)); + + if ((rc = vbdev_ocf_core_unregister(core_ctx, _module_fini_core_unregister_cb, core))) { + SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during module stop: %s\n", + ocf_core_get_name(core), spdk_strerror(-rc)); + return rc; + } + + return rc; +} + +static int +_module_fini_cache_visit(ocf_cache_t cache, void *cb_arg) +{ + int rc = 0; + + if ((rc = ocf_core_visit(cache, _module_fini_core_visit, cb_arg, false))) { // only opened cores ? + SPDK_ERRLOG("OCF: failed to iterate over core bdevs: %s\n", spdk_strerror(-rc)); + return rc; } + + return rc; } static void vbdev_ocf_module_fini_start(void) { - struct vbdev_ocf_cache *cache; + struct vbdev_ocf_core *core_ctx; + int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF: initiating module stop\n"); - if (STAILQ_EMPTY(&g_vbdev_ocf_caches)) { + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + if (vbdev_ocf_core_is_base_attached(core_ctx)) { + vbdev_ocf_core_base_detach(core_ctx); + } + } + + if (!ocf_mngt_cache_get_count(vbdev_ocf_ctx)) { + spdk_bdev_module_fini_start_done(); + return; + } + + if ((rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _module_fini_cache_visit, NULL))) { + SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); spdk_bdev_module_fini_start_done(); return; } - /* Stop all OCF caches before unregistering all bdevs. */ - vbdev_ocf_foreach_cache(cache) { - ocf_mngt_cache_lock(cache->ocf_cache, _cache_stop_module_fini_lock_cb, cache); + // async fuckup ? (move to unregister_cb and do by refcnt) + if ((rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _module_fini_cache_visit_stop, NULL))) { + SPDK_ERRLOG("OCF: failed to iterate over cache bdevs: %s\n", spdk_strerror(-rc)); + spdk_bdev_module_fini_start_done(); + return; } } static void vbdev_ocf_module_fini(void) { - struct vbdev_ocf_cache *cache; - struct vbdev_ocf_core *core; + struct vbdev_ocf_core *core_ctx; SPDK_DEBUGLOG(vbdev_ocf, "OCF: finishing module stop\n"); - while ((core = STAILQ_FIRST(&g_vbdev_ocf_incomplete_cores))) { - vbdev_ocf_core_remove_incomplete(core); - vbdev_ocf_core_destroy(core); - } - while ((cache = STAILQ_FIRST(&g_vbdev_ocf_caches))) { - if (vbdev_ocf_cache_is_incomplete(cache)) { - /* If cache is incomplete it won't be started yet and won't have - * any cores attached, thus just removing init params here. */ - vbdev_ocf_cache_remove_incomplete(cache); - } else { - while ((core = STAILQ_FIRST(&cache->cores))) { - vbdev_ocf_core_remove_from_cache(core); - vbdev_ocf_core_destroy(core); - } - } - vbdev_ocf_cache_destroy(cache); + while ((core_ctx = STAILQ_FIRST(&g_vbdev_ocf_core_waitlist))) { + STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); + vbdev_ocf_core_destroy(core_ctx); } vbdev_ocf_volume_cleanup(); @@ -223,102 +313,141 @@ vbdev_ocf_module_get_ctx_size(void) } static void -_io_device_unregister_cb(void *io_device) +vbdev_ocf_module_examine_config(struct spdk_bdev *bdev) { - struct vbdev_ocf_core *core = io_device; + //const char *bdev_name = spdk_bdev_get_name(bdev); + //struct vbdev_ocf_cache *cache; + //struct vbdev_ocf_core *core; + //int rc; - vbdev_ocf_core_base_detach(core); + //vbdev_ocf_foreach_cache(cache) { + // if (!vbdev_ocf_cache_is_incomplete(cache)) { + // continue; + // } + // if (strcmp(bdev_name, cache->init_params->bdev_name)) { + // continue; + // } - /* This one finally calls the callback from spdk_bdev_unregister_by_name(). */ - spdk_bdev_destruct_done(&core->ocf_vbdev, 0); -} + // assert(!vbdev_ocf_cache_is_base_attached(cache)); -// remove this wrapper ? -static void -vbdev_ocf_destruct_done(struct vbdev_ocf_core *core) -{ - SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': finishing destruct\n", core->ocf_vbdev.name); + // if ((rc = vbdev_ocf_cache_base_attach(cache, bdev_name))) { + // SPDK_ERRLOG("OCF cache '%s': failed to attach base bdev '%s'\n", + // cache->name, bdev_name); + // } + + // printf("*** START OCF cache\n"); + + // vbdev_ocf_cache_remove_incomplete(cache); + + // goto end; + //} + + //vbdev_ocf_foreach_core_in_waitlist(core) { + // if (strcmp(bdev_name, core->init_params->bdev_name)) { + // continue; + // } + + // assert(!vbdev_ocf_core_is_base_attached(core)); - spdk_io_device_unregister(core, _io_device_unregister_cb); + // if ((rc = vbdev_ocf_core_base_attach(core, bdev_name))) { + // SPDK_ERRLOG("OCF core '%s': failed to attach base bdev '%s'\n", + // core->name, bdev_name); + // } + + // cache = vbdev_ocf_cache_get_by_name(core->init_params->cache_name); + // if (!cache || vbdev_ocf_cache_is_incomplete(cache)) { + // SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache '%s'\n", + // core->name, core->init_params->cache_name); + // goto end; + // } + + // printf("*** ADD OCF core\n"); + + // vbdev_ocf_core_add_to_cache(core, cache); + // vbdev_ocf_core_remove_incomplete(core); + + // goto end; + //} + +//end: + spdk_bdev_module_examine_done(&ocf_if); } static void -_core_remove_destruct_remove_cb(void *cb_arg, int error) +_destruct_core_detach_cb(ocf_core_t core, void *cb_arg, int error) { - struct vbdev_ocf_core *core = cb_arg; - struct vbdev_ocf_cache *cache = vbdev_ocf_core_get_cache(core); - - SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': finishing remove of OCF core\n", - core->ocf_vbdev.name); + ocf_cache_t cache = cb_arg; + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); - ocf_mngt_cache_unlock(cache->ocf_cache); + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': finishing detach of OCF core\n", ocf_core_get_name(core)); + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': finishing destruct\n", ocf_core_get_name(core)); if (error) { SPDK_ERRLOG("OCF vbdev '%s': failed to remove OCF core device (OCF error: %d)\n", - core->ocf_vbdev.name, error); - spdk_bdev_destruct_done(&core->ocf_vbdev, error); - return; + ocf_core_get_name(core), error); + //spdk_bdev_destruct_done(&core->ocf_vbdev, error); } - vbdev_ocf_core_remove_from_cache(core); + ocf_mngt_cache_unlock(cache); + vbdev_ocf_core_base_detach(core_ctx); - vbdev_ocf_destruct_done(core); + /* This one finally calls the callback from spdk_bdev_unregister_by_name(). */ + spdk_bdev_destruct_done(&core_ctx->ocf_vbdev, 0); } static void -_core_remove_destruct_flush_cb(ocf_core_t ocf_core, void *cb_arg, int error) +_destruct_core_flush_cb(ocf_core_t core, void *cb_arg, int error) { - struct vbdev_ocf_core *core = cb_arg; - - assert(ocf_core == core->ocf_core); - - if (error) { // WARN only ? (may be hotremoved) + if (error) { SPDK_ERRLOG("OCF vbdev '%s': failed to flush OCF core device (OCF error: %d)\n", - core->ocf_vbdev.name, error); - ocf_mngt_cache_unlock(vbdev_ocf_core_get_cache(core)->ocf_cache); - spdk_bdev_destruct_done(&core->ocf_vbdev, error); - return; + ocf_core_get_name(core), error); + //ocf_mngt_cache_unlock(cache); + //spdk_bdev_destruct_done(&core_ctx->ocf_vbdev, error); + //return; } - ocf_mngt_cache_remove_core(ocf_core, _core_remove_destruct_remove_cb, core); + ocf_mngt_cache_detach_core(core, _destruct_core_detach_cb, cb_arg); } static void -_core_remove_destruct_lock_cb(ocf_cache_t ocf_cache, void *lock_arg, int lock_err) +_destruct_cache_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_core *core = lock_arg; + ocf_core_t core = cb_arg; - SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': initiating remove of OCF core\n", - core->ocf_vbdev.name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': initiating detach of OCF core\n", + ocf_core_get_name(core)); - if (lock_err) { + if (error) { SPDK_ERRLOG("OCF vbdev '%s': failed to acquire OCF cache lock (OCF error: %d)\n", - core->ocf_vbdev.name, lock_err); - spdk_bdev_destruct_done(&core->ocf_vbdev, lock_err); - return; + ocf_core_get_name(core), error); + //spdk_bdev_destruct_done(&core_ctx->ocf_vbdev, error); + //return; } - if (ocf_mngt_core_is_dirty(core->ocf_core)) { - ocf_mngt_core_flush(core->ocf_core, _core_remove_destruct_flush_cb, core); + if (ocf_mngt_core_is_dirty(core)) { + ocf_mngt_core_flush(core, _destruct_core_flush_cb, cache); } else { - ocf_mngt_cache_remove_core(core->ocf_core, _core_remove_destruct_remove_cb, core); + ocf_mngt_cache_detach_core(core, _destruct_core_detach_cb, cache); } } -/* This is called internally by SPDK during vbdev_ocf_core_unregister(). */ +static void +_destruct_io_device_unregister_cb(void *io_device) +{ + ocf_core_t core = io_device; + + ocf_mngt_cache_lock(ocf_core_get_cache(core), _destruct_cache_lock_cb, core); +} + +/* This is called internally by SPDK during spdk_bdev_unregister_by_name(). */ static int vbdev_ocf_fn_destruct(void *ctx) { - struct vbdev_ocf_core *core = ctx; + ocf_core_t core = ctx; - SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': initiating destruct\n", core->ocf_vbdev.name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': initiating destruct\n", ocf_core_get_name(core)); - if (vbdev_ocf_core_cache_is_started(core)) { - ocf_mngt_cache_lock(vbdev_ocf_core_get_cache(core)->ocf_cache, - _core_remove_destruct_lock_cb, core); - } else { - vbdev_ocf_destruct_done(core); - } + spdk_io_device_unregister(core, _destruct_io_device_unregister_cb); /* Return one to indicate async destruct. */ return 1; @@ -331,14 +460,15 @@ _vbdev_ocf_submit_io_cb(ocf_io_t io, void *priv1, void *priv2, int error) // rm ? SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': finishing submit of IO request\n", - bdev_io->bdev->name); + spdk_bdev_get_name(bdev_io->bdev)); ocf_io_put(io); if (error == -OCF_ERR_NO_MEM) { spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); } else if (error) { - SPDK_ERRLOG("OCF vbdev '%s': failed to complete OCF IO\n", bdev_io->bdev->name); + SPDK_ERRLOG("OCF vbdev '%s': failed to complete OCF IO\n", + spdk_bdev_get_name(bdev_io->bdev)); spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); } else { spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); @@ -351,20 +481,20 @@ static void vbdev_ocf_submit_io(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, uint64_t offset, uint32_t len, uint32_t dir, uint64_t flags, submit_io_to_ocf_fn submit_io_fn) { - struct vbdev_ocf_core *core = bdev_io->bdev->ctxt; + ocf_core_t core = bdev_io->bdev->ctxt; struct vbdev_ocf_data *data = (struct vbdev_ocf_data *)bdev_io->driver_ctx; struct vbdev_ocf_core_io_channel_ctx *ch_ctx = spdk_io_channel_get_ctx(ch); ocf_io_t io = NULL; // impossible to be true ? - if (!core->ocf_core) { + if (!core) { SPDK_ERRLOG("OCF vbdev '%s': failed to submit IO - no OCF core device\n", - bdev_io->bdev->name); + spdk_bdev_get_name(bdev_io->bdev)); spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); return; } - io = ocf_volume_new_io(ocf_core_get_front_volume(core->ocf_core), ch_ctx->queue, + io = ocf_volume_new_io(ocf_core_get_front_volume(core), ch_ctx->queue, offset, len, dir, 0, flags); if (!io) { spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); @@ -389,7 +519,7 @@ _io_read_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bo if (!success) { SPDK_ERRLOG("OCF vbdev '%s': failed to allocate IO buffer - size of the " "buffer to allocate might be greater than the permitted maximum\n", - bdev_io->bdev->name); + spdk_bdev_get_name(bdev_io->bdev)); spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); return; } @@ -405,7 +535,7 @@ vbdev_ocf_fn_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bde // rm ? SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': initiating submit of IO request\n", - bdev_io->bdev->name); + spdk_bdev_get_name(bdev_io->bdev)); switch (bdev_io->type) { case SPDK_BDEV_IO_TYPE_READ: @@ -423,7 +553,7 @@ vbdev_ocf_fn_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bde vbdev_ocf_submit_io(ch, bdev_io, 0, 0, OCF_WRITE, OCF_WRITE_FLUSH, ocf_core_submit_flush); break; default: - SPDK_ERRLOG("OCF vbdev '%s': unsupported IO type: %s\n", bdev_io->bdev->name, + SPDK_ERRLOG("OCF vbdev '%s': unsupported IO type: %s\n", spdk_bdev_get_name(bdev_io->bdev), spdk_bdev_get_io_type_name(bdev_io->type)); spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); } @@ -432,17 +562,18 @@ vbdev_ocf_fn_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bde static bool vbdev_ocf_fn_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) { - struct vbdev_ocf_core *core = ctx; + ocf_core_t core = ctx; + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': checking if IO type '%s' is supported\n", - core->ocf_vbdev.name, spdk_bdev_get_io_type_name(io_type)); + spdk_bdev_get_name(&core_ctx->ocf_vbdev), spdk_bdev_get_io_type_name(io_type)); switch (io_type) { case SPDK_BDEV_IO_TYPE_READ: case SPDK_BDEV_IO_TYPE_WRITE: case SPDK_BDEV_IO_TYPE_UNMAP: case SPDK_BDEV_IO_TYPE_FLUSH: - return spdk_bdev_io_type_supported(core->base.bdev, io_type); + return spdk_bdev_io_type_supported(core_ctx->base.bdev, io_type); default: return false; } @@ -451,9 +582,10 @@ vbdev_ocf_fn_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) static struct spdk_io_channel * vbdev_ocf_fn_get_io_channel(void *ctx) // ctx == ocf_vbdev.ctxt { - struct vbdev_ocf_core *core = ctx; + ocf_core_t core = ctx; - SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': got request for IO channel\n", core->ocf_vbdev.name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': got request for IO channel\n", + spdk_bdev_get_name(&(((struct vbdev_ocf_core *)ocf_core_get_priv(core))->ocf_vbdev))); return spdk_get_io_channel(core); } @@ -470,149 +602,134 @@ vbdev_ocf_fn_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ct } static void -_cache_stop_err_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +_cache_start_rpc_err_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_cache *cache = cb_arg; - - ocf_mngt_cache_unlock(ocf_cache); - if (error) { - SPDK_ERRLOG("OCF cache '%s': failed to stop OCF cache (OCF error: %d)\n", - cache->name, error); - return; + SPDK_ERRLOG("OCF cache '%s': failed to stop OCF cache properly (OCF error: %d)\n", + ocf_cache_get_name(cache), error); } -} -static void -vbdev_ocf_cache_start_rollback(struct vbdev_ocf_cache *cache) -{ - vbdev_ocf_queue_put(cache->ocf_cache_mngt_q); - ocf_mngt_cache_stop(cache->ocf_cache, _cache_stop_err_cb, cache); - vbdev_ocf_cache_base_detach(cache); vbdev_ocf_cache_destroy(cache); + ocf_mngt_cache_unlock(cache); // use in error path only ? + //ocf_mngt_cache_put(cache); // no need ? (it's already stopped) } static void -_cache_start_rpc_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +_cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_cache_start_ctx *cache_start_ctx = cb_arg; - struct vbdev_ocf_cache *cache = cache_start_ctx->cache; - int rc = 0; // rm ? + struct vbdev_ocf_mngt_ctx *cache_start_ctx = cb_arg; + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing start\n", cache->name); - - ocf_volume_destroy(cache->ocf_cache_att_cfg.device.volume); + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing start\n", ocf_cache_get_name(cache)); if (error) { SPDK_ERRLOG("OCF cache '%s': failed to attach OCF cache device\n", - cache->name); - // here or in ocf_mngt_cache_start() ? + ocf_cache_get_name(cache)); if (error == -OCF_ERR_NO_MEM) { uint64_t mem_needed, volume_size; - volume_size = cache->base.bdev->blockcnt * cache->base.bdev->blocklen; - mem_needed = ocf_mngt_get_ram_needed(ocf_cache, volume_size); + volume_size = cache_ctx->base.bdev->blockcnt * cache_ctx->base.bdev->blocklen; + mem_needed = ocf_mngt_get_ram_needed(cache, volume_size); SPDK_ERRLOG("Not enough memory. Try to increase hugepage memory size or cache line size.\n"); SPDK_NOTICELOG("Needed memory to start cache in this configuration " "(device size: %"PRIu64", cache line size: %"PRIu64"): %"PRIu64"\n", - volume_size, cache->ocf_cache_cfg.cache_line_size, mem_needed); + volume_size, cache_ctx->cache_cfg.cache_line_size, mem_needed); + } + + vbdev_ocf_cache_base_detach(cache); + vbdev_ocf_queue_put(cache_ctx->cache_mngt_q); + ocf_mngt_cache_stop(cache, _cache_start_rpc_err_cb, NULL); + if (cache_start_ctx->rpc_cb_fn) { + cache_start_ctx->rpc_cb_fn(NULL, cache_start_ctx->rpc_cb_arg, error); // cache name } - vbdev_ocf_cache_start_rollback(cache); - cache_start_ctx->rpc_cb_fn(NULL, cache_start_ctx->rpc_cb_arg, error); free(cache_start_ctx); + return; } - SPDK_NOTICELOG("OCF cache '%s': started\n", cache->name); + SPDK_NOTICELOG("OCF cache '%s': started\n", ocf_cache_get_name(cache)); - // check for cores in g_vbdev_ocf_incomplete_cores + // check for cores in g_vbdev_ocf_core_waitlist // check if (cache_block_size > core_block_size) - ocf_mngt_cache_unlock(ocf_cache); - cache_start_ctx->rpc_cb_fn(cache, cache_start_ctx->rpc_cb_arg, rc); + ocf_mngt_cache_unlock(cache); + if (cache_start_ctx->rpc_cb_fn) { + cache_start_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_start_ctx->rpc_cb_arg, 0); + } free(cache_start_ctx); } +static void +_cache_start_rpc_metadata_probe_cb(void *priv, int error, struct ocf_metadata_probe_status *status) +{ + struct vbdev_ocf_mngt_ctx *cache_start_ctx = priv; + ocf_cache_t cache = cache_start_ctx->cache; + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + + ocf_volume_close(cache_ctx->cache_att_cfg.device.volume); + + if (error && error != -OCF_ERR_NO_METADATA) { + SPDK_ERRLOG("OCF cache '%s': failed to probe metadata\n", ocf_cache_get_name(cache)); + _cache_start_rpc_attach_cb(cache, cache_start_ctx, error); + } + + if (error == -OCF_ERR_NO_METADATA) { + SPDK_NOTICELOG("OCF cache '%s': metadata not found - starting new cache\n", + ocf_cache_get_name(cache)); + //cache_ctx->cache_att_cfg.force = true; // needed ? + ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, + _cache_start_rpc_attach_cb, cache_start_ctx); + } else { + SPDK_NOTICELOG("OCF cache '%s': metadata found - loading cache from metadata\n", + ocf_cache_get_name(cache)); + // check status for cache_name/mode/line_size/dirty ? + ocf_mngt_cache_load(cache, &cache_ctx->cache_att_cfg, + _cache_start_rpc_attach_cb, cache_start_ctx); + } +} + /* RPC entry point. */ void -vbdev_ocf_cache_start(const char *cache_name, const char *bdev_name, const char *cache_mode, - const uint8_t cache_line_size, vbdev_ocf_cache_start_cb rpc_cb_fn, void *rpc_cb_arg) +vbdev_ocf_cache_start(const char *cache_name, const char *base_name, + const char *cache_mode, const uint8_t cache_line_size, bool no_load, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { - struct vbdev_ocf_cache *cache; - struct vbdev_ocf_cache_start_ctx *cache_start_ctx; - struct ocf_volume_uuid volume_uuid; + ocf_cache_t cache; + struct vbdev_ocf_cache *cache_ctx; + struct vbdev_ocf_mngt_ctx *cache_start_ctx; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating start\n", cache_name); - if (vbdev_ocf_device_exists(cache_name)) { - SPDK_ERRLOG("OCF: device '%s' already exists\n", cache_name); + if (vbdev_ocf_bdev_exists(cache_name)) { + SPDK_ERRLOG("OCF: bdev '%s' already exists\n", cache_name); rc = -EEXIST; goto err_create; } - /* Allocate memory for cache struct and put it on the global cache list. */ - if ((rc = vbdev_ocf_cache_create(cache_name, &cache))) { + if ((rc = vbdev_ocf_cache_create(&cache, cache_name, cache_mode, cache_line_size, no_load))) { SPDK_ERRLOG("OCF cache '%s': failed to create cache\n", cache_name); goto err_create; } - /* Create OCF configs for cache and attach. */ - if ((rc = vbdev_ocf_cache_set_config(cache, cache_mode, cache_line_size))) { - SPDK_ERRLOG("OCF cache '%s': failed to create OCF config\n", cache_name); - goto err_base; + if ((rc = vbdev_ocf_cache_mngt_queue_create(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to create management queue\n", cache_name); + goto err_queue; } - /* Check if base device for this cache is already present. */ - if ((rc = vbdev_ocf_cache_base_attach(cache, bdev_name))) { + if ((rc = vbdev_ocf_cache_base_attach(cache, base_name))) { if (rc == -ENODEV) { SPDK_NOTICELOG("OCF cache '%s': start deferred - waiting for base bdev '%s'\n", - cache_name, bdev_name); - - /* If not, just save cache init params for use in examine and exit. */ - if ((rc = vbdev_ocf_cache_add_incomplete(cache, bdev_name))) { - SPDK_ERRLOG("OCF cache '%s': failed to save init params - removing cache\n", - cache_name); - goto err_base; - } - rpc_cb_fn(cache, rpc_cb_arg, -ENODEV); + cache_name, base_name); + ocf_mngt_cache_unlock(cache); + rpc_cb_fn(cache_name, rpc_cb_arg, -ENODEV); return; } - SPDK_ERRLOG("OCF cache '%s': failed to open base bdev '%s'\n", cache_name, bdev_name); + SPDK_ERRLOG("OCF cache '%s': failed to open base bdev '%s'\n", cache_name, base_name); goto err_base; } - /* Start OCF cache. */ - - if ((rc = ocf_uuid_set_str(&volume_uuid, cache->name))) { // set spdk uuid ? - SPDK_ERRLOG("OCF cache '%s': failed to set OCF volume uuid\n", cache_name); - goto err_volume; - } - - // move elsewhere - cache->ocf_cache_att_cfg.device.volume_params = &cache->base; // for ocf_volume_open() in ocf_mngt_cache_attach() - - // do it right after set_config() and then ocf_volume_destroy() in all callbacks ? - if ((rc = ocf_ctx_volume_create(vbdev_ocf_ctx, &cache->ocf_cache_att_cfg.device.volume, - &volume_uuid, SPDK_OBJECT))) { - SPDK_ERRLOG("OCF cache '%s': failed to create OCF volume\n", cache_name); - goto err_volume; - } - - // maybe this as well for incomplete cache ? - // but then how to check for incomplition? is_cache_detached() ? - // in examine only attach (both in start_rpc and attach_rpc) ? - if ((rc = ocf_mngt_cache_start(vbdev_ocf_ctx, &cache->ocf_cache, &cache->ocf_cache_cfg, cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to start OCF cache\n", cache_name); - goto err_start; - } - - if ((rc = vbdev_ocf_cache_mngt_queue_create(cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to create management queue\n", cache_name); - goto err_queue; - } - - cache_start_ctx = calloc(1, sizeof(struct vbdev_ocf_cache_start_ctx)); + cache_start_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!cache_start_ctx) { SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache start context\n", cache_name); @@ -623,498 +740,763 @@ vbdev_ocf_cache_start(const char *cache_name, const char *bdev_name, const char cache_start_ctx->rpc_cb_fn = rpc_cb_fn; cache_start_ctx->rpc_cb_arg = rpc_cb_arg; - ocf_mngt_cache_attach(cache->ocf_cache, &cache->ocf_cache_att_cfg, - _cache_start_rpc_cb, cache_start_ctx); + cache_ctx = ocf_cache_get_priv(cache); + + if (no_load) { + ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, _cache_start_rpc_attach_cb, + cache_start_ctx); + return; + } + + if ((rc = ocf_volume_open(cache_ctx->cache_att_cfg.device.volume, &cache_ctx->base))) { + SPDK_ERRLOG("OCF cache '%s': failed to open volume\n", ocf_cache_get_name(cache)); + goto err_open; + } + + ocf_metadata_probe(vbdev_ocf_ctx, cache_ctx->cache_att_cfg.device.volume, + _cache_start_rpc_metadata_probe_cb, cache_start_ctx); return; +err_open: + free(cache_start_ctx); err_alloc: - vbdev_ocf_queue_put(cache->ocf_cache_mngt_q); -err_queue: - ocf_mngt_cache_stop(cache->ocf_cache, _cache_stop_err_cb, cache); -err_start: - ocf_volume_destroy(cache->ocf_cache_att_cfg.device.volume); -err_volume: vbdev_ocf_cache_base_detach(cache); err_base: - vbdev_ocf_cache_destroy(cache); + vbdev_ocf_queue_put(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); +err_queue: + ocf_mngt_cache_stop(cache, _cache_start_rpc_err_cb, NULL); err_create: - rpc_cb_fn(NULL, rpc_cb_arg, rc); + rpc_cb_fn(cache_name, rpc_cb_arg, rc); +} + +static void +_cache_stop_rpc_stop_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *cache_stop_ctx = cb_arg; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", + ocf_cache_get_name(cache)); + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop\n", + ocf_cache_get_name(cache)); + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to stop OCF cache (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + } else { + SPDK_NOTICELOG("OCF cache '%s': stopped\n", ocf_cache_get_name(cache)); + vbdev_ocf_cache_base_detach(cache); + vbdev_ocf_cache_destroy(cache); + } + + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); + cache_stop_ctx->rpc_cb_fn(NULL, cache_stop_ctx->rpc_cb_arg, error); // cache name + free(cache_stop_ctx); +} + +static void +_cache_stop_rpc_flush_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to flush OCF cache (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + } + + ocf_mngt_cache_stop(cache, _cache_stop_rpc_stop_cb, cb_arg); +} + +static void +_cache_stop_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop of OCF cache\n", + ocf_cache_get_name(cache)); + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + } + + // no need to manually flush ? + if (ocf_mngt_cache_is_dirty(cache)) { + ocf_mngt_cache_flush(cache, _cache_stop_rpc_flush_cb, cb_arg); + } else { + ocf_mngt_cache_stop(cache, _cache_stop_rpc_stop_cb, cb_arg); + } } static void -_core_unregister_cache_stop_cb(void *cb_arg, int error) +_cache_stop_rpc_core_unregister_cb(void *cb_arg, int error) { - struct vbdev_ocf_core *core = cb_arg; + struct vbdev_ocf_mngt_ctx *cache_stop_ctx = cb_arg; + ocf_cache_t cache = cache_stop_ctx->cache; + ocf_core_t core = cache_stop_ctx->core; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", - core->name); + ocf_core_get_name(core)); if (error) { SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during cache stop: %s\n", - core->name, spdk_strerror(-error)); + ocf_core_get_name(core), spdk_strerror(-error)); } - // destroy core despite the error ? - vbdev_ocf_core_destroy(core); + printf("*** (core unregister) cache refcnt before put: %d\n", ocf_cache_get_refcnt(cache)); + ocf_mngt_cache_put(cache); + printf("*** (core unregister) cache refcnt after put: %d\n", ocf_cache_get_refcnt(cache)); + + if (ocf_cache_get_refcnt(cache) < 2) { + ocf_mngt_cache_lock(cache, _cache_stop_rpc_lock_cb, cb_arg); + } } -static void -_cache_stop_rpc_stop_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +static int +_cache_stop_rpc_core_visit(ocf_core_t core, void *cb_arg) { - struct vbdev_ocf_cache_stop_ctx *cache_stop_ctx = cb_arg; - struct vbdev_ocf_cache *cache = cache_stop_ctx->cache; - struct vbdev_ocf_core *core; + struct vbdev_ocf_mngt_ctx *cache_stop_ctx = cb_arg; + ocf_cache_t cache = ocf_core_get_cache(core); + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); int rc = 0; - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", cache->name); - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop\n", cache->name); + // async fuckup ? + cache_stop_ctx->core = core; - ocf_mngt_cache_unlock(ocf_cache); + printf("*** (core unregister) cache refcnt before get: %d\n", ocf_cache_get_refcnt(cache)); + if ((rc = ocf_mngt_cache_get(cache))) { + SPDK_ERRLOG("OCF core '%s': failed to increment cache '%s' ref count: %s\n", + ocf_core_get_name(core), ocf_cache_get_name(cache), spdk_strerror(-rc)); + return rc; + } + printf("*** (core unregister) cache refcnt after get: %d\n", ocf_cache_get_refcnt(cache)); - if (error) { - SPDK_ERRLOG("OCF cache '%s': failed to stop OCF cache\n", - cache->name); - cache_stop_ctx->rpc_cb_fn(cache_stop_ctx->rpc_cb_arg, error); - free(cache_stop_ctx); - return; + if ((rc = vbdev_ocf_core_unregister(core_ctx, _cache_stop_rpc_core_unregister_cb, cache_stop_ctx))) { + SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during cache stop: %s\n", + ocf_core_get_name(core), spdk_strerror(-rc)); + return rc; } - SPDK_NOTICELOG("OCF cache '%s': stopped\n", cache->name); + return rc; +} + +/* RPC entry point. */ +void +vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) +{ + ocf_cache_t cache; + struct vbdev_ocf_mngt_ctx *cache_stop_ctx; + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop\n", cache_name); + + // increments cache refcnt !!! + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { + SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); + rc = -ENXIO; + goto err_cache; + } + + // DONE ? + // TODO: send hotremove to each core opener first - check how spdk_bdev_unregister does that + // take from bdev_unregister_unsafe() + // or stop cache after unregister ? - vbdev_ocf_foreach_core_in_cache(core, cache) { - /* It's important to set ocf_core back to NULL before unregistering. - * This is an indicator for destruct that OCF cache isn't there any more. */ - core->ocf_core = NULL; + cache_stop_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!cache_stop_ctx) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache stop context\n", + cache_name); + rc = -ENOMEM; + goto err_alloc; + } + cache_stop_ctx->cache = cache; + cache_stop_ctx->rpc_cb_fn = rpc_cb_fn; + cache_stop_ctx->rpc_cb_arg = rpc_cb_arg; - if ((rc = vbdev_ocf_core_unregister(core, _core_unregister_cache_stop_cb, core))) { - SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during cache stop: %s\n", - core->name, spdk_strerror(-rc)); - // destroy core despite the error ? - vbdev_ocf_core_destroy(core); + if (ocf_cache_get_core_count(cache) > 0) { + if ((rc = ocf_core_visit(cache, _cache_stop_rpc_core_visit, cache_stop_ctx, true))) { + SPDK_ERRLOG("OCF: failed to iterate over core bdevs: %s\n", spdk_strerror(-rc)); + // ocf_mngt_cache_lock(...) ? } + } else { + ocf_mngt_cache_lock(cache, _cache_stop_rpc_lock_cb, cache_stop_ctx); } - vbdev_ocf_cache_base_detach(cache); - vbdev_ocf_cache_destroy(cache); + return; - cache_stop_ctx->rpc_cb_fn(cache_stop_ctx->rpc_cb_arg, rc); - free(cache_stop_ctx); +err_alloc: + ocf_mngt_cache_put(cache); +err_cache: + rpc_cb_fn(cache_name, rpc_cb_arg, rc); } static void -_cache_stop_rpc_flush_cb(ocf_cache_t ocf_cache, void *cb_arg, int error) +_cache_detach_rpc_detach_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_cache_stop_ctx *cache_stop_ctx = cb_arg; - struct vbdev_ocf_cache *cache = cache_stop_ctx->cache; + struct vbdev_ocf_mngt_ctx *cache_detach_ctx = cb_arg; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing device detach\n", + ocf_cache_get_name(cache)); if (error) { - SPDK_ERRLOG("OCF cache '%s': failed to flush OCF cache\n", - cache->name); - ocf_mngt_cache_unlock(ocf_cache); - cache_stop_ctx->rpc_cb_fn(cache_stop_ctx->rpc_cb_arg, error); - free(cache_stop_ctx); - return; + SPDK_ERRLOG("OCF cache '%s': failed to detach OCF cache device (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + } else { + vbdev_ocf_cache_base_detach(cache); + SPDK_NOTICELOG("OCF cache '%s': detached\n", ocf_cache_get_name(cache)); } - ocf_mngt_cache_stop(ocf_cache, _cache_stop_rpc_stop_cb, cache_stop_ctx); + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); + cache_detach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_detach_ctx->rpc_cb_arg, error); + free(cache_detach_ctx); } static void -_cache_stop_rpc_lock_cb(ocf_cache_t ocf_cache, void *lock_arg, int lock_err) +_cache_detach_rpc_flush_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_cache_stop_ctx *cache_stop_ctx = lock_arg; - struct vbdev_ocf_cache *cache = cache_stop_ctx->cache; + struct vbdev_ocf_mngt_ctx *cache_detach_ctx = cb_arg; + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to flush OCF cache (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); + cache_detach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_detach_ctx->rpc_cb_arg, error); + free(cache_detach_ctx); + return; + } - assert(ocf_cache == cache->ocf_cache); + ocf_mngt_cache_detach(cache, _cache_detach_rpc_detach_cb, cache_detach_ctx); +} - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop of OCF cache\n", cache->name); +static void +_cache_detach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *cache_detach_ctx = cb_arg; - if (lock_err) { - SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock\n", - cache->name); - cache_stop_ctx->rpc_cb_fn(cache_stop_ctx->rpc_cb_arg, lock_err); - free(cache_stop_ctx); + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + ocf_mngt_cache_put(cache); + cache_detach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_detach_ctx->rpc_cb_arg, error); + free(cache_detach_ctx); return; } - if (ocf_mngt_cache_is_dirty(ocf_cache)) { - ocf_mngt_cache_flush(ocf_cache, _cache_stop_rpc_flush_cb, cache_stop_ctx); + // no need to check dirty ? + if (ocf_mngt_cache_is_dirty(cache)) { + ocf_mngt_cache_flush(cache, _cache_detach_rpc_flush_cb, cache_detach_ctx); } else { - ocf_mngt_cache_stop(ocf_cache, _cache_stop_rpc_stop_cb, cache_stop_ctx); + ocf_mngt_cache_detach(cache, _cache_detach_rpc_detach_cb, cache_detach_ctx); } } /* RPC entry point. */ void -vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_cache_stop_cb rpc_cb_fn, void *rpc_cb_arg) +vbdev_ocf_cache_detach(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { - struct vbdev_ocf_cache *cache; - struct vbdev_ocf_cache_stop_ctx *cache_stop_ctx; + ocf_cache_t cache; + struct vbdev_ocf_mngt_ctx *cache_detach_ctx; + int rc; - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop\n", cache_name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating device detach\n", cache_name); - cache = vbdev_ocf_cache_get_by_name(cache_name); - if (!cache) { - SPDK_ERRLOG("OCF cache '%s': device not found\n", cache_name); - rpc_cb_fn(rpc_cb_arg, -ENODEV); - return; + // increments cache refcnt !!! + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { + SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); + rc = -ENXIO; + goto err_cache; } - /* If cache was not started yet due to lack of base device, just free its structs and exit. */ - if (vbdev_ocf_cache_is_incomplete(cache)) { - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': removing as incomplete\n", cache->name); - assert(!cache->base.attached); - vbdev_ocf_cache_remove_incomplete(cache); - vbdev_ocf_cache_destroy(cache); - rpc_cb_fn(rpc_cb_arg, 0); - return; + if (ocf_cache_is_detached(cache)) { + SPDK_ERRLOG("OCF cache '%s': device already detached\n", cache_name); + rc = -EALREADY; // better errno ? + goto err_state; } - // TODO: send hotremove to each core opener first - check how spdk_bdev_unregister does that - // take from bdev_unregister_unsafe() - // or stop cache after unregister ? - - cache_stop_ctx = calloc(1, sizeof(struct vbdev_ocf_cache_stop_ctx)); - if (!cache_stop_ctx) { - SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache stop context\n", + cache_detach_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!cache_detach_ctx) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache detach context\n", cache_name); - rpc_cb_fn(rpc_cb_arg, -ENOMEM); - return; + rc = -ENOMEM; + goto err_state; } - cache_stop_ctx->cache = cache; - cache_stop_ctx->rpc_cb_fn = rpc_cb_fn; - cache_stop_ctx->rpc_cb_arg = rpc_cb_arg; + cache_detach_ctx->rpc_cb_fn = rpc_cb_fn; + cache_detach_ctx->rpc_cb_arg = rpc_cb_arg; + + ocf_mngt_cache_lock(cache, _cache_detach_rpc_lock_cb, cache_detach_ctx); - ocf_mngt_cache_lock(cache->ocf_cache, _cache_stop_rpc_lock_cb, cache_stop_ctx); + return; + +err_state: + ocf_mngt_cache_put(cache); +err_cache: + rpc_cb_fn(cache_name, rpc_cb_arg, rc); } static void -_core_remove_err_cb(void *cb_arg, int error) +_cache_attach_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_core *core = cb_arg; - struct vbdev_ocf_cache *cache = vbdev_ocf_core_get_cache(core); + struct vbdev_ocf_mngt_ctx *cache_attach_ctx = cb_arg; - ocf_mngt_cache_unlock(cache->ocf_cache); + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing device attach\n", + ocf_cache_get_name(cache)); if (error) { - SPDK_ERRLOG("OCF core '%s': failed to remove OCF core device (OCF error: %d)\n", - core->name, error); + SPDK_ERRLOG("OCF cache '%s': failed to attach OCF cache device (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + vbdev_ocf_cache_base_detach(cache); + } else { + SPDK_NOTICELOG("OCF cache '%s': attached\n", ocf_cache_get_name(cache)); + } + + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); + cache_attach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_attach_ctx->rpc_cb_arg, error); + free(cache_attach_ctx); +} + +static void +_cache_attach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *cache_attach_ctx = cb_arg; + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + vbdev_ocf_cache_base_detach(cache); + ocf_mngt_cache_put(cache); + cache_attach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_attach_ctx->rpc_cb_arg, error); + free(cache_attach_ctx); return; } - vbdev_ocf_core_remove_from_cache(core); + ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, _cache_attach_rpc_attach_cb, + cache_attach_ctx); +} + +/* RPC entry point. */ +void +vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) +{ + ocf_cache_t cache; + struct vbdev_ocf_cache *cache_ctx; + struct vbdev_ocf_mngt_ctx *cache_attach_ctx; + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating device attach\n", cache_name); + + // increments cache refcnt !!! (put it right away ?) + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { + SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); + rc = -ENXIO; + goto err_cache; + } + + if (!ocf_cache_is_detached(cache)) { + SPDK_ERRLOG("OCF cache '%s': device already attached\n", cache_name); + rc = -EEXIST; // better errno ? + goto err_state; + } + + if ((rc = vbdev_ocf_cache_base_attach(cache, base_name))) { + if (rc == -ENODEV) { + SPDK_NOTICELOG("OCF cache '%s': start deferred - waiting for base bdev '%s'\n", + cache_name, base_name); + rpc_cb_fn(cache_name, rpc_cb_arg, -ENODEV); + return; + } + SPDK_ERRLOG("OCF cache '%s': failed to open base bdev '%s'\n", cache_name, base_name); + goto err_state; + } + + cache_attach_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!cache_attach_ctx) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache attach context\n", + cache_name); + rc = -ENOMEM; + goto err_alloc; + } + cache_attach_ctx->rpc_cb_fn = rpc_cb_fn; + cache_attach_ctx->rpc_cb_arg = rpc_cb_arg; + + cache_ctx = ocf_cache_get_priv(cache); + cache_ctx->cache_att_cfg.force = force; + + ocf_mngt_cache_lock(cache, _cache_attach_rpc_lock_cb, cache_attach_ctx); + + return; + +err_alloc: + vbdev_ocf_cache_base_detach(cache); +err_state: + ocf_mngt_cache_put(cache); +err_cache: + rpc_cb_fn(cache_name, rpc_cb_arg, rc); } static void -vbdev_ocf_core_add_rollback(struct vbdev_ocf_core *core) +_core_add_rpc_err_cb(void *cb_arg, int error) { - ocf_mngt_cache_remove_core(core->ocf_core, _core_remove_err_cb, core); - vbdev_ocf_core_base_detach(core); - vbdev_ocf_core_destroy(core); + struct vbdev_ocf_mngt_ctx *core_add_ctx = cb_arg; + struct vbdev_ocf_core *core_ctx = core_add_ctx->core_ctx; + ocf_cache_t cache = core_add_ctx->cache; + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to remove OCF core device (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), error); + } + + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); + vbdev_ocf_core_base_detach(core_ctx); + vbdev_ocf_core_destroy(core_ctx); + core_add_ctx->rpc_cb_fn(NULL, core_add_ctx->rpc_cb_arg, error); + free(core_add_ctx); } static void -_core_add_rpc_cb(ocf_cache_t ocf_cache, ocf_core_t ocf_core, void *cb_arg, int error) +_core_add_rpc_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int error) { - struct vbdev_ocf_core_add_ctx *core_add_ctx = cb_arg; - struct vbdev_ocf_cache *cache = core_add_ctx->cache; - struct vbdev_ocf_core *core = core_add_ctx->core; + struct vbdev_ocf_mngt_ctx *core_add_ctx = cb_arg; + struct vbdev_ocf_core *core_ctx = core_add_ctx->core_ctx; int rc = 0; - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", core->name); - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add\n", core->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", + vbdev_ocf_core_get_name(core_ctx)); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add\n", vbdev_ocf_core_get_name(core_ctx)); if (error) { SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s'\n", - core->name, cache->name); - ocf_mngt_cache_unlock(ocf_cache); - vbdev_ocf_core_base_detach(core); - vbdev_ocf_core_destroy(core); + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); + vbdev_ocf_core_base_detach(core_ctx); + vbdev_ocf_core_destroy(core_ctx); core_add_ctx->rpc_cb_fn(NULL, core_add_ctx->rpc_cb_arg, error); free(core_add_ctx); return; } - vbdev_ocf_core_add_to_cache(core, cache); - core->ocf_core = ocf_core; + ocf_core_set_priv(core, core_ctx); if ((rc = vbdev_ocf_core_register(core))) { - SPDK_ERRLOG("OCF core '%s': failed to register vbdev\n", core->name); - vbdev_ocf_core_add_rollback(core); - core_add_ctx->rpc_cb_fn(NULL, core_add_ctx->rpc_cb_arg, rc); - free(core_add_ctx); + SPDK_ERRLOG("OCF core '%s': failed to register vbdev\n", ocf_core_get_name(core)); + ocf_mngt_cache_remove_core(core, _core_add_rpc_err_cb, core_add_ctx); return; } - SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n", core->name, cache->name); + SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n", + ocf_core_get_name(core), ocf_cache_get_name(cache)); - ocf_mngt_cache_unlock(ocf_cache); - core_add_ctx->rpc_cb_fn(core, core_add_ctx->rpc_cb_arg, rc); + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); + core_add_ctx->rpc_cb_fn(ocf_core_get_name(core), core_add_ctx->rpc_cb_arg, rc); free(core_add_ctx); } static void -_vbdev_ocf_core_add_rpc_lock_cb(ocf_cache_t ocf_cache, void *lock_arg, int lock_err) +_core_add_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_core_add_ctx *core_add_ctx = lock_arg; - struct vbdev_ocf_cache *cache = core_add_ctx->cache; - struct vbdev_ocf_core *core = core_add_ctx->core; + struct vbdev_ocf_mngt_ctx *core_add_ctx = cb_arg; + struct vbdev_ocf_core *core_ctx = core_add_ctx->core_ctx; - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add of OCF core\n", core->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add of OCF core\n", + vbdev_ocf_core_get_name(core_ctx)); - if (lock_err) { + if (error) { SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock\n", - core->name); - vbdev_ocf_core_base_detach(core); - vbdev_ocf_core_destroy(core); - core_add_ctx->rpc_cb_fn(NULL, core_add_ctx->rpc_cb_arg, lock_err); + vbdev_ocf_core_get_name(core_ctx)); + ocf_mngt_cache_put(cache); + vbdev_ocf_core_base_detach(core_ctx); + vbdev_ocf_core_destroy(core_ctx); + core_add_ctx->rpc_cb_fn(NULL, core_add_ctx->rpc_cb_arg, error); // core name free(core_add_ctx); return; } - ocf_mngt_cache_add_core(cache->ocf_cache, &core->ocf_core_cfg, _core_add_rpc_cb, core_add_ctx); + ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _core_add_rpc_add_cb, core_add_ctx); } /* RPC entry point. */ void -vbdev_ocf_core_add(const char *core_name, const char *bdev_name, const char *cache_name, - vbdev_ocf_core_add_cb rpc_cb_fn, void *rpc_cb_arg) +vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cache_name, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { - struct vbdev_ocf_cache *cache; - struct vbdev_ocf_core *core; - struct vbdev_ocf_core_add_ctx *core_add_ctx; + ocf_cache_t cache; + struct vbdev_ocf_core *core_ctx; + struct vbdev_ocf_mngt_ctx *core_add_ctx; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add\n", core_name); - if (vbdev_ocf_device_exists(core_name)) { - SPDK_ERRLOG("OCF: device '%s' already exists\n", core_name); + if (vbdev_ocf_bdev_exists(core_name)) { + SPDK_ERRLOG("OCF: bdev '%s' already exists\n", core_name); rc = -EEXIST; goto err_create; } - /* Allocate memory for core struct. */ - if ((rc = vbdev_ocf_core_create(core_name, &core))) { + if ((rc = vbdev_ocf_core_create(&core_ctx, core_name, cache_name))) { SPDK_ERRLOG("OCF core '%s': failed to create core\n", core_name); goto err_create; } - /* Create OCF core config. */ - if ((rc = vbdev_ocf_core_set_config(core))) { - SPDK_ERRLOG("OCF core '%s': failed to create OCF config\n", core_name); - goto err_base; - } - /* First, check if base device for this core is already present. */ - if ((rc = vbdev_ocf_core_base_attach(core, bdev_name))) { + if ((rc = vbdev_ocf_core_base_attach(core_ctx, base_name))) { if (rc == -ENODEV) { + /* If not, just put core context on the temporary core wait list and exit. */ SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for base bdev '%s'\n", - core_name, bdev_name); - /* If not, just save core init params for use in examine, - * put core on the temporary incomplete core list and exit. */ - if ((rc = vbdev_ocf_core_add_incomplete(core, bdev_name, cache_name))) { - SPDK_ERRLOG("OCF core '%s': failed to save init params - removing core\n", - core_name); - goto err_base; - } - rpc_cb_fn(core, rpc_cb_arg, -ENODEV); + core_name, base_name); + STAILQ_INSERT_TAIL(&g_vbdev_ocf_core_waitlist, core_ctx, waitlist_entry); + rpc_cb_fn(core_name, rpc_cb_arg, -ENODEV); return; } - SPDK_ERRLOG("OCF core '%s': failed to open base bdev '%s'\n", core_name, bdev_name); + SPDK_ERRLOG("OCF core '%s': failed to open base bdev '%s'\n", core_name, base_name); goto err_base; } - // move elsewhere - core->ocf_core_cfg.volume_params = &core->base; // for ocf_volume_open() in ocf_mngt_cache_add_core() - /* Second, check if OCF cache for this core is already present and started. */ - cache = vbdev_ocf_cache_get_by_name(cache_name); - if (!cache || vbdev_ocf_cache_is_incomplete(cache)) { + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { + /* If not, just put core context on the temporary core wait list and exit. */ SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache '%s'\n", core_name, cache_name); - - /* If not, just save core init params for use in examine, - * put core on the temporary incomplete core list and exit. */ - if ((rc = vbdev_ocf_core_add_incomplete(core, bdev_name, cache_name))) { - SPDK_ERRLOG("OCF core '%s': failed to save init params - removing core\n", - core_name); - goto err_alloc; - } - rpc_cb_fn(core, rpc_cb_arg, -ENODEV); + STAILQ_INSERT_TAIL(&g_vbdev_ocf_core_waitlist, core_ctx, waitlist_entry); + rpc_cb_fn(core_name, rpc_cb_arg, -ENODEV); return; } // check if (cache_block_size > core_block_size) - core_add_ctx = calloc(1, sizeof(struct vbdev_ocf_core_add_ctx)); + core_add_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!core_add_ctx) { - SPDK_ERRLOG("OCF core '%s': failed to allocate memory for vbdev_ocf_core_add_ctx\n", + SPDK_ERRLOG("OCF core '%s': failed to allocate memory for core add context\n", core_name); rc = -ENOMEM; goto err_alloc; } - core_add_ctx->core = core; core_add_ctx->cache = cache; + core_add_ctx->core_ctx = core_ctx; core_add_ctx->rpc_cb_fn = rpc_cb_fn; core_add_ctx->rpc_cb_arg = rpc_cb_arg; - ocf_mngt_cache_lock(cache->ocf_cache, _vbdev_ocf_core_add_rpc_lock_cb, core_add_ctx); + ocf_mngt_cache_lock(cache, _core_add_rpc_lock_cb, core_add_ctx); return; err_alloc: - vbdev_ocf_core_base_detach(core); + ocf_mngt_cache_put(cache); + vbdev_ocf_core_base_detach(core_ctx); err_base: - vbdev_ocf_core_destroy(core); + vbdev_ocf_core_destroy(core_ctx); err_create: - rpc_cb_fn(NULL, rpc_cb_arg, rc); + rpc_cb_fn(core_name, rpc_cb_arg, rc); } static void -_core_unregister_core_rm_cb(void *cb_arg, int error) +_core_remove_rpc_remove_cb(void *cb_arg, int error) { - struct vbdev_ocf_core_remove_ctx *core_rm_ctx = cb_arg; - struct vbdev_ocf_core *core = core_rm_ctx->core; + struct vbdev_ocf_mngt_ctx *core_rm_ctx = cb_arg; + struct vbdev_ocf_core *core_ctx = core_rm_ctx->core_ctx; + ocf_cache_t cache = core_rm_ctx->cache; - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", - core->name); - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing removal\n", core->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing remove of OCF core\n", + vbdev_ocf_core_get_name(core_ctx)); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing removal\n", + vbdev_ocf_core_get_name(core_ctx)); if (error) { - SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during core removal\n", - core->name); + SPDK_ERRLOG("OCF core '%s': failed to remove OCF core device (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), error); } else { - SPDK_NOTICELOG("OCF core '%s': removed from cache\n", core->name); - vbdev_ocf_core_destroy(core); + SPDK_NOTICELOG("OCF core '%s': removed from cache '%s'\n", + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); } - core_rm_ctx->rpc_cb_fn(core_rm_ctx->rpc_cb_arg, error); + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); + vbdev_ocf_core_destroy(core_ctx); + core_rm_ctx->rpc_cb_fn(NULL, core_rm_ctx->rpc_cb_arg, error); // core name free(core_rm_ctx); } +static void +_core_remove_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *core_rm_ctx = cb_arg; + ocf_core_t core = core_rm_ctx->core; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating remove of OCF core\n", + ocf_core_get_name(core)); + + if (error) { + SPDK_ERRLOG("OCF vbdev '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_core_get_name(core), error); + } + + ocf_mngt_cache_remove_core(core, _core_remove_rpc_remove_cb, core_rm_ctx); +} + +static void +_core_remove_rpc_unregister_cb(void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *core_rm_ctx = cb_arg; + ocf_core_t core = core_rm_ctx->core; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", + ocf_core_get_name(core)); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during core removal\n", + ocf_core_get_name(core)); + } + + ocf_mngt_cache_lock(ocf_core_get_cache(core), _core_remove_rpc_lock_cb, core_rm_ctx); +} + /* RPC entry point. */ void -vbdev_ocf_core_remove(const char *core_name, vbdev_ocf_core_remove_cb rpc_cb_fn, void *rpc_cb_arg) +vbdev_ocf_core_remove(const char *core_name, const char *cache_name, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { - struct vbdev_ocf_core *core; - struct vbdev_ocf_core_remove_ctx *core_rm_ctx; + ocf_cache_t cache; + ocf_core_t core; + struct vbdev_ocf_core *core_ctx; + struct vbdev_ocf_mngt_ctx *core_rm_ctx; int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating removal\n", core_name); - core = vbdev_ocf_core_get_by_name(core_name); - if (!core) { - SPDK_ERRLOG("OCF core '%s': device not found\n", core_name); - rpc_cb_fn(rpc_cb_arg, -ENODEV); - return; - } - /* If core was not added yet due to lack of base or cache device, * just free its structs (and detach its base if exists) and exit. */ - if (vbdev_ocf_core_is_incomplete(core)) { - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': removing as incomplete\n", core->name); - if (core->base.attached) { - vbdev_ocf_core_base_detach(core); + core_ctx = vbdev_ocf_core_waitlist_get_by_name(core_name); + if (core_ctx) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': removing from wait list\n", core_name); + if (vbdev_ocf_core_is_base_attached(core_ctx)) { + vbdev_ocf_core_base_detach(core_ctx); } - vbdev_ocf_core_remove_incomplete(core); - vbdev_ocf_core_destroy(core); - rpc_cb_fn(rpc_cb_arg, 0); + vbdev_ocf_core_destroy(core_ctx); + rpc_cb_fn(core_name, rpc_cb_arg, 0); return; } - core_rm_ctx = calloc(1, sizeof(struct vbdev_ocf_core_remove_ctx)); + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { + SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); + rc = -ENXIO; + goto err_cache; + } + + if ((rc = ocf_core_get_by_name(cache, core_name, OCF_CORE_NAME_SIZE, &core))) { + if (rc == -OCF_ERR_CORE_NOT_EXIST) { + SPDK_ERRLOG("OCF core '%s': not exist within cache '%s'\n", core_name, cache_name); + rc = -ENXIO; + } + goto err_core; + } + + core_rm_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!core_rm_ctx) { SPDK_ERRLOG("OCF core '%s': failed to allocate memory for core remove context\n", core_name); - rpc_cb_fn(rpc_cb_arg, -ENOMEM); - return; + rc = -ENOMEM; + goto err_core; } + core_rm_ctx->cache = cache; core_rm_ctx->core = core; + core_rm_ctx->core_ctx = ocf_core_get_priv(core); core_rm_ctx->rpc_cb_fn = rpc_cb_fn; core_rm_ctx->rpc_cb_arg = rpc_cb_arg; - if ((rc = vbdev_ocf_core_unregister(core, _core_unregister_core_rm_cb, core_rm_ctx))) { + if ((rc = vbdev_ocf_core_unregister(core_rm_ctx->core_ctx, _core_remove_rpc_unregister_cb, core_rm_ctx))) { SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during core removal\n", - core->name); - rpc_cb_fn(rpc_cb_arg, rc); - free(core_rm_ctx); + ocf_core_get_name(core)); + goto err_unregister; } -} -static void -_write_cache_info_begin(struct spdk_json_write_ctx *w, struct vbdev_ocf_cache *cache) -{ - spdk_json_write_object_begin(w); - spdk_json_write_named_string(w, "type", "OCF_cache"); - spdk_json_write_named_string(w, "name", cache->name); - spdk_json_write_named_string(w, "base_bdev_name", - cache->base.bdev ? spdk_bdev_get_name(cache->base.bdev) : "" ); - spdk_json_write_named_uint16(w, "cores_count", cache->cores_count); -} + return; -static void -_write_cache_info_end(struct spdk_json_write_ctx *w, struct vbdev_ocf_cache *cache) -{ - spdk_json_write_object_end(w); +err_unregister: + free(core_rm_ctx); +err_core: + ocf_mngt_cache_put(cache); +err_cache: + rpc_cb_fn(core_name, rpc_cb_arg, rc); } -static void -_write_core_info(struct spdk_json_write_ctx *w, struct vbdev_ocf_core *core) -{ - spdk_json_write_object_begin(w); - spdk_json_write_named_string(w, "type", "OCF_core"); - spdk_json_write_named_string(w, "name", core->name); - spdk_json_write_named_string(w, "base_bdev_name", - core->base.bdev ? spdk_bdev_get_name(core->base.bdev) : "" ); - spdk_json_write_named_string(w, "cache_name", vbdev_ocf_core_get_cache(core)->name); - spdk_json_write_object_end(w); -} +//comp +//static void +//_write_cache_info_begin(struct spdk_json_write_ctx *w, struct vbdev_ocf_cache *cache) +//{ +// spdk_json_write_object_begin(w); +// spdk_json_write_named_string(w, "type", "OCF_cache"); +// spdk_json_write_named_string(w, "name", cache->name); +// spdk_json_write_named_string(w, "base_bdev_name", +// cache->base.bdev ? spdk_bdev_get_name(cache->base.bdev) : "" ); +// spdk_json_write_named_uint16(w, "cores_count", cache->cores_count); +//} +// +//static void +//_write_cache_info_end(struct spdk_json_write_ctx *w, struct vbdev_ocf_cache *cache) +//{ +// spdk_json_write_object_end(w); +//} +// +//static void +//_write_core_info(struct spdk_json_write_ctx *w, struct vbdev_ocf_core *core) +//{ +// spdk_json_write_object_begin(w); +// spdk_json_write_named_string(w, "type", "OCF_core"); +// spdk_json_write_named_string(w, "name", core->name); +// spdk_json_write_named_string(w, "base_bdev_name", +// core->base.bdev ? spdk_bdev_get_name(core->base.bdev) : "" ); +// spdk_json_write_named_string(w, "cache_name", vbdev_ocf_core_get_cache(core)->name); +// spdk_json_write_object_end(w); +//} /* RPC entry point. */ void vbdev_ocf_get_bdevs(const char *name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2) { - struct spdk_json_write_ctx *w = rpc_cb_arg1; - struct vbdev_ocf_cache *cache; - struct vbdev_ocf_core *core; - bool found = false; - - if (name) { - vbdev_ocf_foreach_cache(cache) { - vbdev_ocf_foreach_core_in_cache(core, cache) { - if (strcmp(name, core->name)) { - continue; - } - found = true; - - // dump_info_json() instead? - _write_core_info(w, core); - break; - } - if (found) { - break; - } - if (strcmp(name, cache->name)) { - continue; - } - - _write_cache_info_begin(w, cache); - _write_cache_info_end(w, cache); - break; - } - } else { - vbdev_ocf_foreach_cache(cache) { - _write_cache_info_begin(w, cache); - spdk_json_write_named_array_begin(w, "cores"); - vbdev_ocf_foreach_core_in_cache(core, cache) { - _write_core_info(w, core); - } - spdk_json_write_array_end(w); - _write_cache_info_end(w, cache); - } - } - - rpc_cb_fn(rpc_cb_arg1, rpc_cb_arg2); + //comp + //struct spdk_json_write_ctx *w = rpc_cb_arg1; + //struct vbdev_ocf_cache *cache; + //struct vbdev_ocf_core *core; + //bool found = false; + + //if (name) { + // vbdev_ocf_foreach_cache(cache) { + // vbdev_ocf_foreach_core_in_cache(core, cache) { + // if (strcmp(name, core->name)) { + // continue; + // } + // found = true; + + // // dump_info_json() instead? + // _write_core_info(w, core); + // break; + // } + // if (found) { + // break; + // } + // if (strcmp(name, cache->name)) { + // continue; + // } + + // _write_cache_info_begin(w, cache); + // _write_cache_info_end(w, cache); + // break; + // } + //} else { + // vbdev_ocf_foreach_cache(cache) { + // _write_cache_info_begin(w, cache); + // spdk_json_write_named_array_begin(w, "cores"); + // vbdev_ocf_foreach_core_in_cache(core, cache) { + // _write_core_info(w, core); + // } + // spdk_json_write_array_end(w); + // _write_cache_info_end(w, cache); + // } + //} + + //rpc_cb_fn(rpc_cb_arg1, rpc_cb_arg2); } SPDK_LOG_REGISTER_COMPONENT(vbdev_ocf) diff --git a/module/bdev/ocf/vbdev_ocf.h b/module/bdev/ocf/vbdev_ocf.h index 48db95a29e4..e11bfa9f513 100644 --- a/module/bdev/ocf/vbdev_ocf.h +++ b/module/bdev/ocf/vbdev_ocf.h @@ -9,67 +9,56 @@ #include "vbdev_ocf_cache.h" #include "vbdev_ocf_core.h" -typedef void (*vbdev_ocf_cache_start_cb)(struct vbdev_ocf_cache *vbdev_cache, void *cb_arg, int error); -typedef void (*vbdev_ocf_cache_stop_cb)(void *cb_arg, int error); -typedef void (*vbdev_ocf_core_add_cb)(struct vbdev_ocf_core *vbdev_core, void *cb_arg, int error); -typedef void (*vbdev_ocf_core_remove_cb)(void *cb_arg, int error); +typedef void (*vbdev_ocf_rpc_mngt_cb)(const char *bdev_name, void *cb_arg, int error); typedef void (*vbdev_ocf_get_bdevs_cb)(void *cb_arg1, void *cb_arg2); -// merge into one for all start/stop/add/remove ? -struct vbdev_ocf_cache_start_ctx { - struct vbdev_ocf_cache * cache; - vbdev_ocf_cache_start_cb rpc_cb_fn; +struct vbdev_ocf_mngt_ctx { + ocf_cache_t cache; + ocf_core_t core; + struct vbdev_ocf_core * core_ctx; + ocf_mngt_cache_attach_end_t attach_cb_fn; + vbdev_ocf_rpc_mngt_cb rpc_cb_fn; void * rpc_cb_arg; }; -struct vbdev_ocf_cache_stop_ctx { - struct vbdev_ocf_cache * cache; - vbdev_ocf_cache_stop_cb rpc_cb_fn; - void * rpc_cb_arg; -}; - -struct vbdev_ocf_core_add_ctx { - struct vbdev_ocf_cache * cache; - struct vbdev_ocf_core * core; - vbdev_ocf_core_add_cb rpc_cb_fn; - void * rpc_cb_arg; -}; +/* RPC entry points. */ -struct vbdev_ocf_core_remove_ctx { - struct vbdev_ocf_core * core; - vbdev_ocf_core_remove_cb rpc_cb_fn; - void * rpc_cb_arg; -}; - -/* RPC entry point. */ void vbdev_ocf_cache_start(const char *cache_name, - const char *bdev_name, + const char *base_name, const char *cache_mode, const uint8_t cache_line_size, - vbdev_ocf_cache_start_cb cb_fn, - void *cb_arg); + bool no_load, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, + void *rpc_cb_arg); -/* RPC entry point. */ void vbdev_ocf_cache_stop(const char *cache_name, - vbdev_ocf_cache_stop_cb cb_fn, - void *cb_arg); + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, + void *rpc_cb_arg); + +void vbdev_ocf_cache_attach(const char *cache_name, + const char *base_name, + bool force, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, + void *rpc_cb_arg); + +void vbdev_ocf_cache_detach(const char *cache_name, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, + void *rpc_cb_arg); -/* RPC entry point. */ void vbdev_ocf_core_add(const char *core_name, - const char *bdev_name, + const char *base_name, const char *cache_name, - vbdev_ocf_core_add_cb cb_fn, - void *cb_arg); + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, + void *rpc_cb_arg); -/* RPC entry point. */ void vbdev_ocf_core_remove(const char *core_name, - vbdev_ocf_core_remove_cb cb_fn, - void *cb_arg); + const char *cache_name, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, + void *rpc_cb_arg); -/* RPC entry point. */ void vbdev_ocf_get_bdevs(const char *name, - vbdev_ocf_get_bdevs_cb cb_fn, - void *cb_arg1, - void *cb_arg2); + vbdev_ocf_get_bdevs_cb rpc_cb_fn, + void *rpc_cb_arg1, + void *rpc_cb_arg2); #endif diff --git a/module/bdev/ocf/vbdev_ocf_cache.c b/module/bdev/ocf/vbdev_ocf_cache.c index 0f21a66be49..47dddd40f9e 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.c +++ b/module/bdev/ocf/vbdev_ocf_cache.c @@ -7,34 +7,60 @@ #include "ctx.h" #include "utils.h" -struct vbdev_ocf_caches_head g_vbdev_ocf_caches = STAILQ_HEAD_INITIALIZER(g_vbdev_ocf_caches); - int -vbdev_ocf_cache_create(const char *cache_name, struct vbdev_ocf_cache **out) +vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cache_mode, + const uint8_t cache_line_size, bool no_load) { - struct vbdev_ocf_cache *cache; + ocf_cache_t cache; + struct ocf_mngt_cache_config *cache_cfg; + struct ocf_mngt_cache_attach_config *cache_att_cfg; + struct vbdev_ocf_cache *cache_ctx; int rc = 0; - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': allocating vbdev_ocf_cache and adding it to cache list\n", + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': allocating structs and starting cache\n", cache_name); - cache = calloc(1, sizeof(struct vbdev_ocf_cache)); - if (!cache) { - SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for vbdev_ocf_cache\n", + cache_ctx = calloc(1, sizeof(struct vbdev_ocf_cache)); + if (!cache_ctx) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache context\n", cache_name); return -ENOMEM; } - cache->name = strdup(cache_name); - if (!cache->name) { - SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for vbdev_ocf_cache name\n", - cache_name); - free(cache); - return -ENOMEM; + cache_cfg = &cache_ctx->cache_cfg; + cache_att_cfg = &cache_ctx->cache_att_cfg; + + ocf_mngt_cache_config_set_default(cache_cfg); + ocf_mngt_cache_attach_config_set_default(cache_att_cfg); + + strncpy(cache_cfg->name, cache_name, OCF_CACHE_NAME_SIZE); + if (cache_mode) { + cache_cfg->cache_mode = ocf_get_cache_mode(cache_mode); + } + if (cache_line_size) { + cache_cfg->cache_line_size = cache_line_size * KiB; + cache_att_cfg->cache_line_size = cache_line_size * KiB; + } + cache_cfg->locked = true; + cache_att_cfg->open_cores = false; + cache_att_cfg->discard_on_start = false; // needed ? + cache_att_cfg->device.perform_test = false; // needed ? + cache_att_cfg->force = no_load; + + if ((rc = ocf_mngt_cache_start(vbdev_ocf_ctx, &cache, cache_cfg, cache_ctx))) { + SPDK_ERRLOG("OCF cache '%s': failed to start OCF cache\n", cache_name); + free(cache_ctx); + return rc; } - STAILQ_INIT(&cache->cores); - STAILQ_INSERT_TAIL(&g_vbdev_ocf_caches, cache, link); + // needed ? + //if ((rc = ocf_mngt_cache_get(cache))) { + // SPDK_ERRLOG("OCF cache '%s': failed to increment cache ref count: %s\n", + // cache_name, spdk_strerror(-rc)); + // ocf_mngt_cache_stop(cache, NULL, NULL); // needs callback (_cache_start_err_cb ?) + // free(cache_ctx); + // return rc; + //} *out = cache; @@ -42,168 +68,131 @@ vbdev_ocf_cache_create(const char *cache_name, struct vbdev_ocf_cache **out) } void -vbdev_ocf_cache_destroy(struct vbdev_ocf_cache *cache) -{ - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': deallocating vbdev_ocf_cache and removing it from cache list\n", - cache->name); - - STAILQ_REMOVE(&g_vbdev_ocf_caches, cache, vbdev_ocf_cache, link); - free(cache->name); - free(cache); -} - -int -vbdev_ocf_cache_set_config(struct vbdev_ocf_cache *cache, const char *cache_mode, - const uint8_t cache_line_size) +vbdev_ocf_cache_destroy(ocf_cache_t cache) { - struct ocf_mngt_cache_config *ocf_cache_cfg = &cache->ocf_cache_cfg; - struct ocf_mngt_cache_attach_config *ocf_cache_att_cfg = &cache->ocf_cache_att_cfg; - int rc = 0; + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': setting OCF config\n", cache->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': deallocating structs\n", + ocf_cache_get_name(cache)); - ocf_mngt_cache_config_set_default(ocf_cache_cfg); - ocf_mngt_cache_attach_config_set_default(ocf_cache_att_cfg); - - strncpy(ocf_cache_cfg->name, cache->name, OCF_CACHE_NAME_SIZE); - if (cache_mode) { - ocf_cache_cfg->cache_mode = ocf_get_cache_mode(cache_mode); - } - if (cache_line_size) { - ocf_cache_cfg->cache_line_size = cache_line_size * KiB; - ocf_cache_att_cfg->cache_line_size = cache_line_size * KiB; - } - ocf_cache_cfg->locked = true; - ocf_cache_att_cfg->open_cores = false; - ocf_cache_att_cfg->discard_on_start = false; // needed ? - ocf_cache_att_cfg->device.perform_test = false; // needed ? - // TODO: add load option - ocf_cache_att_cfg->force = true; - - return rc; // rm? + free(cache_ctx); } // vbdev_ocf_cache_detach() instead of this ? static void vbdev_ocf_cache_hotremove(struct spdk_bdev *bdev, void *event_ctx) { - struct vbdev_ocf_cache *cache = event_ctx; + ocf_cache_t cache = event_ctx; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': hot removal of base bdev '%s'\n", - cache->name, bdev->name); + ocf_cache_get_name(cache), spdk_bdev_get_name(bdev)); - assert(bdev == cache->base.bdev); + assert(bdev == ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->base.bdev); - if (vbdev_ocf_cache_is_running(cache)) { - // OCF cache flush + if (ocf_cache_is_device_attached(cache)) { // always true? // OCF cache detach // (to comply with SPDK hotremove support) } - vbdev_ocf_cache_base_detach(cache); + vbdev_ocf_cache_base_detach(cache); // in detach callback ? } static void _vbdev_ocf_cache_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) { - struct vbdev_ocf_cache *cache = event_ctx; + ocf_cache_t cache = event_ctx; switch (type) { case SPDK_BDEV_EVENT_REMOVE: vbdev_ocf_cache_hotremove(bdev, event_ctx); break; default: - SPDK_NOTICELOG("OCF cache '%s': unsupported bdev event type: %d\n", cache->name, type); + SPDK_NOTICELOG("OCF cache '%s': unsupported bdev event type: %d\n", + ocf_cache_get_name(cache), type); } } int -vbdev_ocf_cache_base_attach(struct vbdev_ocf_cache *cache, const char *bdev_name) +vbdev_ocf_cache_base_attach(ocf_cache_t cache, const char *base_name) { + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + struct vbdev_ocf_base *base = &cache_ctx->base; + struct ocf_volume_uuid volume_uuid; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': attaching base bdev '%s'\n", - cache->name, bdev_name); + ocf_cache_get_name(cache), base_name); - if ((rc = spdk_bdev_open_ext(bdev_name, true, _vbdev_ocf_cache_event_cb, cache, &cache->base.desc))) { + //strncpy(base->name, base_name, OCF_CACHE_NAME_SIZE); + + if ((rc = spdk_bdev_open_ext(base_name, true, _vbdev_ocf_cache_event_cb, cache, &base->desc))) { return rc; } - if ((rc = spdk_bdev_module_claim_bdev_desc(cache->base.desc, - SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, + if ((rc = spdk_bdev_module_claim_bdev_desc(base->desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, NULL, &ocf_if))) { - SPDK_ERRLOG("OCF cache '%s': failed to claim base bdev '%s'\n", cache->name, bdev_name); - spdk_bdev_close(cache->base.desc); + SPDK_ERRLOG("OCF cache '%s': failed to claim base bdev '%s'\n", + ocf_cache_get_name(cache), base_name); + spdk_bdev_close(base->desc); return rc; } - cache->base.mngt_ch = spdk_bdev_get_io_channel(cache->base.desc); - if (!cache->base.mngt_ch) { + base->mngt_ch = spdk_bdev_get_io_channel(base->desc); + if (!base->mngt_ch) { SPDK_ERRLOG("OCF cache '%s': failed to get IO channel for base bdev '%s'\n", - cache->name, bdev_name); - spdk_bdev_close(cache->base.desc); + ocf_cache_get_name(cache), base_name); + spdk_bdev_close(base->desc); return -ENOMEM; } - cache->base.bdev = spdk_bdev_desc_get_bdev(cache->base.desc); - cache->base.thread = spdk_get_thread(); - cache->base.is_cache = true; - cache->base.attached = true; + base->bdev = spdk_bdev_desc_get_bdev(base->desc); + base->thread = spdk_get_thread(); + base->is_cache = true; + base->attached = true; + + if ((rc = ocf_uuid_set_str(&volume_uuid, (char *)base_name))) { + SPDK_ERRLOG("OCF cache '%s': failed to set OCF volume uuid\n", + ocf_cache_get_name(cache)); + spdk_put_io_channel(base->mngt_ch); + spdk_bdev_close(base->desc); + return rc; + } + + if ((rc = ocf_ctx_volume_create(vbdev_ocf_ctx, &cache_ctx->cache_att_cfg.device.volume, + &volume_uuid, SPDK_OBJECT))) { + SPDK_ERRLOG("OCF cache '%s': failed to create OCF volume\n", ocf_cache_get_name(cache)); + spdk_put_io_channel(base->mngt_ch); + spdk_bdev_close(base->desc); + return rc; + } + + // for ocf_volume_open() in ocf_mngt_cache_attach/load() + cache_ctx->cache_att_cfg.device.volume_params = base; // why not ? what is it then ?!? - //assert(__bdev_to_io_dev(cache->base.bdev) == cache->base.bdev->ctxt); + //assert(__bdev_to_io_dev(base->bdev) == base->bdev->ctxt); return rc; } void -vbdev_ocf_cache_base_detach(struct vbdev_ocf_cache *cache) +vbdev_ocf_cache_base_detach(ocf_cache_t cache) { - struct vbdev_ocf_base *base = &cache->base; + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + struct vbdev_ocf_base *base = &cache_ctx->base; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': detaching base bdev '%s'\n", - cache->name, base->bdev->name); - - vbdev_ocf_base_detach(base); -} - -int -vbdev_ocf_cache_add_incomplete(struct vbdev_ocf_cache *cache, const char *bdev_name) -{ - struct vbdev_ocf_cache_init_params *init_params; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': allocating init params\n", cache->name); - - init_params = calloc(1, sizeof(struct vbdev_ocf_cache_init_params)); - if (!init_params) { - SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for init params\n", - cache->name); - return -ENOMEM; - } - - init_params->bdev_name = strdup(bdev_name); - if (!init_params->bdev_name) { - SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for base bdev name\n", - cache->name); - free(init_params); - return -ENOMEM; - } + ocf_cache_get_name(cache), spdk_bdev_get_name(base->bdev)); - cache->init_params = init_params; - return 0; -} - -void -vbdev_ocf_cache_remove_incomplete(struct vbdev_ocf_cache *cache) -{ - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': deallocating init params\n", cache->name); + //if (cache_ctx->cache_att_cfg.device.volume) { + // ocf_volume_destroy(cache_ctx->cache_att_cfg.device.volume); + // cache_ctx->cache_att_cfg.device.volume = NULL; + // //cache_ctx->cache_att_cfg.device.volume_params = NULL; // ? + //} + ocf_volume_destroy(cache_ctx->cache_att_cfg.device.volume); - free(cache->init_params->bdev_name); - free(cache->init_params); - /* It's important to set init_params back to NULL. - * This is an indicator that cache is not incomplete any more. */ - cache->init_params = NULL; + vbdev_ocf_base_detach(base); } static void @@ -221,7 +210,7 @@ vbdev_ocf_cache_mngt_queue_stop(ocf_queue_t queue) struct vbdev_ocf_cache_mngt_queue_ctx *mngt_q_ctx = ocf_queue_get_priv(queue); SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': destroying OCF management queue\n", - mngt_q_ctx->cache->name); + ocf_cache_get_name(mngt_q_ctx->cache)); if (mngt_q_ctx->thread && mngt_q_ctx->thread != spdk_get_thread()) { spdk_thread_send_msg(mngt_q_ctx->thread, _cache_mngt_queue_stop, mngt_q_ctx); @@ -242,31 +231,35 @@ const struct ocf_queue_ops cache_mngt_queue_ops = { }; int -vbdev_ocf_cache_mngt_queue_create(struct vbdev_ocf_cache *cache) +vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache) { + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); struct vbdev_ocf_cache_mngt_queue_ctx *mngt_q_ctx; int rc = 0; - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': creating OCF management queue\n", cache->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': creating OCF management queue\n", + ocf_cache_get_name(cache)); mngt_q_ctx = calloc(1, sizeof(struct vbdev_ocf_cache_mngt_queue_ctx)); if (!mngt_q_ctx) { SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for management queue context\n", - cache->name); + ocf_cache_get_name(cache)); return -ENOMEM; } - if ((rc = vbdev_ocf_queue_create_mngt(cache->ocf_cache, &cache->ocf_cache_mngt_q, &cache_mngt_queue_ops))) { - SPDK_ERRLOG("OCF cache '%s': failed to create OCF management queue\n", cache->name); + if ((rc = vbdev_ocf_queue_create_mngt(cache, &cache_ctx->cache_mngt_q, &cache_mngt_queue_ops))) { + SPDK_ERRLOG("OCF cache '%s': failed to create OCF management queue\n", + ocf_cache_get_name(cache)); free(mngt_q_ctx); return rc; } - ocf_queue_set_priv(cache->ocf_cache_mngt_q, mngt_q_ctx); + ocf_queue_set_priv(cache_ctx->cache_mngt_q, mngt_q_ctx); - mngt_q_ctx->poller = SPDK_POLLER_REGISTER(vbdev_ocf_queue_poller, cache->ocf_cache_mngt_q, 1000); + mngt_q_ctx->poller = SPDK_POLLER_REGISTER(vbdev_ocf_queue_poller, cache_ctx->cache_mngt_q, 1000); if (!mngt_q_ctx->poller) { - SPDK_ERRLOG("OCF cache '%s': failed to create management queue poller\n", cache->name); - vbdev_ocf_queue_put(cache->ocf_cache_mngt_q); + SPDK_ERRLOG("OCF cache '%s': failed to create management queue poller\n", + ocf_cache_get_name(cache)); + vbdev_ocf_queue_put(cache_ctx->cache_mngt_q); return -ENOMEM; } @@ -276,33 +269,25 @@ vbdev_ocf_cache_mngt_queue_create(struct vbdev_ocf_cache *cache) return rc; } -struct vbdev_ocf_cache * -vbdev_ocf_cache_get_by_name(const char *cache_name) -{ - struct vbdev_ocf_cache *cache; - - vbdev_ocf_foreach_cache(cache) { - if (strcmp(cache_name, cache->name)) { - continue; - } - return cache; - } - return NULL; -} - bool -vbdev_ocf_cache_is_running(struct vbdev_ocf_cache *cache) +vbdev_ocf_cache_is_base_attached(ocf_cache_t cache) { - ocf_cache_t ocf_cache = cache->ocf_cache; + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); - if (ocf_cache && ocf_cache_is_running(ocf_cache)) { - return true; - } - return false; + return cache_ctx->base.attached; } -bool -vbdev_ocf_cache_is_incomplete(struct vbdev_ocf_cache *cache) -{ - return !!cache->init_params; -} +//bool +//vbdev_ocf_any_cache_started(void) +//{ +// /* OCF context is created with refcount set to 1 and any started cache +// * will increment it further. So, if context refcount equals 1, it means +// * that it's just created without any started caches. */ +// //return (ocf_ctx_get_refcnt(vbdev_ocf_ctx) > 1) ? true : false; +// +// int refcnt; +// +// refcnt = ocf_ctx_get_refcnt(vbdev_ocf_ctx); +// printf("*** CTX refcnt: %d\n", refcnt); +// return (refcnt > 1) ? true : false; +//} diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h index 8a7afd0408d..6684f67c802 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.h +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -10,57 +10,32 @@ #include "spdk/bdev_module.h" #include "volume.h" -/* Global list of all caches (started and incomplete). */ -extern STAILQ_HEAD(vbdev_ocf_caches_head, vbdev_ocf_cache) g_vbdev_ocf_caches; - /* OCF module interface. */ extern struct spdk_bdev_module ocf_if; -#define vbdev_ocf_foreach_cache(cache) \ - STAILQ_FOREACH(cache, &g_vbdev_ocf_caches, link) - -struct vbdev_ocf_cache_init_params { - char * bdev_name; -}; - struct vbdev_ocf_cache { - char * name; - struct spdk_uuid uuid; + struct ocf_mngt_cache_config cache_cfg; + struct ocf_mngt_cache_attach_config cache_att_cfg; + ocf_queue_t cache_mngt_q; + bool load; struct vbdev_ocf_base base; - - STAILQ_HEAD(, vbdev_ocf_core) cores; - uint16_t cores_count; - - ocf_cache_t ocf_cache; - ocf_queue_t ocf_cache_mngt_q; - struct ocf_mngt_cache_config ocf_cache_cfg; - struct ocf_mngt_cache_attach_config ocf_cache_att_cfg; - - struct vbdev_ocf_cache_init_params * init_params; - - STAILQ_ENTRY(vbdev_ocf_cache) link; }; struct vbdev_ocf_cache_mngt_queue_ctx { struct spdk_poller * poller; struct spdk_thread * thread; /* Currently kept only for its name used in debug log. */ - struct vbdev_ocf_cache * cache; // rm ? + ocf_cache_t cache; // rm ? }; -int vbdev_ocf_cache_create(const char *cache_name, struct vbdev_ocf_cache **out); -void vbdev_ocf_cache_destroy(struct vbdev_ocf_cache *cache); -int vbdev_ocf_cache_set_config(struct vbdev_ocf_cache *cache, const char *cache_mode, - const uint8_t cache_line_size); -int vbdev_ocf_cache_base_attach(struct vbdev_ocf_cache *cache, const char *bdev_name); -void vbdev_ocf_cache_base_detach(struct vbdev_ocf_cache *cache); -int vbdev_ocf_cache_add_incomplete(struct vbdev_ocf_cache *cache, const char *bdev_name); -void vbdev_ocf_cache_remove_incomplete(struct vbdev_ocf_cache *cache); -int vbdev_ocf_cache_mngt_queue_create(struct vbdev_ocf_cache *cache); +int vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cache_mode, + const uint8_t cache_line_size, bool no_load); +void vbdev_ocf_cache_destroy(ocf_cache_t cache); +int vbdev_ocf_cache_base_attach(ocf_cache_t cache, const char *base_name); +void vbdev_ocf_cache_base_detach(ocf_cache_t cache); +int vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache); -struct vbdev_ocf_cache *vbdev_ocf_cache_get_by_name(const char *cache_name); -bool vbdev_ocf_cache_is_running(struct vbdev_ocf_cache *cache); -bool vbdev_ocf_cache_is_incomplete(struct vbdev_ocf_cache *cache); +bool vbdev_ocf_cache_is_base_attached(ocf_cache_t cache); #endif diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c index ff4344dcb7e..c745b5bce65 100644 --- a/module/bdev/ocf/vbdev_ocf_core.c +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -9,250 +9,156 @@ #include "vbdev_ocf_cache.h" #include "ctx.h" -struct vbdev_ocf_incomplete_cores_head g_vbdev_ocf_incomplete_cores = - STAILQ_HEAD_INITIALIZER(g_vbdev_ocf_incomplete_cores); +struct vbdev_ocf_core_waitlist_head g_vbdev_ocf_core_waitlist = + STAILQ_HEAD_INITIALIZER(g_vbdev_ocf_core_waitlist); int -vbdev_ocf_core_create(const char *core_name, struct vbdev_ocf_core **out) +vbdev_ocf_core_create(struct vbdev_ocf_core **out, const char *core_name, const char *cache_name) { - struct vbdev_ocf_core *core; + struct vbdev_ocf_core *core_ctx; + struct ocf_mngt_core_config *core_cfg; - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': allocating vbdev_ocf_core and adding it to core list\n", + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': allocating structs\n", core_name); - core = calloc(1, sizeof(struct vbdev_ocf_core)); - if (!core) { - SPDK_ERRLOG("OCF core '%s': failed to allocate memory for vbdev_ocf_core\n", + core_ctx = calloc(1, sizeof(struct vbdev_ocf_core)); + if (!core_ctx) { + SPDK_ERRLOG("OCF core '%s': failed to allocate memory for core context\n", core_name); return -ENOMEM; } - core->name = strdup(core_name); - if (!core->name) { - SPDK_ERRLOG("OCF core '%s': failed to allocate memory for vbdev_ocf_core name\n", - core_name); - free(core); - return -ENOMEM; - } + core_cfg = &core_ctx->core_cfg; + ocf_mngt_core_config_set_default(core_cfg); + strncpy(core_cfg->name, core_name, OCF_CORE_NAME_SIZE); + strncpy(core_ctx->cache_name, cache_name, OCF_CACHE_NAME_SIZE); - *out = core; + *out = core_ctx; return 0; } void -vbdev_ocf_core_destroy(struct vbdev_ocf_core *core) +vbdev_ocf_core_destroy(struct vbdev_ocf_core *core_ctx) { - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': deallocating vbdev_ocf_core\n", - core->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': deallocating structs\n", + vbdev_ocf_core_get_name(core_ctx)); - free(core->name); - free(core); -} - -int -vbdev_ocf_core_set_config(struct vbdev_ocf_core *core) -{ - struct ocf_mngt_core_config *ocf_core_cfg = &core->ocf_core_cfg; - int rc = 0; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': setting OCF config\n", core->name); - - ocf_mngt_core_config_set_default(ocf_core_cfg); - - strncpy(ocf_core_cfg->name, core->name, OCF_CORE_NAME_SIZE); - if ((rc = ocf_uuid_set_str(&ocf_core_cfg->uuid, core->name))) { // set spdk uuid ? - return rc; - } - ocf_core_cfg->volume_type = SPDK_OBJECT; - - return rc; -} - -static void -_core_unregister_core_hotrm_cb(void *cb_arg, int error) -{ - struct vbdev_ocf_core *core = cb_arg; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", - core->name); - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing hot removal of base bdev\n", - core->name); - - if (error) { - SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during hot removal: %s\n", - core->name, spdk_strerror(-error)); - } else { - /* Do not destroy core struct in hotremove; it will be needed in examine. - * Just indicate that it's not connected to cache anymore. */ - core->ocf_core = NULL; - } + free(core_ctx); } static void vbdev_ocf_core_hotremove(struct spdk_bdev *bdev, void *event_ctx) { - struct vbdev_ocf_core *core = event_ctx; + struct vbdev_ocf_core *core_ctx = event_ctx; + struct vbdev_ocf_core *core_ctx_waitlist; int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating hot removal of base bdev '%s'\n", - core->name, bdev->name); - - assert(bdev == core->base.bdev); - - if (vbdev_ocf_core_is_incomplete(core)) { - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': hot removing as incomplete\n", core->name); - vbdev_ocf_core_base_detach(core); - return; + vbdev_ocf_core_get_name(core_ctx), spdk_bdev_get_name(bdev)); + + assert(bdev == core_ctx->base.bdev); + + vbdev_ocf_foreach_core_in_waitlist(core_ctx_waitlist) { + if (core_ctx == core_ctx_waitlist) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': hot removing from wait list\n", + vbdev_ocf_core_get_name(core_ctx)); + // always true ? + if (vbdev_ocf_core_is_base_attached(core_ctx)) { + vbdev_ocf_core_base_detach(core_ctx); + return; + } + } } - if ((rc = vbdev_ocf_core_unregister(core, _core_unregister_core_hotrm_cb, core))) { + if ((rc = vbdev_ocf_core_unregister(core_ctx, NULL, NULL))) { SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during core hot removal: %s\n", - core->name, spdk_strerror(-rc)); + vbdev_ocf_core_get_name(core_ctx), spdk_strerror(-rc)); // detach base despite the error ? - vbdev_ocf_core_base_detach(core); - return; + vbdev_ocf_core_base_detach(core_ctx); } } static void _vbdev_ocf_core_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) { - struct vbdev_ocf_core *core = event_ctx; + struct vbdev_ocf_core *core_ctx = event_ctx; switch (type) { case SPDK_BDEV_EVENT_REMOVE: vbdev_ocf_core_hotremove(bdev, event_ctx); break; default: - SPDK_NOTICELOG("OCF core '%s': unsupported bdev event type: %d\n", core->name, type); + SPDK_NOTICELOG("OCF core '%s': unsupported bdev event type: %d\n", + vbdev_ocf_core_get_name(core_ctx), type); } } int -vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core, const char *bdev_name) +vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core_ctx, const char *base_name) { + struct vbdev_ocf_base *base = &core_ctx->base; + struct ocf_mngt_core_config *core_cfg = &core_ctx->core_cfg; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': attaching base bdev '%s'\n", - core->name, bdev_name); + vbdev_ocf_core_get_name(core_ctx), base_name); - if ((rc = spdk_bdev_open_ext(bdev_name, true, _vbdev_ocf_core_event_cb, core, &core->base.desc))) { + if ((rc = spdk_bdev_open_ext(base_name, true, _vbdev_ocf_core_event_cb, core_ctx, &base->desc))) { return rc; } - if ((rc = spdk_bdev_module_claim_bdev_desc(core->base.desc, - SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, + if ((rc = spdk_bdev_module_claim_bdev_desc(base->desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, NULL, &ocf_if))) { - SPDK_ERRLOG("OCF core '%s': failed to claim base bdev '%s'\n", core->name, bdev_name); - spdk_bdev_close(core->base.desc); + SPDK_ERRLOG("OCF core '%s': failed to claim base bdev '%s'\n", + vbdev_ocf_core_get_name(core_ctx), base_name); + spdk_bdev_close(base->desc); return rc; } - core->base.mngt_ch = spdk_bdev_get_io_channel(core->base.desc); - if (!core->base.mngt_ch) { + base->mngt_ch = spdk_bdev_get_io_channel(base->desc); + if (!base->mngt_ch) { SPDK_ERRLOG("OCF core '%s': failed to get IO channel for base bdev '%s'\n", - core->name, bdev_name); - spdk_bdev_close(core->base.desc); + vbdev_ocf_core_get_name(core_ctx), base_name); + spdk_bdev_close(base->desc); return -ENOMEM; } - core->base.bdev = spdk_bdev_desc_get_bdev(core->base.desc); - core->base.thread = spdk_get_thread(); - core->base.is_cache = false; - core->base.attached = true; + base->bdev = spdk_bdev_desc_get_bdev(base->desc); + base->thread = spdk_get_thread(); + base->is_cache = false; + base->attached = true; + + // alloc uuid ? + if ((rc = ocf_uuid_set_str(&core_cfg->uuid, (char *)base_name))) { + SPDK_ERRLOG("OCF core '%s': failed to set OCF volume uuid\n", + vbdev_ocf_core_get_name(core_ctx)); + spdk_put_io_channel(base->mngt_ch); + spdk_bdev_close(base->desc); + return rc; + } + + core_cfg->volume_type = SPDK_OBJECT; + // for ocf_volume_open() in ocf_mngt_cache_add_core() + core_cfg->volume_params = base; // why not ? what is it then ?!? - //assert(__bdev_to_io_dev(core->base.bdev) == core->base.bdev->ctxt); + //assert(__bdev_to_io_dev(base->bdev) == base->bdev->ctxt); return rc; } void -vbdev_ocf_core_base_detach(struct vbdev_ocf_core *core) +vbdev_ocf_core_base_detach(struct vbdev_ocf_core *core_ctx) { - struct vbdev_ocf_base *base = &core->base; + struct vbdev_ocf_base *base = &core_ctx->base; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': detaching base bdev '%s'\n", - core->name, base->bdev->name); - - vbdev_ocf_base_detach(base); -} - -int -vbdev_ocf_core_add_incomplete(struct vbdev_ocf_core *core, const char *bdev_name, - const char *cache_name) -{ - struct vbdev_ocf_core_init_params *init_params; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': allocating init params and adding core to incomplete list\n", - core->name); + vbdev_ocf_core_get_name(core_ctx), spdk_bdev_get_name(base->bdev)); - init_params = calloc(1, sizeof(struct vbdev_ocf_core_init_params)); - if (!init_params) { - SPDK_ERRLOG("OCF core '%s': failed to allocate memory for init params\n", - core->name); - return -ENOMEM; - } - - init_params->bdev_name = strdup(bdev_name); - if (!init_params->bdev_name) { - SPDK_ERRLOG("OCF core '%s': failed to allocate memory for base bdev name\n", - core->name); - free(init_params); - return -ENOMEM; - } - - init_params->cache_name = strdup(cache_name); - if (!init_params->cache_name) { - SPDK_ERRLOG("OCF core '%s': failed to allocate memory for cache bdev name\n", - core->name); - free(init_params->bdev_name); - free(init_params); - return -ENOMEM; - } - - core->init_params = init_params; - STAILQ_INSERT_TAIL(&g_vbdev_ocf_incomplete_cores, core, link); - - return 0; -} - -void -vbdev_ocf_core_remove_incomplete(struct vbdev_ocf_core *core) -{ - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': deallocating init params and removing core from incomplete list\n", - core->name); - - free(core->init_params->bdev_name); - free(core->init_params->cache_name); - free(core->init_params); - /* It's important to set init_params back to NULL. - * This is an indicator that core is not incomplete any more. */ - core->init_params = NULL; - STAILQ_REMOVE(&g_vbdev_ocf_incomplete_cores, core, vbdev_ocf_core, link); -} - -void -vbdev_ocf_core_add_to_cache(struct vbdev_ocf_core *core, struct vbdev_ocf_cache *cache) -{ - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': adding core to list in cache\n", core->name); + // dealloc uuid ? - core->cache = cache; - cache->cores_count++; - STAILQ_INSERT_TAIL(&cache->cores, core, link); -} - -void -vbdev_ocf_core_remove_from_cache(struct vbdev_ocf_core *core) -{ - struct vbdev_ocf_cache *cache; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': removing core from list in cache\n", core->name); - - cache = vbdev_ocf_core_get_cache(core); - cache->cores_count--; - STAILQ_REMOVE(&cache->cores, core, vbdev_ocf_core, link); + vbdev_ocf_base_detach(base); } static void @@ -272,7 +178,7 @@ vbdev_ocf_core_io_queue_stop(ocf_queue_t queue) struct vbdev_ocf_core_io_channel_ctx *ch_ctx = ocf_queue_get_priv(queue); SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': deallocating external IO channel context\n", - ch_ctx->core->ocf_vbdev.name); + spdk_bdev_get_name(&((struct vbdev_ocf_core *)ocf_core_get_priv(ch_ctx->core))->ocf_vbdev)); if (ch_ctx->thread && ch_ctx->thread != spdk_get_thread()) { spdk_thread_send_msg(ch_ctx->thread, _core_io_queue_stop, ch_ctx); @@ -295,14 +201,19 @@ const struct ocf_queue_ops core_io_queue_ops = { static int _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) { - struct vbdev_ocf_core *core = io_device; - struct vbdev_ocf_cache *cache = vbdev_ocf_core_get_cache(core); + ocf_core_t core = io_device; + ocf_cache_t cache = ocf_core_get_cache(core); + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + struct vbdev_ocf_base *core_base = &core_ctx->base; + struct vbdev_ocf_base *cache_base = &cache_ctx->base; struct vbdev_ocf_core_io_channel_ctx *ch_destroy_ctx = ctx_buf; struct vbdev_ocf_core_io_channel_ctx *ch_ctx; + const char *vbdev_name = spdk_bdev_get_name(&core_ctx->ocf_vbdev); int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': creating IO channel and allocating external context\n", - core->ocf_vbdev.name); + vbdev_name); /* Do not use provided buffer for IO channel context, as it will be freed * when this channel is destroyed. Instead allocate our own and keep it @@ -311,29 +222,29 @@ _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) ch_ctx = calloc(1, sizeof(struct vbdev_ocf_core_io_channel_ctx)); if (!ch_ctx) { SPDK_ERRLOG("OCF vbdev '%s': failed to allocate memory for IO channel context\n", - core->ocf_vbdev.name); + vbdev_name); return -ENOMEM; } - if ((rc = vbdev_ocf_queue_create(cache->ocf_cache, &ch_ctx->queue, &core_io_queue_ops))) { - SPDK_ERRLOG("OCF vbdev '%s': failed to create OCF queue\n", core->ocf_vbdev.name); + if ((rc = vbdev_ocf_queue_create(cache, &ch_ctx->queue, &core_io_queue_ops))) { + SPDK_ERRLOG("OCF vbdev '%s': failed to create OCF queue\n", vbdev_name); free(ch_ctx); return rc; } ocf_queue_set_priv(ch_ctx->queue, ch_ctx); - ch_ctx->cache_ch = spdk_bdev_get_io_channel(cache->base.desc); + ch_ctx->cache_ch = spdk_bdev_get_io_channel(cache_base->desc); if (!ch_ctx->cache_ch) { SPDK_ERRLOG("OCF vbdev '%s': failed to create IO channel for base bdev '%s'\n", - core->ocf_vbdev.name, cache->base.bdev->name); + vbdev_name, spdk_bdev_get_name(cache_base->bdev)); vbdev_ocf_queue_put(ch_ctx->queue); return -ENOMEM; } - ch_ctx->core_ch = spdk_bdev_get_io_channel(core->base.desc); + ch_ctx->core_ch = spdk_bdev_get_io_channel(core_base->desc); if (!ch_ctx->core_ch) { SPDK_ERRLOG("OCF vbdev '%s': failed to create IO channel for base bdev '%s'\n", - core->ocf_vbdev.name, core->base.bdev->name); + vbdev_name, spdk_bdev_get_name(core_base->bdev)); spdk_put_io_channel(ch_ctx->cache_ch); vbdev_ocf_queue_put(ch_ctx->queue); return -ENOMEM; @@ -341,7 +252,7 @@ _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) ch_ctx->poller = SPDK_POLLER_REGISTER(vbdev_ocf_queue_poller, ch_ctx->queue, 0); if (!ch_ctx->poller) { - SPDK_ERRLOG("OCF vbdev '%s': failed to create IO queue poller\n", core->ocf_vbdev.name); + SPDK_ERRLOG("OCF vbdev '%s': failed to create IO queue poller\n", vbdev_name); spdk_put_io_channel(ch_ctx->core_ch); spdk_put_io_channel(ch_ctx->cache_ch); vbdev_ocf_queue_put(ch_ctx->queue); @@ -362,40 +273,47 @@ _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) static void _vbdev_ocf_ch_destroy_cb(void *io_device, void *ctx_buf) { - struct vbdev_ocf_core *core = io_device; struct vbdev_ocf_core_io_channel_ctx *ch_destroy_ctx = ctx_buf; - SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': destroying IO channel\n", core->ocf_vbdev.name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': destroying IO channel\n", + spdk_bdev_get_name(&(((struct vbdev_ocf_core *)ocf_core_get_priv( + (ocf_core_t)io_device))->ocf_vbdev))); vbdev_ocf_queue_put(ch_destroy_ctx->queue); } int -vbdev_ocf_core_register(struct vbdev_ocf_core *core) +vbdev_ocf_core_register(ocf_core_t core) { + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + struct vbdev_ocf_base *base = &core_ctx->base; + struct spdk_bdev *ocf_vbdev = &core_ctx->ocf_vbdev; int rc = 0; - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': registering OCF vbdev in SPDK bdev layer\n", core->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': registering OCF vbdev in SPDK bdev layer\n", + ocf_core_get_name(core)); - core->ocf_vbdev.ctxt = core; - core->ocf_vbdev.name = core->name; - core->ocf_vbdev.product_name = "OCF_disk"; - core->ocf_vbdev.write_cache = core->base.bdev->write_cache; - core->ocf_vbdev.blocklen = core->base.bdev->blocklen; - core->ocf_vbdev.blockcnt = core->base.bdev->blockcnt; + ocf_vbdev->ctxt = core; + ocf_vbdev->name = (char *)ocf_core_get_name(core); + ocf_vbdev->product_name = "OCF_disk"; + ocf_vbdev->write_cache = base->bdev->write_cache; + ocf_vbdev->blocklen = base->bdev->blocklen; + ocf_vbdev->blockcnt = base->bdev->blockcnt; // ? - //core->ocf_vbdev.required_alignment = core->base.bdev->required_alignment; - //core->ocf_vbdev.optimal_io_boundary = core->base.bdev->optimal_io_boundary; + //ocf_vbdev->required_alignment = base->bdev->required_alignment; + //ocf_vbdev->optimal_io_boundary = base->bdev->optimal_io_boundary; // generate UUID based on namespace UUID + base bdev UUID (take from old module?) - core->ocf_vbdev.fn_table = &vbdev_ocf_fn_table; - core->ocf_vbdev.module = &ocf_if; + ocf_vbdev->fn_table = &vbdev_ocf_fn_table; + ocf_vbdev->module = &ocf_if; spdk_io_device_register(core, _vbdev_ocf_ch_create_cb, _vbdev_ocf_ch_destroy_cb, - sizeof(struct vbdev_ocf_core_io_channel_ctx), core->name); - SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': io_device created at 0x%p\n", core->ocf_vbdev.name, core); + sizeof(struct vbdev_ocf_core_io_channel_ctx), ocf_core_get_name(core)); + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': io_device created at 0x%p\n", + spdk_bdev_get_name(ocf_vbdev), core); - if ((rc = spdk_bdev_register(&core->ocf_vbdev))) { // needs to be called from SPDK app thread - SPDK_ERRLOG("OCF vbdev '%s': failed to register SPDK bdev\n", core->ocf_vbdev.name); + if ((rc = spdk_bdev_register(ocf_vbdev))) { // needs to be called from SPDK app thread + SPDK_ERRLOG("OCF vbdev '%s': failed to register SPDK bdev\n", + spdk_bdev_get_name(ocf_vbdev)); spdk_io_device_unregister(core, NULL); return rc; } @@ -404,59 +322,37 @@ vbdev_ocf_core_register(struct vbdev_ocf_core *core) } int -vbdev_ocf_core_unregister(struct vbdev_ocf_core *core, spdk_bdev_unregister_cb cb_fn, void *cb_arg) +vbdev_ocf_core_unregister(struct vbdev_ocf_core *core_ctx, spdk_bdev_unregister_cb cb_fn, void *cb_arg) { - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating unregister of OCF vbdev\n", core->name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating unregister of OCF vbdev\n", + vbdev_ocf_core_get_name(core_ctx)); - return spdk_bdev_unregister_by_name(core->ocf_vbdev.name, &ocf_if, cb_fn, cb_arg); + return spdk_bdev_unregister_by_name(spdk_bdev_get_name(&core_ctx->ocf_vbdev), + &ocf_if, cb_fn, cb_arg); } -struct vbdev_ocf_cache * -vbdev_ocf_core_get_cache(struct vbdev_ocf_core *core) +char * +vbdev_ocf_core_get_name(struct vbdev_ocf_core *core_ctx) { - return core->cache; + return core_ctx->core_cfg.name; } struct vbdev_ocf_core * -vbdev_ocf_core_get_by_name(const char *core_name) +vbdev_ocf_core_waitlist_get_by_name(const char *core_name) { - struct vbdev_ocf_cache *cache; - struct vbdev_ocf_core *core; + struct vbdev_ocf_core *core_ctx; - vbdev_ocf_foreach_core_incomplete(core) { - if (strcmp(core_name, core->name)) { - continue; - } - return core; - } - vbdev_ocf_foreach_cache(cache) { - vbdev_ocf_foreach_core_in_cache(core, cache) { - if (strcmp(core_name, core->name)) { - continue; - } - return core; + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + if (!strcmp(core_name, vbdev_ocf_core_get_name(core_ctx))) { + return core_ctx; } } - return NULL; -} -bool -vbdev_ocf_core_cache_is_started(struct vbdev_ocf_core *core) -{ - ocf_core_t ocf_core = core->ocf_core; - ocf_cache_t ocf_cache; - - if (ocf_core) { - ocf_cache = ocf_core_get_cache(ocf_core); - if (ocf_cache_is_running(ocf_cache) || ocf_cache_is_detached(ocf_cache)) { // smth else ? - return true; - } - } - return false; + return NULL; } bool -vbdev_ocf_core_is_incomplete(struct vbdev_ocf_core *core) +vbdev_ocf_core_is_base_attached(struct vbdev_ocf_core *core_ctx) { - return !!core->init_params; + return core_ctx->base.attached; } diff --git a/module/bdev/ocf/vbdev_ocf_core.h b/module/bdev/ocf/vbdev_ocf_core.h index a868ad87d9a..30ed7959d25 100644 --- a/module/bdev/ocf/vbdev_ocf_core.h +++ b/module/bdev/ocf/vbdev_ocf_core.h @@ -10,44 +10,29 @@ #include "spdk/bdev_module.h" #include "volume.h" -/* Global list of all incomplete cores (not added to - * the cache yet due to lack of cache or base device. */ -extern STAILQ_HEAD(vbdev_ocf_incomplete_cores_head, vbdev_ocf_core) g_vbdev_ocf_incomplete_cores; - /* OCF module interface. */ extern struct spdk_bdev_module ocf_if; /* Function table of exposed OCF vbdev. */ extern struct spdk_bdev_fn_table vbdev_ocf_fn_table; -#define vbdev_ocf_foreach_core_incomplete(core) \ - STAILQ_FOREACH(core, &g_vbdev_ocf_incomplete_cores, link) - -#define vbdev_ocf_foreach_core_in_cache(core, cache) \ - STAILQ_FOREACH(core, &cache->cores, link) +/* Global wait list of all incomplete cores (not added to + * the cache yet due to lack of cache or base device. */ +extern STAILQ_HEAD(vbdev_ocf_core_waitlist_head, vbdev_ocf_core) g_vbdev_ocf_core_waitlist; -struct vbdev_ocf_core_init_params { - char * bdev_name; - char * cache_name; -}; +#define vbdev_ocf_foreach_core_in_waitlist(core_ctx) \ + STAILQ_FOREACH(core_ctx, &g_vbdev_ocf_core_waitlist, waitlist_entry) struct vbdev_ocf_core { - char * name; - struct spdk_uuid uuid; + struct ocf_mngt_core_config core_cfg; - struct vbdev_ocf_base base; - - struct vbdev_ocf_cache * cache; - - ocf_core_t ocf_core; - struct ocf_mngt_core_config ocf_core_cfg; - - /* Exposed OCF vbdev; the one which is registered in bdev layer for usage */ + /* Exposed OCF vbdev; the one which is registered in bdev layer for usage. */ struct spdk_bdev ocf_vbdev; - struct vbdev_ocf_core_init_params * init_params; + struct vbdev_ocf_base base; + char cache_name[OCF_CACHE_NAME_SIZE]; - STAILQ_ENTRY(vbdev_ocf_core) link; + STAILQ_ENTRY(vbdev_ocf_core) waitlist_entry; }; struct vbdev_ocf_core_io_channel_ctx { @@ -57,25 +42,18 @@ struct vbdev_ocf_core_io_channel_ctx { struct spdk_poller * poller; struct spdk_thread * thread; /* Currently kept only for its name used in debug log. */ - struct vbdev_ocf_core * core; // rm ? + ocf_core_t core; // rm ? }; -int vbdev_ocf_core_create(const char *core_name, struct vbdev_ocf_core **out); -void vbdev_ocf_core_destroy(struct vbdev_ocf_core *core); -int vbdev_ocf_core_set_config(struct vbdev_ocf_core *core); -int vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core, const char *bdev_name); -void vbdev_ocf_core_base_detach(struct vbdev_ocf_core *core); -int vbdev_ocf_core_add_incomplete(struct vbdev_ocf_core *core, const char *bdev_name, - const char *cache_name); -void vbdev_ocf_core_remove_incomplete(struct vbdev_ocf_core *core); -void vbdev_ocf_core_add_to_cache(struct vbdev_ocf_core *core, struct vbdev_ocf_cache *cache); -void vbdev_ocf_core_remove_from_cache(struct vbdev_ocf_core *core); -int vbdev_ocf_core_register(struct vbdev_ocf_core *core); -int vbdev_ocf_core_unregister(struct vbdev_ocf_core *core, spdk_bdev_unregister_cb cb_fn, void *cb_arg); +int vbdev_ocf_core_create(struct vbdev_ocf_core **out, const char *core_name, const char *cache_name); +void vbdev_ocf_core_destroy(struct vbdev_ocf_core *core_ctx); +int vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core_ctx, const char *base_name); +void vbdev_ocf_core_base_detach(struct vbdev_ocf_core *core_ctx); +int vbdev_ocf_core_register(ocf_core_t core); +int vbdev_ocf_core_unregister(struct vbdev_ocf_core *core_ctx, spdk_bdev_unregister_cb cb_fn, void *cb_arg); -struct vbdev_ocf_cache *vbdev_ocf_core_get_cache(struct vbdev_ocf_core *core); -struct vbdev_ocf_core *vbdev_ocf_core_get_by_name(const char *core_name); -bool vbdev_ocf_core_cache_is_started(struct vbdev_ocf_core *core); -bool vbdev_ocf_core_is_incomplete(struct vbdev_ocf_core *core); +char *vbdev_ocf_core_get_name(struct vbdev_ocf_core *core_ctx); +struct vbdev_ocf_core *vbdev_ocf_core_waitlist_get_by_name(const char *core_name); +bool vbdev_ocf_core_is_base_attached(struct vbdev_ocf_core *core_ctx); #endif diff --git a/module/bdev/ocf/vbdev_ocf_rpc.c b/module/bdev/ocf/vbdev_ocf_rpc.c index 45a2ff68ef5..8ca0ece632c 100644 --- a/module/bdev/ocf/vbdev_ocf_rpc.c +++ b/module/bdev/ocf/vbdev_ocf_rpc.c @@ -9,27 +9,29 @@ struct rpc_bdev_ocf_start_cache { char *cache_name; - char *bdev_name; + char *base_name; char *cache_mode; uint8_t cache_line_size; + bool no_load; }; static void free_rpc_bdev_ocf_start_cache(struct rpc_bdev_ocf_start_cache *r) { free(r->cache_name); - free(r->bdev_name); + free(r->base_name); free(r->cache_mode); } static const struct spdk_json_object_decoder rpc_bdev_ocf_start_cache_decoders[] = { {"cache_name", offsetof(struct rpc_bdev_ocf_start_cache, cache_name), spdk_json_decode_string}, - {"bdev_name", offsetof(struct rpc_bdev_ocf_start_cache, bdev_name), spdk_json_decode_string}, + {"base_name", offsetof(struct rpc_bdev_ocf_start_cache, base_name), spdk_json_decode_string}, {"cache_mode", offsetof(struct rpc_bdev_ocf_start_cache, cache_mode), spdk_json_decode_string, true}, {"cache_line_size", offsetof(struct rpc_bdev_ocf_start_cache, cache_line_size), spdk_json_decode_uint8, true}, + {"no_load", offsetof(struct rpc_bdev_ocf_start_cache, no_load), spdk_json_decode_bool, true}, }; static void -rpc_bdev_ocf_start_cache_cb(struct vbdev_ocf_cache *vbdev_cache, void *cb_arg, int error) +rpc_bdev_ocf_start_cache_cb(const char *bdev_name, void *cb_arg, int error) { struct spdk_jsonrpc_request *request = cb_arg; struct spdk_json_write_ctx *w; @@ -42,7 +44,7 @@ rpc_bdev_ocf_start_cache_cb(struct vbdev_ocf_cache *vbdev_cache, void *cb_arg, i } w = spdk_jsonrpc_begin_result(request); - spdk_json_write_string(w, vbdev_cache->name); + spdk_json_write_string(w, bdev_name); spdk_jsonrpc_end_result(request, w); } @@ -60,8 +62,8 @@ rpc_bdev_ocf_start_cache(struct spdk_jsonrpc_request *request, const struct spdk goto cleanup; } - vbdev_ocf_cache_start(req.cache_name, req.bdev_name, req.cache_mode, req.cache_line_size, - rpc_bdev_ocf_start_cache_cb, request); + vbdev_ocf_cache_start(req.cache_name, req.base_name, req.cache_mode, req.cache_line_size, + req.no_load, rpc_bdev_ocf_start_cache_cb, request); cleanup: free_rpc_bdev_ocf_start_cache(&req); @@ -82,7 +84,7 @@ static const struct spdk_json_object_decoder rpc_bdev_ocf_stop_cache_decoders[] }; static void -rpc_bdev_ocf_stop_cache_cb(void *cb_arg, int error) +rpc_bdev_ocf_stop_cache_cb(const char *bdev_name, void *cb_arg, int error) { struct spdk_jsonrpc_request *request = cb_arg; @@ -115,27 +117,131 @@ rpc_bdev_ocf_stop_cache(struct spdk_jsonrpc_request *request, const struct spdk_ } SPDK_RPC_REGISTER("bdev_ocf_stop_cache", rpc_bdev_ocf_stop_cache, SPDK_RPC_RUNTIME) +struct rpc_bdev_ocf_detach_cache { + char *cache_name; +}; + +static void +free_rpc_bdev_ocf_detach_cache(struct rpc_bdev_ocf_detach_cache *r) { + free(r->cache_name); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_detach_cache_decoders[] = { + {"cache_name", offsetof(struct rpc_bdev_ocf_detach_cache, cache_name), spdk_json_decode_string}, +}; + +static void +rpc_bdev_ocf_detach_cache_cb(const char *bdev_name, void *cb_arg, int error) +{ + struct spdk_jsonrpc_request *request = cb_arg; + + if (error) { + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Could not detach OCF cache: %s", + spdk_strerror(-error)); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); +} + +static void +rpc_bdev_ocf_detach_cache(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_bdev_ocf_detach_cache req = {}; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_detach_cache_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_detach_cache_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parametes"); + goto cleanup; + } + + vbdev_ocf_cache_detach(req.cache_name, rpc_bdev_ocf_detach_cache_cb, request); + +cleanup: + free_rpc_bdev_ocf_detach_cache(&req); +} +SPDK_RPC_REGISTER("bdev_ocf_detach_cache", rpc_bdev_ocf_detach_cache, SPDK_RPC_RUNTIME) + +struct rpc_bdev_ocf_attach_cache { + char *cache_name; + char *base_name; + bool force; +}; + +static void +free_rpc_bdev_ocf_attach_cache(struct rpc_bdev_ocf_attach_cache *r) { + free(r->cache_name); + free(r->base_name); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_attach_cache_decoders[] = { + {"cache_name", offsetof(struct rpc_bdev_ocf_attach_cache, cache_name), spdk_json_decode_string}, + {"base_name", offsetof(struct rpc_bdev_ocf_attach_cache, base_name), spdk_json_decode_string}, + {"force", offsetof(struct rpc_bdev_ocf_attach_cache, force), spdk_json_decode_bool, true}, +}; + +static void +rpc_bdev_ocf_attach_cache_cb(const char *bdev_name, void *cb_arg, int error) +{ + struct spdk_jsonrpc_request *request = cb_arg; + + if (error && error != -ENODEV) { + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Could not attach OCF cache: %s", + spdk_strerror(-error)); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); +} + +static void +rpc_bdev_ocf_attach_cache(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_bdev_ocf_attach_cache req = {}; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_attach_cache_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_attach_cache_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parametes"); + goto cleanup; + } + + vbdev_ocf_cache_attach(req.cache_name, req.base_name, req.force, + rpc_bdev_ocf_attach_cache_cb, request); + +cleanup: + free_rpc_bdev_ocf_attach_cache(&req); +} +SPDK_RPC_REGISTER("bdev_ocf_attach_cache", rpc_bdev_ocf_attach_cache, SPDK_RPC_RUNTIME) + struct rpc_bdev_ocf_add_core { char *core_name; - char *bdev_name; + char *base_name; char *cache_name; }; static void free_rpc_bdev_ocf_add_core(struct rpc_bdev_ocf_add_core *r) { free(r->core_name); - free(r->bdev_name); + free(r->base_name); free(r->cache_name); } static const struct spdk_json_object_decoder rpc_bdev_ocf_add_core_decoders[] = { {"core_name", offsetof(struct rpc_bdev_ocf_add_core, core_name), spdk_json_decode_string}, - {"bdev_name", offsetof(struct rpc_bdev_ocf_add_core, bdev_name), spdk_json_decode_string}, + {"base_name", offsetof(struct rpc_bdev_ocf_add_core, base_name), spdk_json_decode_string}, {"cache_name", offsetof(struct rpc_bdev_ocf_add_core, cache_name), spdk_json_decode_string}, }; static void -rpc_bdev_ocf_add_core_cb(struct vbdev_ocf_core *vbdev_core, void *cb_arg, int error) +rpc_bdev_ocf_add_core_cb(const char *bdev_name, void *cb_arg, int error) { struct spdk_jsonrpc_request *request = cb_arg; struct spdk_json_write_ctx *w; @@ -148,7 +254,7 @@ rpc_bdev_ocf_add_core_cb(struct vbdev_ocf_core *vbdev_core, void *cb_arg, int er } w = spdk_jsonrpc_begin_result(request); - spdk_json_write_string(w, vbdev_core->name); + spdk_json_write_string(w, bdev_name); spdk_jsonrpc_end_result(request, w); } @@ -166,7 +272,7 @@ rpc_bdev_ocf_add_core(struct spdk_jsonrpc_request *request, const struct spdk_js goto cleanup; } - vbdev_ocf_core_add(req.core_name, req.bdev_name, req.cache_name, rpc_bdev_ocf_add_core_cb, request); + vbdev_ocf_core_add(req.core_name, req.base_name, req.cache_name, rpc_bdev_ocf_add_core_cb, request); cleanup: free_rpc_bdev_ocf_add_core(&req); @@ -175,19 +281,22 @@ SPDK_RPC_REGISTER("bdev_ocf_add_core", rpc_bdev_ocf_add_core, SPDK_RPC_RUNTIME) struct rpc_bdev_ocf_remove_core { char *core_name; + char *cache_name; }; static void free_rpc_bdev_ocf_remove_core(struct rpc_bdev_ocf_remove_core *r) { free(r->core_name); + free(r->cache_name); } static const struct spdk_json_object_decoder rpc_bdev_ocf_remove_core_decoders[] = { {"core_name", offsetof(struct rpc_bdev_ocf_remove_core, core_name), spdk_json_decode_string}, + {"cache_name", offsetof(struct rpc_bdev_ocf_remove_core, cache_name), spdk_json_decode_string}, }; static void -rpc_bdev_ocf_remove_core_cb(void *cb_arg, int error) +rpc_bdev_ocf_remove_core_cb(const char *bdev_name, void *cb_arg, int error) { struct spdk_jsonrpc_request *request = cb_arg; @@ -213,7 +322,7 @@ rpc_bdev_ocf_remove_core(struct spdk_jsonrpc_request *request, const struct spdk goto cleanup; } - vbdev_ocf_core_remove(req.core_name, rpc_bdev_ocf_remove_core_cb, request); + vbdev_ocf_core_remove(req.core_name, req.cache_name, rpc_bdev_ocf_remove_core_cb, request); cleanup: free_rpc_bdev_ocf_remove_core(&req); diff --git a/module/bdev/ocf/volume.h b/module/bdev/ocf/volume.h index 461e14ca582..b122930dd21 100644 --- a/module/bdev/ocf/volume.h +++ b/module/bdev/ocf/volume.h @@ -8,6 +8,7 @@ #define VBDEV_OCF_VOLUME_H struct vbdev_ocf_base { + //char name[spdk_max(OCF_CACHE_NAME_SIZE, OCF_CORE_NAME_SIZE)]; bool is_cache; bool attached; struct spdk_bdev * bdev; diff --git a/python/spdk/cli/bdev.py b/python/spdk/cli/bdev.py index 7a3583415c0..f9e7dba51be 100644 --- a/python/spdk/cli/bdev.py +++ b/python/spdk/cli/bdev.py @@ -106,19 +106,21 @@ def bdev_crypto_delete(args): def bdev_ocf_start_cache(args): print_json(args.client.bdev_ocf_start_cache( cache_name=args.cache_name, - bdev_name=args.bdev_name, + base_name=args.base_name, cache_mode=args.cache_mode, - cache_line_size=args.cache_line_size)) + cache_line_size=args.cache_line_size, + no_load=args.no_load)) p = subparsers.add_parser('bdev_ocf_start_cache', help='Start OCF cache instance') p.add_argument('cache_name', help='name for the new OCF cache vbdev') - p.add_argument('bdev_name', help='name of the bdev to use as cache') - p.add_argument('--cache-mode', + p.add_argument('base_name', help='name of the base bdev to use as cache') + p.add_argument('-m', '--cache-mode', help='choose between {wt|wb|wa|wo|wi|pt}; default wt (Write-Through)', choices=['wt', 'wb', 'wa', 'wo', 'wi', 'pt']) - p.add_argument('--cache-line-size', + p.add_argument('-l', '--cache-line-size', help='choose between {4|8|16|32|64}; default 4 [KiB]', type=int, choices=[4, 8, 16, 32, 64]) + p.add_argument('-n', '--no-load', action='store_true', help='do not load previous cache instance from metadata and force starting a new one instead (WARNING: all cache metadata will be discarded!)') p.set_defaults(func=bdev_ocf_start_cache) def bdev_ocf_stop_cache(args): @@ -127,21 +129,39 @@ def bdev_ocf_stop_cache(args): p.add_argument('cache_name', help='name of the cache vbdev to stop') p.set_defaults(func=bdev_ocf_stop_cache) + def bdev_ocf_detach_cache(args): + args.client.bdev_ocf_detach_cache(cache_name=args.cache_name) + p = subparsers.add_parser('bdev_ocf_detach_cache', help='Detach caching device from OCF cache') + p.add_argument('cache_name', help='name of the cache vbdev to detach device from') + p.set_defaults(func=bdev_ocf_detach_cache) + + def bdev_ocf_attach_cache(args): + args.client.bdev_ocf_attach_cache( + cache_name=args.cache_name, + base_name=args.base_name, + force=args.force) + p = subparsers.add_parser('bdev_ocf_attach_cache', help='Attach caching device to OCF cache') + p.add_argument('cache_name', help='name of the cache vbdev to attach device to') + p.add_argument('base_name', help='name of the base bdev to use as caching device') + p.add_argument('-f', '--force', action='store_true', help='discard cache metadata if present on device') + p.set_defaults(func=bdev_ocf_attach_cache) + def bdev_ocf_add_core(args): print_json(args.client.bdev_ocf_add_core( core_name=args.core_name, - bdev_name=args.bdev_name, + base_name=args.base_name, cache_name=args.cache_name)) p = subparsers.add_parser('bdev_ocf_add_core', help='Add core device to OCF cache') p.add_argument('core_name', help='name for the new OCF core vbdev') - p.add_argument('bdev_name', help='name of the bdev to use as core') + p.add_argument('base_name', help='name of the base bdev to use as core') p.add_argument('cache_name', help='name of already started OCF cache vbdev') p.set_defaults(func=bdev_ocf_add_core) def bdev_ocf_remove_core(args): - args.client.bdev_ocf_remove_core(core_name=args.core_name) + args.client.bdev_ocf_remove_core(core_name=args.core_name, cache_name=args.cache_name) p = subparsers.add_parser('bdev_ocf_remove_core', help='Remove core device from OCF cache') p.add_argument('core_name', help='name of the core vbdev to remove from OCF cache instance') + p.add_argument('cache_name', help='name of the cache vbdev to remove core from') p.set_defaults(func=bdev_ocf_remove_core) def bdev_ocf_get_bdevs(args): diff --git a/test/ocf/common.sh b/test/ocf/common.sh index 44ffbf86a87..e1196345e78 100644 --- a/test/ocf/common.sh +++ b/test/ocf/common.sh @@ -1,24 +1,31 @@ +# # SPDX-License-Identifier: BSD-3-Clause -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2025 Huawei Technologies # All rights reserved. # -source $rootdir/scripts/common.sh -source $rootdir/test/common/autotest_common.sh +source "$rootdir/test/common/autotest_common.sh" + +rpc_py="$rootdir/scripts/rpc.py" -rpc_py=$rootdir/scripts/rpc.py +start_spdk() { + $SPDK_BIN_DIR/spdk_tgt "$@" & + spdk_pid=$! + trap 'killprocess $spdk_pid; exit 1' SIGINT SIGTERM EXIT + waitforlisten $spdk_pid +} -function clear_nvme() { - mapfile -t bdf < <(get_first_nvme_bdf) +stop_spdk() { + trap - SIGINT SIGTERM EXIT + killprocess $spdk_pid +} - # Clear metadata on NVMe device - $rootdir/scripts/setup.sh reset +# Convert an array of items to comma separated list of items. +array_to_comma_list() { + # $1: name of array variable + # stdout: string with comma separated elements - name=$(get_nvme_name_from_bdf "${bdf[0]}") - mountpoints=$(lsblk /dev/$name --output MOUNTPOINT -n | wc -w) - if [ "$mountpoints" != "0" ]; then - exit 1 - fi - dd if=/dev/zero of=/dev/$name bs=1M count=1000 oflag=direct - $rootdir/scripts/setup.sh + IFS="," + declare -n array="$1" + echo "${array[*]}" } From a5b59d23df3da92f2f8165d47313224337ca207c Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Mon, 12 May 2025 10:06:51 +0200 Subject: [PATCH 04/34] fix memleaks Change-Id: I82aae275e79444afbda292bd0cc8211d18fd5c0e Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 259 ++++++++++++++---------------- module/bdev/ocf/vbdev_ocf_cache.c | 4 +- module/bdev/ocf/vbdev_ocf_core.c | 27 ++-- module/bdev/ocf/vbdev_ocf_core.h | 1 + 4 files changed, 144 insertions(+), 147 deletions(-) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index d61da72e5e1..19f80232074 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -132,6 +132,8 @@ vbdev_ocf_module_init(void) static void _module_fini_cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) { + ocf_queue_t cache_mngt_q = ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q; + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", ocf_cache_get_name(cache)); @@ -140,10 +142,14 @@ _module_fini_cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_cache_get_name(cache), error); } - vbdev_ocf_cache_base_detach(cache); + if (vbdev_ocf_cache_is_base_attached(cache)) { + vbdev_ocf_cache_base_detach(cache); + } else { + /* If device was not attached to cache, then cache stop won't 'put' its management queue. */ + ocf_queue_put(cache_mngt_q); + } vbdev_ocf_cache_destroy(cache); ocf_mngt_cache_unlock(cache); - ocf_mngt_cache_put(cache); if (!ocf_mngt_cache_get_count(vbdev_ocf_ctx)) { spdk_bdev_module_fini_start_done(); @@ -164,6 +170,9 @@ _module_fini_cache_flush_cb(ocf_cache_t cache, void *cb_arg, int error) static void _module_fini_cache_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop of OCF cache\n", + ocf_cache_get_name(cache)); + if (error) { SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", ocf_cache_get_name(cache), error); @@ -176,32 +185,12 @@ _module_fini_cache_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } } -static int -_module_fini_cache_visit_stop(ocf_cache_t cache, void *cb_arg) -{ - int rc = 0; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop of OCF cache\n", - ocf_cache_get_name(cache)); - - printf("*** (cache stop) cache refcnt before get: %d\n", ocf_cache_get_refcnt(cache)); - if ((rc = ocf_mngt_cache_get(cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to increment ref count: %s\n", - ocf_cache_get_name(cache), spdk_strerror(-rc)); - return rc; - } - printf("*** (cache stop) cache refcnt after get: %d\n", ocf_cache_get_refcnt(cache)); - - ocf_mngt_cache_lock(cache, _module_fini_cache_lock_cb, cb_arg); - - return rc; -} - static void _module_fini_core_unregister_cb(void *cb_arg, int error) { ocf_core_t core = cb_arg; ocf_cache_t cache = ocf_core_get_cache(core); + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", ocf_core_get_name(core)); @@ -211,13 +200,12 @@ _module_fini_core_unregister_cb(void *cb_arg, int error) ocf_core_get_name(core), spdk_strerror(-error)); } - printf("*** (core unregister) cache refcnt before put: %d\n", ocf_cache_get_refcnt(cache)); - ocf_mngt_cache_put(cache); // needed ? - printf("*** (core unregister) cache refcnt after put: %d\n", ocf_cache_get_refcnt(cache)); + ocf_mngt_cache_put(cache); + vbdev_ocf_core_destroy(core_ctx); - //if (ocf_cache_get_refcnt(cache) < 2) { - // ocf_mngt_cache_lock(cache, _module_fini_cache_lock_cb, cb_arg); - //} + if (ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { + ocf_mngt_cache_lock(cache, _module_fini_cache_lock_cb, cb_arg); + } } static int @@ -227,13 +215,13 @@ _module_fini_core_visit(ocf_core_t core, void *cb_arg) struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); int rc = 0; - printf("*** (core unregister) cache refcnt before get: %d\n", ocf_cache_get_refcnt(cache)); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': module stop visit\n", ocf_core_get_name(core)); + if ((rc = ocf_mngt_cache_get(cache))) { SPDK_ERRLOG("OCF core '%s': failed to increment cache '%s' ref count: %s\n", ocf_core_get_name(core), ocf_cache_get_name(cache), spdk_strerror(-rc)); return rc; } - printf("*** (core unregister) cache refcnt after get: %d\n", ocf_cache_get_refcnt(cache)); if ((rc = vbdev_ocf_core_unregister(core_ctx, _module_fini_core_unregister_cb, core))) { SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during module stop: %s\n", @@ -249,6 +237,13 @@ _module_fini_cache_visit(ocf_cache_t cache, void *cb_arg) { int rc = 0; + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': module stop visit\n", ocf_cache_get_name(cache)); + + if (!ocf_cache_get_core_count(cache)) { + ocf_mngt_cache_lock(cache, _module_fini_cache_lock_cb, cb_arg); + return rc; + } + if ((rc = ocf_core_visit(cache, _module_fini_core_visit, cb_arg, false))) { // only opened cores ? SPDK_ERRLOG("OCF: failed to iterate over core bdevs: %s\n", spdk_strerror(-rc)); return rc; @@ -281,13 +276,6 @@ vbdev_ocf_module_fini_start(void) spdk_bdev_module_fini_start_done(); return; } - - // async fuckup ? (move to unregister_cb and do by refcnt) - if ((rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _module_fini_cache_visit_stop, NULL))) { - SPDK_ERRLOG("OCF: failed to iterate over cache bdevs: %s\n", spdk_strerror(-rc)); - spdk_bdev_module_fini_start_done(); - return; - } } static void @@ -637,8 +625,9 @@ _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) } vbdev_ocf_cache_base_detach(cache); - vbdev_ocf_queue_put(cache_ctx->cache_mngt_q); + ocf_queue_put(cache_ctx->cache_mngt_q); // needed ? ocf_mngt_cache_stop(cache, _cache_start_rpc_err_cb, NULL); + // move to ocf_mngt_cache_stop callback ? if (cache_start_ctx->rpc_cb_fn) { cache_start_ctx->rpc_cb_fn(NULL, cache_start_ctx->rpc_cb_arg, error); // cache name } @@ -682,6 +671,7 @@ _cache_start_rpc_metadata_probe_cb(void *priv, int error, struct ocf_metadata_pr } else { SPDK_NOTICELOG("OCF cache '%s': metadata found - loading cache from metadata\n", ocf_cache_get_name(cache)); + SPDK_NOTICELOG("(start cache with '--no-load' flag to start new cache instead of loading config from metadata)\n"); // check status for cache_name/mode/line_size/dirty ? ocf_mngt_cache_load(cache, &cache_ctx->cache_att_cfg, _cache_start_rpc_attach_cb, cache_start_ctx); @@ -704,7 +694,7 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, if (vbdev_ocf_bdev_exists(cache_name)) { SPDK_ERRLOG("OCF: bdev '%s' already exists\n", cache_name); rc = -EEXIST; - goto err_create; + goto err_exist; } if ((rc = vbdev_ocf_cache_create(&cache, cache_name, cache_mode, cache_line_size, no_load))) { @@ -743,6 +733,7 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, cache_ctx = ocf_cache_get_priv(cache); if (no_load) { + SPDK_NOTICELOG("'--no-load' flag specified - starting new cache without looking for metadata\n"); ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, _cache_start_rpc_attach_cb, cache_start_ctx); return; @@ -763,10 +754,11 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, err_alloc: vbdev_ocf_cache_base_detach(cache); err_base: - vbdev_ocf_queue_put(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); + ocf_queue_put(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); err_queue: ocf_mngt_cache_stop(cache, _cache_start_rpc_err_cb, NULL); err_create: +err_exist: rpc_cb_fn(cache_name, rpc_cb_arg, rc); } @@ -774,6 +766,7 @@ static void _cache_stop_rpc_stop_cb(ocf_cache_t cache, void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *cache_stop_ctx = cb_arg; + ocf_queue_t cache_mngt_q = ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", ocf_cache_get_name(cache)); @@ -785,7 +778,12 @@ _cache_stop_rpc_stop_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_cache_get_name(cache), error); } else { SPDK_NOTICELOG("OCF cache '%s': stopped\n", ocf_cache_get_name(cache)); - vbdev_ocf_cache_base_detach(cache); + if (vbdev_ocf_cache_is_base_attached(cache)) { + vbdev_ocf_cache_base_detach(cache); + } else { + /* If device was not attached to cache, then cache stop won't 'put' its management queue. */ + ocf_queue_put(cache_mngt_q); + } vbdev_ocf_cache_destroy(cache); } @@ -828,9 +826,10 @@ _cache_stop_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) static void _cache_stop_rpc_core_unregister_cb(void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *cache_stop_ctx = cb_arg; - ocf_cache_t cache = cache_stop_ctx->cache; - ocf_core_t core = cache_stop_ctx->core; + ocf_core_t core = cb_arg; + ocf_cache_t cache = ocf_core_get_cache(core); + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + struct vbdev_ocf_mngt_ctx *cache_stop_ctx = core_ctx->mngt_ctx; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", ocf_core_get_name(core)); @@ -840,12 +839,11 @@ _cache_stop_rpc_core_unregister_cb(void *cb_arg, int error) ocf_core_get_name(core), spdk_strerror(-error)); } - printf("*** (core unregister) cache refcnt before put: %d\n", ocf_cache_get_refcnt(cache)); ocf_mngt_cache_put(cache); - printf("*** (core unregister) cache refcnt after put: %d\n", ocf_cache_get_refcnt(cache)); + vbdev_ocf_core_destroy(core_ctx); - if (ocf_cache_get_refcnt(cache) < 2) { - ocf_mngt_cache_lock(cache, _cache_stop_rpc_lock_cb, cb_arg); + if (ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { + ocf_mngt_cache_lock(cache, _cache_stop_rpc_lock_cb, cache_stop_ctx); } } @@ -857,18 +855,17 @@ _cache_stop_rpc_core_visit(ocf_core_t core, void *cb_arg) struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); int rc = 0; - // async fuckup ? - cache_stop_ctx->core = core; + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': cache stop visit\n", ocf_core_get_name(core)); + + core_ctx->mngt_ctx = cache_stop_ctx; - printf("*** (core unregister) cache refcnt before get: %d\n", ocf_cache_get_refcnt(cache)); if ((rc = ocf_mngt_cache_get(cache))) { SPDK_ERRLOG("OCF core '%s': failed to increment cache '%s' ref count: %s\n", ocf_core_get_name(core), ocf_cache_get_name(cache), spdk_strerror(-rc)); return rc; } - printf("*** (core unregister) cache refcnt after get: %d\n", ocf_cache_get_refcnt(cache)); - if ((rc = vbdev_ocf_core_unregister(core_ctx, _cache_stop_rpc_core_unregister_cb, cache_stop_ctx))) { + if ((rc = vbdev_ocf_core_unregister(core_ctx, _cache_stop_rpc_core_unregister_cb, core))) { SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during cache stop: %s\n", ocf_core_get_name(core), spdk_strerror(-rc)); return rc; @@ -887,7 +884,6 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop\n", cache_name); - // increments cache refcnt !!! if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; @@ -910,7 +906,7 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo cache_stop_ctx->rpc_cb_fn = rpc_cb_fn; cache_stop_ctx->rpc_cb_arg = rpc_cb_arg; - if (ocf_cache_get_core_count(cache) > 0) { + if (ocf_cache_get_core_count(cache)) { if ((rc = ocf_core_visit(cache, _cache_stop_rpc_core_visit, cache_stop_ctx, true))) { SPDK_ERRLOG("OCF: failed to iterate over core bdevs: %s\n", spdk_strerror(-rc)); // ocf_mngt_cache_lock(...) ? @@ -940,9 +936,12 @@ _cache_detach_rpc_detach_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_cache_get_name(cache), error); } else { vbdev_ocf_cache_base_detach(cache); - SPDK_NOTICELOG("OCF cache '%s': detached\n", ocf_cache_get_name(cache)); + SPDK_NOTICELOG("OCF cache '%s': device detached\n", ocf_cache_get_name(cache)); } + /* Increment queue refcount to prevent destroying management queue after cache device detach. */ + ocf_queue_get(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); + ocf_mngt_cache_unlock(cache); ocf_mngt_cache_put(cache); cache_detach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_detach_ctx->rpc_cb_arg, error); @@ -972,6 +971,9 @@ _cache_detach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *cache_detach_ctx = cb_arg; + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': detaching OCF cache device\n", + ocf_cache_get_name(cache)); + if (error) { SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", ocf_cache_get_name(cache), error); @@ -999,14 +1001,13 @@ vbdev_ocf_cache_detach(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating device detach\n", cache_name); - // increments cache refcnt !!! if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; goto err_cache; } - if (ocf_cache_is_detached(cache)) { + if (!ocf_cache_is_device_attached(cache)) { SPDK_ERRLOG("OCF cache '%s': device already detached\n", cache_name); rc = -EALREADY; // better errno ? goto err_state; @@ -1017,7 +1018,7 @@ vbdev_ocf_cache_detach(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache detach context\n", cache_name); rc = -ENOMEM; - goto err_state; + goto err_alloc; } cache_detach_ctx->rpc_cb_fn = rpc_cb_fn; cache_detach_ctx->rpc_cb_arg = rpc_cb_arg; @@ -1026,6 +1027,7 @@ vbdev_ocf_cache_detach(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, return; +err_alloc: err_state: ocf_mngt_cache_put(cache); err_cache: @@ -1045,7 +1047,7 @@ _cache_attach_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_cache_get_name(cache), error); vbdev_ocf_cache_base_detach(cache); } else { - SPDK_NOTICELOG("OCF cache '%s': attached\n", ocf_cache_get_name(cache)); + SPDK_NOTICELOG("OCF cache '%s': device attached\n", ocf_cache_get_name(cache)); } ocf_mngt_cache_unlock(cache); @@ -1060,6 +1062,9 @@ _cache_attach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) struct vbdev_ocf_mngt_ctx *cache_attach_ctx = cb_arg; struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': attaching OCF cache device\n", + ocf_cache_get_name(cache)); + if (error) { SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", ocf_cache_get_name(cache), error); @@ -1107,7 +1112,7 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force return; } SPDK_ERRLOG("OCF cache '%s': failed to open base bdev '%s'\n", cache_name, base_name); - goto err_state; + goto err_base; } cache_attach_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); @@ -1129,6 +1134,7 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force err_alloc: vbdev_ocf_cache_base_detach(cache); +err_base: err_state: ocf_mngt_cache_put(cache); err_cache: @@ -1233,7 +1239,7 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac if (vbdev_ocf_bdev_exists(core_name)) { SPDK_ERRLOG("OCF: bdev '%s' already exists\n", core_name); rc = -EEXIST; - goto err_create; + goto err_exist; } if ((rc = vbdev_ocf_core_create(&core_ctx, core_name, cache_name))) { @@ -1289,6 +1295,7 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac err_base: vbdev_ocf_core_destroy(core_ctx); err_create: +err_exist: rpc_cb_fn(core_name, rpc_cb_arg, rc); } @@ -1371,6 +1378,8 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, core_ctx = vbdev_ocf_core_waitlist_get_by_name(core_name); if (core_ctx) { SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': removing from wait list\n", core_name); + + STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); if (vbdev_ocf_core_is_base_attached(core_ctx)) { vbdev_ocf_core_base_detach(core_ctx); } @@ -1397,7 +1406,7 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, if (!core_rm_ctx) { SPDK_ERRLOG("OCF core '%s': failed to allocate memory for core remove context\n", core_name); rc = -ENOMEM; - goto err_core; + goto err_alloc; } core_rm_ctx->cache = cache; core_rm_ctx->core = core; @@ -1415,88 +1424,70 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, err_unregister: free(core_rm_ctx); +err_alloc: err_core: ocf_mngt_cache_put(cache); err_cache: rpc_cb_fn(core_name, rpc_cb_arg, rc); } -//comp -//static void -//_write_cache_info_begin(struct spdk_json_write_ctx *w, struct vbdev_ocf_cache *cache) -//{ -// spdk_json_write_object_begin(w); -// spdk_json_write_named_string(w, "type", "OCF_cache"); -// spdk_json_write_named_string(w, "name", cache->name); -// spdk_json_write_named_string(w, "base_bdev_name", -// cache->base.bdev ? spdk_bdev_get_name(cache->base.bdev) : "" ); -// spdk_json_write_named_uint16(w, "cores_count", cache->cores_count); -//} -// -//static void -//_write_cache_info_end(struct spdk_json_write_ctx *w, struct vbdev_ocf_cache *cache) -//{ -// spdk_json_write_object_end(w); -//} -// -//static void -//_write_core_info(struct spdk_json_write_ctx *w, struct vbdev_ocf_core *core) -//{ -// spdk_json_write_object_begin(w); -// spdk_json_write_named_string(w, "type", "OCF_core"); -// spdk_json_write_named_string(w, "name", core->name); -// spdk_json_write_named_string(w, "base_bdev_name", -// core->base.bdev ? spdk_bdev_get_name(core->base.bdev) : "" ); -// spdk_json_write_named_string(w, "cache_name", vbdev_ocf_core_get_cache(core)->name); -// spdk_json_write_object_end(w); -//} +// handle errors ? +static int +_get_bdevs_core_visit(ocf_core_t core, void *cb_arg) +{ + struct spdk_json_write_ctx *w = cb_arg; + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "type", "OCF_core"); + spdk_json_write_named_string(w, "name", ocf_core_get_name(core)); + spdk_json_write_named_bool(w, "base_attached", vbdev_ocf_core_is_base_attached(core_ctx)); + spdk_json_write_named_string(w, "base_name", vbdev_ocf_core_is_base_attached(core_ctx) ? + spdk_bdev_get_name(core_ctx->base.bdev) : "" ); + spdk_json_write_object_end(w); + + return 0; +} + +static int +_get_bdevs_cache_visit(ocf_cache_t cache, void *cb_arg) +{ + struct spdk_json_write_ctx *w = cb_arg; + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + int rc; + + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "type", "OCF_cache"); + spdk_json_write_named_string(w, "name", ocf_cache_get_name(cache)); + spdk_json_write_named_bool(w, "base_attached", ocf_cache_is_device_attached(cache)); + spdk_json_write_named_string(w, "base_name", ocf_cache_is_device_attached(cache) ? + spdk_bdev_get_name(cache_ctx->base.bdev) : "" ); + spdk_json_write_named_uint16(w, "cores_count", ocf_cache_get_core_count(cache)); + spdk_json_write_named_array_begin(w, "cores"); + if ((rc = ocf_core_visit(cache, _get_bdevs_core_visit, w, false))) { + return rc; + } + spdk_json_write_array_end(w); + spdk_json_write_object_end(w); + + return 0; +} /* RPC entry point. */ void vbdev_ocf_get_bdevs(const char *name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2) { - //comp - //struct spdk_json_write_ctx *w = rpc_cb_arg1; - //struct vbdev_ocf_cache *cache; - //struct vbdev_ocf_core *core; - //bool found = false; - - //if (name) { - // vbdev_ocf_foreach_cache(cache) { - // vbdev_ocf_foreach_core_in_cache(core, cache) { - // if (strcmp(name, core->name)) { - // continue; - // } - // found = true; - - // // dump_info_json() instead? - // _write_core_info(w, core); - // break; - // } - // if (found) { - // break; - // } - // if (strcmp(name, cache->name)) { - // continue; - // } - - // _write_cache_info_begin(w, cache); - // _write_cache_info_end(w, cache); - // break; - // } - //} else { - // vbdev_ocf_foreach_cache(cache) { - // _write_cache_info_begin(w, cache); - // spdk_json_write_named_array_begin(w, "cores"); - // vbdev_ocf_foreach_core_in_cache(core, cache) { - // _write_core_info(w, core); - // } - // spdk_json_write_array_end(w); - // _write_cache_info_end(w, cache); - // } - //} + struct spdk_json_write_ctx *w = rpc_cb_arg1; + int rc; + + // TODO: dump info about 'name' bdev only + // TODO: dump info about cores in wait list + + if ((rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _get_bdevs_cache_visit, w))) { + SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + } - //rpc_cb_fn(rpc_cb_arg1, rpc_cb_arg2); + rpc_cb_fn(rpc_cb_arg1, rpc_cb_arg2); } SPDK_LOG_REGISTER_COMPONENT(vbdev_ocf) diff --git a/module/bdev/ocf/vbdev_ocf_cache.c b/module/bdev/ocf/vbdev_ocf_cache.c index 47dddd40f9e..5b063ed308f 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.c +++ b/module/bdev/ocf/vbdev_ocf_cache.c @@ -247,7 +247,7 @@ vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache) return -ENOMEM; } - if ((rc = vbdev_ocf_queue_create_mngt(cache, &cache_ctx->cache_mngt_q, &cache_mngt_queue_ops))) { + if ((rc = ocf_queue_create_mngt(cache, &cache_ctx->cache_mngt_q, &cache_mngt_queue_ops))) { SPDK_ERRLOG("OCF cache '%s': failed to create OCF management queue\n", ocf_cache_get_name(cache)); free(mngt_q_ctx); @@ -259,7 +259,7 @@ vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache) if (!mngt_q_ctx->poller) { SPDK_ERRLOG("OCF cache '%s': failed to create management queue poller\n", ocf_cache_get_name(cache)); - vbdev_ocf_queue_put(cache_ctx->cache_mngt_q); + ocf_queue_put(cache_ctx->cache_mngt_q); return -ENOMEM; } diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c index c745b5bce65..92bdc2e4429 100644 --- a/module/bdev/ocf/vbdev_ocf_core.c +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -167,7 +167,9 @@ _core_io_queue_stop(void *ctx) struct vbdev_ocf_core_io_channel_ctx *ch_ctx = ctx; spdk_poller_unregister(&ch_ctx->poller); - spdk_put_io_channel(ch_ctx->cache_ch); + if (ch_ctx->cache_ch) { + spdk_put_io_channel(ch_ctx->cache_ch); + } spdk_put_io_channel(ch_ctx->core_ch); free(ch_ctx); } @@ -226,19 +228,21 @@ _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) return -ENOMEM; } - if ((rc = vbdev_ocf_queue_create(cache, &ch_ctx->queue, &core_io_queue_ops))) { + if ((rc = ocf_queue_create(cache, &ch_ctx->queue, &core_io_queue_ops))) { SPDK_ERRLOG("OCF vbdev '%s': failed to create OCF queue\n", vbdev_name); free(ch_ctx); return rc; } ocf_queue_set_priv(ch_ctx->queue, ch_ctx); - ch_ctx->cache_ch = spdk_bdev_get_io_channel(cache_base->desc); - if (!ch_ctx->cache_ch) { - SPDK_ERRLOG("OCF vbdev '%s': failed to create IO channel for base bdev '%s'\n", - vbdev_name, spdk_bdev_get_name(cache_base->bdev)); - vbdev_ocf_queue_put(ch_ctx->queue); - return -ENOMEM; + if (!ocf_cache_is_detached(cache)) { + ch_ctx->cache_ch = spdk_bdev_get_io_channel(cache_base->desc); + if (!ch_ctx->cache_ch) { + SPDK_ERRLOG("OCF vbdev '%s': failed to create IO channel for base bdev '%s'\n", + vbdev_name, spdk_bdev_get_name(cache_base->bdev)); + ocf_queue_put(ch_ctx->queue); + return -ENOMEM; + } } ch_ctx->core_ch = spdk_bdev_get_io_channel(core_base->desc); @@ -246,7 +250,7 @@ _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) SPDK_ERRLOG("OCF vbdev '%s': failed to create IO channel for base bdev '%s'\n", vbdev_name, spdk_bdev_get_name(core_base->bdev)); spdk_put_io_channel(ch_ctx->cache_ch); - vbdev_ocf_queue_put(ch_ctx->queue); + ocf_queue_put(ch_ctx->queue); return -ENOMEM; } @@ -255,7 +259,7 @@ _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) SPDK_ERRLOG("OCF vbdev '%s': failed to create IO queue poller\n", vbdev_name); spdk_put_io_channel(ch_ctx->core_ch); spdk_put_io_channel(ch_ctx->cache_ch); - vbdev_ocf_queue_put(ch_ctx->queue); + ocf_queue_put(ch_ctx->queue); return -ENOMEM; } @@ -279,7 +283,7 @@ _vbdev_ocf_ch_destroy_cb(void *io_device, void *ctx_buf) spdk_bdev_get_name(&(((struct vbdev_ocf_core *)ocf_core_get_priv( (ocf_core_t)io_device))->ocf_vbdev))); - vbdev_ocf_queue_put(ch_destroy_ctx->queue); + ocf_queue_put(ch_destroy_ctx->queue); } int @@ -343,6 +347,7 @@ vbdev_ocf_core_waitlist_get_by_name(const char *core_name) struct vbdev_ocf_core *core_ctx; vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + // if (core_ctx && ...) ? if (!strcmp(core_name, vbdev_ocf_core_get_name(core_ctx))) { return core_ctx; } diff --git a/module/bdev/ocf/vbdev_ocf_core.h b/module/bdev/ocf/vbdev_ocf_core.h index 30ed7959d25..952829f9543 100644 --- a/module/bdev/ocf/vbdev_ocf_core.h +++ b/module/bdev/ocf/vbdev_ocf_core.h @@ -31,6 +31,7 @@ struct vbdev_ocf_core { struct vbdev_ocf_base base; char cache_name[OCF_CACHE_NAME_SIZE]; + struct vbdev_ocf_mngt_ctx * mngt_ctx; STAILQ_ENTRY(vbdev_ocf_core) waitlist_entry; }; From cff45c1513eeac0de1d3223d8b037767c58ddd4c Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Mon, 19 May 2025 11:04:06 +0200 Subject: [PATCH 05/34] Implement bdev examine and hotremove Change-Id: I74ef4446df4f51d662b31e2774a8d1c7f1c11df0 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 586 ++++++++++++++++++++++++------ module/bdev/ocf/vbdev_ocf_cache.c | 130 ++++--- module/bdev/ocf/vbdev_ocf_cache.h | 4 +- module/bdev/ocf/vbdev_ocf_core.c | 26 +- module/bdev/ocf/vbdev_ocf_rpc.c | 4 +- module/bdev/ocf/volume.h | 2 +- 6 files changed, 587 insertions(+), 165 deletions(-) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 19f80232074..a088d4c14c2 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -21,6 +21,7 @@ static void vbdev_ocf_module_fini_start(void); static void vbdev_ocf_module_fini(void); static int vbdev_ocf_module_get_ctx_size(void); static void vbdev_ocf_module_examine_config(struct spdk_bdev *bdev); +static void vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev); struct spdk_bdev_module ocf_if = { .name = "OCF", @@ -29,7 +30,7 @@ struct spdk_bdev_module ocf_if = { .module_fini = vbdev_ocf_module_fini, .get_ctx_size = vbdev_ocf_module_get_ctx_size, .examine_config = vbdev_ocf_module_examine_config, - .examine_disk = NULL, // todo ? + .examine_disk = vbdev_ocf_module_examine_disk, .async_fini_start = true, }; @@ -217,15 +218,17 @@ _module_fini_core_visit(ocf_core_t core, void *cb_arg) SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': module stop visit\n", ocf_core_get_name(core)); - if ((rc = ocf_mngt_cache_get(cache))) { - SPDK_ERRLOG("OCF core '%s': failed to increment cache '%s' ref count: %s\n", - ocf_core_get_name(core), ocf_cache_get_name(cache), spdk_strerror(-rc)); - return rc; + /* If core is detached it's already unregistered, so just free its data and exit. */ + if (!vbdev_ocf_core_is_base_attached(core_ctx)) { + ocf_mngt_cache_put(cache); + vbdev_ocf_core_destroy(core_ctx); + return 0; } if ((rc = vbdev_ocf_core_unregister(core_ctx, _module_fini_core_unregister_cb, core))) { SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during module stop: %s\n", ocf_core_get_name(core), spdk_strerror(-rc)); + ocf_mngt_cache_put(cache); return rc; } @@ -235,18 +238,35 @@ _module_fini_core_visit(ocf_core_t core, void *cb_arg) static int _module_fini_cache_visit(ocf_cache_t cache, void *cb_arg) { - int rc = 0; + int i, rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': module stop visit\n", ocf_cache_get_name(cache)); - if (!ocf_cache_get_core_count(cache)) { + /* Increment cache refcount to not destroy cache structs before destroying all cores. */ + printf("--- cores count: %d\n", ocf_cache_get_core_count(cache)); + printf("--- cores inactive count: %d\n", ocf_cache_get_core_inactive_count(cache)); + printf("*** cache refcnt before get: %d\n", ocf_cache_get_refcnt(cache)); + for (i = ocf_cache_get_core_count(cache); i > 0; i--) { + printf("*** INC refcnt for core %d\n", i); + if ((rc = ocf_mngt_cache_get(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to increment refcount: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-rc)); + } + } + printf("*** cache refcnt after get: %d\n", ocf_cache_get_refcnt(cache)); + + if (!ocf_cache_get_core_count(cache) || + ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { + /* If there are no cores or all of them are detached, + * then cache stop can be triggered already. */ ocf_mngt_cache_lock(cache, _module_fini_cache_lock_cb, cb_arg); - return rc; } - if ((rc = ocf_core_visit(cache, _module_fini_core_visit, cb_arg, false))) { // only opened cores ? - SPDK_ERRLOG("OCF: failed to iterate over core bdevs: %s\n", spdk_strerror(-rc)); - return rc; + printf("*** cache refcnt before visit: %d\n", ocf_cache_get_refcnt(cache)); + if ((rc = ocf_core_visit(cache, _module_fini_core_visit, cb_arg, false))) { + SPDK_ERRLOG("OCF cache '%s': failed to iterate over core bdevs: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-rc)); + // ocf_mngt_cache_lock(...) ? } return rc; @@ -300,67 +320,292 @@ vbdev_ocf_module_get_ctx_size(void) return sizeof(struct vbdev_ocf_data); } +static int +_examine_config_core_visit(ocf_core_t core, void *cb_arg) +{ + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + char *bdev_name = cb_arg; + int rc = 0; + + if (strcmp(bdev_name, core_ctx->base.name)) { + return 0; + } + + SPDK_NOTICELOG("OCF core '%s': base bdev '%s' found\n", + ocf_core_get_name(core), bdev_name); + + assert(!vbdev_ocf_core_is_base_attached(core_ctx)); + + if ((rc = vbdev_ocf_core_base_attach(core_ctx, bdev_name))) { + SPDK_ERRLOG("OCF core '%s': failed to attach base bdev '%s'\n", + vbdev_ocf_core_get_name(core_ctx), bdev_name); + return rc; + } + + core_ctx->core_cfg.try_add = true; + + return -EEXIST; +} + +static int +_examine_config_cache_visit(ocf_cache_t cache, void *cb_arg) +{ + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + char *bdev_name = cb_arg; + int rc = 0; + + if (strcmp(bdev_name, cache_ctx->base.name)) { + return ocf_core_visit(cache, _examine_config_core_visit, bdev_name, false); + } + + SPDK_NOTICELOG("OCF cache '%s': base bdev '%s' found\n", + ocf_cache_get_name(cache), bdev_name); + + assert(!ocf_cache_is_device_attached(cache)); + assert(!vbdev_ocf_cache_is_base_attached(cache)); + + if ((rc = vbdev_ocf_cache_base_attach(cache, bdev_name))) { + SPDK_ERRLOG("OCF cache '%s': failed to attach base bdev '%s'\n", + ocf_cache_get_name(cache), bdev_name); + return rc; + } + + if ((rc = vbdev_ocf_cache_config_volume_create(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to create config volume\n", + ocf_cache_get_name(cache)); + vbdev_ocf_cache_base_detach(cache); + return rc; + } + + return -EEXIST; +} + static void vbdev_ocf_module_examine_config(struct spdk_bdev *bdev) { - //const char *bdev_name = spdk_bdev_get_name(bdev); - //struct vbdev_ocf_cache *cache; - //struct vbdev_ocf_core *core; - //int rc; + struct vbdev_ocf_core *core_ctx; + char *bdev_name = (char *)spdk_bdev_get_name(bdev); + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF: looking for vbdevs waiting for '%s'\n", bdev_name); + + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + if (strcmp(bdev_name, core_ctx->base.name)) { + continue; + } + + SPDK_NOTICELOG("OCF core '%s': base bdev '%s' found\n", + vbdev_ocf_core_get_name(core_ctx), bdev_name); + + assert(!vbdev_ocf_core_is_base_attached(core_ctx)); + + if ((rc = vbdev_ocf_core_base_attach(core_ctx, bdev_name))) { + SPDK_ERRLOG("OCF core '%s': failed to attach base bdev '%s'\n", + vbdev_ocf_core_get_name(core_ctx), bdev_name); + } + + spdk_bdev_module_examine_done(&ocf_if); + return; + } + + rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _examine_config_cache_visit, bdev_name); + if (rc && rc != -EEXIST) { + SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + } + + spdk_bdev_module_examine_done(&ocf_if); +} + +static void +_core_add_examine_err_cb(void *cb_arg, int error) +{ + ocf_cache_t cache = cb_arg; + + if (error) { + SPDK_ERRLOG("OCF core: failed to remove OCF core device (OCF error: %d)\n", error); + } + + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); + spdk_bdev_module_examine_done(&ocf_if); +} + +static void +_core_add_examine_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int error) +{ + struct vbdev_ocf_core *core_ctx = cb_arg; + struct vbdev_ocf_core *core_ctx_waitlist; + int rc = 0; - //vbdev_ocf_foreach_cache(cache) { - // if (!vbdev_ocf_cache_is_incomplete(cache)) { - // continue; - // } - // if (strcmp(bdev_name, cache->init_params->bdev_name)) { - // continue; - // } + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", + vbdev_ocf_core_get_name(core_ctx)); - // assert(!vbdev_ocf_cache_is_base_attached(cache)); + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s' (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache), error); + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); + vbdev_ocf_core_base_detach(core_ctx); + spdk_bdev_module_examine_done(&ocf_if); + return; + } - // if ((rc = vbdev_ocf_cache_base_attach(cache, bdev_name))) { - // SPDK_ERRLOG("OCF cache '%s': failed to attach base bdev '%s'\n", - // cache->name, bdev_name); - // } + ocf_core_set_priv(core, core_ctx); - // printf("*** START OCF cache\n"); + if ((rc = vbdev_ocf_core_register(core))) { + SPDK_ERRLOG("OCF core '%s': failed to register vbdev\n", ocf_core_get_name(core)); + ocf_mngt_cache_remove_core(core, _core_add_examine_err_cb, cache); + vbdev_ocf_core_base_detach(core_ctx); // move to callback (pass core_ctx ?) + return; + } - // vbdev_ocf_cache_remove_incomplete(cache); + SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n", + ocf_core_get_name(core), ocf_cache_get_name(cache)); + + /* If core was taken from waitlist, remove it from there. */ + vbdev_ocf_foreach_core_in_waitlist(core_ctx_waitlist) { + if (strcmp(vbdev_ocf_core_get_name(core_ctx), vbdev_ocf_core_get_name(core_ctx_waitlist))) { + continue; + } - // goto end; - //} + STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); + } + + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); + spdk_bdev_module_examine_done(&ocf_if); +} - //vbdev_ocf_foreach_core_in_waitlist(core) { - // if (strcmp(bdev_name, core->init_params->bdev_name)) { - // continue; - // } +static void +_core_add_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_core *core_ctx = cb_arg; - // assert(!vbdev_ocf_core_is_base_attached(core)); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add of OCF core\n", + vbdev_ocf_core_get_name(core_ctx)); - // if ((rc = vbdev_ocf_core_base_attach(core, bdev_name))) { - // SPDK_ERRLOG("OCF core '%s': failed to attach base bdev '%s'\n", - // core->name, bdev_name); - // } + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), error); + ocf_mngt_cache_put(cache); + vbdev_ocf_core_base_detach(core_ctx); + spdk_bdev_module_examine_done(&ocf_if); + return; + } - // cache = vbdev_ocf_cache_get_by_name(core->init_params->cache_name); - // if (!cache || vbdev_ocf_cache_is_incomplete(cache)) { - // SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache '%s'\n", - // core->name, core->init_params->cache_name); - // goto end; - // } + ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _core_add_examine_add_cb, core_ctx); +} - // printf("*** ADD OCF core\n"); +static void +_cache_attach_examine_attach_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing device attach\n", + ocf_cache_get_name(cache)); - // vbdev_ocf_core_add_to_cache(core, cache); - // vbdev_ocf_core_remove_incomplete(core); + /* At this point volume was either moved to ocf_cache_t struct + * or is no longer needed due to some errors, so we need to deallocate it. */ + vbdev_ocf_cache_config_volume_destroy(cache); - // goto end; - //} + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to attach OCF cache device (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + vbdev_ocf_cache_base_detach(cache); + } else { + SPDK_NOTICELOG("OCF cache '%s': device attached\n", ocf_cache_get_name(cache)); + } -//end: + ocf_mngt_cache_unlock(cache); spdk_bdev_module_examine_done(&ocf_if); } +static void +_cache_attach_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': attaching OCF cache device\n", + ocf_cache_get_name(cache)); + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + vbdev_ocf_cache_config_volume_destroy(cache); + vbdev_ocf_cache_base_detach(cache); + spdk_bdev_module_examine_done(&ocf_if); + return; + } + + ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, _cache_attach_examine_attach_cb, NULL); +} + +static int +_examine_disk_core_visit(ocf_core_t core, void *cb_arg) +{ + ocf_cache_t cache = ocf_core_get_cache(core); + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + char *bdev_name = cb_arg; + + if (strcmp(bdev_name, core_ctx->base.name)) { + return 0; + } + + ocf_mngt_cache_get(cache); + ocf_mngt_cache_lock(cache, _core_add_examine_lock_cb, core_ctx); + + return -EEXIST; +} + +static int +_examine_disk_cache_visit(ocf_cache_t cache, void *cb_arg) +{ + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + char *bdev_name = cb_arg; + + if (strcmp(bdev_name, cache_ctx->base.name)) { + return ocf_core_visit(cache, _examine_disk_core_visit, bdev_name, false); + } + + assert(!ocf_cache_is_device_attached(cache)); + + ocf_mngt_cache_lock(cache, _cache_attach_examine_lock_cb, NULL); + + return -EEXIST; +} + +static void +vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev) +{ + ocf_cache_t cache; + struct vbdev_ocf_core *core_ctx; + char *bdev_name = (char *)spdk_bdev_get_name(bdev); + int rc; + + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + if (strcmp(bdev_name, core_ctx->base.name)) { + continue; + } + + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, core_ctx->cache_name, + OCF_CACHE_NAME_SIZE, &cache)) { + SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache '%s'\n", + vbdev_ocf_core_get_name(core_ctx), core_ctx->cache_name); + spdk_bdev_module_examine_done(&ocf_if); + return; + } + + ocf_mngt_cache_lock(cache, _core_add_examine_lock_cb, core_ctx); + return; + } + + rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _examine_disk_cache_visit, bdev_name); + if (rc && rc != -EEXIST) { + SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + } else if (!rc) { + /* No visitor matched this new bdev, so no one called _examine_done(). */ + spdk_bdev_module_examine_done(&ocf_if); // ??? + } +} + static void _destruct_core_detach_cb(ocf_core_t core, void *cb_arg, int error) { @@ -589,6 +834,66 @@ vbdev_ocf_fn_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ct { } +static void +_cache_start_rpc_core_add_err_cb(void *cb_arg, int error) +{ + ocf_cache_t cache = cb_arg; + + if (error) { + SPDK_ERRLOG("OCF core: failed to remove OCF core device (OCF error: %d)\n", error); + } + + ocf_mngt_cache_unlock(cache); +} + +static void +_cache_start_rpc_core_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int error) +{ + struct vbdev_ocf_core *core_ctx = cb_arg; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", + vbdev_ocf_core_get_name(core_ctx)); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s'\n", + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); + ocf_mngt_cache_unlock(cache); + return; + } + + ocf_core_set_priv(core, core_ctx); + + if ((rc = vbdev_ocf_core_register(core))) { + SPDK_ERRLOG("OCF core '%s': failed to register vbdev\n", ocf_core_get_name(core)); + ocf_mngt_cache_remove_core(core, _cache_start_rpc_core_add_err_cb, cache); + return; + } + + SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n", + ocf_core_get_name(core), ocf_cache_get_name(cache)); + + ocf_mngt_cache_unlock(cache); + STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); +} + +static void +_cache_start_rpc_core_add_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_core *core_ctx = cb_arg; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add of OCF core\n", + vbdev_ocf_core_get_name(core_ctx)); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock\n", + vbdev_ocf_core_get_name(core_ctx)); + return; + } + + ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _cache_start_rpc_core_add_cb, core_ctx); +} + static void _cache_start_rpc_err_cb(ocf_cache_t cache, void *cb_arg, int error) { @@ -599,7 +904,7 @@ _cache_start_rpc_err_cb(ocf_cache_t cache, void *cb_arg, int error) vbdev_ocf_cache_destroy(cache); ocf_mngt_cache_unlock(cache); // use in error path only ? - //ocf_mngt_cache_put(cache); // no need ? (it's already stopped) + //ocf_mngt_cache_put(cache); // no need ? (check refcnt) } static void @@ -607,12 +912,18 @@ _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *cache_start_ctx = cb_arg; struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + struct vbdev_ocf_core *core_ctx; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing start\n", ocf_cache_get_name(cache)); + /* At this point volume was either moved to ocf_cache_t struct + * or is no longer needed due to some errors, so we need to deallocate it. */ + vbdev_ocf_cache_config_volume_destroy(cache); + if (error) { SPDK_ERRLOG("OCF cache '%s': failed to attach OCF cache device\n", ocf_cache_get_name(cache)); + if (error == -OCF_ERR_NO_MEM) { uint64_t mem_needed, volume_size; @@ -627,25 +938,31 @@ _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) vbdev_ocf_cache_base_detach(cache); ocf_queue_put(cache_ctx->cache_mngt_q); // needed ? ocf_mngt_cache_stop(cache, _cache_start_rpc_err_cb, NULL); - // move to ocf_mngt_cache_stop callback ? - if (cache_start_ctx->rpc_cb_fn) { - cache_start_ctx->rpc_cb_fn(NULL, cache_start_ctx->rpc_cb_arg, error); // cache name - } + cache_start_ctx->rpc_cb_fn(NULL, cache_start_ctx->rpc_cb_arg, error); // cache name free(cache_start_ctx); return; } SPDK_NOTICELOG("OCF cache '%s': started\n", ocf_cache_get_name(cache)); - - // check for cores in g_vbdev_ocf_core_waitlist - // check if (cache_block_size > core_block_size) ocf_mngt_cache_unlock(cache); - if (cache_start_ctx->rpc_cb_fn) { - cache_start_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_start_ctx->rpc_cb_arg, 0); - } + cache_start_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_start_ctx->rpc_cb_arg, 0); free(cache_start_ctx); + + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + if (strcmp(ocf_cache_get_name(cache), core_ctx->cache_name) || + !vbdev_ocf_core_is_base_attached(core_ctx)) { + continue; + } + + SPDK_NOTICELOG("OCF core '%s': adding from waitlist to cache '%s'\n", + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); + + // check if (cache_block_size > core_block_size) + + ocf_mngt_cache_lock(cache, _cache_start_rpc_core_add_lock_cb, core_ctx); + } } static void @@ -663,15 +980,15 @@ _cache_start_rpc_metadata_probe_cb(void *priv, int error, struct ocf_metadata_pr } if (error == -OCF_ERR_NO_METADATA) { - SPDK_NOTICELOG("OCF cache '%s': metadata not found - starting new cache\n", + SPDK_NOTICELOG("OCF cache '%s': metadata not found - starting new cache instance\n", ocf_cache_get_name(cache)); - //cache_ctx->cache_att_cfg.force = true; // needed ? + //cache_ctx->cache_att_cfg.force = true; // needed for later ? ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, _cache_start_rpc_attach_cb, cache_start_ctx); } else { - SPDK_NOTICELOG("OCF cache '%s': metadata found - loading cache from metadata\n", + SPDK_NOTICELOG("OCF cache '%s': metadata found - loading previous cache instance\n", ocf_cache_get_name(cache)); - SPDK_NOTICELOG("(start cache with '--no-load' flag to start new cache instead of loading config from metadata)\n"); + SPDK_NOTICELOG("(start cache with '--no-load' flag to create new cache instead of loading config from metadata)\n"); // check status for cache_name/mode/line_size/dirty ? ocf_mngt_cache_load(cache, &cache_ctx->cache_att_cfg, _cache_start_rpc_attach_cb, cache_start_ctx); @@ -697,7 +1014,8 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, goto err_exist; } - if ((rc = vbdev_ocf_cache_create(&cache, cache_name, cache_mode, cache_line_size, no_load))) { + if ((rc = vbdev_ocf_cache_create(&cache, cache_name, cache_mode, + cache_line_size, no_load))) { SPDK_ERRLOG("OCF cache '%s': failed to create cache\n", cache_name); goto err_create; } @@ -715,10 +1033,15 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, rpc_cb_fn(cache_name, rpc_cb_arg, -ENODEV); return; } - SPDK_ERRLOG("OCF cache '%s': failed to open base bdev '%s'\n", cache_name, base_name); + SPDK_ERRLOG("OCF cache '%s': failed to attach base bdev '%s'\n", cache_name, base_name); goto err_base; } + if ((rc = vbdev_ocf_cache_config_volume_create(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to create config volume\n", cache_name); + goto err_volume; + } + cache_start_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!cache_start_ctx) { SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache start context\n", @@ -752,6 +1075,8 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, err_open: free(cache_start_ctx); err_alloc: + vbdev_ocf_cache_config_volume_destroy(cache); +err_volume: vbdev_ocf_cache_base_detach(cache); err_base: ocf_queue_put(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); @@ -788,7 +1113,7 @@ _cache_stop_rpc_stop_cb(ocf_cache_t cache, void *cb_arg, int error) } ocf_mngt_cache_unlock(cache); - ocf_mngt_cache_put(cache); + //ocf_mngt_cache_put(cache); // no need to put !!! (check mergability with module fini) cache_stop_ctx->rpc_cb_fn(NULL, cache_stop_ctx->rpc_cb_arg, error); // cache name free(cache_stop_ctx); } @@ -857,17 +1182,19 @@ _cache_stop_rpc_core_visit(ocf_core_t core, void *cb_arg) SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': cache stop visit\n", ocf_core_get_name(core)); - core_ctx->mngt_ctx = cache_stop_ctx; - - if ((rc = ocf_mngt_cache_get(cache))) { - SPDK_ERRLOG("OCF core '%s': failed to increment cache '%s' ref count: %s\n", - ocf_core_get_name(core), ocf_cache_get_name(cache), spdk_strerror(-rc)); - return rc; + /* If core is detached it's already unregistered, so just free its data and exit. */ + if (!vbdev_ocf_core_is_base_attached(core_ctx)) { + ocf_mngt_cache_put(cache); + vbdev_ocf_core_destroy(core_ctx); + return 0; } + core_ctx->mngt_ctx = cache_stop_ctx; + if ((rc = vbdev_ocf_core_unregister(core_ctx, _cache_stop_rpc_core_unregister_cb, core))) { SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during cache stop: %s\n", ocf_core_get_name(core), spdk_strerror(-rc)); + ocf_mngt_cache_put(cache); return rc; } @@ -880,7 +1207,7 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo { ocf_cache_t cache; struct vbdev_ocf_mngt_ctx *cache_stop_ctx; - int rc; + int i, rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop\n", cache_name); @@ -906,17 +1233,38 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo cache_stop_ctx->rpc_cb_fn = rpc_cb_fn; cache_stop_ctx->rpc_cb_arg = rpc_cb_arg; - if (ocf_cache_get_core_count(cache)) { - if ((rc = ocf_core_visit(cache, _cache_stop_rpc_core_visit, cache_stop_ctx, true))) { - SPDK_ERRLOG("OCF: failed to iterate over core bdevs: %s\n", spdk_strerror(-rc)); - // ocf_mngt_cache_lock(...) ? + /* Increment cache refcount to not destroy cache structs before destroying all cores. */ + printf("--- cores count: %d\n", ocf_cache_get_core_count(cache)); + printf("--- cores inactive count: %d\n", ocf_cache_get_core_inactive_count(cache)); + printf("*** cache refcnt before get: %d\n", ocf_cache_get_refcnt(cache)); + for (i = ocf_cache_get_core_count(cache); i > 0; i--) { + printf("*** INC refcnt for core %d\n", i); + if ((rc = ocf_mngt_cache_get(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to increment refcount\n", + ocf_cache_get_name(cache)); + goto err_get; } - } else { + } + printf("*** cache refcnt after get: %d\n", ocf_cache_get_refcnt(cache)); + + if (!ocf_cache_get_core_count(cache) || + ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { + /* If there are no cores or all of them are detached, + * then cache stop can be triggered already. */ ocf_mngt_cache_lock(cache, _cache_stop_rpc_lock_cb, cache_stop_ctx); } + printf("*** cache refcnt before visit: %d\n", ocf_cache_get_refcnt(cache)); + if ((rc = ocf_core_visit(cache, _cache_stop_rpc_core_visit, cache_stop_ctx, false))) { + SPDK_ERRLOG("OCF cache '%s': failed to iterate over core bdevs\n", + ocf_cache_get_name(cache)); + // ocf_mngt_cache_lock() or rpc_cb_fn() ? + } + return; +err_get: + free(cache_stop_ctx); err_alloc: ocf_mngt_cache_put(cache); err_cache: @@ -1042,6 +1390,10 @@ _cache_attach_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing device attach\n", ocf_cache_get_name(cache)); + /* At this point volume was either moved to ocf_cache_t struct + * or is no longer needed due to some errors, so we need to deallocate it. */ + vbdev_ocf_cache_config_volume_destroy(cache); + if (error) { SPDK_ERRLOG("OCF cache '%s': failed to attach OCF cache device (OCF error: %d)\n", ocf_cache_get_name(cache), error); @@ -1068,6 +1420,7 @@ _cache_attach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (error) { SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", ocf_cache_get_name(cache), error); + vbdev_ocf_cache_config_volume_destroy(cache); vbdev_ocf_cache_base_detach(cache); ocf_mngt_cache_put(cache); cache_attach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_attach_ctx->rpc_cb_arg, error); @@ -1091,7 +1444,7 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating device attach\n", cache_name); - // increments cache refcnt !!! (put it right away ?) + // ocf_mngt_cache_put() right away to merge cache_start/attach callbacks if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; @@ -1104,6 +1457,9 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force goto err_state; } + cache_ctx = ocf_cache_get_priv(cache); + cache_ctx->no_load = force; + if ((rc = vbdev_ocf_cache_base_attach(cache, base_name))) { if (rc == -ENODEV) { SPDK_NOTICELOG("OCF cache '%s': start deferred - waiting for base bdev '%s'\n", @@ -1111,10 +1467,15 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force rpc_cb_fn(cache_name, rpc_cb_arg, -ENODEV); return; } - SPDK_ERRLOG("OCF cache '%s': failed to open base bdev '%s'\n", cache_name, base_name); + SPDK_ERRLOG("OCF cache '%s': failed to attach base bdev '%s'\n", cache_name, base_name); goto err_base; } + if ((rc = vbdev_ocf_cache_config_volume_create(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to create config volume\n", cache_name); + goto err_volume; + } + cache_attach_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!cache_attach_ctx) { SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache attach context\n", @@ -1125,14 +1486,13 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force cache_attach_ctx->rpc_cb_fn = rpc_cb_fn; cache_attach_ctx->rpc_cb_arg = rpc_cb_arg; - cache_ctx = ocf_cache_get_priv(cache); - cache_ctx->cache_att_cfg.force = force; - ocf_mngt_cache_lock(cache, _cache_attach_rpc_lock_cb, cache_attach_ctx); return; err_alloc: + vbdev_ocf_cache_config_volume_destroy(cache); +err_volume: vbdev_ocf_cache_base_detach(cache); err_base: err_state: @@ -1257,7 +1617,7 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac rpc_cb_fn(core_name, rpc_cb_arg, -ENODEV); return; } - SPDK_ERRLOG("OCF core '%s': failed to open base bdev '%s'\n", core_name, base_name); + SPDK_ERRLOG("OCF core '%s': failed to attach base bdev '%s'\n", core_name, base_name); goto err_base; } @@ -1402,6 +1762,8 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, goto err_core; } + core_ctx = ocf_core_get_priv(core); + core_rm_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!core_rm_ctx) { SPDK_ERRLOG("OCF core '%s': failed to allocate memory for core remove context\n", core_name); @@ -1410,11 +1772,17 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, } core_rm_ctx->cache = cache; core_rm_ctx->core = core; - core_rm_ctx->core_ctx = ocf_core_get_priv(core); + core_rm_ctx->core_ctx = core_ctx; core_rm_ctx->rpc_cb_fn = rpc_cb_fn; core_rm_ctx->rpc_cb_arg = rpc_cb_arg; - if ((rc = vbdev_ocf_core_unregister(core_rm_ctx->core_ctx, _core_remove_rpc_unregister_cb, core_rm_ctx))) { + if (!vbdev_ocf_core_is_base_attached(core_ctx)) { + // DEBUGLOG for all unregister skipping + ocf_mngt_cache_lock(cache, _core_remove_rpc_lock_cb, core_rm_ctx); + return; + } + + if ((rc = vbdev_ocf_core_unregister(core_ctx, _core_remove_rpc_unregister_cb, core_rm_ctx))) { SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during core removal\n", ocf_core_get_name(core)); goto err_unregister; @@ -1432,6 +1800,13 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, } // handle errors ? +static void +_get_bdevs_core_dump(struct vbdev_ocf_core *core_ctx, struct spdk_json_write_ctx *w) +{ + spdk_json_write_named_string(w, "base_name", core_ctx->base.name); + spdk_json_write_named_bool(w, "base_attached", vbdev_ocf_core_is_base_attached(core_ctx)); +} + static int _get_bdevs_core_visit(ocf_core_t core, void *cb_arg) { @@ -1439,11 +1814,8 @@ _get_bdevs_core_visit(ocf_core_t core, void *cb_arg) struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); spdk_json_write_object_begin(w); - spdk_json_write_named_string(w, "type", "OCF_core"); spdk_json_write_named_string(w, "name", ocf_core_get_name(core)); - spdk_json_write_named_bool(w, "base_attached", vbdev_ocf_core_is_base_attached(core_ctx)); - spdk_json_write_named_string(w, "base_name", vbdev_ocf_core_is_base_attached(core_ctx) ? - spdk_bdev_get_name(core_ctx->base.bdev) : "" ); + _get_bdevs_core_dump(core_ctx, w); spdk_json_write_object_end(w); return 0; @@ -1457,20 +1829,18 @@ _get_bdevs_cache_visit(ocf_cache_t cache, void *cb_arg) int rc; spdk_json_write_object_begin(w); - spdk_json_write_named_string(w, "type", "OCF_cache"); spdk_json_write_named_string(w, "name", ocf_cache_get_name(cache)); + spdk_json_write_named_string(w, "base_name", cache_ctx->base.name); spdk_json_write_named_bool(w, "base_attached", ocf_cache_is_device_attached(cache)); - spdk_json_write_named_string(w, "base_name", ocf_cache_is_device_attached(cache) ? - spdk_bdev_get_name(cache_ctx->base.bdev) : "" ); spdk_json_write_named_uint16(w, "cores_count", ocf_cache_get_core_count(cache)); spdk_json_write_named_array_begin(w, "cores"); - if ((rc = ocf_core_visit(cache, _get_bdevs_core_visit, w, false))) { - return rc; - } + + rc = ocf_core_visit(cache, _get_bdevs_core_visit, w, false); + spdk_json_write_array_end(w); spdk_json_write_object_end(w); - return 0; + return rc; } /* RPC entry point. */ @@ -1478,14 +1848,26 @@ void vbdev_ocf_get_bdevs(const char *name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2) { struct spdk_json_write_ctx *w = rpc_cb_arg1; + struct vbdev_ocf_core *core_ctx; int rc; // TODO: dump info about 'name' bdev only - // TODO: dump info about cores in wait list + spdk_json_write_named_array_begin(w, "cores_waitlist"); + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "name", vbdev_ocf_core_get_name(core_ctx)); + spdk_json_write_named_string(w, "cache_name", core_ctx->cache_name); + _get_bdevs_core_dump(core_ctx, w); + spdk_json_write_object_end(w); + } + spdk_json_write_array_end(w); + + spdk_json_write_named_array_begin(w, "caches"); if ((rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _get_bdevs_cache_visit, w))) { SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); } + spdk_json_write_array_end(w); rpc_cb_fn(rpc_cb_arg1, rpc_cb_arg2); } diff --git a/module/bdev/ocf/vbdev_ocf_cache.c b/module/bdev/ocf/vbdev_ocf_cache.c index 5b063ed308f..00e11037ca3 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.c +++ b/module/bdev/ocf/vbdev_ocf_cache.c @@ -13,7 +13,6 @@ vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cac { ocf_cache_t cache; struct ocf_mngt_cache_config *cache_cfg; - struct ocf_mngt_cache_attach_config *cache_att_cfg; struct vbdev_ocf_cache *cache_ctx; int rc = 0; @@ -27,25 +26,20 @@ vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cac return -ENOMEM; } - cache_cfg = &cache_ctx->cache_cfg; - cache_att_cfg = &cache_ctx->cache_att_cfg; + cache_ctx->no_load = no_load; + cache_cfg = &cache_ctx->cache_cfg; ocf_mngt_cache_config_set_default(cache_cfg); - ocf_mngt_cache_attach_config_set_default(cache_att_cfg); strncpy(cache_cfg->name, cache_name, OCF_CACHE_NAME_SIZE); + if (cache_mode) { cache_cfg->cache_mode = ocf_get_cache_mode(cache_mode); } if (cache_line_size) { cache_cfg->cache_line_size = cache_line_size * KiB; - cache_att_cfg->cache_line_size = cache_line_size * KiB; } cache_cfg->locked = true; - cache_att_cfg->open_cores = false; - cache_att_cfg->discard_on_start = false; // needed ? - cache_att_cfg->device.perform_test = false; // needed ? - cache_att_cfg->force = no_load; if ((rc = ocf_mngt_cache_start(vbdev_ocf_ctx, &cache, cache_cfg, cache_ctx))) { SPDK_ERRLOG("OCF cache '%s': failed to start OCF cache\n", cache_name); @@ -78,23 +72,56 @@ vbdev_ocf_cache_destroy(ocf_cache_t cache) free(cache_ctx); } -// vbdev_ocf_cache_detach() instead of this ? +static void +_cache_hotremove_detach_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing hot removal\n", + ocf_cache_get_name(cache)); + + // how to detach base despite the error (detached core) and not cause use-after-free on module fini? + //vbdev_ocf_cache_base_detach(cache); + + ocf_mngt_cache_unlock(cache); + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to detach OCF cache device (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + return; + } + + SPDK_NOTICELOG("OCF cache '%s': device detached\n", ocf_cache_get_name(cache)); + + vbdev_ocf_cache_base_detach(cache); + /* Increment queue refcount to prevent destroying management queue after cache device detach. */ + ocf_queue_get(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); +} + +static void +_cache_hotremove_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': detaching OCF cache device\n", + ocf_cache_get_name(cache)); + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + } + + ocf_mngt_cache_detach(cache, _cache_hotremove_detach_cb, NULL); +} + static void vbdev_ocf_cache_hotremove(struct spdk_bdev *bdev, void *event_ctx) { ocf_cache_t cache = event_ctx; - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': hot removal of base bdev '%s'\n", + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating hot removal of base bdev '%s'\n", ocf_cache_get_name(cache), spdk_bdev_get_name(bdev)); assert(bdev == ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->base.bdev); + assert(ocf_cache_is_device_attached(cache)); - if (ocf_cache_is_device_attached(cache)) { // always true? - // OCF cache detach - // (to comply with SPDK hotremove support) - } - - vbdev_ocf_cache_base_detach(cache); // in detach callback ? + ocf_mngt_cache_lock(cache, _cache_hotremove_lock_cb, NULL); } static void @@ -117,13 +144,12 @@ vbdev_ocf_cache_base_attach(ocf_cache_t cache, const char *base_name) { struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); struct vbdev_ocf_base *base = &cache_ctx->base; - struct ocf_volume_uuid volume_uuid; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': attaching base bdev '%s'\n", ocf_cache_get_name(cache), base_name); - //strncpy(base->name, base_name, OCF_CACHE_NAME_SIZE); + strncpy(base->name, base_name, OCF_CACHE_NAME_SIZE); if ((rc = spdk_bdev_open_ext(base_name, true, _vbdev_ocf_cache_event_cb, cache, &base->desc))) { return rc; @@ -150,25 +176,6 @@ vbdev_ocf_cache_base_attach(ocf_cache_t cache, const char *base_name) base->is_cache = true; base->attached = true; - if ((rc = ocf_uuid_set_str(&volume_uuid, (char *)base_name))) { - SPDK_ERRLOG("OCF cache '%s': failed to set OCF volume uuid\n", - ocf_cache_get_name(cache)); - spdk_put_io_channel(base->mngt_ch); - spdk_bdev_close(base->desc); - return rc; - } - - if ((rc = ocf_ctx_volume_create(vbdev_ocf_ctx, &cache_ctx->cache_att_cfg.device.volume, - &volume_uuid, SPDK_OBJECT))) { - SPDK_ERRLOG("OCF cache '%s': failed to create OCF volume\n", ocf_cache_get_name(cache)); - spdk_put_io_channel(base->mngt_ch); - spdk_bdev_close(base->desc); - return rc; - } - - // for ocf_volume_open() in ocf_mngt_cache_attach/load() - cache_ctx->cache_att_cfg.device.volume_params = base; - // why not ? what is it then ?!? //assert(__bdev_to_io_dev(base->bdev) == base->bdev->ctxt); @@ -184,15 +191,48 @@ vbdev_ocf_cache_base_detach(ocf_cache_t cache) SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': detaching base bdev '%s'\n", ocf_cache_get_name(cache), spdk_bdev_get_name(base->bdev)); + vbdev_ocf_base_detach(base); +} - //if (cache_ctx->cache_att_cfg.device.volume) { - // ocf_volume_destroy(cache_ctx->cache_att_cfg.device.volume); - // cache_ctx->cache_att_cfg.device.volume = NULL; - // //cache_ctx->cache_att_cfg.device.volume_params = NULL; // ? - //} - ocf_volume_destroy(cache_ctx->cache_att_cfg.device.volume); +int +vbdev_ocf_cache_config_volume_create(ocf_cache_t cache) +{ + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + struct ocf_mngt_cache_attach_config *cache_att_cfg = &cache_ctx->cache_att_cfg; + struct ocf_volume_uuid volume_uuid; + int rc = 0; - vbdev_ocf_base_detach(base); + ocf_mngt_cache_attach_config_set_default(cache_att_cfg); + cache_att_cfg->cache_line_size = cache_ctx->cache_cfg.cache_line_size; + cache_att_cfg->open_cores = false; + cache_att_cfg->discard_on_start = false; // needed ? + cache_att_cfg->device.perform_test = false; // needed ? + // for ocf_volume_open() in ocf_mngt_cache_attach/load() + cache_att_cfg->device.volume_params = &cache_ctx->base; + cache_att_cfg->force = cache_ctx->no_load; + + if ((rc = ocf_uuid_set_str(&volume_uuid, (char *)ocf_cache_get_name(cache)))) { + SPDK_ERRLOG("OCF cache '%s': failed to set OCF volume uuid\n", + ocf_cache_get_name(cache)); + return rc; + } + + if ((rc = ocf_ctx_volume_create(vbdev_ocf_ctx, &cache_att_cfg->device.volume, + &volume_uuid, SPDK_OBJECT))) { + SPDK_ERRLOG("OCF cache '%s': failed to create OCF volume\n", + ocf_cache_get_name(cache)); + return rc; + } + + return rc; +} + +void +vbdev_ocf_cache_config_volume_destroy(ocf_cache_t cache) +{ + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + + ocf_volume_destroy(cache_ctx->cache_att_cfg.device.volume); } static void diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h index 6684f67c802..e1e5c5deea8 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.h +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -17,7 +17,7 @@ struct vbdev_ocf_cache { struct ocf_mngt_cache_config cache_cfg; struct ocf_mngt_cache_attach_config cache_att_cfg; ocf_queue_t cache_mngt_q; - bool load; + bool no_load; struct vbdev_ocf_base base; }; @@ -34,6 +34,8 @@ int vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char void vbdev_ocf_cache_destroy(ocf_cache_t cache); int vbdev_ocf_cache_base_attach(ocf_cache_t cache, const char *base_name); void vbdev_ocf_cache_base_detach(ocf_cache_t cache); +int vbdev_ocf_cache_config_volume_create(ocf_cache_t cache); +void vbdev_ocf_cache_config_volume_destroy(ocf_cache_t cache); int vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache); bool vbdev_ocf_cache_is_base_attached(ocf_cache_t cache); diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c index 92bdc2e4429..839c69307a6 100644 --- a/module/bdev/ocf/vbdev_ocf_core.c +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -17,6 +17,7 @@ vbdev_ocf_core_create(struct vbdev_ocf_core **out, const char *core_name, const { struct vbdev_ocf_core *core_ctx; struct ocf_mngt_core_config *core_cfg; + int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': allocating structs\n", core_name); @@ -30,12 +31,19 @@ vbdev_ocf_core_create(struct vbdev_ocf_core **out, const char *core_name, const core_cfg = &core_ctx->core_cfg; ocf_mngt_core_config_set_default(core_cfg); + strncpy(core_cfg->name, core_name, OCF_CORE_NAME_SIZE); strncpy(core_ctx->cache_name, cache_name, OCF_CACHE_NAME_SIZE); + if ((rc = ocf_uuid_set_str(&core_cfg->uuid, core_cfg->name))) { + SPDK_ERRLOG("OCF core '%s': failed to set OCF volume uuid\n", core_name); + free(core_ctx); + return rc; + } + *out = core_ctx; - return 0; + return rc; } void @@ -104,6 +112,8 @@ vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core_ctx, const char *base_nam SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': attaching base bdev '%s'\n", vbdev_ocf_core_get_name(core_ctx), base_name); + strncpy(base->name, base_name, OCF_CORE_NAME_SIZE); + if ((rc = spdk_bdev_open_ext(base_name, true, _vbdev_ocf_core_event_cb, core_ctx, &base->desc))) { return rc; } @@ -128,16 +138,6 @@ vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core_ctx, const char *base_nam base->thread = spdk_get_thread(); base->is_cache = false; base->attached = true; - - // alloc uuid ? - if ((rc = ocf_uuid_set_str(&core_cfg->uuid, (char *)base_name))) { - SPDK_ERRLOG("OCF core '%s': failed to set OCF volume uuid\n", - vbdev_ocf_core_get_name(core_ctx)); - spdk_put_io_channel(base->mngt_ch); - spdk_bdev_close(base->desc); - return rc; - } - core_cfg->volume_type = SPDK_OBJECT; // for ocf_volume_open() in ocf_mngt_cache_add_core() core_cfg->volume_params = base; @@ -156,8 +156,6 @@ vbdev_ocf_core_base_detach(struct vbdev_ocf_core *core_ctx) SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': detaching base bdev '%s'\n", vbdev_ocf_core_get_name(core_ctx), spdk_bdev_get_name(base->bdev)); - // dealloc uuid ? - vbdev_ocf_base_detach(base); } @@ -312,7 +310,7 @@ vbdev_ocf_core_register(ocf_core_t core) spdk_io_device_register(core, _vbdev_ocf_ch_create_cb, _vbdev_ocf_ch_destroy_cb, sizeof(struct vbdev_ocf_core_io_channel_ctx), ocf_core_get_name(core)); - SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': io_device created at 0x%p\n", + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': io_device created at %p\n", spdk_bdev_get_name(ocf_vbdev), core); if ((rc = spdk_bdev_register(ocf_vbdev))) { // needs to be called from SPDK app thread diff --git a/module/bdev/ocf/vbdev_ocf_rpc.c b/module/bdev/ocf/vbdev_ocf_rpc.c index 8ca0ece632c..cdacaae0234 100644 --- a/module/bdev/ocf/vbdev_ocf_rpc.c +++ b/module/bdev/ocf/vbdev_ocf_rpc.c @@ -348,7 +348,7 @@ rpc_bdev_ocf_get_bdevs_cb(void *cb_arg1, void *cb_arg2) struct spdk_json_write_ctx *w = cb_arg1; struct spdk_jsonrpc_request *request = cb_arg2; - spdk_json_write_array_end(w); + spdk_json_write_object_end(w); spdk_jsonrpc_end_result(request, w); } @@ -368,7 +368,7 @@ rpc_bdev_ocf_get_bdevs(struct spdk_jsonrpc_request *request, const struct spdk_j } w = spdk_jsonrpc_begin_result(request); - spdk_json_write_array_begin(w); + spdk_json_write_object_begin(w); vbdev_ocf_get_bdevs(params ? req.name : NULL, rpc_bdev_ocf_get_bdevs_cb, w, request); diff --git a/module/bdev/ocf/volume.h b/module/bdev/ocf/volume.h index b122930dd21..dda02a6f8e0 100644 --- a/module/bdev/ocf/volume.h +++ b/module/bdev/ocf/volume.h @@ -8,7 +8,7 @@ #define VBDEV_OCF_VOLUME_H struct vbdev_ocf_base { - //char name[spdk_max(OCF_CACHE_NAME_SIZE, OCF_CORE_NAME_SIZE)]; + char name[spdk_max(OCF_CACHE_NAME_SIZE, OCF_CORE_NAME_SIZE)]; bool is_cache; bool attached; struct spdk_bdev * bdev; From 83cc3ae9284e305839ce3fadf8e3058187fcb27c Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Thu, 5 Jun 2025 17:10:04 +0200 Subject: [PATCH 06/34] Update OCF to 25.03 Change-Id: I6cd3262aed2b309929df68d24fac5b431ecfc006 Signed-off-by: Rafal Stefanowski --- lib/env_ocf/ocf_env_headers.h | 2 +- lib/env_ocf/ocf_env_refcnt.c | 81 +++++++++++++++++++++++++++++++++++ lib/env_ocf/ocf_env_refcnt.h | 54 +++++++++++++++++++++++ ocf | 2 +- 4 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 lib/env_ocf/ocf_env_refcnt.c create mode 100644 lib/env_ocf/ocf_env_refcnt.h diff --git a/lib/env_ocf/ocf_env_headers.h b/lib/env_ocf/ocf_env_headers.h index 403c0bdcb21..21ddc372a97 100644 --- a/lib/env_ocf/ocf_env_headers.h +++ b/lib/env_ocf/ocf_env_headers.h @@ -8,7 +8,7 @@ #include "spdk/stdinc.h" -#define OCF_VERSION_MAIN 20 +#define OCF_VERSION_MAIN 25 #define OCF_VERSION_MAJOR 3 #define OCF_VERSION_MINOR 0 diff --git a/lib/env_ocf/ocf_env_refcnt.c b/lib/env_ocf/ocf_env_refcnt.c new file mode 100644 index 00000000000..69045e6ee32 --- /dev/null +++ b/lib/env_ocf/ocf_env_refcnt.c @@ -0,0 +1,81 @@ +/* + * Copyright(c) 2019-2021 Intel Corporation + * Copyright(c) 2024 Huawei Technologies + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "ocf_env_refcnt.h" + +int env_refcnt_init(struct env_refcnt *rc, const char *name, size_t name_len) +{ + env_atomic_set(&rc->counter, 0); + env_atomic_set(&rc->freeze, 0); + env_atomic_set(&rc->callback, 0); + rc->cb = NULL; + + return 0; +} + +void env_refcnt_deinit(struct env_refcnt *rc) +{ + +} + +void env_refcnt_dec(struct env_refcnt *rc) +{ + int val = env_atomic_dec_return(&rc->counter); + ENV_BUG_ON(val < 0); + + if (!val && env_atomic_cmpxchg(&rc->callback, 1, 0)) + rc->cb(rc->priv); +} + +bool env_refcnt_inc(struct env_refcnt *rc) +{ + int val; + + if (!env_atomic_read(&rc->freeze)) { + val = env_atomic_inc_return(&rc->counter); + if (!env_atomic_read(&rc->freeze)) + return !!val; + else + env_refcnt_dec(rc); + } + + return 0; +} + + +void env_refcnt_freeze(struct env_refcnt *rc) +{ + env_atomic_inc(&rc->freeze); +} + +void env_refcnt_register_zero_cb(struct env_refcnt *rc, env_refcnt_cb_t cb, + void *priv) +{ + ENV_BUG_ON(!env_atomic_read(&rc->freeze)); + ENV_BUG_ON(env_atomic_read(&rc->callback)); + + env_atomic_inc(&rc->counter); + rc->cb = cb; + rc->priv = priv; + env_atomic_set(&rc->callback, 1); + env_refcnt_dec(rc); +} + +void env_refcnt_unfreeze(struct env_refcnt *rc) +{ + int val = env_atomic_dec_return(&rc->freeze); + ENV_BUG_ON(val < 0); +} + +bool env_refcnt_frozen(struct env_refcnt *rc) +{ + return !!env_atomic_read(&rc->freeze); +} + +bool env_refcnt_zeroed(struct env_refcnt *rc) +{ + return (env_atomic_read(&rc->counter) == 0); +} diff --git a/lib/env_ocf/ocf_env_refcnt.h b/lib/env_ocf/ocf_env_refcnt.h new file mode 100644 index 00000000000..0395ed6e47b --- /dev/null +++ b/lib/env_ocf/ocf_env_refcnt.h @@ -0,0 +1,54 @@ +/* + * Copyright(c) 2019-2021 Intel Corporation + * Copyright(c) 2024-2025 Huawei Technologies + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __OCF_ENV_REFCNT_H__ +#define __OCF_ENV_REFCNT_H__ + +#include "ocf_env.h" + +typedef void (*env_refcnt_cb_t)(void *priv); + +struct env_refcnt +{ + env_atomic counter __attribute__((aligned(64))); + env_atomic freeze; + env_atomic callback; + env_refcnt_cb_t cb; + void *priv; +}; + +/* Initialize reference counter */ +int env_refcnt_init(struct env_refcnt *rc, const char *name, size_t name_len); + +void env_refcnt_deinit(struct env_refcnt *rc); + +/* Try to increment counter. Returns true if successfull, false + * if counter is frozen */ +bool env_refcnt_inc(struct env_refcnt *rc); + +/* Decrement reference counter */ +void env_refcnt_dec(struct env_refcnt *rc); + +/* Disallow incrementing of underlying counter - attempts to increment counter + * will be failing until env_refcnt_unfreeze is calleed. + * It's ok to call freeze multiple times, in which case counter is frozen + * until all freeze calls are offset by a corresponding unfreeze.*/ +void env_refcnt_freeze(struct env_refcnt *rc); + +/* Cancel the effect of single env_refcnt_freeze call */ +void env_refcnt_unfreeze(struct env_refcnt *rc); + +bool env_refcnt_frozen(struct env_refcnt *rc); + +bool env_refcnt_zeroed(struct env_refcnt *rc); + +/* Register callback to be called when reference counter drops to 0. + * Must be called after counter is frozen. + * Cannot be called until previously regsitered callback had fired. */ +void env_refcnt_register_zero_cb(struct env_refcnt *rc, env_refcnt_cb_t cb, + void *priv); + +#endif // __OCF_ENV_REFCNT_H__ diff --git a/ocf b/ocf index 6ad1007e6fa..6a019be7f59 160000 --- a/ocf +++ b/ocf @@ -1 +1 @@ -Subproject commit 6ad1007e6fa47ff6468eaba0b304d77334c34d4a +Subproject commit 6a019be7f593c0978fa98c1b3814e6646113a276 From aef6dd15d5377c1d656f0cb1145f2ad37047f532 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Thu, 5 Jun 2025 18:18:34 +0200 Subject: [PATCH 07/34] Implement cache load Change-Id: Ie509dc0f48e76cd9c01cb7e07ef8698dfa3bab14 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 292 +++++++++++++++++++++--------- module/bdev/ocf/vbdev_ocf.h | 9 - module/bdev/ocf/vbdev_ocf_cache.c | 59 +++++- module/bdev/ocf/vbdev_ocf_cache.h | 12 ++ module/bdev/ocf/vbdev_ocf_core.c | 39 ++++ module/bdev/ocf/vbdev_ocf_core.h | 1 + module/bdev/ocf/volume.c | 6 +- 7 files changed, 314 insertions(+), 104 deletions(-) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index a088d4c14c2..1a0512bbd29 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -61,6 +61,13 @@ static int _bdev_exists_core_visit(ocf_core_t core, void *cb_arg) { const char *name = cb_arg; + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + + if (!core_ctx) { + /* Skip this core. If there is no context, it means that this core + * was added from metadata during cache load and it's just an empty shell. */ + return 0; + } if (!strcmp(name, ocf_core_get_name(core))) { return -EEXIST; @@ -218,6 +225,12 @@ _module_fini_core_visit(ocf_core_t core, void *cb_arg) SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': module stop visit\n", ocf_core_get_name(core)); + if (!core_ctx) { + /* Skip this core. If there is no context, it means that this core + * was added from metadata during cache load and it's just an empty shell. */ + return 0; + } + /* If core is detached it's already unregistered, so just free its data and exit. */ if (!vbdev_ocf_core_is_base_attached(core_ctx)) { ocf_mngt_cache_put(cache); @@ -243,17 +256,12 @@ _module_fini_cache_visit(ocf_cache_t cache, void *cb_arg) SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': module stop visit\n", ocf_cache_get_name(cache)); /* Increment cache refcount to not destroy cache structs before destroying all cores. */ - printf("--- cores count: %d\n", ocf_cache_get_core_count(cache)); - printf("--- cores inactive count: %d\n", ocf_cache_get_core_inactive_count(cache)); - printf("*** cache refcnt before get: %d\n", ocf_cache_get_refcnt(cache)); for (i = ocf_cache_get_core_count(cache); i > 0; i--) { - printf("*** INC refcnt for core %d\n", i); if ((rc = ocf_mngt_cache_get(cache))) { SPDK_ERRLOG("OCF cache '%s': failed to increment refcount: %s\n", ocf_cache_get_name(cache), spdk_strerror(-rc)); } } - printf("*** cache refcnt after get: %d\n", ocf_cache_get_refcnt(cache)); if (!ocf_cache_get_core_count(cache) || ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { @@ -262,7 +270,6 @@ _module_fini_cache_visit(ocf_cache_t cache, void *cb_arg) ocf_mngt_cache_lock(cache, _module_fini_cache_lock_cb, cb_arg); } - printf("*** cache refcnt before visit: %d\n", ocf_cache_get_refcnt(cache)); if ((rc = ocf_core_visit(cache, _module_fini_core_visit, cb_arg, false))) { SPDK_ERRLOG("OCF cache '%s': failed to iterate over core bdevs: %s\n", ocf_cache_get_name(cache), spdk_strerror(-rc)); @@ -327,6 +334,12 @@ _examine_config_core_visit(ocf_core_t core, void *cb_arg) char *bdev_name = cb_arg; int rc = 0; + if (!core_ctx) { + /* Skip this core. If there is no context, it means that this core + * was added from metadata during cache load and it's just an empty shell. */ + return 0; + } + if (strcmp(bdev_name, core_ctx->base.name)) { return 0; } @@ -342,6 +355,10 @@ _examine_config_core_visit(ocf_core_t core, void *cb_arg) return rc; } + /* This whole situation with core being present in cache without base bdev attached + * is only possible when core was previously hot removed from SPDK. + * In such case it was detached from cache (not removed), so set 'try_add' in core + * config to 'true' to indicate that this core is still in cache metadata. */ core_ctx->core_cfg.try_add = true; return -EEXIST; @@ -496,9 +513,73 @@ _core_add_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _core_add_examine_add_cb, core_ctx); } +static void +_cache_attach_examine_core_add_err_cb(void *cb_arg, int error) +{ + ocf_cache_t cache = cb_arg; + + if (error) { + SPDK_ERRLOG("OCF core: failed to remove OCF core device (OCF error: %d)\n", error); + } + + ocf_mngt_cache_unlock(cache); +} + +static void +_cache_attach_examine_core_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int error) +{ + struct vbdev_ocf_core *core_ctx = cb_arg; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", + vbdev_ocf_core_get_name(core_ctx)); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s' (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache), error); + ocf_mngt_cache_unlock(cache); + return; + } + + ocf_core_set_priv(core, core_ctx); + + if ((rc = vbdev_ocf_core_register(core))) { + SPDK_ERRLOG("OCF core '%s': failed to register vbdev: %s\n", + ocf_core_get_name(core), spdk_strerror(-rc)); + ocf_mngt_cache_remove_core(core, _cache_attach_examine_core_add_err_cb, cache); + return; + } + + SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n", + ocf_core_get_name(core), ocf_cache_get_name(cache)); + + ocf_mngt_cache_unlock(cache); + STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); +} + +static void +_cache_attach_examine_core_add_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_core *core_ctx = cb_arg; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add of OCF core\n", + vbdev_ocf_core_get_name(core_ctx)); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), error); + return; + } + + ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _cache_attach_examine_core_add_cb, core_ctx); +} + static void _cache_attach_examine_attach_cb(ocf_cache_t cache, void *cb_arg, int error) { + struct vbdev_ocf_mngt_ctx *examine_attach_ctx = cb_arg; + struct vbdev_ocf_core *core_ctx; + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing device attach\n", ocf_cache_get_name(cache)); @@ -515,13 +596,31 @@ _cache_attach_examine_attach_cb(ocf_cache_t cache, void *cb_arg, int error) } ocf_mngt_cache_unlock(cache); - spdk_bdev_module_examine_done(&ocf_if); + free(examine_attach_ctx); + spdk_bdev_module_examine_done(&ocf_if); // move after adding cores from waitlist ? + + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + if (strcmp(ocf_cache_get_name(cache), core_ctx->cache_name) || + !vbdev_ocf_core_is_base_attached(core_ctx)) { + continue; + } + + SPDK_NOTICELOG("OCF core '%s': adding from waitlist to cache '%s'\n", + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); + + // check if (cache_block_size > core_block_size) + + core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx)); + + ocf_mngt_cache_lock(cache, _cache_attach_examine_core_add_lock_cb, core_ctx); + } } static void _cache_attach_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + struct vbdev_ocf_mngt_ctx *examine_attach_ctx; + int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': attaching OCF cache device\n", ocf_cache_get_name(cache)); @@ -529,13 +628,34 @@ _cache_attach_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (error) { SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", ocf_cache_get_name(cache), error); - vbdev_ocf_cache_config_volume_destroy(cache); - vbdev_ocf_cache_base_detach(cache); - spdk_bdev_module_examine_done(&ocf_if); - return; + goto err_lock; } - ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, _cache_attach_examine_attach_cb, NULL); + examine_attach_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!examine_attach_ctx) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for examine attach context: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-ENOMEM)); + goto err_alloc; + } + examine_attach_ctx->cache = cache; + examine_attach_ctx->att_cb_fn = _cache_attach_examine_attach_cb; + + if ((rc = vbdev_ocf_cache_volume_attach(cache, examine_attach_ctx))) { + SPDK_ERRLOG("OCF cache '%s': failed to attach volume (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_attach; + } + + return; + +err_attach: + free(examine_attach_ctx); +err_alloc: + ocf_mngt_cache_unlock(cache); +err_lock: + vbdev_ocf_cache_config_volume_destroy(cache); + vbdev_ocf_cache_base_detach(cache); + spdk_bdev_module_examine_done(&ocf_if); } static int @@ -545,10 +665,17 @@ _examine_disk_core_visit(ocf_core_t core, void *cb_arg) struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); char *bdev_name = cb_arg; + if (!core_ctx) { + /* Skip this core. If there is no context, it means that this core + * was added from metadata during cache load and it's just an empty shell. */ + return 0; + } + if (strcmp(bdev_name, core_ctx->base.name)) { return 0; } + /* Get cache once to be in sync with adding core from waitlist scenario. */ ocf_mngt_cache_get(cache); ocf_mngt_cache_lock(cache, _core_add_examine_lock_cb, core_ctx); @@ -593,6 +720,17 @@ vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev) return; } + if (!ocf_cache_is_device_attached(cache)) { + SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache device '%s'\n", + vbdev_ocf_core_get_name(core_ctx), + ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->base.name); + ocf_mngt_cache_put(cache); // ? + spdk_bdev_module_examine_done(&ocf_if); + return; + } + + core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx)); + ocf_mngt_cache_lock(cache, _core_add_examine_lock_cb, core_ctx); return; } @@ -602,7 +740,7 @@ vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev) SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); } else if (!rc) { /* No visitor matched this new bdev, so no one called _examine_done(). */ - spdk_bdev_module_examine_done(&ocf_if); // ??? + spdk_bdev_module_examine_done(&ocf_if); } } @@ -856,8 +994,8 @@ _cache_start_rpc_core_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, i vbdev_ocf_core_get_name(core_ctx)); if (error) { - SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s'\n", - vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); + SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s' (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache), error); ocf_mngt_cache_unlock(cache); return; } @@ -865,7 +1003,8 @@ _cache_start_rpc_core_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, i ocf_core_set_priv(core, core_ctx); if ((rc = vbdev_ocf_core_register(core))) { - SPDK_ERRLOG("OCF core '%s': failed to register vbdev\n", ocf_core_get_name(core)); + SPDK_ERRLOG("OCF core '%s': failed to register vbdev: %s\n", + ocf_core_get_name(core), spdk_strerror(-rc)); ocf_mngt_cache_remove_core(core, _cache_start_rpc_core_add_err_cb, cache); return; } @@ -886,8 +1025,8 @@ _cache_start_rpc_core_add_lock_cb(ocf_cache_t cache, void *cb_arg, int error) vbdev_ocf_core_get_name(core_ctx)); if (error) { - SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock\n", - vbdev_ocf_core_get_name(core_ctx)); + SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), error); return; } @@ -961,37 +1100,9 @@ _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) // check if (cache_block_size > core_block_size) - ocf_mngt_cache_lock(cache, _cache_start_rpc_core_add_lock_cb, core_ctx); - } -} - -static void -_cache_start_rpc_metadata_probe_cb(void *priv, int error, struct ocf_metadata_probe_status *status) -{ - struct vbdev_ocf_mngt_ctx *cache_start_ctx = priv; - ocf_cache_t cache = cache_start_ctx->cache; - struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); - - ocf_volume_close(cache_ctx->cache_att_cfg.device.volume); + core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx)); - if (error && error != -OCF_ERR_NO_METADATA) { - SPDK_ERRLOG("OCF cache '%s': failed to probe metadata\n", ocf_cache_get_name(cache)); - _cache_start_rpc_attach_cb(cache, cache_start_ctx, error); - } - - if (error == -OCF_ERR_NO_METADATA) { - SPDK_NOTICELOG("OCF cache '%s': metadata not found - starting new cache instance\n", - ocf_cache_get_name(cache)); - //cache_ctx->cache_att_cfg.force = true; // needed for later ? - ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, - _cache_start_rpc_attach_cb, cache_start_ctx); - } else { - SPDK_NOTICELOG("OCF cache '%s': metadata found - loading previous cache instance\n", - ocf_cache_get_name(cache)); - SPDK_NOTICELOG("(start cache with '--no-load' flag to create new cache instead of loading config from metadata)\n"); - // check status for cache_name/mode/line_size/dirty ? - ocf_mngt_cache_load(cache, &cache_ctx->cache_att_cfg, - _cache_start_rpc_attach_cb, cache_start_ctx); + ocf_mngt_cache_lock(cache, _cache_start_rpc_core_add_lock_cb, core_ctx); } } @@ -1002,7 +1113,6 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { ocf_cache_t cache; - struct vbdev_ocf_cache *cache_ctx; struct vbdev_ocf_mngt_ctx *cache_start_ctx; int rc = 0; @@ -1050,29 +1160,18 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, goto err_alloc; } cache_start_ctx->cache = cache; + cache_start_ctx->att_cb_fn = _cache_start_rpc_attach_cb; cache_start_ctx->rpc_cb_fn = rpc_cb_fn; cache_start_ctx->rpc_cb_arg = rpc_cb_arg; - cache_ctx = ocf_cache_get_priv(cache); - - if (no_load) { - SPDK_NOTICELOG("'--no-load' flag specified - starting new cache without looking for metadata\n"); - ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, _cache_start_rpc_attach_cb, - cache_start_ctx); - return; - } - - if ((rc = ocf_volume_open(cache_ctx->cache_att_cfg.device.volume, &cache_ctx->base))) { - SPDK_ERRLOG("OCF cache '%s': failed to open volume\n", ocf_cache_get_name(cache)); - goto err_open; + if ((rc = vbdev_ocf_cache_volume_attach(cache, cache_start_ctx))) { + SPDK_ERRLOG("OCF cache '%s': failed to attach volume\n", cache_name); + goto err_attach; } - ocf_metadata_probe(vbdev_ocf_ctx, cache_ctx->cache_att_cfg.device.volume, - _cache_start_rpc_metadata_probe_cb, cache_start_ctx); - return; -err_open: +err_attach: free(cache_start_ctx); err_alloc: vbdev_ocf_cache_config_volume_destroy(cache); @@ -1182,6 +1281,12 @@ _cache_stop_rpc_core_visit(ocf_core_t core, void *cb_arg) SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': cache stop visit\n", ocf_core_get_name(core)); + if (!core_ctx) { + /* Skip this core. If there is no context, it means that this core + * was added from metadata during cache load and it's just an empty shell. */ + return 0; + } + /* If core is detached it's already unregistered, so just free its data and exit. */ if (!vbdev_ocf_core_is_base_attached(core_ctx)) { ocf_mngt_cache_put(cache); @@ -1234,18 +1339,13 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo cache_stop_ctx->rpc_cb_arg = rpc_cb_arg; /* Increment cache refcount to not destroy cache structs before destroying all cores. */ - printf("--- cores count: %d\n", ocf_cache_get_core_count(cache)); - printf("--- cores inactive count: %d\n", ocf_cache_get_core_inactive_count(cache)); - printf("*** cache refcnt before get: %d\n", ocf_cache_get_refcnt(cache)); for (i = ocf_cache_get_core_count(cache); i > 0; i--) { - printf("*** INC refcnt for core %d\n", i); if ((rc = ocf_mngt_cache_get(cache))) { SPDK_ERRLOG("OCF cache '%s': failed to increment refcount\n", ocf_cache_get_name(cache)); goto err_get; } } - printf("*** cache refcnt after get: %d\n", ocf_cache_get_refcnt(cache)); if (!ocf_cache_get_core_count(cache) || ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { @@ -1254,7 +1354,6 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo ocf_mngt_cache_lock(cache, _cache_stop_rpc_lock_cb, cache_stop_ctx); } - printf("*** cache refcnt before visit: %d\n", ocf_cache_get_refcnt(cache)); if ((rc = ocf_core_visit(cache, _cache_stop_rpc_core_visit, cache_stop_ctx, false))) { SPDK_ERRLOG("OCF cache '%s': failed to iterate over core bdevs\n", ocf_cache_get_name(cache)); @@ -1444,7 +1543,7 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating device attach\n", cache_name); - // ocf_mngt_cache_put() right away to merge cache_start/attach callbacks + // ocf_mngt_cache_put() right away to merge cache_start/attach callbacks ? if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; @@ -1621,7 +1720,7 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac goto err_base; } - /* Second, check if OCF cache for this core is already present and started. */ + /* Second, check if OCF cache for this core is already started. */ if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { /* If not, just put core context on the temporary core wait list and exit. */ SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache '%s'\n", @@ -1631,6 +1730,25 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac return; } + /* And finally, check if OCF cache device is already attached. + * We need to have cache device attached to know if cache was loaded or attached + * and then set 'try_add' in core config accordingly. */ + if (!ocf_cache_is_device_attached(cache)) { + SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache device '%s'\n", + core_name, ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->base.name); + //ocf_mngt_cache_put(cache); // ? + STAILQ_INSERT_TAIL(&g_vbdev_ocf_core_waitlist, core_ctx, waitlist_entry); + rpc_cb_fn(core_name, rpc_cb_arg, -ENODEV); + return; + } + + // DONE + // find a better way to check if cache was loaded or attached + //core_ctx->core_cfg.try_add = cache_ctx->cache_att_cfg.force ? false : true; + core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(core_name); + + // open (and then close) cache-load-added core volumes (with 'base' to save it in priv) ? + // check if (cache_block_size > core_block_size) core_add_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); @@ -1763,6 +1881,13 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, } core_ctx = ocf_core_get_priv(core); + if (!core_ctx) { + /* Skip this core. If there is no context, it means that this core + * was added from metadata during cache load and it's just an empty shell. */ + SPDK_ERRLOG("OCF core '%s': not exist within cache '%s'\n", core_name, cache_name); + rc = -ENXIO; + goto err_ctx; + } core_rm_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!core_rm_ctx) { @@ -1777,7 +1902,8 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, core_rm_ctx->rpc_cb_arg = rpc_cb_arg; if (!vbdev_ocf_core_is_base_attached(core_ctx)) { - // DEBUGLOG for all unregister skipping + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': removing detached (no unregister)\n", core_name); + ocf_mngt_cache_lock(cache, _core_remove_rpc_lock_cb, core_rm_ctx); return; } @@ -1793,6 +1919,7 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, err_unregister: free(core_rm_ctx); err_alloc: +err_ctx: err_core: ocf_mngt_cache_put(cache); err_cache: @@ -1800,13 +1927,6 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, } // handle errors ? -static void -_get_bdevs_core_dump(struct vbdev_ocf_core *core_ctx, struct spdk_json_write_ctx *w) -{ - spdk_json_write_named_string(w, "base_name", core_ctx->base.name); - spdk_json_write_named_bool(w, "base_attached", vbdev_ocf_core_is_base_attached(core_ctx)); -} - static int _get_bdevs_core_visit(ocf_core_t core, void *cb_arg) { @@ -1815,7 +1935,10 @@ _get_bdevs_core_visit(ocf_core_t core, void *cb_arg) spdk_json_write_object_begin(w); spdk_json_write_named_string(w, "name", ocf_core_get_name(core)); - _get_bdevs_core_dump(core_ctx, w); + spdk_json_write_named_string(w, "base_name", core_ctx ? core_ctx->base.name : ""); + spdk_json_write_named_bool(w, "base_attached", + core_ctx ? vbdev_ocf_core_is_base_attached(core_ctx) : false); + spdk_json_write_named_bool(w, "loading", !core_ctx); spdk_json_write_object_end(w); return 0; @@ -1858,7 +1981,8 @@ vbdev_ocf_get_bdevs(const char *name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rp spdk_json_write_object_begin(w); spdk_json_write_named_string(w, "name", vbdev_ocf_core_get_name(core_ctx)); spdk_json_write_named_string(w, "cache_name", core_ctx->cache_name); - _get_bdevs_core_dump(core_ctx, w); + spdk_json_write_named_string(w, "base_name", core_ctx->base.name); + spdk_json_write_named_bool(w, "base_attached", vbdev_ocf_core_is_base_attached(core_ctx)); spdk_json_write_object_end(w); } spdk_json_write_array_end(w); diff --git a/module/bdev/ocf/vbdev_ocf.h b/module/bdev/ocf/vbdev_ocf.h index e11bfa9f513..e1004d56a19 100644 --- a/module/bdev/ocf/vbdev_ocf.h +++ b/module/bdev/ocf/vbdev_ocf.h @@ -12,15 +12,6 @@ typedef void (*vbdev_ocf_rpc_mngt_cb)(const char *bdev_name, void *cb_arg, int error); typedef void (*vbdev_ocf_get_bdevs_cb)(void *cb_arg1, void *cb_arg2); -struct vbdev_ocf_mngt_ctx { - ocf_cache_t cache; - ocf_core_t core; - struct vbdev_ocf_core * core_ctx; - ocf_mngt_cache_attach_end_t attach_cb_fn; - vbdev_ocf_rpc_mngt_cb rpc_cb_fn; - void * rpc_cb_arg; -}; - /* RPC entry points. */ void vbdev_ocf_cache_start(const char *cache_name, diff --git a/module/bdev/ocf/vbdev_ocf_cache.c b/module/bdev/ocf/vbdev_ocf_cache.c index 00e11037ca3..19ae95546f1 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.c +++ b/module/bdev/ocf/vbdev_ocf_cache.c @@ -47,15 +47,6 @@ vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cac return rc; } - // needed ? - //if ((rc = ocf_mngt_cache_get(cache))) { - // SPDK_ERRLOG("OCF cache '%s': failed to increment cache ref count: %s\n", - // cache_name, spdk_strerror(-rc)); - // ocf_mngt_cache_stop(cache, NULL, NULL); // needs callback (_cache_start_err_cb ?) - // free(cache_ctx); - // return rc; - //} - *out = cache; return rc; @@ -235,6 +226,56 @@ vbdev_ocf_cache_config_volume_destroy(ocf_cache_t cache) ocf_volume_destroy(cache_ctx->cache_att_cfg.device.volume); } +static void +_volume_attach_metadata_probe_cb(void *priv, int error, struct ocf_metadata_probe_status *status) +{ + struct vbdev_ocf_mngt_ctx *ctx = priv; + ocf_cache_t cache = ctx->cache; + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + + ocf_volume_close(cache_ctx->cache_att_cfg.device.volume); + + if (error && error != -OCF_ERR_NO_METADATA) { + SPDK_ERRLOG("OCF cache '%s': failed to probe metadata\n", ocf_cache_get_name(cache)); + ctx->att_cb_fn(cache, ctx, error); + } + + if (error == -OCF_ERR_NO_METADATA) { + SPDK_NOTICELOG("OCF cache '%s': metadata not found - starting new cache instance\n", + ocf_cache_get_name(cache)); + ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, ctx->att_cb_fn, ctx); + } else { + SPDK_NOTICELOG("OCF cache '%s': metadata found - loading previous cache instance\n", + ocf_cache_get_name(cache)); + SPDK_NOTICELOG("(start cache with 'no-load' flag to create new cache instead of loading config from metadata)\n"); + // check status for cache_name/mode/line_size/dirty ? + ocf_mngt_cache_load(cache, &cache_ctx->cache_att_cfg, ctx->att_cb_fn, ctx); + } +} + +int +vbdev_ocf_cache_volume_attach(ocf_cache_t cache, struct vbdev_ocf_mngt_ctx *ctx) +{ + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + int rc; + + if (cache_ctx->no_load) { + SPDK_NOTICELOG("'no-load' flag specified - starting new cache without looking for metadata\n"); + ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, ctx->att_cb_fn, ctx); + return 0; + } + + if ((rc = ocf_volume_open(cache_ctx->cache_att_cfg.device.volume, &cache_ctx->base))) { + SPDK_ERRLOG("OCF cache '%s': failed to open volume\n", ocf_cache_get_name(cache)); + return rc; + } + + ocf_metadata_probe(vbdev_ocf_ctx, cache_ctx->cache_att_cfg.device.volume, + _volume_attach_metadata_probe_cb, ctx); + + return 0; +} + static void _cache_mngt_queue_stop(void *ctx) { diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h index e1e5c5deea8..49c96025ad0 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.h +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -22,6 +22,17 @@ struct vbdev_ocf_cache { struct vbdev_ocf_base base; }; +typedef void (*vbdev_ocf_rpc_mngt_cb)(const char *bdev_name, void *cb_arg, int error); + +struct vbdev_ocf_mngt_ctx { + ocf_cache_t cache; + ocf_core_t core; + struct vbdev_ocf_core * core_ctx; + ocf_mngt_cache_attach_end_t att_cb_fn; + vbdev_ocf_rpc_mngt_cb rpc_cb_fn; + void * rpc_cb_arg; +}; + struct vbdev_ocf_cache_mngt_queue_ctx { struct spdk_poller * poller; struct spdk_thread * thread; @@ -36,6 +47,7 @@ int vbdev_ocf_cache_base_attach(ocf_cache_t cache, const char *base_name); void vbdev_ocf_cache_base_detach(ocf_cache_t cache); int vbdev_ocf_cache_config_volume_create(ocf_cache_t cache); void vbdev_ocf_cache_config_volume_destroy(ocf_cache_t cache); +int vbdev_ocf_cache_volume_attach(ocf_cache_t cache, struct vbdev_ocf_mngt_ctx *ctx); int vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache); bool vbdev_ocf_cache_is_base_attached(ocf_cache_t cache); diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c index 839c69307a6..791da74cc17 100644 --- a/module/bdev/ocf/vbdev_ocf_core.c +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -359,3 +359,42 @@ vbdev_ocf_core_is_base_attached(struct vbdev_ocf_core *core_ctx) { return core_ctx->base.attached; } + +static int +_core_is_loaded_core_visit(ocf_core_t core, void *cb_arg) +{ + const char *core_name = cb_arg; + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + + if (!strcmp(core_name, ocf_core_get_name(core)) && !core_ctx) { + /* Core context is assigned only after manual core add (either right away + * if all devices are present, after corresponding cache start, or base bdev + * appearance). + * If there is no context, it means that this core was added from metadata + * during cache load. */ + return -EEXIST; + } + + return 0; +} + +static int +_core_is_loaded_cache_visit(ocf_cache_t cache, void *cb_arg) +{ + return ocf_core_visit(cache, _core_is_loaded_core_visit, cb_arg, false); +} + +bool +vbdev_ocf_core_is_loaded(const char *core_name) +{ + int rc; + + rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _core_is_loaded_cache_visit, (char *)core_name); + if (rc == -EEXIST) { + return true; + } else if (rc) { + SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + } + + return false; +} diff --git a/module/bdev/ocf/vbdev_ocf_core.h b/module/bdev/ocf/vbdev_ocf_core.h index 952829f9543..ddbecc00013 100644 --- a/module/bdev/ocf/vbdev_ocf_core.h +++ b/module/bdev/ocf/vbdev_ocf_core.h @@ -56,5 +56,6 @@ int vbdev_ocf_core_unregister(struct vbdev_ocf_core *core_ctx, spdk_bdev_unregis char *vbdev_ocf_core_get_name(struct vbdev_ocf_core *core_ctx); struct vbdev_ocf_core *vbdev_ocf_core_waitlist_get_by_name(const char *core_name); bool vbdev_ocf_core_is_base_attached(struct vbdev_ocf_core *core_ctx); +bool vbdev_ocf_core_is_loaded(const char *core_name); #endif diff --git a/module/bdev/ocf/volume.c b/module/bdev/ocf/volume.c index 075131d73f0..e09f824fdfd 100644 --- a/module/bdev/ocf/volume.c +++ b/module/bdev/ocf/volume.c @@ -23,9 +23,11 @@ vbdev_ocf_volume_open(ocf_volume_t volume, void *opts) // refactor (like ocf_core_volume in ocf_core.c ?) - assert(opts); + if (opts) { + *priv = opts; + } - *priv = opts; + assert(*priv); return 0; } From a9039b9dabfb9104d8a57bb5813883124cac3c37 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Tue, 10 Jun 2025 10:40:51 +0200 Subject: [PATCH 08/34] Management tests Change-Id: I91828704921fb74ff538ba37e6f6a5a8cbd218bf Signed-off-by: Rafal Stefanowski --- test/ocf/common.sh | 165 +++ test/ocf/management/detach-attach.sh | 369 +++++ test/ocf/management/hotremove.sh | 1347 ++++++++++++++++++ test/ocf/management/run.sh | 16 + test/ocf/management/start-stop/basic.sh | 181 +++ test/ocf/management/start-stop/examine.sh | 361 +++++ test/ocf/management/start-stop/incomplete.sh | 172 +++ test/ocf/management/start-stop/load.sh | 531 +++++++ test/ocf/management/start-stop/run.sh | 17 + test/ocf/ocf.sh | 17 +- 10 files changed, 3166 insertions(+), 10 deletions(-) create mode 100755 test/ocf/management/detach-attach.sh create mode 100755 test/ocf/management/hotremove.sh create mode 100755 test/ocf/management/run.sh create mode 100755 test/ocf/management/start-stop/basic.sh create mode 100755 test/ocf/management/start-stop/examine.sh create mode 100755 test/ocf/management/start-stop/incomplete.sh create mode 100755 test/ocf/management/start-stop/load.sh create mode 100755 test/ocf/management/start-stop/run.sh diff --git a/test/ocf/common.sh b/test/ocf/common.sh index e1196345e78..28a9f2b2a4f 100644 --- a/test/ocf/common.sh +++ b/test/ocf/common.sh @@ -7,6 +7,7 @@ source "$rootdir/test/common/autotest_common.sh" rpc_py="$rootdir/scripts/rpc.py" +cache_dev_persistent_addr=(05:00.0 06:00.0 08:00.0) start_spdk() { $SPDK_BIN_DIR/spdk_tgt "$@" & @@ -20,6 +21,170 @@ stop_spdk() { killprocess $spdk_pid } +create_caches() { + for i in {1..3}; do + $rpc_py bdev_malloc_create -b Cache_dev$i 100 512 + done + + # Give it some time to settle before checking, as there might be + # some cores in waitlist that needs to be moved to started caches. + sleep 3 + + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +destroy_caches() { + for i in {1..3}; do + $rpc_py bdev_malloc_delete Cache_dev$i + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +create_caches_persistent() { + for i in {1..3}; do + $rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[i-1]}" -d blk Cache_dev$i + done + + # Give a bit more time to settle, because this will be called after + # caches load with attached cores in waitlist and it may take more time + # to move all of them to loaded caches. + sleep 15 + + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +destroy_caches_persistent() { + for i in {1..3}; do + $rpc_py bdev_virtio_detach_controller Cache_dev$i + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +create_cores() { + for i in {1..3}; do + for j in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-$j 200 512 + done + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +destroy_cores() { + for i in {1..3}; do + for j in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-$j + done + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +start_caches() { + for i in {1..3}; do + $rpc_py bdev_ocf_start_cache Ocf_cache$i Cache_dev$i --no-load + done + # executed before cores from waitlist are added + sleep 3 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +start_caches_try_load() { + for i in {1..3}; do + $rpc_py bdev_ocf_start_cache Ocf_cache$i Cache_dev$i + done + # executed before cores from waitlist are added + sleep 3 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +stop_caches() { + for i in {1..3}; do + $rpc_py bdev_ocf_stop_cache Ocf_cache$i + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +detach_caches() { + for i in {1..3}; do + $rpc_py bdev_ocf_detach_cache Ocf_cache$i + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +attach_caches() { + for i in {1..3}; do + $rpc_py bdev_ocf_attach_cache Ocf_cache$i Cache_dev$i + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +add_cores() { + for i in {1..3}; do + for j in {1..3}; do + $rpc_py bdev_ocf_add_core Ocf_core$i-$j Core_dev$i-$j Ocf_cache$i + done + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +remove_cores() { + for i in {1..3}; do + for j in {1..3}; do + $rpc_py bdev_ocf_remove_core Ocf_core$i-$j Ocf_cache$i + done + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +__check_caches_empty() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 0' +} + +__check_caches_attached() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].base_attached' +} + +__check_caches_detached() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].base_attached | not' +} + +__check_cores_empty() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 0' +} + +__check_cores_attached() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].loading | not' +} + +__check_cores_detached() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].loading | not' +} + +__check_cores_loading() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].loading' +} + +__check_cores_waitlist_empty() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 0' +} + +__check_cores_waitlist_attached() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 9' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[].base_attached' +} + +__check_cores_waitlist_detached() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 9' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[].base_attached | not' +} + # Convert an array of items to comma separated list of items. array_to_comma_list() { # $1: name of array variable diff --git a/test/ocf/management/detach-attach.sh b/test/ocf/management/detach-attach.sh new file mode 100755 index 00000000000..79e9f1f4ea3 --- /dev/null +++ b/test/ocf/management/detach-attach.sh @@ -0,0 +1,369 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f $curdir/../../..) +source "$rootdir/test/ocf/common.sh" + +# caches only: + +start_spdk +create_caches +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +attach_caches +__check_caches_attached +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +attach_caches +__check_caches_attached +stop_caches +__check_caches_empty +stop_spdk + +# add cores before detaching caches: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +detach_caches +__check_caches_detached +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +detach_caches +__check_caches_detached +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +detach_caches +__check_caches_detached +__check_cores_attached +attach_caches +__check_caches_attached +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +detach_caches +__check_caches_detached +__check_cores_attached +attach_caches +__check_caches_attached +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +# add cores after detaching caches: + +start_spdk +create_caches +create_cores +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +add_cores +__check_caches_detached +__check_cores_waitlist_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +add_cores +__check_caches_detached +__check_cores_waitlist_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +add_cores +__check_caches_detached +__check_cores_waitlist_attached +attach_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +add_cores +__check_caches_detached +__check_cores_waitlist_attached +attach_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +# add cores before and remove after detaching caches: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +detach_caches +__check_caches_detached +__check_cores_attached +remove_cores +__check_caches_detached +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +detach_caches +__check_caches_detached +__check_cores_attached +remove_cores +__check_caches_detached +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +detach_caches +__check_caches_detached +__check_cores_attached +remove_cores +__check_caches_detached +__check_cores_empty +attach_caches +__check_caches_attached +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +detach_caches +__check_caches_detached +__check_cores_attached +remove_cores +__check_caches_detached +__check_cores_empty +attach_caches +__check_caches_attached +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +# add cores after detaching caches and remove (before/after/without) attaching: + +start_spdk +create_caches +create_cores +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_attached +remove_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_attached +remove_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +add_cores +__check_caches_detached +__check_cores_waitlist_attached +remove_cores +__check_caches_detached +__check_cores_waitlist_empty +attach_caches +__check_caches_attached +__check_cores_empty +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +add_cores +__check_caches_detached +__check_cores_waitlist_attached +remove_cores +__check_caches_detached +__check_cores_waitlist_empty +attach_caches +__check_caches_attached +__check_cores_empty +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_attached +attach_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +remove_cores +__check_caches_attached +__check_cores_empty +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +__check_caches_attached +detach_caches +__check_caches_detached +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_attached +attach_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +remove_cores +__check_caches_attached +__check_cores_empty +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk diff --git a/test/ocf/management/hotremove.sh b/test/ocf/management/hotremove.sh new file mode 100755 index 00000000000..7b209a1ddc6 --- /dev/null +++ b/test/ocf/management/hotremove.sh @@ -0,0 +1,1347 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f $curdir/../../..) +source "$rootdir/test/ocf/common.sh" + +__check_caches_detached_only_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' +} + +__check_caches_detached_all_but_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached | not' +} + +__check_cores_detached_only_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[0].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[0].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[1].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[1].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[2].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[2].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[0].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[0].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[1].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[1].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[2].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[2].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[0].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[0].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[1].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[1].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[2].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[2].loading | not' +} + +__check_cores_detached_all_but_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[0].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[0].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[1].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[1].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[2].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[2].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[0].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[0].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[1].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[1].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[2].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[2].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[0].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[0].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[1].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[1].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[2].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[2].loading | not' +} + +__check_cores_waitlist_detached_only_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 9' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[0].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[1].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[2].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[3].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[4].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[5].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[6].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[7].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[8].base_attached' +} + +__check_cores_waitlist_detached_all_but_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 9' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[0].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[1].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[2].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[3].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[4].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[5].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[6].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[7].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[8].base_attached | not' +} + +# cores in waitlist: + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +destroy_cores +__check_cores_waitlist_detached +stop_spdk + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +destroy_cores +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +stop_spdk + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +destroy_cores +__check_cores_waitlist_detached +remove_cores +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +destroy_cores +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +remove_cores +__check_cores_waitlist_empty +stop_spdk + +# cores in waitlist (destroying only first): + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_detached_only_first +stop_spdk + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_detached_only_first +for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_attached +stop_spdk + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_detached_only_first +remove_cores +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_detached_only_first +for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_attached +remove_cores +__check_cores_waitlist_empty +stop_spdk + +# cores in waitlist (destroying all but first): + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_detached_all_but_first +stop_spdk + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_detached_all_but_first +for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 + $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_attached +stop_spdk + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_detached_all_but_first +remove_cores +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_detached_all_but_first +for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 + $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_cores_waitlist_attached +remove_cores +__check_cores_waitlist_empty +stop_spdk + +# caches only: + +start_spdk +create_caches +start_caches +__check_caches_attached +destroy_caches +__check_caches_detached +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +destroy_caches +__check_caches_detached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +destroy_caches +__check_caches_detached +create_caches +__check_caches_attached +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +destroy_caches +__check_caches_detached +create_caches +__check_caches_attached +stop_caches +__check_caches_empty +stop_spdk + +# caches only (destroying only first): + +start_spdk +create_caches +start_caches +__check_caches_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +$rpc_py bdev_malloc_create -b Cache_dev1 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +$rpc_py bdev_malloc_create -b Cache_dev1 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +stop_caches +__check_caches_empty +stop_spdk + +# caches only (destroying all but first): + +start_spdk +create_caches +start_caches +__check_caches_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +$rpc_py bdev_malloc_create -b Cache_dev2 100 512 +$rpc_py bdev_malloc_create -b Cache_dev3 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +stop_spdk + +start_spdk +create_caches +start_caches +__check_caches_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +$rpc_py bdev_malloc_create -b Cache_dev2 100 512 +$rpc_py bdev_malloc_create -b Cache_dev3 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +stop_caches +__check_caches_empty +stop_spdk + +# hotremove caches: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +__check_cores_attached +create_caches +__check_caches_attached +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +__check_cores_attached +create_caches +__check_caches_attached +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +__check_cores_attached +remove_cores +__check_caches_detached +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +__check_cores_attached +remove_cores +__check_caches_detached +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +__check_cores_attached +remove_cores +__check_caches_detached +__check_cores_empty +create_caches +__check_caches_attached +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +__check_cores_attached +remove_cores +__check_caches_detached +__check_cores_empty +create_caches +__check_caches_attached +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +# hotremove caches (destroying only first): + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +__check_cores_attached +$rpc_py bdev_malloc_create -b Cache_dev1 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +__check_cores_attached +$rpc_py bdev_malloc_create -b Cache_dev1 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +__check_cores_attached +remove_cores +__check_caches_detached_only_first +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +__check_cores_attached +remove_cores +__check_caches_detached_only_first +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +__check_cores_attached +remove_cores +__check_caches_detached_only_first +__check_cores_empty +$rpc_py bdev_malloc_create -b Cache_dev1 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev1 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_only_first +__check_cores_attached +remove_cores +__check_caches_detached_only_first +__check_cores_empty +$rpc_py bdev_malloc_create -b Cache_dev1 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +# hotremove caches (destroying all but first): + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +__check_cores_attached +$rpc_py bdev_malloc_create -b Cache_dev2 100 512 +$rpc_py bdev_malloc_create -b Cache_dev3 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +__check_cores_attached +$rpc_py bdev_malloc_create -b Cache_dev2 100 512 +$rpc_py bdev_malloc_create -b Cache_dev3 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +__check_cores_attached +remove_cores +__check_caches_detached_all_but_first +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +__check_cores_attached +remove_cores +__check_caches_detached_all_but_first +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +__check_cores_attached +remove_cores +__check_caches_detached_all_but_first +__check_cores_empty +$rpc_py bdev_malloc_create -b Cache_dev2 100 512 +$rpc_py bdev_malloc_create -b Cache_dev3 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +$rpc_py bdev_malloc_delete Cache_dev2 +$rpc_py bdev_malloc_delete Cache_dev3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_detached_all_but_first +__check_cores_attached +remove_cores +__check_caches_detached_all_but_first +__check_cores_empty +$rpc_py bdev_malloc_create -b Cache_dev2 100 512 +$rpc_py bdev_malloc_create -b Cache_dev3 100 512 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +# hotremove cores: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_cores +__check_caches_attached +__check_cores_detached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_cores +__check_caches_attached +__check_cores_detached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_cores +__check_caches_attached +__check_cores_detached +create_cores +__check_caches_attached +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_cores +__check_caches_attached +__check_cores_detached +create_cores +__check_caches_attached +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_cores +__check_caches_attached +__check_cores_detached +remove_cores +__check_caches_attached +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_cores +__check_caches_attached +__check_cores_detached +remove_cores +__check_caches_attached +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +# hotremove cores (destroying only first): + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_only_first +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_only_first +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_only_first +for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_only_first +for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_only_first +remove_cores +__check_caches_attached +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_only_first +remove_cores +__check_caches_attached +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +# hotremove cores (destroying all but first): + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_all_but_first +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_all_but_first +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_all_but_first +for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 + $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_all_but_first +for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 + $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_all_but_first +remove_cores +__check_caches_attached +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +__check_caches_attached +__check_cores_detached_all_but_first +remove_cores +__check_caches_attached +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +# hotremove caches and cores: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +destroy_cores +__check_cores_detached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +destroy_cores +__check_cores_detached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +destroy_cores +__check_cores_detached +create_caches +__check_caches_attached +create_cores +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +destroy_cores +__check_cores_detached +create_caches +__check_caches_attached +create_cores +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +destroy_cores +__check_cores_detached +remove_cores +__check_caches_detached +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +destroy_cores +__check_cores_detached +remove_cores +__check_caches_detached +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +destroy_cores +__check_cores_detached +remove_cores +__check_caches_detached +__check_cores_empty +create_caches +__check_caches_attached +__check_cores_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +destroy_cores +__check_cores_detached +remove_cores +__check_caches_detached +__check_cores_empty +create_caches +__check_caches_attached +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk + +# hotremove caches and cores (inverse create after destroy order) + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +destroy_cores +__check_cores_detached +create_cores +__check_cores_attached +create_caches +__check_caches_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +destroy_caches +__check_caches_detached +destroy_cores +__check_cores_detached +create_cores +__check_cores_attached +create_caches +__check_caches_attached +stop_caches +__check_caches_empty +stop_spdk diff --git a/test/ocf/management/run.sh b/test/ocf/management/run.sh new file mode 100755 index 00000000000..d58cd724aa6 --- /dev/null +++ b/test/ocf/management/run.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f $curdir/../../..) + +source $rootdir/test/common/autotest_common.sh + +run_test "start-stop" "$curdir/start-stop/run.sh" +run_test "detach-attach" "$curdir/detach-attach.sh" +run_test "hotremove" "$curdir/hotremove.sh" diff --git a/test/ocf/management/start-stop/basic.sh b/test/ocf/management/start-stop/basic.sh new file mode 100755 index 00000000000..61cfe7880a2 --- /dev/null +++ b/test/ocf/management/start-stop/basic.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f $curdir/../../../..) +source "$rootdir/test/ocf/common.sh" + +# start caches and shutdown app: + +start_spdk +create_caches +start_caches +__check_caches_attached +stop_spdk + +# add cores and shutdown app: + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +stop_spdk + +# start and stop caches: + +start_spdk +create_caches +start_caches +__check_caches_attached +stop_caches +stop_spdk + +# add and remove cores: + +start_spdk +create_cores +add_cores +__check_cores_waitlist_attached +remove_cores +__check_cores_waitlist_empty +stop_spdk + +# start caches, add cores and shutdown app: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +stop_spdk + +# add cores, start caches and shutdown app: + +start_spdk +create_caches +create_cores +add_cores +__check_cores_waitlist_attached +start_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +# start caches, add cores and stop caches: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +stop_caches +__check_caches_empty +__check_cores_waitlist_empty +stop_spdk + +# add cores, start caches and stop caches: + +start_spdk +create_caches +create_cores +add_cores +__check_cores_waitlist_attached +start_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +__check_cores_waitlist_empty +stop_spdk + +# restart all caches with cores: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +stop_caches +__check_caches_empty +start_caches +__check_caches_attached +add_cores +__check_cores_attached +stop_caches +__check_caches_empty +__check_cores_waitlist_empty +stop_spdk + +# remove first core and shutdown app: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_ocf_remove_core Ocf_core$i-1 Ocf_cache$i +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 2' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached' +stop_spdk + +# remove all cores but first and shutdown app: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +for i in {1..3}; do + $rpc_py bdev_ocf_remove_core Ocf_core$i-2 Ocf_cache$i + $rpc_py bdev_ocf_remove_core Ocf_core$i-3 Ocf_cache$i +done +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 1' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached' +stop_spdk + +# remove all cores and shutdown app: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +remove_cores +__check_cores_empty +stop_spdk + +# remove all cores and stop all caches: + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +remove_cores +__check_cores_empty +stop_caches +__check_caches_empty +stop_spdk diff --git a/test/ocf/management/start-stop/examine.sh b/test/ocf/management/start-stop/examine.sh new file mode 100755 index 00000000000..423b480e211 --- /dev/null +++ b/test/ocf/management/start-stop/examine.sh @@ -0,0 +1,361 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f $curdir/../../../..) +source "$rootdir/test/ocf/common.sh" + +# examine partial configuration (not all devices present): + +start_spdk +start_caches +__check_caches_detached +create_caches +__check_caches_attached +stop_spdk + +start_spdk +start_caches +__check_caches_detached +create_caches +__check_caches_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_waitlist_detached +create_caches +__check_caches_attached +__check_cores_waitlist_detached +stop_spdk + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_waitlist_detached +create_caches +__check_caches_attached +__check_cores_waitlist_detached +stop_caches +__check_caches_empty +__check_cores_waitlist_detached +stop_spdk + +start_spdk +add_cores +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +stop_spdk + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_cores +__check_cores_empty +__check_cores_waitlist_attached +stop_spdk + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_cores +__check_cores_empty +__check_cores_waitlist_attached +stop_caches +__check_caches_empty +__check_cores_waitlist_attached +stop_spdk + +start_spdk +add_cores +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +start_caches +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_attached +stop_spdk + +start_spdk +add_cores +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +start_caches +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_attached +stop_caches +__check_caches_empty +__check_cores_waitlist_attached +stop_spdk + +start_spdk +add_cores +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +start_caches +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_attached +remove_cores +__check_cores_waitlist_empty +stop_spdk + +start_spdk +add_cores +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +start_caches +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_attached +remove_cores +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +add_cores +start_caches +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +stop_spdk + +start_spdk +add_cores +start_caches +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +stop_caches +__check_caches_empty +__check_cores_waitlist_attached +stop_spdk + +# examine full configuration with inversing order: + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_caches +__check_caches_attached +__check_cores_empty +create_cores +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_caches +__check_caches_attached +__check_cores_empty +create_cores +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +__check_cores_waitlist_empty +stop_spdk + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +create_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +create_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +add_cores +start_caches +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_caches +__check_caches_attached +__check_cores_empty +create_cores +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +start_spdk +add_cores +start_caches +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_caches +__check_caches_attached +__check_cores_empty +create_cores +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +add_cores +start_caches +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +create_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +start_spdk +add_cores +start_caches +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_detached +create_cores +__check_cores_waitlist_attached +create_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +# examine found core (not yet added or hotremoved) on detached cache: + +start_spdk +create_caches +start_caches +add_cores +__check_caches_attached +__check_cores_waitlist_detached +detach_caches +__check_caches_detached +create_cores +__check_caches_detached +__check_cores_waitlist_attached +attach_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_caches +start_caches +add_cores +__check_caches_attached +__check_cores_waitlist_detached +detach_caches +__check_caches_detached +create_cores +__check_caches_detached +__check_cores_waitlist_attached +attach_caches +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +detach_caches +__check_caches_detached +destroy_cores +__check_cores_detached +create_cores +__check_caches_detached +__check_cores_attached +attach_caches +__check_caches_attached +__check_cores_attached +stop_spdk + +start_spdk +create_caches +create_cores +start_caches +add_cores +__check_caches_attached +__check_cores_attached +detach_caches +__check_caches_detached +destroy_cores +__check_cores_detached +create_cores +__check_caches_detached +__check_cores_attached +attach_caches +__check_caches_attached +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk diff --git a/test/ocf/management/start-stop/incomplete.sh b/test/ocf/management/start-stop/incomplete.sh new file mode 100755 index 00000000000..283c8acaf72 --- /dev/null +++ b/test/ocf/management/start-stop/incomplete.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f $curdir/../../../..) +source "$rootdir/test/ocf/common.sh" + +# incomplete caches: + +start_spdk +start_caches +__check_caches_detached +stop_spdk + +start_spdk +start_caches +__check_caches_detached +stop_caches +__check_caches_empty +stop_spdk + +# incomplete cores: + +start_spdk +add_cores +__check_cores_waitlist_detached +stop_spdk + +start_spdk +add_cores +__check_cores_waitlist_detached +remove_cores +__check_cores_waitlist_empty +stop_spdk + +# incomplete caches and incomplete cores: + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_waitlist_detached +stop_spdk + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_waitlist_detached +stop_caches +__check_caches_empty +__check_cores_waitlist_detached +stop_spdk + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_waitlist_detached +remove_cores +__check_cores_waitlist_empty +stop_spdk + +start_spdk +start_caches +add_cores +__check_caches_detached +__check_cores_waitlist_detached +remove_cores +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +# complete caches and incomplete cores: + +start_spdk +create_caches +start_caches +add_cores +__check_caches_attached +__check_cores_empty +__check_cores_waitlist_detached +stop_spdk + +start_spdk +create_caches +start_caches +add_cores +__check_caches_attached +__check_cores_empty +__check_cores_waitlist_detached +stop_caches +__check_caches_empty +__check_cores_waitlist_detached +stop_spdk + +start_spdk +create_caches +start_caches +add_cores +__check_caches_attached +__check_cores_empty +__check_cores_waitlist_detached +remove_cores +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_caches +start_caches +add_cores +__check_caches_attached +__check_cores_empty +__check_cores_waitlist_detached +remove_cores +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +# incomplete caches and complete cores: + +start_spdk +create_cores +start_caches +__check_caches_detached +add_cores +__check_cores_empty +__check_cores_waitlist_attached +stop_spdk + +start_spdk +create_cores +start_caches +__check_caches_detached +add_cores +__check_cores_empty +__check_cores_waitlist_attached +stop_caches +__check_caches_empty +__check_cores_waitlist_attached +stop_spdk + +start_spdk +create_cores +start_caches +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_attached +remove_cores +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_cores +start_caches +add_cores +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_attached +remove_cores +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk diff --git a/test/ocf/management/start-stop/load.sh b/test/ocf/management/start-stop/load.sh new file mode 100755 index 00000000000..6d3a82f3766 --- /dev/null +++ b/test/ocf/management/start-stop/load.sh @@ -0,0 +1,531 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f $curdir/../../../..) +source "$rootdir/test/ocf/common.sh" + +create_caches_with_metadata() { + create_caches + start_caches + __check_caches_attached + stop_caches + __check_caches_empty +} + +create_caches_with_metadata_with_cores() { + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + stop_caches + __check_caches_empty + destroy_cores +} + +create_caches_with_metadata_persistent() { + create_caches_persistent + start_caches + __check_caches_attached + stop_caches + __check_caches_empty +} + +create_caches_with_metadata_with_cores_persistent() { + create_caches_persistent + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + stop_caches + __check_caches_empty + destroy_cores +} + +# load caches without cores: + +start_spdk +create_caches_with_metadata +start_caches_try_load +__check_caches_attached +stop_spdk + +start_spdk +create_caches_with_metadata +start_caches_try_load +__check_caches_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches_with_metadata +create_cores +start_caches_try_load +__check_caches_attached +__check_cores_empty +add_cores +__check_cores_attached +stop_spdk + +start_spdk +create_caches_with_metadata +create_cores +start_caches_try_load +__check_caches_attached +__check_cores_empty +add_cores +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +# load caches with cores: + +start_spdk +create_caches_with_metadata_with_cores +start_caches_try_load +__check_caches_attached +__check_cores_loading +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores +start_caches_try_load +__check_caches_attached +__check_cores_loading +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores +create_cores +start_caches_try_load +__check_caches_attached +__check_cores_loading +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores +create_cores +start_caches_try_load +__check_caches_attached +__check_cores_loading +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores +create_cores +start_caches_try_load +__check_caches_attached +__check_cores_loading +add_cores +__check_cores_attached +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores +create_cores +start_caches_try_load +__check_caches_attached +__check_cores_loading +add_cores +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores +add_cores +__check_cores_waitlist_detached +start_caches_try_load +__check_caches_attached +__check_cores_loading +__check_cores_waitlist_detached +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores +add_cores +__check_cores_waitlist_detached +start_caches_try_load +__check_caches_attached +__check_cores_loading +__check_cores_waitlist_detached +stop_caches +__check_caches_empty +stop_spdk + +# examine found cache device with metadata: + +start_spdk +create_caches_with_metadata_persistent +destroy_caches_persistent +start_caches_try_load +__check_caches_detached +create_caches_persistent +__check_caches_attached +stop_spdk + +start_spdk +create_caches_with_metadata_persistent +destroy_caches_persistent +start_caches_try_load +__check_caches_detached +create_caches_persistent +__check_caches_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +destroy_caches_persistent +start_caches_try_load +__check_caches_detached +__check_cores_empty +create_caches_persistent +__check_caches_attached +__check_cores_loading +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +destroy_caches_persistent +start_caches_try_load +__check_caches_detached +__check_cores_empty +create_caches_persistent +__check_caches_attached +__check_cores_loading +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +create_cores +destroy_caches_persistent +start_caches_try_load +__check_caches_detached +__check_cores_empty +create_caches_persistent +__check_caches_attached +__check_cores_loading +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +create_cores +destroy_caches_persistent +start_caches_try_load +__check_caches_detached +__check_cores_empty +create_caches_persistent +__check_caches_attached +__check_cores_loading +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +create_cores +destroy_caches_persistent +start_caches_try_load +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_empty +create_caches_persistent +__check_caches_attached +__check_cores_loading +__check_cores_waitlist_empty +add_cores +__check_cores_attached +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +create_cores +destroy_caches_persistent +start_caches_try_load +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_empty +create_caches_persistent +__check_caches_attached +__check_cores_loading +__check_cores_waitlist_empty +add_cores +__check_cores_attached +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +create_cores +destroy_caches_persistent +start_caches_try_load +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_empty +add_cores +__check_cores_empty +__check_cores_waitlist_attached +create_caches_persistent +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +create_cores +destroy_caches_persistent +start_caches_try_load +__check_caches_detached +__check_cores_empty +__check_cores_waitlist_empty +add_cores +__check_cores_empty +__check_cores_waitlist_attached +create_caches_persistent +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +create_cores +destroy_caches_persistent +add_cores +__check_cores_waitlist_attached +start_caches_try_load +__check_caches_detached +__check_cores_empty +create_caches_persistent +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +create_cores +destroy_caches_persistent +add_cores +__check_cores_waitlist_attached +start_caches_try_load +__check_caches_detached +__check_cores_empty +create_caches_persistent +__check_caches_attached +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +# examine found core after cache load: + +start_spdk +create_caches_with_metadata +start_caches_try_load +__check_caches_attached +__check_cores_empty +add_cores +__check_cores_empty +__check_cores_waitlist_detached +create_cores +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_caches_with_metadata +start_caches_try_load +__check_caches_attached +__check_cores_empty +add_cores +__check_cores_empty +__check_cores_waitlist_detached +create_cores +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores +start_caches_try_load +__check_caches_attached +__check_cores_loading +add_cores +__check_cores_loading +__check_cores_waitlist_detached +create_cores +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores +start_caches_try_load +__check_caches_attached +__check_cores_loading +add_cores +__check_cores_loading +__check_cores_waitlist_detached +create_cores +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +# examine found cache with metadata and core after cache load: + +start_spdk +create_caches_with_metadata_with_cores_persistent +destroy_caches_persistent +add_cores +__check_cores_waitlist_detached +start_caches_try_load +__check_caches_detached +__check_cores_empty +create_caches_persistent +__check_caches_attached +__check_cores_loading +__check_cores_waitlist_detached +create_cores +__check_cores_attached +__check_cores_waitlist_empty +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +destroy_caches_persistent +add_cores +__check_cores_waitlist_detached +start_caches_try_load +__check_caches_detached +__check_cores_empty +create_caches_persistent +__check_caches_attached +__check_cores_loading +__check_cores_waitlist_detached +create_cores +__check_cores_attached +__check_cores_waitlist_empty +stop_caches +__check_caches_empty +stop_spdk + +# partially found cache and core devices: + +start_spdk +create_caches_with_metadata_with_cores_persistent +destroy_caches_persistent +add_cores +__check_cores_waitlist_detached +start_caches_try_load +__check_caches_detached +__check_cores_empty +$rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[1]}" -d blk Cache_dev2 +$rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[2]}" -d blk Cache_dev3 +sleep 3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' +__check_cores_waitlist_detached +for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev2-$i 200 512 +done +sleep 3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' +$rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 6' +$rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[].base_attached | not' +stop_spdk + +start_spdk +create_caches_with_metadata_with_cores_persistent +destroy_caches_persistent +add_cores +__check_cores_waitlist_detached +start_caches_try_load +__check_caches_detached +__check_cores_empty +$rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[1]}" -d blk Cache_dev2 +$rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[2]}" -d blk Cache_dev3 +sleep 3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' +__check_cores_waitlist_detached +for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev2-$i 200 512 +done +sleep 3 +$rpc_py bdev_ocf_get_bdevs | jq -e '.' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' +$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' +$rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 6' +$rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[].base_attached | not' +stop_caches +__check_caches_empty +stop_spdk diff --git a/test/ocf/management/start-stop/run.sh b/test/ocf/management/start-stop/run.sh new file mode 100755 index 00000000000..e5019223a92 --- /dev/null +++ b/test/ocf/management/start-stop/run.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f $curdir/../../../..) + +source $rootdir/test/common/autotest_common.sh + +run_test "basic" "$curdir/basic.sh" +run_test "incomplete" "$curdir/incomplete.sh" +run_test "examine" "$curdir/examine.sh" +run_test "load" "$curdir/load.sh" diff --git a/test/ocf/ocf.sh b/test/ocf/ocf.sh index 4d01a42bb38..cb45074fbe6 100755 --- a/test/ocf/ocf.sh +++ b/test/ocf/ocf.sh @@ -1,18 +1,15 @@ #!/usr/bin/env bash + +# # SPDX-License-Identifier: BSD-3-Clause # Copyright (C) 2018 Intel Corporation +# Copyright (C) 2025 Huawei Technologies # All rights reserved. # -testdir=$(readlink -f $(dirname $0)) -rootdir=$(readlink -f $testdir/../..) + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f $curdir/../..) source $rootdir/test/common/autotest_common.sh -run_test "ocf_fio_modes" "$testdir/integrity/fio-modes.sh" -run_test "ocf_bdevperf_iotypes" "$testdir/integrity/bdevperf-iotypes.sh" -run_test "ocf_stats" "$testdir/integrity/stats.sh" -run_test "ocf_flush" "$testdir/integrity/flush.sh" -run_test "ocf_create_destruct" "$testdir/management/create-destruct.sh" -run_test "ocf_multicore" "$testdir/management/multicore.sh" -run_test "ocf_remove" "$testdir/management/remove.sh" -run_test "ocf_configuration_change" "$testdir/management/configuration-change.sh" +run_test "ocf_management" "$curdir/management/run.sh" From c4e64605c63918d0bb36ee645dcb129bd7009858 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Tue, 10 Jun 2025 15:30:53 +0200 Subject: [PATCH 09/34] Multiple fixes - check cache/core block size - merge cache stop paths - unify adding cores from waitlist - error handling - comments - cleanup Change-Id: Ib111089a9e5fe1ec155487db3b2c4541d1ce9520 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 386 ++++++------------------------ module/bdev/ocf/vbdev_ocf_cache.h | 26 +- module/bdev/ocf/vbdev_ocf_core.c | 91 +++++++ module/bdev/ocf/vbdev_ocf_core.h | 49 ++-- 4 files changed, 226 insertions(+), 326 deletions(-) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 1a0512bbd29..a23765b35a2 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -6,7 +6,7 @@ #include #include "spdk/bdev_module.h" -#include "spdk/string.h" // rm ? +#include "spdk/string.h" #include "vbdev_ocf.h" #include "ctx.h" @@ -138,8 +138,9 @@ vbdev_ocf_module_init(void) } static void -_module_fini_cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) +_cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) { + struct vbdev_ocf_mngt_ctx *cache_stop_ctx = cb_arg; ocf_queue_t cache_mngt_q = ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", @@ -148,35 +149,50 @@ _module_fini_cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) if (error) { SPDK_ERRLOG("OCF cache '%s': failed to stop OCF cache (OCF error: %d)\n", ocf_cache_get_name(cache), error); + } else { + SPDK_NOTICELOG("OCF cache '%s': stopped\n", ocf_cache_get_name(cache)); } - if (vbdev_ocf_cache_is_base_attached(cache)) { - vbdev_ocf_cache_base_detach(cache); - } else { - /* If device was not attached to cache, then cache stop won't 'put' its management queue. */ - ocf_queue_put(cache_mngt_q); + /* In module fini (no cache stop context) do the cleanup despite the error. */ + if (!error || !cache_stop_ctx) { + if (vbdev_ocf_cache_is_base_attached(cache)) { + vbdev_ocf_cache_base_detach(cache); + } else { + /* If device was not attached to cache, then + * cache stop won't put its management queue. */ + ocf_queue_put(cache_mngt_q); + } + vbdev_ocf_cache_destroy(cache); } - vbdev_ocf_cache_destroy(cache); + ocf_mngt_cache_unlock(cache); - if (!ocf_mngt_cache_get_count(vbdev_ocf_ctx)) { + if (cache_stop_ctx) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop\n", + ocf_cache_get_name(cache)); + + cache_stop_ctx->rpc_cb_fn(NULL, cache_stop_ctx->rpc_cb_arg, error); // cache name + free(cache_stop_ctx); + } else if (!ocf_mngt_cache_get_count(vbdev_ocf_ctx)) { + /* In module fini (no cache stop context) call spdk_bdev_module_fini_start_done() + * if there are no caches left to stop. */ spdk_bdev_module_fini_start_done(); } } static void -_module_fini_cache_flush_cb(ocf_cache_t cache, void *cb_arg, int error) +_cache_stop_flush_cb(ocf_cache_t cache, void *cb_arg, int error) { if (error) { SPDK_ERRLOG("OCF cache '%s': failed to flush OCF cache (OCF error: %d)\n", ocf_cache_get_name(cache), error); } - ocf_mngt_cache_stop(cache, _module_fini_cache_stop_cb, cb_arg); + ocf_mngt_cache_stop(cache, _cache_stop_cb, cb_arg); } static void -_module_fini_cache_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +_cache_stop_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop of OCF cache\n", ocf_cache_get_name(cache)); @@ -186,25 +202,27 @@ _module_fini_cache_lock_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_cache_get_name(cache), error); } + // no need to manually flush ? if (ocf_mngt_cache_is_dirty(cache)) { - ocf_mngt_cache_flush(cache, _module_fini_cache_flush_cb, cb_arg); + ocf_mngt_cache_flush(cache, _cache_stop_flush_cb, cb_arg); } else { - ocf_mngt_cache_stop(cache, _module_fini_cache_stop_cb, cb_arg); + ocf_mngt_cache_stop(cache, _cache_stop_cb, cb_arg); } } static void -_module_fini_core_unregister_cb(void *cb_arg, int error) +_cache_stop_core_unregister_cb(void *cb_arg, int error) { ocf_core_t core = cb_arg; ocf_cache_t cache = ocf_core_get_cache(core); struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + struct vbdev_ocf_mngt_ctx *cache_stop_ctx = core_ctx->mngt_ctx; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", ocf_core_get_name(core)); if (error) { - SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during module stop: %s\n", + SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev: %s\n", ocf_core_get_name(core), spdk_strerror(-error)); } @@ -212,22 +230,26 @@ _module_fini_core_unregister_cb(void *cb_arg, int error) vbdev_ocf_core_destroy(core_ctx); if (ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { - ocf_mngt_cache_lock(cache, _module_fini_cache_lock_cb, cb_arg); + /* All cores in this cache were already unregistered + * and detached, so proceed with stopping the cache. */ + ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, cache_stop_ctx); } } static int -_module_fini_core_visit(ocf_core_t core, void *cb_arg) +_cache_stop_core_visit(ocf_core_t core, void *cb_arg) { + struct vbdev_ocf_mngt_ctx *cache_stop_ctx = cb_arg; ocf_cache_t cache = ocf_core_get_cache(core); struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); int rc = 0; - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': module stop visit\n", ocf_core_get_name(core)); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': cache stop visit\n", ocf_core_get_name(core)); if (!core_ctx) { /* Skip this core. If there is no context, it means that this core * was added from metadata during cache load and it's just an empty shell. */ + ocf_mngt_cache_put(cache); return 0; } @@ -238,8 +260,10 @@ _module_fini_core_visit(ocf_core_t core, void *cb_arg) return 0; } - if ((rc = vbdev_ocf_core_unregister(core_ctx, _module_fini_core_unregister_cb, core))) { - SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during module stop: %s\n", + core_ctx->mngt_ctx = cache_stop_ctx; + + if ((rc = vbdev_ocf_core_unregister(core_ctx, _cache_stop_core_unregister_cb, core))) { + SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev: %s\n", ocf_core_get_name(core), spdk_strerror(-rc)); ocf_mngt_cache_put(cache); return rc; @@ -267,16 +291,17 @@ _module_fini_cache_visit(ocf_cache_t cache, void *cb_arg) ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { /* If there are no cores or all of them are detached, * then cache stop can be triggered already. */ - ocf_mngt_cache_lock(cache, _module_fini_cache_lock_cb, cb_arg); + ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, NULL); } - if ((rc = ocf_core_visit(cache, _module_fini_core_visit, cb_arg, false))) { + if ((rc = ocf_core_visit(cache, _cache_stop_core_visit, NULL, false))) { SPDK_ERRLOG("OCF cache '%s': failed to iterate over core bdevs: %s\n", ocf_cache_get_name(cache), spdk_strerror(-rc)); - // ocf_mngt_cache_lock(...) ? + ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, NULL); } - return rc; + /* In module fini return 0 despite any errors to keep shutting down all caches. */ + return 0; } static void @@ -513,72 +538,10 @@ _core_add_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _core_add_examine_add_cb, core_ctx); } -static void -_cache_attach_examine_core_add_err_cb(void *cb_arg, int error) -{ - ocf_cache_t cache = cb_arg; - - if (error) { - SPDK_ERRLOG("OCF core: failed to remove OCF core device (OCF error: %d)\n", error); - } - - ocf_mngt_cache_unlock(cache); -} - -static void -_cache_attach_examine_core_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int error) -{ - struct vbdev_ocf_core *core_ctx = cb_arg; - int rc = 0; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", - vbdev_ocf_core_get_name(core_ctx)); - - if (error) { - SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s' (OCF error: %d)\n", - vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache), error); - ocf_mngt_cache_unlock(cache); - return; - } - - ocf_core_set_priv(core, core_ctx); - - if ((rc = vbdev_ocf_core_register(core))) { - SPDK_ERRLOG("OCF core '%s': failed to register vbdev: %s\n", - ocf_core_get_name(core), spdk_strerror(-rc)); - ocf_mngt_cache_remove_core(core, _cache_attach_examine_core_add_err_cb, cache); - return; - } - - SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n", - ocf_core_get_name(core), ocf_cache_get_name(cache)); - - ocf_mngt_cache_unlock(cache); - STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); -} - -static void -_cache_attach_examine_core_add_lock_cb(ocf_cache_t cache, void *cb_arg, int error) -{ - struct vbdev_ocf_core *core_ctx = cb_arg; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add of OCF core\n", - vbdev_ocf_core_get_name(core_ctx)); - - if (error) { - SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock (OCF error: %d)\n", - vbdev_ocf_core_get_name(core_ctx), error); - return; - } - - ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _cache_attach_examine_core_add_cb, core_ctx); -} - static void _cache_attach_examine_attach_cb(ocf_cache_t cache, void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *examine_attach_ctx = cb_arg; - struct vbdev_ocf_core *core_ctx; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing device attach\n", ocf_cache_get_name(cache)); @@ -599,21 +562,7 @@ _cache_attach_examine_attach_cb(ocf_cache_t cache, void *cb_arg, int error) free(examine_attach_ctx); spdk_bdev_module_examine_done(&ocf_if); // move after adding cores from waitlist ? - vbdev_ocf_foreach_core_in_waitlist(core_ctx) { - if (strcmp(ocf_cache_get_name(cache), core_ctx->cache_name) || - !vbdev_ocf_core_is_base_attached(core_ctx)) { - continue; - } - - SPDK_NOTICELOG("OCF core '%s': adding from waitlist to cache '%s'\n", - vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); - - // check if (cache_block_size > core_block_size) - - core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx)); - - ocf_mngt_cache_lock(cache, _cache_attach_examine_core_add_lock_cb, core_ctx); - } + vbdev_ocf_core_add_from_waitlist(cache); } static void @@ -829,7 +778,6 @@ _vbdev_ocf_submit_io_cb(ocf_io_t io, void *priv1, void *priv2, int error) { struct spdk_bdev_io *bdev_io = priv1; - // rm ? SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': finishing submit of IO request\n", spdk_bdev_get_name(bdev_io->bdev)); @@ -904,7 +852,6 @@ vbdev_ocf_fn_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bde uint64_t offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen; uint32_t len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen; - // rm ? SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': initiating submit of IO request\n", spdk_bdev_get_name(bdev_io->bdev)); @@ -972,67 +919,6 @@ vbdev_ocf_fn_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ct { } -static void -_cache_start_rpc_core_add_err_cb(void *cb_arg, int error) -{ - ocf_cache_t cache = cb_arg; - - if (error) { - SPDK_ERRLOG("OCF core: failed to remove OCF core device (OCF error: %d)\n", error); - } - - ocf_mngt_cache_unlock(cache); -} - -static void -_cache_start_rpc_core_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int error) -{ - struct vbdev_ocf_core *core_ctx = cb_arg; - int rc = 0; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", - vbdev_ocf_core_get_name(core_ctx)); - - if (error) { - SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s' (OCF error: %d)\n", - vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache), error); - ocf_mngt_cache_unlock(cache); - return; - } - - ocf_core_set_priv(core, core_ctx); - - if ((rc = vbdev_ocf_core_register(core))) { - SPDK_ERRLOG("OCF core '%s': failed to register vbdev: %s\n", - ocf_core_get_name(core), spdk_strerror(-rc)); - ocf_mngt_cache_remove_core(core, _cache_start_rpc_core_add_err_cb, cache); - return; - } - - SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n", - ocf_core_get_name(core), ocf_cache_get_name(cache)); - - ocf_mngt_cache_unlock(cache); - STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); -} - -static void -_cache_start_rpc_core_add_lock_cb(ocf_cache_t cache, void *cb_arg, int error) -{ - struct vbdev_ocf_core *core_ctx = cb_arg; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add of OCF core\n", - vbdev_ocf_core_get_name(core_ctx)); - - if (error) { - SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock (OCF error: %d)\n", - vbdev_ocf_core_get_name(core_ctx), error); - return; - } - - ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _cache_start_rpc_core_add_cb, core_ctx); -} - static void _cache_start_rpc_err_cb(ocf_cache_t cache, void *cb_arg, int error) { @@ -1051,7 +937,6 @@ _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *cache_start_ctx = cb_arg; struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); - struct vbdev_ocf_core *core_ctx; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing start\n", ocf_cache_get_name(cache)); @@ -1089,21 +974,7 @@ _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) cache_start_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_start_ctx->rpc_cb_arg, 0); free(cache_start_ctx); - vbdev_ocf_foreach_core_in_waitlist(core_ctx) { - if (strcmp(ocf_cache_get_name(cache), core_ctx->cache_name) || - !vbdev_ocf_core_is_base_attached(core_ctx)) { - continue; - } - - SPDK_NOTICELOG("OCF core '%s': adding from waitlist to cache '%s'\n", - vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); - - // check if (cache_block_size > core_block_size) - - core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx)); - - ocf_mngt_cache_lock(cache, _cache_start_rpc_core_add_lock_cb, core_ctx); - } + vbdev_ocf_core_add_from_waitlist(cache); } /* RPC entry point. */ @@ -1186,126 +1057,6 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, rpc_cb_fn(cache_name, rpc_cb_arg, rc); } -static void -_cache_stop_rpc_stop_cb(ocf_cache_t cache, void *cb_arg, int error) -{ - struct vbdev_ocf_mngt_ctx *cache_stop_ctx = cb_arg; - ocf_queue_t cache_mngt_q = ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", - ocf_cache_get_name(cache)); - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop\n", - ocf_cache_get_name(cache)); - - if (error) { - SPDK_ERRLOG("OCF cache '%s': failed to stop OCF cache (OCF error: %d)\n", - ocf_cache_get_name(cache), error); - } else { - SPDK_NOTICELOG("OCF cache '%s': stopped\n", ocf_cache_get_name(cache)); - if (vbdev_ocf_cache_is_base_attached(cache)) { - vbdev_ocf_cache_base_detach(cache); - } else { - /* If device was not attached to cache, then cache stop won't 'put' its management queue. */ - ocf_queue_put(cache_mngt_q); - } - vbdev_ocf_cache_destroy(cache); - } - - ocf_mngt_cache_unlock(cache); - //ocf_mngt_cache_put(cache); // no need to put !!! (check mergability with module fini) - cache_stop_ctx->rpc_cb_fn(NULL, cache_stop_ctx->rpc_cb_arg, error); // cache name - free(cache_stop_ctx); -} - -static void -_cache_stop_rpc_flush_cb(ocf_cache_t cache, void *cb_arg, int error) -{ - if (error) { - SPDK_ERRLOG("OCF cache '%s': failed to flush OCF cache (OCF error: %d)\n", - ocf_cache_get_name(cache), error); - } - - ocf_mngt_cache_stop(cache, _cache_stop_rpc_stop_cb, cb_arg); -} - -static void -_cache_stop_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) -{ - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop of OCF cache\n", - ocf_cache_get_name(cache)); - - if (error) { - SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", - ocf_cache_get_name(cache), error); - } - - // no need to manually flush ? - if (ocf_mngt_cache_is_dirty(cache)) { - ocf_mngt_cache_flush(cache, _cache_stop_rpc_flush_cb, cb_arg); - } else { - ocf_mngt_cache_stop(cache, _cache_stop_rpc_stop_cb, cb_arg); - } -} - -static void -_cache_stop_rpc_core_unregister_cb(void *cb_arg, int error) -{ - ocf_core_t core = cb_arg; - ocf_cache_t cache = ocf_core_get_cache(core); - struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); - struct vbdev_ocf_mngt_ctx *cache_stop_ctx = core_ctx->mngt_ctx; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", - ocf_core_get_name(core)); - - if (error) { - SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during cache stop: %s\n", - ocf_core_get_name(core), spdk_strerror(-error)); - } - - ocf_mngt_cache_put(cache); - vbdev_ocf_core_destroy(core_ctx); - - if (ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { - ocf_mngt_cache_lock(cache, _cache_stop_rpc_lock_cb, cache_stop_ctx); - } -} - -static int -_cache_stop_rpc_core_visit(ocf_core_t core, void *cb_arg) -{ - struct vbdev_ocf_mngt_ctx *cache_stop_ctx = cb_arg; - ocf_cache_t cache = ocf_core_get_cache(core); - struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); - int rc = 0; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': cache stop visit\n", ocf_core_get_name(core)); - - if (!core_ctx) { - /* Skip this core. If there is no context, it means that this core - * was added from metadata during cache load and it's just an empty shell. */ - return 0; - } - - /* If core is detached it's already unregistered, so just free its data and exit. */ - if (!vbdev_ocf_core_is_base_attached(core_ctx)) { - ocf_mngt_cache_put(cache); - vbdev_ocf_core_destroy(core_ctx); - return 0; - } - - core_ctx->mngt_ctx = cache_stop_ctx; - - if ((rc = vbdev_ocf_core_unregister(core_ctx, _cache_stop_rpc_core_unregister_cb, core))) { - SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during cache stop: %s\n", - ocf_core_get_name(core), spdk_strerror(-rc)); - ocf_mngt_cache_put(cache); - return rc; - } - - return rc; -} - /* RPC entry point. */ void vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) @@ -1343,6 +1094,9 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo if ((rc = ocf_mngt_cache_get(cache))) { SPDK_ERRLOG("OCF cache '%s': failed to increment refcount\n", ocf_cache_get_name(cache)); + for (i = ocf_cache_get_core_count(cache) - i; i > 0; i--) { + ocf_mngt_cache_put(cache); + } goto err_get; } } @@ -1351,17 +1105,19 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { /* If there are no cores or all of them are detached, * then cache stop can be triggered already. */ - ocf_mngt_cache_lock(cache, _cache_stop_rpc_lock_cb, cache_stop_ctx); + ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, cache_stop_ctx); } - if ((rc = ocf_core_visit(cache, _cache_stop_rpc_core_visit, cache_stop_ctx, false))) { + if ((rc = ocf_core_visit(cache, _cache_stop_core_visit, cache_stop_ctx, false))) { SPDK_ERRLOG("OCF cache '%s': failed to iterate over core bdevs\n", ocf_cache_get_name(cache)); - // ocf_mngt_cache_lock() or rpc_cb_fn() ? + // ocf_mngt_cache_put() ? how many times ? + goto err_visit; } return; +err_visit: err_get: free(cache_stop_ctx); err_alloc: @@ -1505,6 +1261,8 @@ _cache_attach_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_mngt_cache_put(cache); cache_attach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_attach_ctx->rpc_cb_arg, error); free(cache_attach_ctx); + + vbdev_ocf_core_add_from_waitlist(cache); } static void @@ -1689,8 +1447,10 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { ocf_cache_t cache; + struct vbdev_ocf_cache *cache_ctx; struct vbdev_ocf_core *core_ctx; struct vbdev_ocf_mngt_ctx *core_add_ctx; + uint32_t cache_block_size, core_block_size; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add\n", core_name); @@ -1730,18 +1490,29 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac return; } + cache_ctx = ocf_cache_get_priv(cache); + /* And finally, check if OCF cache device is already attached. * We need to have cache device attached to know if cache was loaded or attached * and then set 'try_add' in core config accordingly. */ if (!ocf_cache_is_device_attached(cache)) { SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache device '%s'\n", - core_name, ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->base.name); - //ocf_mngt_cache_put(cache); // ? + core_name, cache_ctx->base.name); + ocf_mngt_cache_put(cache); STAILQ_INSERT_TAIL(&g_vbdev_ocf_core_waitlist, core_ctx, waitlist_entry); rpc_cb_fn(core_name, rpc_cb_arg, -ENODEV); return; } + cache_block_size = cache_ctx->base.bdev->blocklen; + core_block_size = core_ctx->base.bdev->blocklen; + if (cache_block_size > core_block_size) { + SPDK_ERRLOG("OCF core '%s': block size (%d) is less than cache '%s' block size (%d)\n", + core_name, core_block_size, cache_name, cache_block_size); + rc = -ENOTSUP; + goto err_bsize; + } + // DONE // find a better way to check if cache was loaded or attached //core_ctx->core_cfg.try_add = cache_ctx->cache_att_cfg.force ? false : true; @@ -1749,8 +1520,6 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac // open (and then close) cache-load-added core volumes (with 'base' to save it in priv) ? - // check if (cache_block_size > core_block_size) - core_add_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!core_add_ctx) { SPDK_ERRLOG("OCF core '%s': failed to allocate memory for core add context\n", @@ -1768,6 +1537,7 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac return; err_alloc: +err_bsize: ocf_mngt_cache_put(cache); vbdev_ocf_core_base_detach(core_ctx); err_base: diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h index 49c96025ad0..4918a3825dc 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.h +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -13,17 +13,23 @@ /* OCF module interface. */ extern struct spdk_bdev_module ocf_if; +/* Cache context (assigned as cache priv). */ struct vbdev_ocf_cache { + /* Base bdev data. */ + struct vbdev_ocf_base base; + /* Cache start config. */ struct ocf_mngt_cache_config cache_cfg; + /* Cache device attach config. */ struct ocf_mngt_cache_attach_config cache_att_cfg; + /* Cache management queue. */ ocf_queue_t cache_mngt_q; + /* Indicator for cache start to ignore cache metadata found on device. */ bool no_load; - - struct vbdev_ocf_base base; }; typedef void (*vbdev_ocf_rpc_mngt_cb)(const char *bdev_name, void *cb_arg, int error); +/* Temporary context for management operations. */ struct vbdev_ocf_mngt_ctx { ocf_cache_t cache; ocf_core_t core; @@ -33,23 +39,35 @@ struct vbdev_ocf_mngt_ctx { void * rpc_cb_arg; }; +/* Cache management queue context. */ struct vbdev_ocf_cache_mngt_queue_ctx { + /* Registered poller. */ struct spdk_poller * poller; + /* Thread on which poller was registered. */ struct spdk_thread * thread; /* Currently kept only for its name used in debug log. */ ocf_cache_t cache; // rm ? }; +/* Check if cache base is already attached. */ +bool vbdev_ocf_cache_is_base_attached(ocf_cache_t cache); + +/* Create cache context, fill config and start OCF cache. */ int vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cache_mode, const uint8_t cache_line_size, bool no_load); +/* Free cache context. */ void vbdev_ocf_cache_destroy(ocf_cache_t cache); +/* Open, claim and create IO channel for base bdev. */ int vbdev_ocf_cache_base_attach(ocf_cache_t cache, const char *base_name); +/* Close base bdev and IO channel. */ void vbdev_ocf_cache_base_detach(ocf_cache_t cache); +/* Create OCF cache volume and fill device config. */ int vbdev_ocf_cache_config_volume_create(ocf_cache_t cache); +/* Destroy OCF cache volume from config. */ void vbdev_ocf_cache_config_volume_destroy(ocf_cache_t cache); +/* Attach or load cache device. */ int vbdev_ocf_cache_volume_attach(ocf_cache_t cache, struct vbdev_ocf_mngt_ctx *ctx); +/* Create cache management queue poller. */ int vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache); -bool vbdev_ocf_cache_is_base_attached(ocf_cache_t cache); - #endif diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c index 791da74cc17..93fba991a30 100644 --- a/module/bdev/ocf/vbdev_ocf_core.c +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -333,6 +333,97 @@ vbdev_ocf_core_unregister(struct vbdev_ocf_core *core_ctx, spdk_bdev_unregister_ &ocf_if, cb_fn, cb_arg); } +static void +_core_add_from_waitlist_err_cb(void *cb_arg, int error) +{ + ocf_cache_t cache = cb_arg; + + if (error) { + SPDK_ERRLOG("OCF core: failed to remove OCF core device (OCF error: %d)\n", error); + } + + ocf_mngt_cache_unlock(cache); +} + +static void +_core_add_from_waitlist_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int error) +{ + struct vbdev_ocf_core *core_ctx = cb_arg; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", + vbdev_ocf_core_get_name(core_ctx)); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s' (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache), error); + ocf_mngt_cache_unlock(cache); + return; + } + + ocf_core_set_priv(core, core_ctx); + + if ((rc = vbdev_ocf_core_register(core))) { + SPDK_ERRLOG("OCF core '%s': failed to register vbdev: %s\n", + ocf_core_get_name(core), spdk_strerror(-rc)); + ocf_mngt_cache_remove_core(core, _core_add_from_waitlist_err_cb, cache); + return; + } + + SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n", + ocf_core_get_name(core), ocf_cache_get_name(cache)); + + ocf_mngt_cache_unlock(cache); + STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); +} + +static void +_core_add_from_waitlist_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_core *core_ctx = cb_arg; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add of OCF core\n", + vbdev_ocf_core_get_name(core_ctx)); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), error); + return; + } + + ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _core_add_from_waitlist_add_cb, core_ctx); +} + +void +vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache) +{ + struct vbdev_ocf_core *core_ctx; + uint32_t cache_block_size = ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->base.bdev->blocklen; + uint32_t core_block_size; + + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + if (strcmp(ocf_cache_get_name(cache), core_ctx->cache_name) || + !vbdev_ocf_core_is_base_attached(core_ctx)) { + continue; + } + + SPDK_NOTICELOG("OCF core '%s': adding from waitlist to cache '%s'\n", + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); + + core_block_size = core_ctx->base.bdev->blocklen; + if (cache_block_size > core_block_size) { + SPDK_ERRLOG("OCF core '%s': failed to add to cache '%s': cache block size (%d) is greater than core block size (%d)\n", + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache), + cache_block_size, core_block_size); + continue; + } + + core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx)); + + ocf_mngt_cache_lock(cache, _core_add_from_waitlist_lock_cb, core_ctx); + } +} + char * vbdev_ocf_core_get_name(struct vbdev_ocf_core *core_ctx) { diff --git a/module/bdev/ocf/vbdev_ocf_core.h b/module/bdev/ocf/vbdev_ocf_core.h index ddbecc00013..26184b22d7d 100644 --- a/module/bdev/ocf/vbdev_ocf_core.h +++ b/module/bdev/ocf/vbdev_ocf_core.h @@ -23,39 +23,60 @@ extern STAILQ_HEAD(vbdev_ocf_core_waitlist_head, vbdev_ocf_core) g_vbdev_ocf_cor #define vbdev_ocf_foreach_core_in_waitlist(core_ctx) \ STAILQ_FOREACH(core_ctx, &g_vbdev_ocf_core_waitlist, waitlist_entry) +/* Core context (assigned as core priv). */ struct vbdev_ocf_core { - struct ocf_mngt_core_config core_cfg; + /* Exposed OCF vbdev; the one that is registered in bdev layer for usage. */ + struct spdk_bdev ocf_vbdev; + /* Base bdev data. */ + struct vbdev_ocf_base base; + /* Core add config. */ + struct ocf_mngt_core_config core_cfg; + /* Name of cache that this core is assigned to. */ + char cache_name[OCF_CACHE_NAME_SIZE]; + /* Context for cache stop management operation. */ + struct vbdev_ocf_mngt_ctx * mngt_ctx; - /* Exposed OCF vbdev; the one which is registered in bdev layer for usage. */ - struct spdk_bdev ocf_vbdev; - - struct vbdev_ocf_base base; - char cache_name[OCF_CACHE_NAME_SIZE]; - struct vbdev_ocf_mngt_ctx * mngt_ctx; - - STAILQ_ENTRY(vbdev_ocf_core) waitlist_entry; + STAILQ_ENTRY(vbdev_ocf_core) waitlist_entry; }; +/* Core IO channel context. */ struct vbdev_ocf_core_io_channel_ctx { + /* OCF queue. */ ocf_queue_t queue; + /* Cache base bdev IO channel. */ struct spdk_io_channel * cache_ch; + /* Core base bdev IO channel. */ struct spdk_io_channel * core_ch; + /* Registered poller. */ struct spdk_poller * poller; + /* Thread on which poller was registered and channels were opened. */ struct spdk_thread * thread; /* Currently kept only for its name used in debug log. */ ocf_core_t core; // rm ? }; +/* Return core name string from given core context. */ +char *vbdev_ocf_core_get_name(struct vbdev_ocf_core *core_ctx); +/* Get core from wait list by its name. */ +struct vbdev_ocf_core *vbdev_ocf_core_waitlist_get_by_name(const char *core_name); +/* Check if core base is already attached. */ +bool vbdev_ocf_core_is_base_attached(struct vbdev_ocf_core *core_ctx); +/* Check if core was added to cache from metadata during cache load. */ +bool vbdev_ocf_core_is_loaded(const char *core_name); + +/* Create core context and fill config. */ int vbdev_ocf_core_create(struct vbdev_ocf_core **out, const char *core_name, const char *cache_name); +/* Free core context. */ void vbdev_ocf_core_destroy(struct vbdev_ocf_core *core_ctx); +/* Open, claim and create IO channel for base bdev. */ int vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core_ctx, const char *base_name); +/* Close base bdev and IO channel. */ void vbdev_ocf_core_base_detach(struct vbdev_ocf_core *core_ctx); +/* Register core in SPDK bdev layer for usage. */ int vbdev_ocf_core_register(ocf_core_t core); +/* Unregister core from SPDK bdev layer. */ int vbdev_ocf_core_unregister(struct vbdev_ocf_core *core_ctx, spdk_bdev_unregister_cb cb_fn, void *cb_arg); - -char *vbdev_ocf_core_get_name(struct vbdev_ocf_core *core_ctx); -struct vbdev_ocf_core *vbdev_ocf_core_waitlist_get_by_name(const char *core_name); -bool vbdev_ocf_core_is_base_attached(struct vbdev_ocf_core *core_ctx); -bool vbdev_ocf_core_is_loaded(const char *core_name); +/* Add all cores from wait list assigned to given cache. */ +void vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache); #endif From 1dc8f8e21065d13624f33e49962b3d3adb5435a9 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Fri, 13 Jun 2025 12:14:46 +0200 Subject: [PATCH 10/34] Parameterize tests Change-Id: I1d4ecf9555f9996d236c4985ce460a8839b02e2f Signed-off-by: Rafal Stefanowski --- test/ocf/common.sh | 4 + test/ocf/management/detach-attach.sh | 503 ++---- test/ocf/management/hotremove.sh | 1652 +++++------------- test/ocf/management/start-stop/basic.sh | 314 ++-- test/ocf/management/start-stop/examine.sh | 564 +++--- test/ocf/management/start-stop/incomplete.sh | 224 +-- test/ocf/management/start-stop/load.sh | 735 +++----- test/ocf/ocf.sh | 2 +- 8 files changed, 1325 insertions(+), 2673 deletions(-) diff --git a/test/ocf/common.sh b/test/ocf/common.sh index 28a9f2b2a4f..cec6c4bf623 100644 --- a/test/ocf/common.sh +++ b/test/ocf/common.sh @@ -151,22 +151,26 @@ __check_caches_detached() { __check_cores_empty() { $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 0' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 0' } __check_cores_attached() { $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 3' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].loading | not' } __check_cores_detached() { $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 3' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached | not' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].loading | not' } __check_cores_loading() { $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 3' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached | not' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].loading' } diff --git a/test/ocf/management/detach-attach.sh b/test/ocf/management/detach-attach.sh index 79e9f1f4ea3..977ebbed0bf 100755 --- a/test/ocf/management/detach-attach.sh +++ b/test/ocf/management/detach-attach.sh @@ -12,358 +12,169 @@ source "$rootdir/test/ocf/common.sh" # caches only: -start_spdk -create_caches -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -attach_caches -__check_caches_attached -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -attach_caches -__check_caches_attached -stop_caches -__check_caches_empty -stop_spdk +for attach_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + start_caches + __check_caches_attached + detach_caches + __check_caches_detached + if [ $attach_caches = true ]; then + attach_caches + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done # add cores before detaching caches: -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -detach_caches -__check_caches_detached -__check_cores_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -detach_caches -__check_caches_detached -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -detach_caches -__check_caches_detached -__check_cores_attached -attach_caches -__check_caches_attached -__check_cores_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -detach_caches -__check_caches_detached -__check_cores_attached -attach_caches -__check_caches_attached -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk +for attach_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + detach_caches + __check_caches_detached + __check_cores_attached + if [ $attach_caches = true ]; then + attach_caches + __check_caches_attached + __check_cores_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done # add cores after detaching caches: -start_spdk -create_caches -create_cores -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -add_cores -__check_caches_detached -__check_cores_waitlist_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -add_cores -__check_caches_detached -__check_cores_waitlist_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -add_cores -__check_caches_detached -__check_cores_waitlist_attached -attach_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -add_cores -__check_caches_detached -__check_cores_waitlist_attached -attach_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +for attach_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + __check_caches_attached + detach_caches + __check_caches_detached + add_cores + __check_caches_detached + __check_cores_waitlist_attached + if [ $attach_caches = true ]; then + attach_caches + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done # add cores before and remove after detaching caches: -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -detach_caches -__check_caches_detached -__check_cores_attached -remove_cores -__check_caches_detached -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -detach_caches -__check_caches_detached -__check_cores_attached -remove_cores -__check_caches_detached -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -detach_caches -__check_caches_detached -__check_cores_attached -remove_cores -__check_caches_detached -__check_cores_empty -attach_caches -__check_caches_attached -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -detach_caches -__check_caches_detached -__check_cores_attached -remove_cores -__check_caches_detached -__check_cores_empty -attach_caches -__check_caches_attached -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk - -# add cores after detaching caches and remove (before/after/without) attaching: - -start_spdk -create_caches -create_cores -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_attached -remove_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_attached -remove_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -add_cores -__check_caches_detached -__check_cores_waitlist_attached -remove_cores -__check_caches_detached -__check_cores_waitlist_empty -attach_caches -__check_caches_attached -__check_cores_empty -__check_cores_waitlist_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -add_cores -__check_caches_detached -__check_cores_waitlist_attached -remove_cores -__check_caches_detached -__check_cores_waitlist_empty -attach_caches -__check_caches_attached -__check_cores_empty -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_attached -attach_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -remove_cores -__check_caches_attached -__check_cores_empty -__check_cores_waitlist_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -__check_caches_attached -detach_caches -__check_caches_detached -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_attached -attach_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -remove_cores -__check_caches_attached -__check_cores_empty -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +for attach_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + detach_caches + __check_caches_detached + __check_cores_attached + remove_cores + __check_caches_detached + __check_cores_empty + if [ $attach_caches = true ]; then + attach_caches + __check_caches_attached + __check_cores_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done + +# add cores after detaching caches and remove before/without attaching: + +for attach_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + __check_caches_attached + detach_caches + __check_caches_detached + add_cores + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_attached + remove_cores + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_empty + if [ $attach_caches = true ]; then + attach_caches + __check_caches_attached + __check_cores_empty + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done + +# add cores after detaching caches and remove after attaching: + +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + __check_caches_attached + detach_caches + __check_caches_detached + add_cores + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_attached + attach_caches + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + remove_cores + __check_caches_attached + __check_cores_empty + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done diff --git a/test/ocf/management/hotremove.sh b/test/ocf/management/hotremove.sh index 7b209a1ddc6..605892d4c6a 100755 --- a/test/ocf/management/hotremove.sh +++ b/test/ocf/management/hotremove.sh @@ -25,51 +25,25 @@ __check_caches_detached_all_but_first() { } __check_cores_detached_only_first() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[0].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[0].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[1].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[1].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[2].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[2].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[0].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[0].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[1].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[1].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[2].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[2].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[0].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[0].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[1].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[1].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[2].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[2].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[0].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[0].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[1].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[1].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[2].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[2].loading | not' } __check_cores_detached_all_but_first() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[0].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[0].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[1].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[1].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[2].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores[2].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[0].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[0].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[1].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[1].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[2].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[2].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[0].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[0].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[1].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[1].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[2].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[2].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[0].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[0].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[1].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[1].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[2].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[2].loading | not' } __check_cores_waitlist_detached_only_first() { @@ -100,1248 +74,458 @@ __check_cores_waitlist_detached_all_but_first() { # cores in waitlist: -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -destroy_cores -__check_cores_waitlist_detached -stop_spdk - -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -destroy_cores -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -stop_spdk - -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -destroy_cores -__check_cores_waitlist_detached -remove_cores -__check_cores_waitlist_empty -stop_spdk - -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -destroy_cores -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -remove_cores -__check_cores_waitlist_empty -stop_spdk - -# cores in waitlist (destroying only first): - -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 +for remove_cores in false true; do +for create_cores in false true; do + start_spdk + create_cores + add_cores + __check_cores_waitlist_attached + destroy_cores + __check_cores_waitlist_detached + if [ $create_cores = true ]; then + create_cores + __check_cores_waitlist_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + stop_spdk done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_detached_only_first -stop_spdk - -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 -done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_detached_only_first -for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_attached -stop_spdk -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 -done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_detached_only_first -remove_cores -__check_cores_waitlist_empty -stop_spdk +# cores in waitlist (destroying only first): -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 +for remove_cores in false true; do +for create_cores in false true; do + start_spdk + create_cores + add_cores + __check_cores_waitlist_attached + for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_cores_waitlist_detached_only_first + if [ $create_cores = true ]; then + for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_cores_waitlist_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + stop_spdk done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_detached_only_first -for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_attached -remove_cores -__check_cores_waitlist_empty -stop_spdk # cores in waitlist (destroying all but first): -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 -done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_detached_all_but_first -stop_spdk - -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 +for remove_cores in false true; do +for create_cores in false true; do + start_spdk + create_cores + add_cores + __check_cores_waitlist_attached + for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_cores_waitlist_detached_all_but_first + if [ $create_cores = true ]; then + for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 + $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_cores_waitlist_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + stop_spdk done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_detached_all_but_first -for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 - $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_attached -stop_spdk -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 -done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_detached_all_but_first -remove_cores -__check_cores_waitlist_empty -stop_spdk +# caches only: -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 +for create_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + start_caches + __check_caches_attached + destroy_caches + __check_caches_detached + if [ $create_caches = true ]; then + create_caches + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_detached_all_but_first -for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 - $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_cores_waitlist_attached -remove_cores -__check_cores_waitlist_empty -stop_spdk - -# caches only: - -start_spdk -create_caches -start_caches -__check_caches_attached -destroy_caches -__check_caches_detached -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -destroy_caches -__check_caches_detached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -destroy_caches -__check_caches_detached -create_caches -__check_caches_attached -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -destroy_caches -__check_caches_detached -create_caches -__check_caches_attached -stop_caches -__check_caches_empty -stop_spdk # caches only (destroying only first): -start_spdk -create_caches -start_caches -__check_caches_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -$rpc_py bdev_malloc_create -b Cache_dev1 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -$rpc_py bdev_malloc_create -b Cache_dev1 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -stop_caches -__check_caches_empty -stop_spdk +for create_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + start_caches + __check_caches_attached + $rpc_py bdev_malloc_delete Cache_dev1 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_detached_only_first + if [ $create_caches = true ]; then + $rpc_py bdev_malloc_create -b Cache_dev1 100 512 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done # caches only (destroying all but first): -start_spdk -create_caches -start_caches -__check_caches_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -$rpc_py bdev_malloc_create -b Cache_dev2 100 512 -$rpc_py bdev_malloc_create -b Cache_dev3 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -stop_spdk - -start_spdk -create_caches -start_caches -__check_caches_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -$rpc_py bdev_malloc_create -b Cache_dev2 100 512 -$rpc_py bdev_malloc_create -b Cache_dev3 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -stop_caches -__check_caches_empty -stop_spdk +for create_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + start_caches + __check_caches_attached + $rpc_py bdev_malloc_delete Cache_dev2 + $rpc_py bdev_malloc_delete Cache_dev3 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_detached_all_but_first + if [ $create_caches = true ]; then + $rpc_py bdev_malloc_create -b Cache_dev2 100 512 + $rpc_py bdev_malloc_create -b Cache_dev3 100 512 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done # hotremove caches: -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -__check_cores_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -__check_cores_attached -create_caches -__check_caches_attached -__check_cores_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -__check_cores_attached -create_caches -__check_caches_attached -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -__check_cores_attached -remove_cores -__check_caches_detached -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -__check_cores_attached -remove_cores -__check_caches_detached -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -__check_cores_attached -remove_cores -__check_caches_detached -__check_cores_empty -create_caches -__check_caches_attached -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -__check_cores_attached -remove_cores -__check_caches_detached -__check_cores_empty -create_caches -__check_caches_attached -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk +for remove_cores in false true; do +for create_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + destroy_caches + __check_caches_detached + __check_cores_attached + if [ $remove_cores = true ]; then + remove_cores + __check_caches_detached + __check_cores_empty + fi + if [ $create_caches = true ]; then + create_caches + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done +done # hotremove caches (destroying only first): -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -__check_cores_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -__check_cores_attached -$rpc_py bdev_malloc_create -b Cache_dev1 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -__check_cores_attached -$rpc_py bdev_malloc_create -b Cache_dev1 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -__check_cores_attached -remove_cores -__check_caches_detached_only_first -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -__check_cores_attached -remove_cores -__check_caches_detached_only_first -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -__check_cores_attached -remove_cores -__check_caches_detached_only_first -__check_cores_empty -$rpc_py bdev_malloc_create -b Cache_dev1 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev1 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_only_first -__check_cores_attached -remove_cores -__check_caches_detached_only_first -__check_cores_empty -$rpc_py bdev_malloc_create -b Cache_dev1 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk +for remove_cores in false true; do +for create_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + $rpc_py bdev_malloc_delete Cache_dev1 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_detached_only_first + __check_cores_attached + if [ $remove_cores = true ]; then + remove_cores + __check_caches_detached_only_first + __check_cores_empty + fi + if [ $create_caches = true ]; then + $rpc_py bdev_malloc_create -b Cache_dev1 100 512 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done +done # hotremove caches (destroying all but first): -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -__check_cores_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -__check_cores_attached -$rpc_py bdev_malloc_create -b Cache_dev2 100 512 -$rpc_py bdev_malloc_create -b Cache_dev3 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -__check_cores_attached -$rpc_py bdev_malloc_create -b Cache_dev2 100 512 -$rpc_py bdev_malloc_create -b Cache_dev3 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -__check_cores_attached -remove_cores -__check_caches_detached_all_but_first -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -__check_cores_attached -remove_cores -__check_caches_detached_all_but_first -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -__check_cores_attached -remove_cores -__check_caches_detached_all_but_first -__check_cores_empty -$rpc_py bdev_malloc_create -b Cache_dev2 100 512 -$rpc_py bdev_malloc_create -b Cache_dev3 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -$rpc_py bdev_malloc_delete Cache_dev2 -$rpc_py bdev_malloc_delete Cache_dev3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_detached_all_but_first -__check_cores_attached -remove_cores -__check_caches_detached_all_but_first -__check_cores_empty -$rpc_py bdev_malloc_create -b Cache_dev2 100 512 -$rpc_py bdev_malloc_create -b Cache_dev3 100 512 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk +for remove_cores in false true; do +for create_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + $rpc_py bdev_malloc_delete Cache_dev2 + $rpc_py bdev_malloc_delete Cache_dev3 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_detached_all_but_first + __check_cores_attached + if [ $remove_cores = true ]; then + remove_cores + __check_caches_detached_all_but_first + __check_cores_empty + fi + if [ $create_caches = true ]; then + $rpc_py bdev_malloc_create -b Cache_dev2 100 512 + $rpc_py bdev_malloc_create -b Cache_dev3 100 512 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done +done # hotremove cores: -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_cores -__check_caches_attached -__check_cores_detached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_cores -__check_caches_attached -__check_cores_detached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_cores -__check_caches_attached -__check_cores_detached -create_cores -__check_caches_attached -__check_cores_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_cores -__check_caches_attached -__check_cores_detached -create_cores -__check_caches_attached -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_cores -__check_caches_attached -__check_cores_detached -remove_cores -__check_caches_attached -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_cores -__check_caches_attached -__check_cores_detached -remove_cores -__check_caches_attached -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk - -# hotremove cores (destroying only first): - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 -done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_only_first -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 +for create_cores in false true; do +for remove_cores in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + destroy_cores + __check_caches_attached + __check_cores_detached + if [ $create_cores = true ]; then + create_cores + __check_caches_attached + __check_cores_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_caches_attached + __check_cores_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_only_first -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_only_first -for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_attached -stop_spdk -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 -done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_only_first -for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 -done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk +# hotremove cores (destroying only first): -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 +for create_cores in false true; do +for remove_cores in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + __check_cores_detached_only_first + if [ $create_cores = true ]; then + for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + __check_cores_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_caches_attached + __check_cores_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_only_first -remove_cores -__check_caches_attached -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_only_first -remove_cores -__check_caches_attached -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk # hotremove cores (destroying all but first): -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 -done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_all_but_first -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 +for create_cores in false true; do +for remove_cores in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + __check_cores_detached_all_but_first + if [ $create_cores = true ]; then + for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 + $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + __check_cores_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_caches_attached + __check_cores_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_all_but_first -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_all_but_first -for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 - $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_attached -stop_spdk -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 +# hotremove caches and cores: + +for create_caches_and_cores in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + destroy_caches + __check_caches_detached + destroy_cores + __check_cores_detached + if [ $create_caches_and_cores = true ]; then + create_caches + __check_caches_attached + create_cores + __check_cores_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_all_but_first -for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 - $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 +# hotremove caches and cores, then manually remove cores: + +for create_caches in false true; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + destroy_caches + __check_caches_detached + destroy_cores + __check_cores_detached + remove_cores + __check_caches_detached + __check_cores_empty + if [ $create_caches = true ]; then + create_caches + __check_caches_attached + __check_cores_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_all_but_first -remove_cores -__check_caches_attached -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 done -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -__check_caches_attached -__check_cores_detached_all_but_first -remove_cores -__check_caches_attached -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk - -# hotremove caches and cores: -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -destroy_cores -__check_cores_detached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -destroy_cores -__check_cores_detached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -destroy_cores -__check_cores_detached -create_caches -__check_caches_attached -create_cores -__check_cores_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -destroy_cores -__check_cores_detached -create_caches -__check_caches_attached -create_cores -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -destroy_cores -__check_cores_detached -remove_cores -__check_caches_detached -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -destroy_cores -__check_cores_detached -remove_cores -__check_caches_detached -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -destroy_cores -__check_cores_detached -remove_cores -__check_caches_detached -__check_cores_empty -create_caches -__check_caches_attached -__check_cores_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -destroy_cores -__check_cores_detached -remove_cores -__check_caches_detached -__check_cores_empty -create_caches -__check_caches_attached -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk - -# hotremove caches and cores (inverse create after destroy order) - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -destroy_cores -__check_cores_detached -create_cores -__check_cores_attached -create_caches -__check_caches_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -destroy_caches -__check_caches_detached -destroy_cores -__check_cores_detached -create_cores -__check_cores_attached -create_caches -__check_caches_attached -stop_caches -__check_caches_empty -stop_spdk +# hotremove caches and cores (inverse create after destroy order): + +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + destroy_caches + __check_caches_detached + destroy_cores + __check_cores_detached + create_cores + __check_cores_attached + create_caches + __check_caches_attached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done diff --git a/test/ocf/management/start-stop/basic.sh b/test/ocf/management/start-stop/basic.sh index 61cfe7880a2..46c2cb1aa10 100755 --- a/test/ocf/management/start-stop/basic.sh +++ b/test/ocf/management/start-stop/basic.sh @@ -10,172 +10,158 @@ curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) rootdir=$(readlink -f $curdir/../../../..) source "$rootdir/test/ocf/common.sh" -# start caches and shutdown app: - -start_spdk -create_caches -start_caches -__check_caches_attached -stop_spdk - -# add cores and shutdown app: - -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -stop_spdk - -# start and stop caches: - -start_spdk -create_caches -start_caches -__check_caches_attached -stop_caches -stop_spdk - -# add and remove cores: - -start_spdk -create_cores -add_cores -__check_cores_waitlist_attached -remove_cores -__check_cores_waitlist_empty -stop_spdk - -# start caches, add cores and shutdown app: - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -stop_spdk - -# add cores, start caches and shutdown app: - -start_spdk -create_caches -create_cores -add_cores -__check_cores_waitlist_attached -start_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk - -# start caches, add cores and stop caches: - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -stop_caches -__check_caches_empty -__check_cores_waitlist_empty -stop_spdk - -# add cores, start caches and stop caches: - -start_spdk -create_caches -create_cores -add_cores -__check_cores_waitlist_attached -start_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -__check_cores_waitlist_empty -stop_spdk +# start caches: + +for stop_caches in false true; do + start_spdk + create_caches + start_caches + __check_caches_attached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done + +# add cores: + +for remove_cores in false true; do + start_spdk + create_cores + add_cores + __check_cores_waitlist_attached + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + stop_spdk +done + +# start caches, then add cores: + +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + __check_cores_waitlist_empty + fi + stop_spdk +done + +# add cores, then start caches: + +for stop_caches in false true; do + start_spdk + create_caches + create_cores + add_cores + __check_cores_waitlist_attached + start_caches + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + __check_cores_waitlist_empty + fi + stop_spdk +done # restart all caches with cores: -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -stop_caches -__check_caches_empty -start_caches -__check_caches_attached -add_cores -__check_cores_attached -stop_caches -__check_caches_empty -__check_cores_waitlist_empty -stop_spdk - -# remove first core and shutdown app: - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_ocf_remove_core Ocf_core$i-1 Ocf_cache$i +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + stop_caches + __check_caches_empty + start_caches + __check_caches_attached + add_cores + __check_cores_attached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + __check_cores_waitlist_empty + fi + stop_spdk +done + +# remove cores: + +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + remove_cores + __check_cores_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk done -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 2' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached' -stop_spdk - -# remove all cores but first and shutdown app: - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -for i in {1..3}; do - $rpc_py bdev_ocf_remove_core Ocf_core$i-2 Ocf_cache$i - $rpc_py bdev_ocf_remove_core Ocf_core$i-3 Ocf_cache$i + +# remove first core: + +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + for i in {1..3}; do + $rpc_py bdev_ocf_remove_core Ocf_core$i-1 Ocf_cache$i + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 2' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 2' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached' + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done + +# remove all cores but first: + +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + for i in {1..3}; do + $rpc_py bdev_ocf_remove_core Ocf_core$i-2 Ocf_cache$i + $rpc_py bdev_ocf_remove_core Ocf_core$i-3 Ocf_cache$i + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 1' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 1' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached' + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk done -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 1' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached' -stop_spdk - -# remove all cores and shutdown app: - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -remove_cores -__check_cores_empty -stop_spdk - -# remove all cores and stop all caches: - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -remove_cores -__check_cores_empty -stop_caches -__check_caches_empty -stop_spdk diff --git a/test/ocf/management/start-stop/examine.sh b/test/ocf/management/start-stop/examine.sh index 423b480e211..80448cc9b4a 100755 --- a/test/ocf/management/start-stop/examine.sh +++ b/test/ocf/management/start-stop/examine.sh @@ -10,352 +10,264 @@ curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) rootdir=$(readlink -f $curdir/../../../..) source "$rootdir/test/ocf/common.sh" -# examine partial configuration (not all devices present): +# found cache device (cache only): -start_spdk -start_caches -__check_caches_detached -create_caches -__check_caches_attached -stop_spdk +for stop_caches in false true; do + start_spdk + start_caches + __check_caches_detached + create_caches + __check_caches_attached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done -start_spdk -start_caches -__check_caches_detached -create_caches -__check_caches_attached -stop_caches -__check_caches_empty -stop_spdk +# found cache device (cache with core): -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_waitlist_detached -create_caches -__check_caches_attached -__check_cores_waitlist_detached -stop_spdk +for stop_caches in false true; do + start_spdk + start_caches + add_cores + __check_caches_detached + __check_cores_waitlist_detached + create_caches + __check_caches_attached + __check_cores_waitlist_detached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + __check_cores_waitlist_detached + fi + stop_spdk +done -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_waitlist_detached -create_caches -__check_caches_attached -__check_cores_waitlist_detached -stop_caches -__check_caches_empty -__check_cores_waitlist_detached -stop_spdk +# found core device (cores only): -start_spdk -add_cores -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -stop_spdk +for remove_cores in false true; do + start_spdk + add_cores + __check_cores_waitlist_detached + create_cores + __check_cores_waitlist_attached + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + stop_spdk +done -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_cores -__check_cores_empty -__check_cores_waitlist_attached -stop_spdk +# found core device (cache with core): -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_cores -__check_cores_empty -__check_cores_waitlist_attached -stop_caches -__check_caches_empty -__check_cores_waitlist_attached -stop_spdk +for stop_caches in false true; do + start_spdk + start_caches + add_cores + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_detached + create_cores + __check_cores_empty + __check_cores_waitlist_attached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + __check_cores_waitlist_attached + fi + stop_spdk +done -start_spdk -add_cores -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -start_caches -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_attached -stop_spdk +# found core device (cache with core, inverse add order): -start_spdk -add_cores -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -start_caches -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_attached -stop_caches -__check_caches_empty -__check_cores_waitlist_attached -stop_spdk +for stop_caches in false true; do + start_spdk + add_cores + start_caches + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_detached + create_cores + __check_cores_empty + __check_cores_waitlist_attached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + __check_cores_waitlist_attached + fi + stop_spdk +done -start_spdk -add_cores -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -start_caches -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_attached -remove_cores -__check_cores_waitlist_empty -stop_spdk +# found core device, then start cache: -start_spdk -add_cores -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -start_caches -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_attached -remove_cores -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +for remove_cores in false true; do +for stop_caches in false true; do + start_spdk + add_cores + __check_cores_waitlist_detached + create_cores + __check_cores_waitlist_attached + start_caches + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_attached + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done -start_spdk -add_cores -start_caches -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -stop_spdk +# found cache and then core device: -start_spdk -add_cores -start_caches -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -stop_caches -__check_caches_empty -__check_cores_waitlist_attached -stop_spdk +for stop_caches in false true; do + start_spdk + start_caches + add_cores + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_detached + create_caches + __check_caches_attached + __check_cores_empty + create_cores + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + __check_cores_waitlist_empty + fi + stop_spdk +done -# examine full configuration with inversing order: +# found core and then cache device: -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_caches -__check_caches_attached -__check_cores_empty -create_cores -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk +for stop_caches in false true; do + start_spdk + start_caches + add_cores + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_detached + create_cores + __check_cores_empty + __check_cores_waitlist_attached + create_caches + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + __check_cores_waitlist_empty + fi + stop_spdk +done -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_caches -__check_caches_attached -__check_cores_empty -create_cores -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -__check_cores_waitlist_empty -stop_spdk +# found cache and then core device (inverse add order): -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -create_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk +for stop_caches in false true; do + start_spdk + add_cores + start_caches + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_detached + create_caches + __check_caches_attached + __check_cores_empty + create_cores + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + __check_cores_waitlist_empty + fi + stop_spdk +done -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -create_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +# found core and then cache device (inverse add order): -start_spdk -add_cores -start_caches -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_caches -__check_caches_attached -__check_cores_empty -create_cores -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk +for stop_caches in false true; do + start_spdk + add_cores + start_caches + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_detached + create_cores + __check_cores_empty + __check_cores_waitlist_attached + create_caches + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + __check_cores_waitlist_empty + fi + stop_spdk +done -start_spdk -add_cores -start_caches -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_caches -__check_caches_attached -__check_cores_empty -create_cores -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +# found core device (not yet added) on detached cache: -start_spdk -add_cores -start_caches -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -create_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk +for stop_caches in false true; do + start_spdk + create_caches + start_caches + add_cores + __check_caches_attached + __check_cores_empty + __check_cores_waitlist_detached + detach_caches + __check_caches_detached + create_cores + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_attached + attach_caches + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done -start_spdk -add_cores -start_caches -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_detached -create_cores -__check_cores_waitlist_attached -create_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +# found core device (hotremoved) on detached cache: -# examine found core (not yet added or hotremoved) on detached cache: - -start_spdk -create_caches -start_caches -add_cores -__check_caches_attached -__check_cores_waitlist_detached -detach_caches -__check_caches_detached -create_cores -__check_caches_detached -__check_cores_waitlist_attached -attach_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk - -start_spdk -create_caches -start_caches -add_cores -__check_caches_attached -__check_cores_waitlist_detached -detach_caches -__check_caches_detached -create_cores -__check_caches_detached -__check_cores_waitlist_attached -attach_caches -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -detach_caches -__check_caches_detached -destroy_cores -__check_cores_detached -create_cores -__check_caches_detached -__check_cores_attached -attach_caches -__check_caches_attached -__check_cores_attached -stop_spdk - -start_spdk -create_caches -create_cores -start_caches -add_cores -__check_caches_attached -__check_cores_attached -detach_caches -__check_caches_detached -destroy_cores -__check_cores_detached -create_cores -__check_caches_detached -__check_cores_attached -attach_caches -__check_caches_attached -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + detach_caches + __check_caches_detached + destroy_cores + __check_cores_detached + create_cores + __check_caches_detached + __check_cores_attached + attach_caches + __check_caches_attached + __check_cores_attached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done diff --git a/test/ocf/management/start-stop/incomplete.sh b/test/ocf/management/start-stop/incomplete.sh index 283c8acaf72..525af974dda 100755 --- a/test/ocf/management/start-stop/incomplete.sh +++ b/test/ocf/management/start-stop/incomplete.sh @@ -12,161 +12,93 @@ source "$rootdir/test/ocf/common.sh" # incomplete caches: -start_spdk -start_caches -__check_caches_detached -stop_spdk - -start_spdk -start_caches -__check_caches_detached -stop_caches -__check_caches_empty -stop_spdk +for stop_caches in false true; do + start_spdk + start_caches + __check_caches_detached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done # incomplete cores: -start_spdk -add_cores -__check_cores_waitlist_detached -stop_spdk - -start_spdk -add_cores -__check_cores_waitlist_detached -remove_cores -__check_cores_waitlist_empty -stop_spdk +for remove_cores in false true; do + start_spdk + add_cores + __check_cores_waitlist_detached + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + stop_spdk +done # incomplete caches and incomplete cores: -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_waitlist_detached -stop_spdk - -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_waitlist_detached -stop_caches -__check_caches_empty -__check_cores_waitlist_detached -stop_spdk - -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_waitlist_detached -remove_cores -__check_cores_waitlist_empty -stop_spdk - -start_spdk -start_caches -add_cores -__check_caches_detached -__check_cores_waitlist_detached -remove_cores -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +for remove_cores in false true; do +for stop_caches in false true; do + start_spdk + start_caches + add_cores + __check_caches_detached + __check_cores_waitlist_detached + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done # complete caches and incomplete cores: -start_spdk -create_caches -start_caches -add_cores -__check_caches_attached -__check_cores_empty -__check_cores_waitlist_detached -stop_spdk - -start_spdk -create_caches -start_caches -add_cores -__check_caches_attached -__check_cores_empty -__check_cores_waitlist_detached -stop_caches -__check_caches_empty -__check_cores_waitlist_detached -stop_spdk - -start_spdk -create_caches -start_caches -add_cores -__check_caches_attached -__check_cores_empty -__check_cores_waitlist_detached -remove_cores -__check_cores_waitlist_empty -stop_spdk - -start_spdk -create_caches -start_caches -add_cores -__check_caches_attached -__check_cores_empty -__check_cores_waitlist_detached -remove_cores -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +for remove_cores in false true; do +for stop_caches in false true; do + start_spdk + create_caches + start_caches + add_cores + __check_caches_attached + __check_cores_empty + __check_cores_waitlist_detached + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done # incomplete caches and complete cores: -start_spdk -create_cores -start_caches -__check_caches_detached -add_cores -__check_cores_empty -__check_cores_waitlist_attached -stop_spdk - -start_spdk -create_cores -start_caches -__check_caches_detached -add_cores -__check_cores_empty -__check_cores_waitlist_attached -stop_caches -__check_caches_empty -__check_cores_waitlist_attached -stop_spdk - -start_spdk -create_cores -start_caches -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_attached -remove_cores -__check_cores_waitlist_empty -stop_spdk - -start_spdk -create_cores -start_caches -add_cores -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_attached -remove_cores -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +for remove_cores in false true; do +for stop_caches in false true; do + start_spdk + create_cores + start_caches + add_cores + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_attached + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done diff --git a/test/ocf/management/start-stop/load.sh b/test/ocf/management/start-stop/load.sh index 6d3a82f3766..41959d251e6 100755 --- a/test/ocf/management/start-stop/load.sh +++ b/test/ocf/management/start-stop/load.sh @@ -50,482 +50,305 @@ create_caches_with_metadata_with_cores_persistent() { destroy_cores } -# load caches without cores: +# load caches without cores in metadata: -start_spdk -create_caches_with_metadata -start_caches_try_load -__check_caches_attached -stop_spdk - -start_spdk -create_caches_with_metadata -start_caches_try_load -__check_caches_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches_with_metadata -create_cores -start_caches_try_load -__check_caches_attached -__check_cores_empty -add_cores -__check_cores_attached -stop_spdk - -start_spdk -create_caches_with_metadata -create_cores -start_caches_try_load -__check_caches_attached -__check_cores_empty -add_cores -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk - -# load caches with cores: - -start_spdk -create_caches_with_metadata_with_cores -start_caches_try_load -__check_caches_attached -__check_cores_loading -stop_spdk - -start_spdk -create_caches_with_metadata_with_cores -start_caches_try_load -__check_caches_attached -__check_cores_loading -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches_with_metadata_with_cores -create_cores -start_caches_try_load -__check_caches_attached -__check_cores_loading -stop_spdk - -start_spdk -create_caches_with_metadata_with_cores -create_cores -start_caches_try_load -__check_caches_attached -__check_cores_loading -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches_with_metadata_with_cores -create_cores -start_caches_try_load -__check_caches_attached -__check_cores_loading -add_cores -__check_cores_attached -stop_spdk - -start_spdk -create_caches_with_metadata_with_cores -create_cores -start_caches_try_load -__check_caches_attached -__check_cores_loading -add_cores -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk - -start_spdk -create_caches_with_metadata_with_cores -add_cores -__check_cores_waitlist_detached -start_caches_try_load -__check_caches_attached -__check_cores_loading -__check_cores_waitlist_detached -stop_spdk - -start_spdk -create_caches_with_metadata_with_cores -add_cores -__check_cores_waitlist_detached -start_caches_try_load -__check_caches_attached -__check_cores_loading -__check_cores_waitlist_detached -stop_caches -__check_caches_empty -stop_spdk - -# examine found cache device with metadata: - -start_spdk -create_caches_with_metadata_persistent -destroy_caches_persistent -start_caches_try_load -__check_caches_detached -create_caches_persistent -__check_caches_attached -stop_spdk - -start_spdk -create_caches_with_metadata_persistent -destroy_caches_persistent -start_caches_try_load -__check_caches_detached -create_caches_persistent -__check_caches_attached -stop_caches -__check_caches_empty -stop_spdk +for add_cores in false true; do +for stop_caches in false true; do + start_spdk + create_caches_with_metadata + create_cores + start_caches_try_load + __check_caches_attached + __check_cores_empty + __check_cores_waitlist_empty + if [ $add_cores = true ]; then + add_cores + __check_cores_attached + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done -start_spdk -create_caches_with_metadata_with_cores_persistent -destroy_caches_persistent -start_caches_try_load -__check_caches_detached -__check_cores_empty -create_caches_persistent -__check_caches_attached -__check_cores_loading -stop_spdk +# load caches with cores in metadata: -start_spdk -create_caches_with_metadata_with_cores_persistent -destroy_caches_persistent -start_caches_try_load -__check_caches_detached -__check_cores_empty -create_caches_persistent -__check_caches_attached -__check_cores_loading -stop_caches -__check_caches_empty -stop_spdk +for add_cores in false true; do +for stop_caches in false true; do + start_spdk + create_caches_with_metadata_with_cores + create_cores + start_caches_try_load + __check_caches_attached + __check_cores_loading + __check_cores_waitlist_empty + if [ $add_cores = true ]; then + add_cores + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done -start_spdk -create_caches_with_metadata_with_cores_persistent -create_cores -destroy_caches_persistent -start_caches_try_load -__check_caches_detached -__check_cores_empty -create_caches_persistent -__check_caches_attached -__check_cores_loading -stop_spdk +# load caches with cores in metadata (add cores first): -start_spdk -create_caches_with_metadata_with_cores_persistent -create_cores -destroy_caches_persistent -start_caches_try_load -__check_caches_detached -__check_cores_empty -create_caches_persistent -__check_caches_attached -__check_cores_loading -stop_caches -__check_caches_empty -stop_spdk +for stop_caches in false true; do + start_spdk + create_caches_with_metadata_with_cores + add_cores + __check_cores_waitlist_detached + start_caches_try_load + __check_caches_attached + __check_cores_loading + __check_cores_waitlist_detached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done -start_spdk -create_caches_with_metadata_with_cores_persistent -create_cores -destroy_caches_persistent -start_caches_try_load -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_empty -create_caches_persistent -__check_caches_attached -__check_cores_loading -__check_cores_waitlist_empty -add_cores -__check_cores_attached -stop_spdk +# examine found cache device with metadata (without cores in metadata): -start_spdk -create_caches_with_metadata_with_cores_persistent -create_cores -destroy_caches_persistent -start_caches_try_load -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_empty -create_caches_persistent -__check_caches_attached -__check_cores_loading -__check_cores_waitlist_empty -add_cores -__check_cores_attached -stop_caches -__check_caches_empty -stop_spdk +for stop_caches in false true; do + start_spdk + create_caches_with_metadata_persistent + destroy_caches_persistent + start_caches_try_load + __check_caches_detached + create_caches_persistent + __check_caches_attached + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done -start_spdk -create_caches_with_metadata_with_cores_persistent -create_cores -destroy_caches_persistent -start_caches_try_load -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_empty -add_cores -__check_cores_empty -__check_cores_waitlist_attached -create_caches_persistent -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk +# examine found cache device with metadata (with cores in metadata): -start_spdk -create_caches_with_metadata_with_cores_persistent -create_cores -destroy_caches_persistent -start_caches_try_load -__check_caches_detached -__check_cores_empty -__check_cores_waitlist_empty -add_cores -__check_cores_empty -__check_cores_waitlist_attached -create_caches_persistent -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +for stop_caches in false true; do + start_spdk + create_caches_with_metadata_with_cores_persistent + create_cores + destroy_caches_persistent + start_caches_try_load + __check_caches_detached + __check_cores_empty + create_caches_persistent + __check_caches_attached + __check_cores_loading + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done -start_spdk -create_caches_with_metadata_with_cores_persistent -create_cores -destroy_caches_persistent -add_cores -__check_cores_waitlist_attached -start_caches_try_load -__check_caches_detached -__check_cores_empty -create_caches_persistent -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk +# examine found cache device with metadata (with cores in metadata, added at different stages): -start_spdk -create_caches_with_metadata_with_cores_persistent -create_cores -destroy_caches_persistent -add_cores -__check_cores_waitlist_attached -start_caches_try_load -__check_caches_detached -__check_cores_empty -create_caches_persistent -__check_caches_attached -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +for stop_caches in false true; do + start_spdk + create_caches_with_metadata_with_cores_persistent + create_cores + destroy_caches_persistent + start_caches_try_load + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_empty + create_caches_persistent + __check_caches_attached + __check_cores_loading + __check_cores_waitlist_empty + add_cores + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done -# examine found core after cache load: +for stop_caches in false true; do + start_spdk + create_caches_with_metadata_with_cores_persistent + create_cores + destroy_caches_persistent + start_caches_try_load + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_empty + add_cores + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_attached + create_caches_persistent + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done -start_spdk -create_caches_with_metadata -start_caches_try_load -__check_caches_attached -__check_cores_empty -add_cores -__check_cores_empty -__check_cores_waitlist_detached -create_cores -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk +for stop_caches in false true; do + start_spdk + create_caches_with_metadata_with_cores_persistent + create_cores + destroy_caches_persistent + add_cores + __check_caches_empty + __check_cores_waitlist_attached + start_caches_try_load + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_attached + create_caches_persistent + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done -start_spdk -create_caches_with_metadata -start_caches_try_load -__check_caches_attached -__check_cores_empty -add_cores -__check_cores_empty -__check_cores_waitlist_detached -create_cores -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +# examine found core device after cache load (without cores in metadata): -start_spdk -create_caches_with_metadata_with_cores -start_caches_try_load -__check_caches_attached -__check_cores_loading -add_cores -__check_cores_loading -__check_cores_waitlist_detached -create_cores -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk +for stop_caches in false true; do + start_spdk + create_caches_with_metadata + start_caches_try_load + __check_caches_attached + __check_cores_empty + add_cores + __check_cores_empty + __check_cores_waitlist_detached + create_cores + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done -start_spdk -create_caches_with_metadata_with_cores -start_caches_try_load -__check_caches_attached -__check_cores_loading -add_cores -__check_cores_loading -__check_cores_waitlist_detached -create_cores -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +# examine found core device after cache load (with cores in metadata): -# examine found cache with metadata and core after cache load: +for stop_caches in false true; do + start_spdk + create_caches_with_metadata_with_cores + start_caches_try_load + __check_caches_attached + __check_cores_loading + add_cores + __check_cores_loading + __check_cores_waitlist_detached + create_cores + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done -start_spdk -create_caches_with_metadata_with_cores_persistent -destroy_caches_persistent -add_cores -__check_cores_waitlist_detached -start_caches_try_load -__check_caches_detached -__check_cores_empty -create_caches_persistent -__check_caches_attached -__check_cores_loading -__check_cores_waitlist_detached -create_cores -__check_cores_attached -__check_cores_waitlist_empty -stop_spdk +# examine found cache with metadata and core devices after cache load: -start_spdk -create_caches_with_metadata_with_cores_persistent -destroy_caches_persistent -add_cores -__check_cores_waitlist_detached -start_caches_try_load -__check_caches_detached -__check_cores_empty -create_caches_persistent -__check_caches_attached -__check_cores_loading -__check_cores_waitlist_detached -create_cores -__check_cores_attached -__check_cores_waitlist_empty -stop_caches -__check_caches_empty -stop_spdk +for stop_caches in false true; do + start_spdk + create_caches_with_metadata_with_cores_persistent + destroy_caches_persistent + add_cores + __check_cores_waitlist_detached + start_caches_try_load + __check_caches_detached + __check_cores_empty + create_caches_persistent + __check_caches_attached + __check_cores_loading + __check_cores_waitlist_detached + create_cores + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done # partially found cache and core devices: -start_spdk -create_caches_with_metadata_with_cores_persistent -destroy_caches_persistent -add_cores -__check_cores_waitlist_detached -start_caches_try_load -__check_caches_detached -__check_cores_empty -$rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[1]}" -d blk Cache_dev2 -$rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[2]}" -d blk Cache_dev3 -sleep 3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' -__check_cores_waitlist_detached -for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev2-$i 200 512 -done -sleep 3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' -$rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 6' -$rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[].base_attached | not' -stop_spdk - -start_spdk -create_caches_with_metadata_with_cores_persistent -destroy_caches_persistent -add_cores -__check_cores_waitlist_detached -start_caches_try_load -__check_caches_detached -__check_cores_empty -$rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[1]}" -d blk Cache_dev2 -$rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[2]}" -d blk Cache_dev3 -sleep 3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' -__check_cores_waitlist_detached -for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev2-$i 200 512 +for stop_caches in false true; do + start_spdk + create_caches_with_metadata_with_cores_persistent + destroy_caches_persistent + add_cores + __check_cores_waitlist_detached + start_caches_try_load + __check_caches_detached + __check_cores_empty + $rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[1]}" -d blk Cache_dev2 + $rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[2]}" -d blk Cache_dev3 + sleep 3 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores_count == 0' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores_count == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores_count == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' + __check_cores_waitlist_detached + for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev2-$i 200 512 + done + sleep 3 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores_count == 0' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores_count == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores_count == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 6' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[].base_attached | not' + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk done -sleep 3 -$rpc_py bdev_ocf_get_bdevs | jq -e '.' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' -$rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' -$rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 6' -$rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[].base_attached | not' -stop_caches -__check_caches_empty -stop_spdk diff --git a/test/ocf/ocf.sh b/test/ocf/ocf.sh index cb45074fbe6..d6bcc4b302a 100755 --- a/test/ocf/ocf.sh +++ b/test/ocf/ocf.sh @@ -12,4 +12,4 @@ rootdir=$(readlink -f $curdir/../..) source $rootdir/test/common/autotest_common.sh -run_test "ocf_management" "$curdir/management/run.sh" +run_test "management" "$curdir/management/run.sh" From 50b5214d8ec6f4b2fd29afc77d930f27dc992b63 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Fri, 11 Jul 2025 14:16:45 +0200 Subject: [PATCH 11/34] Implement configuration change RPCs Change-Id: I0049cda213a5d037819b877cd1279286a1a68841 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/utils.c | 112 +++- module/bdev/ocf/utils.h | 28 +- module/bdev/ocf/vbdev_ocf.c | 826 +++++++++++++++++++++++++++++- module/bdev/ocf/vbdev_ocf.h | 23 + module/bdev/ocf/vbdev_ocf_cache.c | 10 +- module/bdev/ocf/vbdev_ocf_cache.h | 38 +- module/bdev/ocf/vbdev_ocf_core.h | 5 + module/bdev/ocf/vbdev_ocf_rpc.c | 341 ++++++++++-- python/spdk/cli/bdev.py | 129 +++-- 9 files changed, 1377 insertions(+), 135 deletions(-) diff --git a/module/bdev/ocf/utils.c b/module/bdev/ocf/utils.c index f3ce5ace56e..c67a6cd70d3 100644 --- a/module/bdev/ocf/utils.c +++ b/module/bdev/ocf/utils.c @@ -4,9 +4,6 @@ * All rights reserved. */ -#include "spdk/stdinc.h" -#include "spdk/log.h" - #include "utils.h" static char *cache_modes[ocf_cache_mode_max] = { @@ -18,6 +15,17 @@ static char *cache_modes[ocf_cache_mode_max] = { [ocf_cache_mode_wo] = "wo", }; +static char *promotion_policies[ocf_promotion_max] = { + [ocf_promotion_always] = "always", + [ocf_promotion_nhit] = "nhit", +}; + +static char *cleaning_policies[ocf_cleaning_max] = { + [ocf_cleaning_nop] = "nop", + [ocf_cleaning_alru] = "alru", + [ocf_cleaning_acp] = "acp", +}; + static char *seqcutoff_policies[ocf_seq_cutoff_policy_max] = { [ocf_seq_cutoff_policy_always] = "always", [ocf_seq_cutoff_policy_full] = "full", @@ -25,13 +33,17 @@ static char *seqcutoff_policies[ocf_seq_cutoff_policy_max] = { }; ocf_cache_mode_t -ocf_get_cache_mode(const char *cache_mode) +vbdev_ocf_cachemode_get_by_name(const char *cache_mode_name) { - int i; + ocf_cache_mode_t cache_mode; - for (i = 0; i < ocf_cache_mode_max; i++) { - if (strcmp(cache_mode, cache_modes[i]) == 0) { - return i; + if (!cache_mode_name) { + return ocf_cache_mode_none; + } + + for (cache_mode = 0; cache_mode < ocf_cache_mode_max; cache_mode++) { + if (!strcmp(cache_mode_name, cache_modes[cache_mode])) { + return cache_mode; } } @@ -39,30 +51,92 @@ ocf_get_cache_mode(const char *cache_mode) } const char * -ocf_get_cache_modename(ocf_cache_mode_t mode) +vbdev_ocf_cachemode_get_name(ocf_cache_mode_t cache_mode) +{ + if (cache_mode > ocf_cache_mode_none && cache_mode < ocf_cache_mode_max) { + return cache_modes[cache_mode]; + } + + return NULL; +} + +ocf_promotion_t +vbdev_ocf_promotion_policy_get_by_name(const char *policy_name) { - if (mode > ocf_cache_mode_none && mode < ocf_cache_mode_max) { - return cache_modes[mode]; - } else { - return NULL; + ocf_promotion_t policy; + + if (!policy_name) { + return -EINVAL; } + + for (policy = 0; policy < ocf_promotion_max; policy++) + if (!strcmp(policy_name, promotion_policies[policy])) { + return policy; + } + + return -EINVAL; } -int -ocf_get_cache_line_size(ocf_cache_t cache) +const char * +vbdev_ocf_promotion_policy_get_name(ocf_promotion_t policy) { - return ocf_cache_get_line_size(cache) / KiB; + if (policy >= ocf_promotion_always && policy < ocf_promotion_max) { + return promotion_policies[policy]; + } + + return NULL; +} + +ocf_cleaning_t +vbdev_ocf_cleaning_policy_get_by_name(const char *policy_name) +{ + ocf_cleaning_t policy; + + if (!policy_name) { + return -EINVAL; + } + + for (policy = 0; policy < ocf_cleaning_max; policy++) + if (!strcmp(policy_name, cleaning_policies[policy])) { + return policy; + } + + return -EINVAL; +} + +const char * +vbdev_ocf_cleaning_policy_get_name(ocf_cleaning_t policy) +{ + if (policy >= ocf_cleaning_nop && policy < ocf_cleaning_max) { + return cleaning_policies[policy]; + } + + return NULL; } ocf_seq_cutoff_policy -ocf_get_seqcutoff_policy(const char *policy_name) +vbdev_ocf_seqcutoff_policy_get_by_name(const char *policy_name) { - int policy; + ocf_seq_cutoff_policy policy; + + if (!policy_name) { + return -EINVAL; + } for (policy = 0; policy < ocf_seq_cutoff_policy_max; policy++) if (!strcmp(policy_name, seqcutoff_policies[policy])) { return policy; } - return ocf_seq_cutoff_policy_max; + return -EINVAL; +} + +const char * +vbdev_ocf_seqcutoff_policy_get_name(ocf_seq_cutoff_policy policy) +{ + if (policy >= ocf_seq_cutoff_policy_always && policy < ocf_seq_cutoff_policy_max) { + return seqcutoff_policies[policy]; + } + + return NULL; } diff --git a/module/bdev/ocf/utils.h b/module/bdev/ocf/utils.h index 7a5003030fd..51c92fa9f84 100644 --- a/module/bdev/ocf/utils.h +++ b/module/bdev/ocf/utils.h @@ -9,12 +9,28 @@ #include -ocf_cache_mode_t ocf_get_cache_mode(const char *cache_mode); -const char *ocf_get_cache_modename(ocf_cache_mode_t mode); +/* Get OCF cache mode by its name. */ +ocf_cache_mode_t vbdev_ocf_cachemode_get_by_name(const char *cache_mode_name); -/* Get cache line size in KiB units */ -int ocf_get_cache_line_size(ocf_cache_t cache); +/* Get the name of OCF cache mode. */ +const char *vbdev_ocf_cachemode_get_name(ocf_cache_mode_t cache_mode); + +/* Get OCF promotion policy by its name. */ +ocf_promotion_t vbdev_ocf_promotion_policy_get_by_name(const char *policy_name); + +/* Get the name of OCF promotion policy. */ +const char *vbdev_ocf_promotion_policy_get_name(ocf_promotion_t policy); + +/* Get OCF cleaning policy by its name. */ +ocf_cleaning_t vbdev_ocf_cleaning_policy_get_by_name(const char *policy_name); + +/* Get the name of OCF cleaning policy. */ +const char *vbdev_ocf_cleaning_policy_get_name(ocf_cleaning_t policy); + +/* Get OCF sequential cut-off policy by its name. */ +ocf_seq_cutoff_policy vbdev_ocf_seqcutoff_policy_get_by_name(const char *policy_name); + +/* Get the name of OCF sequential cut-off policy. */ +const char *vbdev_ocf_seqcutoff_policy_get_name(ocf_seq_cutoff_policy policy); -/* Get sequential cutoff policy by name */ -ocf_seq_cutoff_policy ocf_get_seqcutoff_policy(const char *policy_name); #endif diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index a23765b35a2..f4ba90bafd6 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -11,6 +11,7 @@ #include "vbdev_ocf.h" #include "ctx.h" #include "data.h" +#include "utils.h" #include "volume.h" /* This namespace UUID was generated using uuid_generate() method. */ @@ -587,7 +588,7 @@ _cache_attach_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) goto err_alloc; } examine_attach_ctx->cache = cache; - examine_attach_ctx->att_cb_fn = _cache_attach_examine_attach_cb; + examine_attach_ctx->u.att_cb_fn = _cache_attach_examine_attach_cb; if ((rc = vbdev_ocf_cache_volume_attach(cache, examine_attach_ctx))) { SPDK_ERRLOG("OCF cache '%s': failed to attach volume (OCF error: %d)\n", @@ -1030,10 +1031,10 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, rc = -ENOMEM; goto err_alloc; } - cache_start_ctx->cache = cache; - cache_start_ctx->att_cb_fn = _cache_start_rpc_attach_cb; cache_start_ctx->rpc_cb_fn = rpc_cb_fn; cache_start_ctx->rpc_cb_arg = rpc_cb_arg; + cache_start_ctx->cache = cache; + cache_start_ctx->u.att_cb_fn = _cache_start_rpc_attach_cb; if ((rc = vbdev_ocf_cache_volume_attach(cache, cache_start_ctx))) { SPDK_ERRLOG("OCF cache '%s': failed to attach volume\n", cache_name); @@ -1362,7 +1363,7 @@ static void _core_add_rpc_err_cb(void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *core_add_ctx = cb_arg; - struct vbdev_ocf_core *core_ctx = core_add_ctx->core_ctx; + struct vbdev_ocf_core *core_ctx = core_add_ctx->u.core_ctx; ocf_cache_t cache = core_add_ctx->cache; if (error) { @@ -1382,7 +1383,7 @@ static void _core_add_rpc_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *core_add_ctx = cb_arg; - struct vbdev_ocf_core *core_ctx = core_add_ctx->core_ctx; + struct vbdev_ocf_core *core_ctx = core_add_ctx->u.core_ctx; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", @@ -1422,7 +1423,7 @@ static void _core_add_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *core_add_ctx = cb_arg; - struct vbdev_ocf_core *core_ctx = core_add_ctx->core_ctx; + struct vbdev_ocf_core *core_ctx = core_add_ctx->u.core_ctx; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add of OCF core\n", vbdev_ocf_core_get_name(core_ctx)); @@ -1527,10 +1528,10 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac rc = -ENOMEM; goto err_alloc; } - core_add_ctx->cache = cache; - core_add_ctx->core_ctx = core_ctx; core_add_ctx->rpc_cb_fn = rpc_cb_fn; core_add_ctx->rpc_cb_arg = rpc_cb_arg; + core_add_ctx->cache = cache; + core_add_ctx->u.core_ctx = core_ctx; ocf_mngt_cache_lock(cache, _core_add_rpc_lock_cb, core_add_ctx); @@ -1551,7 +1552,7 @@ static void _core_remove_rpc_remove_cb(void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *core_rm_ctx = cb_arg; - struct vbdev_ocf_core *core_ctx = core_rm_ctx->core_ctx; + struct vbdev_ocf_core *core_ctx = core_rm_ctx->u.core_ctx; ocf_cache_t cache = core_rm_ctx->cache; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing remove of OCF core\n", @@ -1665,11 +1666,11 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, rc = -ENOMEM; goto err_alloc; } - core_rm_ctx->cache = cache; - core_rm_ctx->core = core; - core_rm_ctx->core_ctx = core_ctx; core_rm_ctx->rpc_cb_fn = rpc_cb_fn; core_rm_ctx->rpc_cb_arg = rpc_cb_arg; + core_rm_ctx->cache = cache; + core_rm_ctx->core = core; + core_rm_ctx->u.core_ctx = core_ctx; if (!vbdev_ocf_core_is_base_attached(core_ctx)) { SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': removing detached (no unregister)\n", core_name); @@ -1696,12 +1697,772 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, rpc_cb_fn(core_name, rpc_cb_arg, rc); } +static void +_cache_save_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': saving cache state\n", ocf_cache_get_name(cache)); + + ocf_mngt_cache_unlock(cache); + + if (error) { + SPDK_WARNLOG("OCF cache '%s': failed to save cache state (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + } + + /* Ignore state save error caused by not attached cache volume. */ + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, + error == -OCF_ERR_CACHE_DETACHED ? 0 : error); + ocf_mngt_cache_put(cache); + free(mngt_ctx); +} + +static void +_cache_mode_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *cache_mode_ctx = cb_arg; + int rc; + + if ((rc = error)) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + goto err; + } + + if ((rc = ocf_mngt_cache_set_mode(cache, cache_mode_ctx->u.cache_mode))) { + SPDK_ERRLOG("OCF cache '%s': failed to change cache mode to '%s' (OCF error: %d)\n", + ocf_cache_get_name(cache), + vbdev_ocf_cachemode_get_name(cache_mode_ctx->u.cache_mode), rc); + ocf_mngt_cache_unlock(cache); + goto err; + } + + ocf_mngt_cache_save(cache, _cache_save_cb, cache_mode_ctx); + + return; + +err: + cache_mode_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_mode_ctx->rpc_cb_arg, rc); + ocf_mngt_cache_put(cache); + free(cache_mode_ctx); +} + +/* RPC entry point. */ +void +vbdev_ocf_set_cachemode(const char *cache_name, const char *cache_mode, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) +{ + ocf_cache_t cache; + struct vbdev_ocf_mngt_ctx *cache_mode_ctx; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': setting new cache mode '%s'\n", cache_name, cache_mode); + + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { + SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); + rc = -ENXIO; + goto err_cache; + } + + cache_mode_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!cache_mode_ctx) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache mode change context\n", + cache_name); + rc = -ENOMEM; + goto err_alloc; + } + cache_mode_ctx->u.cache_mode = vbdev_ocf_cachemode_get_by_name(cache_mode); + cache_mode_ctx->rpc_cb_fn = rpc_cb_fn; + cache_mode_ctx->rpc_cb_arg = rpc_cb_arg; + + ocf_mngt_cache_lock(cache, _cache_mode_lock_cb, cache_mode_ctx); + + return; + +err_alloc: + ocf_mngt_cache_put(cache); +err_cache: + rpc_cb_fn(cache_name, rpc_cb_arg, rc); +} + +static void +_promotion_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + int rc; + + if ((rc = error)) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + goto err_lock; + } + + if (mngt_ctx->u.promotion.policy >= ocf_promotion_always && + mngt_ctx->u.promotion.policy < ocf_promotion_max) { + if ((rc = ocf_mngt_cache_promotion_set_policy(cache, mngt_ctx->u.promotion.policy))) { + SPDK_ERRLOG("OCF cache '%s': failed to set promotion policy (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (mngt_ctx->u.promotion.nhit_insertion_threshold >= 0) { + if ((rc = ocf_mngt_cache_promotion_set_param(cache, ocf_promotion_nhit, ocf_nhit_insertion_threshold, + mngt_ctx->u.promotion.nhit_insertion_threshold))) { + SPDK_ERRLOG("OCF cache '%s': failed to set promotion nhit_insertion_threshold param (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (mngt_ctx->u.promotion.nhit_trigger_threshold >= 0) { + if ((rc = ocf_mngt_cache_promotion_set_param(cache, ocf_promotion_nhit, ocf_nhit_trigger_threshold, + mngt_ctx->u.promotion.nhit_trigger_threshold))) { + SPDK_ERRLOG("OCF cache '%s': failed to set promotion nhit_trigger_threshold param (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + ocf_mngt_cache_save(cache, _cache_save_cb, mngt_ctx); + + return; + +err_param: + ocf_mngt_cache_unlock(cache); +err_lock: + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, rc); + ocf_mngt_cache_put(cache); + free(mngt_ctx); +} + +/* RPC entry point. */ +void +vbdev_ocf_set_promotion(const char *cache_name, const char *policy, + int32_t nhit_insertion_threshold, int32_t nhit_trigger_threshold, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) +{ + ocf_cache_t cache; + struct vbdev_ocf_mngt_ctx *mngt_ctx; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': setting promotion params\n", cache_name); + + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { + SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); + rc = -ENXIO; + goto err_cache; + } + + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for promotion set context\n", + cache_name); + rc = -ENOMEM; + goto err_alloc; + } + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->u.promotion.policy = vbdev_ocf_promotion_policy_get_by_name(policy); + mngt_ctx->u.promotion.nhit_insertion_threshold = nhit_insertion_threshold; + mngt_ctx->u.promotion.nhit_trigger_threshold = nhit_trigger_threshold; + + ocf_mngt_cache_lock(cache, _promotion_lock_cb, mngt_ctx); + + return; + +err_alloc: + ocf_mngt_cache_put(cache); +err_cache: + rpc_cb_fn(cache_name, rpc_cb_arg, rc); +} + +static void +_cleaning_policy_cb(void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *cleaning_ctx = cb_arg; + ocf_cache_t cache = cleaning_ctx->cache; + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to set cleaning policy (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + ocf_mngt_cache_unlock(cache); + cleaning_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cleaning_ctx->rpc_cb_arg, error); + ocf_mngt_cache_put(cache); + free(cleaning_ctx); + return; + } + + ocf_mngt_cache_save(cache, _cache_save_cb, cleaning_ctx); +} + +static void +_cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *cleaning_ctx = cb_arg; + int rc; + + if ((rc = error)) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + goto err_lock; + } + + if (cleaning_ctx->u.cleaning.acp_wake_up_time >= 0) { + if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_acp, ocf_acp_wake_up_time, + cleaning_ctx->u.cleaning.acp_wake_up_time))) { + SPDK_ERRLOG("OCF cache '%s': failed to set cleaning acp_wake_up_time param (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (cleaning_ctx->u.cleaning.acp_flush_max_buffers >= 0) { + if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_acp, ocf_acp_flush_max_buffers, + cleaning_ctx->u.cleaning.acp_flush_max_buffers))) { + SPDK_ERRLOG("OCF cache '%s': failed to set cleaning acp_flush_max_buffers param (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (cleaning_ctx->u.cleaning.alru_wake_up_time >= 0) { + if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_wake_up_time, + cleaning_ctx->u.cleaning.alru_wake_up_time))) { + SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_wake_up_time param (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (cleaning_ctx->u.cleaning.alru_flush_max_buffers >= 0) { + if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_flush_max_buffers, + cleaning_ctx->u.cleaning.alru_flush_max_buffers))) { + SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_flush_max_buffers param (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (cleaning_ctx->u.cleaning.alru_staleness_time >= 0) { + if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_stale_buffer_time, + cleaning_ctx->u.cleaning.alru_staleness_time))) { + SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_staleness_time param (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (cleaning_ctx->u.cleaning.alru_activity_threshold >= 0) { + if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_activity_threshold, + cleaning_ctx->u.cleaning.alru_activity_threshold))) { + SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_activity_threshold param (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (cleaning_ctx->u.cleaning.alru_max_dirty_ratio >= 0) { + if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_max_dirty_ratio, + cleaning_ctx->u.cleaning.alru_max_dirty_ratio))) { + SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_max_dirty_ratio param (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (cleaning_ctx->u.cleaning.policy >= ocf_cleaning_nop && + cleaning_ctx->u.cleaning.policy < ocf_cleaning_max) { + ocf_mngt_cache_cleaning_set_policy(cache, cleaning_ctx->u.cleaning.policy, + _cleaning_policy_cb, cleaning_ctx); + } else { + ocf_mngt_cache_save(cache, _cache_save_cb, cleaning_ctx); + } + + return; + +err_param: + ocf_mngt_cache_unlock(cache); +err_lock: + cleaning_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cleaning_ctx->rpc_cb_arg, rc); + ocf_mngt_cache_put(cache); + free(cleaning_ctx); +} + +/* RPC entry point. */ +void +vbdev_ocf_set_cleaning(const char *cache_name, const char *policy, int32_t acp_wake_up_time, + int32_t acp_flush_max_buffers, int32_t alru_wake_up_time, + int32_t alru_flush_max_buffers, int32_t alru_staleness_time, + int32_t alru_activity_threshold, int32_t alru_max_dirty_ratio, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) +{ + ocf_cache_t cache; + struct vbdev_ocf_mngt_ctx *cleaning_ctx; + int rc = 0; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': setting cleaning params\n", cache_name); + + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { + SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); + rc = -ENXIO; + goto err_cache; + } + + cleaning_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!cleaning_ctx) { + SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cleaning set context\n", + cache_name); + rc = -ENOMEM; + goto err_alloc; + } + cleaning_ctx->u.cleaning.policy = vbdev_ocf_cleaning_policy_get_by_name(policy); + cleaning_ctx->u.cleaning.acp_wake_up_time = acp_wake_up_time; + cleaning_ctx->u.cleaning.acp_flush_max_buffers = acp_flush_max_buffers; + cleaning_ctx->u.cleaning.alru_wake_up_time = alru_wake_up_time; + cleaning_ctx->u.cleaning.alru_flush_max_buffers = alru_flush_max_buffers; + cleaning_ctx->u.cleaning.alru_staleness_time = alru_staleness_time; + cleaning_ctx->u.cleaning.alru_activity_threshold = alru_activity_threshold; + cleaning_ctx->u.cleaning.alru_max_dirty_ratio = alru_max_dirty_ratio; + cleaning_ctx->cache = cache; + cleaning_ctx->rpc_cb_fn = rpc_cb_fn; + cleaning_ctx->rpc_cb_arg = rpc_cb_arg; + + ocf_mngt_cache_lock(cache, _cleaning_lock_cb, cleaning_ctx); + + return; + +err_alloc: + ocf_mngt_cache_put(cache); +err_cache: + rpc_cb_fn(cache_name, rpc_cb_arg, rc); +} + +static void +_seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + int rc; + + if ((rc = error)) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + goto err_lock; + } + + if (mngt_ctx->core) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': setting sequential cut-off on core device\n", + mngt_ctx->bdev_name); + + if (mngt_ctx->u.seqcutoff.policy >= ocf_seq_cutoff_policy_always && + mngt_ctx->u.seqcutoff.policy < ocf_seq_cutoff_policy_max) { + if ((rc = ocf_mngt_core_set_seq_cutoff_policy(mngt_ctx->core, + mngt_ctx->u.seqcutoff.policy))) { + SPDK_ERRLOG("OCF core '%s': failed to set sequential cut-off policy (OCF error: %d)\n", + ocf_core_get_name(mngt_ctx->core), rc); + goto err_param; + } + } + + if (mngt_ctx->u.seqcutoff.threshold >= 0) { + if ((rc = ocf_mngt_core_set_seq_cutoff_threshold(mngt_ctx->core, + mngt_ctx->u.seqcutoff.threshold * KiB))) { + SPDK_ERRLOG("OCF core '%s': failed to set sequential cut-off threshold (OCF error: %d)\n", + ocf_core_get_name(mngt_ctx->core), rc); + goto err_param; + } + } + + if (mngt_ctx->u.seqcutoff.promotion_count >= 0) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promotion_count(mngt_ctx->core, + mngt_ctx->u.seqcutoff.promotion_count))) { + SPDK_ERRLOG("OCF core '%s': failed to set sequential cut-off promotion_count (OCF error: %d)\n", + ocf_core_get_name(mngt_ctx->core), rc); + goto err_param; + } + } + + if (mngt_ctx->u.seqcutoff.promote_on_threshold >= 0) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promote_on_threshold(mngt_ctx->core, + mngt_ctx->u.seqcutoff.promote_on_threshold))) { + SPDK_ERRLOG("OCF core '%s': failed to set sequential cut-off promote_on_threshold (OCF error: %d)\n", + ocf_core_get_name(mngt_ctx->core), rc); + goto err_param; + } + } + } else { + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': setting sequential cut-off on all cores in cache device\n", + mngt_ctx->bdev_name); + + if (mngt_ctx->u.seqcutoff.policy >= ocf_seq_cutoff_policy_always && + mngt_ctx->u.seqcutoff.policy < ocf_seq_cutoff_policy_max) { + if ((rc = ocf_mngt_core_set_seq_cutoff_policy_all(cache, + mngt_ctx->u.seqcutoff.policy))) { + SPDK_ERRLOG("OCF cache '%s': failed to set sequential cut-off policy (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (mngt_ctx->u.seqcutoff.threshold >= 0) { + if ((rc = ocf_mngt_core_set_seq_cutoff_threshold_all(cache, + mngt_ctx->u.seqcutoff.threshold * KiB))) { + SPDK_ERRLOG("OCF cache '%s': failed to set sequential cut-off threshold (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (mngt_ctx->u.seqcutoff.promotion_count >= 0) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promotion_count_all(cache, + mngt_ctx->u.seqcutoff.promotion_count))) { + SPDK_ERRLOG("OCF cache '%s': failed to set sequential cut-off promotion_count (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (mngt_ctx->u.seqcutoff.promote_on_threshold >= 0) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promote_on_threshold_all(cache, + mngt_ctx->u.seqcutoff.promote_on_threshold))) { + SPDK_ERRLOG("OCF cache '%s': failed to set sequential cut-off promote_on_threshold (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + } + + /* For compatibility with global _cache_save_cb(). */ + ocf_mngt_cache_get(cache); + + ocf_mngt_cache_save(cache, _cache_save_cb, mngt_ctx); + + return; + +err_param: + ocf_mngt_cache_unlock(cache); +err_lock: + mngt_ctx->rpc_cb_fn(mngt_ctx->bdev_name, mngt_ctx->rpc_cb_arg, rc); + free(mngt_ctx); +} + +static int +_seqcutoff_cache_visitor(ocf_cache_t cache, void *ctx) +{ + ocf_core_t core; + struct vbdev_ocf_mngt_ctx *mngt_ctx = ctx; + int rc; + + if (!strcmp(mngt_ctx->bdev_name, ocf_cache_get_name(cache))) { + ocf_mngt_cache_lock(cache, _seqcutoff_lock_cb, mngt_ctx); + return -EEXIST; + } + + rc = ocf_core_get_by_name(cache, mngt_ctx->bdev_name, OCF_CORE_NAME_SIZE, &core); + if (!rc) { + mngt_ctx->core = core; + ocf_mngt_cache_lock(cache, _seqcutoff_lock_cb, mngt_ctx); + return -EEXIST; + } + + return rc == -OCF_ERR_CORE_NOT_EXIST ? 0 : rc; +} + +/* RPC entry point. */ +void +vbdev_ocf_set_seqcutoff(const char *bdev_name, const char *policy, int32_t threshold, + int32_t promotion_count, int32_t promote_on_threshold, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) +{ + struct vbdev_ocf_mngt_ctx *mngt_ctx; + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': setting sequential cut-off params\n", bdev_name); + + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { + SPDK_ERRLOG("OCF '%s': failed to allocate memory for sequential cut-off set context\n", + bdev_name); + rpc_cb_fn(bdev_name, rpc_cb_arg, -ENOMEM); + return; + } + mngt_ctx->bdev_name = bdev_name; + /* Will be used later if given bdev is a core device. */ + mngt_ctx->core = NULL; + mngt_ctx->u.seqcutoff.policy = vbdev_ocf_seqcutoff_policy_get_by_name(policy); + mngt_ctx->u.seqcutoff.threshold = threshold; + mngt_ctx->u.seqcutoff.promotion_count = promotion_count; + mngt_ctx->u.seqcutoff.promote_on_threshold = promote_on_threshold; + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; + + rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _seqcutoff_cache_visitor, mngt_ctx); + if (rc && rc != -EEXIST) { + SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + rpc_cb_fn(bdev_name, rpc_cb_arg, rc); + } else if (!rc) { + SPDK_ERRLOG("OCF '%s': not exist\n", bdev_name); + rpc_cb_fn(bdev_name, rpc_cb_arg, -ENXIO); + } +} + +static void +_flush_cache_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing flush operation\n", + ocf_cache_get_name(cache)); + + ocf_mngt_cache_read_unlock(cache); + + cache_ctx->flush.error = error; + cache_ctx->flush.in_progress = false; +} + +static void +_flush_core_cb(ocf_core_t core, void *cb_arg, int error) +{ + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing flush operation\n", + ocf_core_get_name(core)); + + ocf_mngt_cache_read_unlock(ocf_core_get_cache(core)); + + core_ctx->flush.error = error; + core_ctx->flush.in_progress = false; +} + +static void +_flush_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *flush_ctx = cb_arg; + struct vbdev_ocf_cache *cache_ctx; + struct vbdev_ocf_core *core_ctx; + int rc; + + if ((rc = error)) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + goto end; + } + + if (flush_ctx->core) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': flushing core device\n", flush_ctx->bdev_name); + + core_ctx = ocf_core_get_priv(flush_ctx->core); + core_ctx->flush.in_progress = true; + ocf_mngt_core_flush(flush_ctx->core, _flush_core_cb, NULL); + } else { + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': flushing cache device\n", flush_ctx->bdev_name); + + cache_ctx = ocf_cache_get_priv(cache); + cache_ctx->flush.in_progress = true; + ocf_mngt_cache_flush(cache, _flush_cache_cb, NULL); + } + +end: + /* Flushing process may take some time to finish, so call + * RPC callback now and leave flush running in background. */ + flush_ctx->rpc_cb_fn(flush_ctx->bdev_name, flush_ctx->rpc_cb_arg, rc); + free(flush_ctx); +} + +static int +_flush_cache_visitor(ocf_cache_t cache, void *ctx) +{ + ocf_core_t core; + struct vbdev_ocf_mngt_ctx *flush_ctx = ctx; + int rc; + + if (!strcmp(flush_ctx->bdev_name, ocf_cache_get_name(cache))) { + ocf_mngt_cache_read_lock(cache, _flush_lock_cb, flush_ctx); + return -EEXIST; + } + + rc = ocf_core_get_by_name(cache, flush_ctx->bdev_name, OCF_CORE_NAME_SIZE, &core); + if (!rc) { + flush_ctx->core = core; + ocf_mngt_cache_read_lock(cache, _flush_lock_cb, flush_ctx); + return -EEXIST; + } + + return rc == -OCF_ERR_CORE_NOT_EXIST ? 0 : rc; +} + +/* RPC entry point. */ +void +vbdev_ocf_flush_start(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) +{ + struct vbdev_ocf_mngt_ctx *flush_ctx; + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': initiating flush operation\n", bdev_name); + + flush_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!flush_ctx) { + SPDK_ERRLOG("OCF '%s': failed to allocate memory for flush context\n", bdev_name); + rpc_cb_fn(bdev_name, rpc_cb_arg, -ENOMEM); + return; + } + flush_ctx->bdev_name = bdev_name; + /* Will be used later if given bdev is a core device. */ + flush_ctx->core = NULL; + flush_ctx->rpc_cb_fn = rpc_cb_fn; + flush_ctx->rpc_cb_arg = rpc_cb_arg; + + rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _flush_cache_visitor, flush_ctx); + if (rc && rc != -EEXIST) { + SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + rpc_cb_fn(bdev_name, rpc_cb_arg, rc); + } else if (!rc) { + SPDK_ERRLOG("OCF '%s': not exist\n", bdev_name); + rpc_cb_fn(bdev_name, rpc_cb_arg, -ENXIO); + } +} + +static int +_dump_promotion_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) +{ + ocf_promotion_t promotion_policy; + uint32_t param_val; + int rc; + + if ((rc = ocf_mngt_cache_promotion_get_policy(cache, &promotion_policy))) { + SPDK_ERRLOG("OCF cache '%s': failed to get promotion policy (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + spdk_json_write_named_string(w, "policy", ""); + return rc; + } + spdk_json_write_named_string(w, "policy", + vbdev_ocf_promotion_policy_get_name(promotion_policy)); + + if (promotion_policy == ocf_promotion_nhit) { + if ((rc = ocf_mngt_cache_promotion_get_param(cache, ocf_promotion_nhit, + ocf_nhit_insertion_threshold, ¶m_val))) { + return rc; + } + spdk_json_write_named_uint32(w, "insertion_threshold", param_val); + + if ((rc = ocf_mngt_cache_promotion_get_param(cache, ocf_promotion_nhit, + ocf_nhit_trigger_threshold, ¶m_val))) { + return rc; + } + spdk_json_write_named_uint32(w, "trigger_threshold", param_val); + + } + + return 0; +} + +static int +_dump_cleaning_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) +{ + ocf_cleaning_t cleaning_policy; + uint32_t param_val; + int rc; + + if ((rc = ocf_mngt_cache_cleaning_get_policy(cache, &cleaning_policy))) { + SPDK_ERRLOG("OCF cache '%s': failed to get cleaning policy (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + spdk_json_write_named_string(w, "policy", ""); + return rc; + } + spdk_json_write_named_string(w, "policy", + vbdev_ocf_cleaning_policy_get_name(cleaning_policy)); + + if (cleaning_policy == ocf_cleaning_acp) { + if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_acp, + ocf_acp_wake_up_time, ¶m_val))) { + return rc; + } + spdk_json_write_named_uint32(w, "wake_up_time", param_val); + + if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_acp, + ocf_acp_flush_max_buffers, ¶m_val))) { + return rc; + } + spdk_json_write_named_uint32(w, "flush_max_buffers", param_val); + + } else if (cleaning_policy == ocf_cleaning_alru) { + if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, + ocf_alru_wake_up_time, ¶m_val))) { + return rc; + } + spdk_json_write_named_uint32(w, "wake_up_time", param_val); + + if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, + ocf_alru_flush_max_buffers, ¶m_val))) { + return rc; + } + spdk_json_write_named_uint32(w, "flush_max_buffers", param_val); + + if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, + ocf_alru_stale_buffer_time, ¶m_val))) { + return rc; + } + spdk_json_write_named_uint32(w, "staleness_time", param_val); + + if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, + ocf_alru_activity_threshold, ¶m_val))) { + return rc; + } + spdk_json_write_named_uint32(w, "activity_threshold", param_val); + + if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, + ocf_alru_max_dirty_ratio, ¶m_val))) { + return rc; + } + spdk_json_write_named_uint32(w, "max_dirty_ratio", param_val); + } + + return 0; +} + +static int +_dump_seqcutoff_info(struct spdk_json_write_ctx *w, ocf_core_t core) +{ + ocf_seq_cutoff_policy seqcutoff_policy; + uint32_t param_val_int; + bool param_val_bool; + int rc; + + if ((rc = ocf_mngt_core_get_seq_cutoff_policy(core, &seqcutoff_policy))) { + SPDK_ERRLOG("OCF core '%s': failed to get sequential cut-off policy (OCF error: %d)\n", + ocf_core_get_name(core), rc); + spdk_json_write_named_string(w, "policy", ""); + return rc; + } + spdk_json_write_named_string(w, "policy", + vbdev_ocf_seqcutoff_policy_get_name(seqcutoff_policy)); + + if ((rc = ocf_mngt_core_get_seq_cutoff_threshold(core, ¶m_val_int))) { + return rc; + } + spdk_json_write_named_uint32(w, "threshold", param_val_int / KiB); + + if ((rc = ocf_mngt_core_get_seq_cutoff_promotion_count(core, ¶m_val_int))) { + return rc; + } + spdk_json_write_named_uint32(w, "promotion_count", param_val_int); + + if ((rc = ocf_mngt_core_get_seq_cutoff_promote_on_threshold(core, ¶m_val_bool))) { + return rc; + } + spdk_json_write_named_bool(w, "promote_on_threshold", param_val_bool); + + return 0; +} + // handle errors ? static int _get_bdevs_core_visit(ocf_core_t core, void *cb_arg) { struct spdk_json_write_ctx *w = cb_arg; struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + int rc; spdk_json_write_object_begin(w); spdk_json_write_named_string(w, "name", ocf_core_get_name(core)); @@ -1709,11 +2470,25 @@ _get_bdevs_core_visit(ocf_core_t core, void *cb_arg) spdk_json_write_named_bool(w, "base_attached", core_ctx ? vbdev_ocf_core_is_base_attached(core_ctx) : false); spdk_json_write_named_bool(w, "loading", !core_ctx); + + spdk_json_write_named_object_begin(w, "seq_cutoff"); + if ((rc = _dump_seqcutoff_info(w, core))) { + SPDK_ERRLOG("OCF core '%s': failed to get sequential cut-off params info (OCF error: %d)\n", + ocf_core_get_name(core), rc); + } + spdk_json_write_object_end(w); + + spdk_json_write_named_object_begin(w, "flush"); + spdk_json_write_named_bool(w, "in_progress", core_ctx ? core_ctx->flush.in_progress : false); + spdk_json_write_named_int32(w, "error", core_ctx ? core_ctx->flush.error : 0); + spdk_json_write_object_end(w); + spdk_json_write_object_end(w); return 0; } +// handle errors ? static int _get_bdevs_cache_visit(ocf_cache_t cache, void *cb_arg) { @@ -1725,12 +2500,35 @@ _get_bdevs_cache_visit(ocf_cache_t cache, void *cb_arg) spdk_json_write_named_string(w, "name", ocf_cache_get_name(cache)); spdk_json_write_named_string(w, "base_name", cache_ctx->base.name); spdk_json_write_named_bool(w, "base_attached", ocf_cache_is_device_attached(cache)); + spdk_json_write_named_string(w, "cache_mode", + vbdev_ocf_cachemode_get_name(ocf_cache_get_mode(cache))); + spdk_json_write_named_uint32(w, "cache_line_size", + ocf_cache_get_line_size(cache) / KiB); + + spdk_json_write_named_object_begin(w, "promotion"); + if ((rc = _dump_promotion_info(w, cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to get promotion params info (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + } + spdk_json_write_object_end(w); + + spdk_json_write_named_object_begin(w, "cleaning"); + if ((rc = _dump_cleaning_info(w, cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to get cleaning params info (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + } + spdk_json_write_object_end(w); + + spdk_json_write_named_object_begin(w, "flush"); + spdk_json_write_named_bool(w, "in_progress", cache_ctx->flush.in_progress); + spdk_json_write_named_int32(w, "error", cache_ctx->flush.error); + spdk_json_write_object_end(w); + spdk_json_write_named_uint16(w, "cores_count", ocf_cache_get_core_count(cache)); spdk_json_write_named_array_begin(w, "cores"); - rc = ocf_core_visit(cache, _get_bdevs_core_visit, w, false); - spdk_json_write_array_end(w); + spdk_json_write_object_end(w); return rc; diff --git a/module/bdev/ocf/vbdev_ocf.h b/module/bdev/ocf/vbdev_ocf.h index e1004d56a19..93177047bf8 100644 --- a/module/bdev/ocf/vbdev_ocf.h +++ b/module/bdev/ocf/vbdev_ocf.h @@ -47,6 +47,29 @@ void vbdev_ocf_core_remove(const char *core_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg); +void vbdev_ocf_set_cachemode(const char *cache_name, + const char *cache_mode, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, + void *rpc_cb_arg); + +void vbdev_ocf_set_promotion(const char *cache_name, const char *policy, + int32_t nhit_insertion_threshold, int32_t nhit_trigger_threshold, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg); + +void vbdev_ocf_set_cleaning(const char *cache_name, const char *policy, int32_t acp_wake_up_time, + int32_t acp_flush_max_buffers, int32_t alru_wake_up_time, + int32_t alru_flush_max_buffers, int32_t alru_staleness_time, + int32_t alru_activity_threshold, int32_t alru_max_dirty_ratio, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg); + +void vbdev_ocf_set_seqcutoff(const char *bdev_name, const char *policy, int32_t threshold, + int32_t promotion_count, int32_t promote_on_threshold, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg); + +void vbdev_ocf_flush_start(const char *bdev_name, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, + void *rpc_cb_arg); + void vbdev_ocf_get_bdevs(const char *name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rpc_cb_arg1, diff --git a/module/bdev/ocf/vbdev_ocf_cache.c b/module/bdev/ocf/vbdev_ocf_cache.c index 19ae95546f1..58eef02fcf1 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.c +++ b/module/bdev/ocf/vbdev_ocf_cache.c @@ -34,7 +34,7 @@ vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cac strncpy(cache_cfg->name, cache_name, OCF_CACHE_NAME_SIZE); if (cache_mode) { - cache_cfg->cache_mode = ocf_get_cache_mode(cache_mode); + cache_cfg->cache_mode = vbdev_ocf_cachemode_get_by_name(cache_mode); } if (cache_line_size) { cache_cfg->cache_line_size = cache_line_size * KiB; @@ -237,19 +237,19 @@ _volume_attach_metadata_probe_cb(void *priv, int error, struct ocf_metadata_prob if (error && error != -OCF_ERR_NO_METADATA) { SPDK_ERRLOG("OCF cache '%s': failed to probe metadata\n", ocf_cache_get_name(cache)); - ctx->att_cb_fn(cache, ctx, error); + ctx->u.att_cb_fn(cache, ctx, error); } if (error == -OCF_ERR_NO_METADATA) { SPDK_NOTICELOG("OCF cache '%s': metadata not found - starting new cache instance\n", ocf_cache_get_name(cache)); - ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, ctx->att_cb_fn, ctx); + ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, ctx->u.att_cb_fn, ctx); } else { SPDK_NOTICELOG("OCF cache '%s': metadata found - loading previous cache instance\n", ocf_cache_get_name(cache)); SPDK_NOTICELOG("(start cache with 'no-load' flag to create new cache instead of loading config from metadata)\n"); // check status for cache_name/mode/line_size/dirty ? - ocf_mngt_cache_load(cache, &cache_ctx->cache_att_cfg, ctx->att_cb_fn, ctx); + ocf_mngt_cache_load(cache, &cache_ctx->cache_att_cfg, ctx->u.att_cb_fn, ctx); } } @@ -261,7 +261,7 @@ vbdev_ocf_cache_volume_attach(ocf_cache_t cache, struct vbdev_ocf_mngt_ctx *ctx) if (cache_ctx->no_load) { SPDK_NOTICELOG("'no-load' flag specified - starting new cache without looking for metadata\n"); - ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, ctx->att_cb_fn, ctx); + ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, ctx->u.att_cb_fn, ctx); return 0; } diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h index 4918a3825dc..c0136c33513 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.h +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -25,18 +25,48 @@ struct vbdev_ocf_cache { ocf_queue_t cache_mngt_q; /* Indicator for cache start to ignore cache metadata found on device. */ bool no_load; + /* Status of cache flush operation. */ + struct { + bool in_progress; + int error; + } flush; }; typedef void (*vbdev_ocf_rpc_mngt_cb)(const char *bdev_name, void *cb_arg, int error); /* Temporary context for management operations. */ struct vbdev_ocf_mngt_ctx { - ocf_cache_t cache; - ocf_core_t core; - struct vbdev_ocf_core * core_ctx; - ocf_mngt_cache_attach_end_t att_cb_fn; vbdev_ocf_rpc_mngt_cb rpc_cb_fn; void * rpc_cb_arg; + const char * bdev_name; + ocf_cache_t cache; + ocf_core_t core; + union { + ocf_mngt_cache_attach_end_t att_cb_fn; + struct vbdev_ocf_core * core_ctx; + ocf_cache_mode_t cache_mode; + struct { + ocf_promotion_t policy; + int32_t nhit_insertion_threshold; + int32_t nhit_trigger_threshold; + } promotion; + struct { + ocf_cleaning_t policy; + int32_t acp_wake_up_time; + int32_t acp_flush_max_buffers; + int32_t alru_wake_up_time; + int32_t alru_flush_max_buffers; + int32_t alru_staleness_time; + int32_t alru_activity_threshold; + int32_t alru_max_dirty_ratio; + } cleaning; + struct { + ocf_seq_cutoff_policy policy; + int32_t threshold; + int32_t promotion_count; + int32_t promote_on_threshold; + } seqcutoff; + } u; }; /* Cache management queue context. */ diff --git a/module/bdev/ocf/vbdev_ocf_core.h b/module/bdev/ocf/vbdev_ocf_core.h index 26184b22d7d..37c131d3685 100644 --- a/module/bdev/ocf/vbdev_ocf_core.h +++ b/module/bdev/ocf/vbdev_ocf_core.h @@ -35,6 +35,11 @@ struct vbdev_ocf_core { char cache_name[OCF_CACHE_NAME_SIZE]; /* Context for cache stop management operation. */ struct vbdev_ocf_mngt_ctx * mngt_ctx; + /* Status of core flush operation. */ + struct { + bool in_progress; + int error; + } flush; STAILQ_ENTRY(vbdev_ocf_core) waitlist_entry; }; diff --git a/module/bdev/ocf/vbdev_ocf_rpc.c b/module/bdev/ocf/vbdev_ocf_rpc.c index cdacaae0234..e6d4f0ecd3c 100644 --- a/module/bdev/ocf/vbdev_ocf_rpc.c +++ b/module/bdev/ocf/vbdev_ocf_rpc.c @@ -16,7 +16,8 @@ struct rpc_bdev_ocf_start_cache { }; static void -free_rpc_bdev_ocf_start_cache(struct rpc_bdev_ocf_start_cache *r) { +free_rpc_bdev_ocf_start_cache(struct rpc_bdev_ocf_start_cache *r) +{ free(r->cache_name); free(r->base_name); free(r->cache_mode); @@ -58,7 +59,7 @@ rpc_bdev_ocf_start_cache(struct spdk_jsonrpc_request *request, const struct spdk &req)) { SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parametes"); + "Invalid parameters"); goto cleanup; } @@ -75,7 +76,8 @@ struct rpc_bdev_ocf_stop_cache { }; static void -free_rpc_bdev_ocf_stop_cache(struct rpc_bdev_ocf_stop_cache *r) { +free_rpc_bdev_ocf_stop_cache(struct rpc_bdev_ocf_stop_cache *r) +{ free(r->cache_name); } @@ -106,7 +108,7 @@ rpc_bdev_ocf_stop_cache(struct spdk_jsonrpc_request *request, const struct spdk_ &req)) { SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parametes"); + "Invalid parameters"); goto cleanup; } @@ -122,7 +124,8 @@ struct rpc_bdev_ocf_detach_cache { }; static void -free_rpc_bdev_ocf_detach_cache(struct rpc_bdev_ocf_detach_cache *r) { +free_rpc_bdev_ocf_detach_cache(struct rpc_bdev_ocf_detach_cache *r) +{ free(r->cache_name); } @@ -155,7 +158,7 @@ rpc_bdev_ocf_detach_cache(struct spdk_jsonrpc_request *request, const struct spd &req)) { SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parametes"); + "Invalid parameters"); goto cleanup; } @@ -173,7 +176,8 @@ struct rpc_bdev_ocf_attach_cache { }; static void -free_rpc_bdev_ocf_attach_cache(struct rpc_bdev_ocf_attach_cache *r) { +free_rpc_bdev_ocf_attach_cache(struct rpc_bdev_ocf_attach_cache *r) +{ free(r->cache_name); free(r->base_name); } @@ -209,7 +213,7 @@ rpc_bdev_ocf_attach_cache(struct spdk_jsonrpc_request *request, const struct spd &req)) { SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parametes"); + "Invalid parameters"); goto cleanup; } @@ -228,7 +232,8 @@ struct rpc_bdev_ocf_add_core { }; static void -free_rpc_bdev_ocf_add_core(struct rpc_bdev_ocf_add_core *r) { +free_rpc_bdev_ocf_add_core(struct rpc_bdev_ocf_add_core *r) +{ free(r->core_name); free(r->base_name); free(r->cache_name); @@ -268,7 +273,7 @@ rpc_bdev_ocf_add_core(struct spdk_jsonrpc_request *request, const struct spdk_js &req)) { SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parametes"); + "Invalid parameters"); goto cleanup; } @@ -285,7 +290,8 @@ struct rpc_bdev_ocf_remove_core { }; static void -free_rpc_bdev_ocf_remove_core(struct rpc_bdev_ocf_remove_core *r) { +free_rpc_bdev_ocf_remove_core(struct rpc_bdev_ocf_remove_core *r) +{ free(r->core_name); free(r->cache_name); } @@ -318,7 +324,7 @@ rpc_bdev_ocf_remove_core(struct spdk_jsonrpc_request *request, const struct spdk &req)) { SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parametes"); + "Invalid parameters"); goto cleanup; } @@ -329,72 +335,288 @@ rpc_bdev_ocf_remove_core(struct spdk_jsonrpc_request *request, const struct spdk } SPDK_RPC_REGISTER("bdev_ocf_remove_core", rpc_bdev_ocf_remove_core, SPDK_RPC_RUNTIME) -struct rpc_bdev_ocf_get_bdevs { - char *name; +struct rpc_bdev_ocf_set_cachemode { + char *cache_name; + char *cache_mode; }; static void -free_rpc_bdev_ocf_get_bdevs(struct rpc_bdev_ocf_get_bdevs *r) { - free(r->name); +free_rpc_bdev_ocf_set_cachemode(struct rpc_bdev_ocf_set_cachemode *r) +{ + free(r->cache_name); + free(r->cache_mode); } -static const struct spdk_json_object_decoder rpc_bdev_ocf_get_bdevs_decoders[] = { - {"name", offsetof(struct rpc_bdev_ocf_get_bdevs, name), spdk_json_decode_string, true}, +static const struct spdk_json_object_decoder rpc_bdev_ocf_set_cachemode_decoders[] = { + {"cache_name", offsetof(struct rpc_bdev_ocf_set_cachemode, cache_name), spdk_json_decode_string}, + {"cache_mode", offsetof(struct rpc_bdev_ocf_set_cachemode, cache_mode), spdk_json_decode_string}, }; static void -rpc_bdev_ocf_get_bdevs_cb(void *cb_arg1, void *cb_arg2) +rpc_bdev_ocf_set_cachemode_cb(const char *bdev_name, void *cb_arg, int error) { - struct spdk_json_write_ctx *w = cb_arg1; - struct spdk_jsonrpc_request *request = cb_arg2; + struct spdk_jsonrpc_request *request = cb_arg; - spdk_json_write_object_end(w); - spdk_jsonrpc_end_result(request, w); + if (error) { + spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); } static void -rpc_bdev_ocf_get_bdevs(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +rpc_bdev_ocf_set_cachemode(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) { - struct rpc_bdev_ocf_get_bdevs req = {}; - struct spdk_json_write_ctx *w; + struct rpc_bdev_ocf_set_cachemode req = {}; - if (params && spdk_json_decode_object(params, rpc_bdev_ocf_get_bdevs_decoders, - SPDK_COUNTOF(rpc_bdev_ocf_get_bdevs_decoders), - &req)) { + if (spdk_json_decode_object(params, rpc_bdev_ocf_set_cachemode_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_set_cachemode_decoders), + &req)) { SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parametes"); + "Invalid parameters"); goto cleanup; } - w = spdk_jsonrpc_begin_result(request); - spdk_json_write_object_begin(w); + vbdev_ocf_set_cachemode(req.cache_name, req.cache_mode, rpc_bdev_ocf_set_cachemode_cb, request); - vbdev_ocf_get_bdevs(params ? req.name : NULL, rpc_bdev_ocf_get_bdevs_cb, w, request); +cleanup: + free_rpc_bdev_ocf_set_cachemode(&req); +} +SPDK_RPC_REGISTER("bdev_ocf_set_cachemode", rpc_bdev_ocf_set_cachemode, SPDK_RPC_RUNTIME) + +struct rpc_bdev_ocf_set_promotion { + char *cache_name; + char *policy; + int32_t nhit_insertion_threshold; + int32_t nhit_trigger_threshold; +}; + +static void +free_rpc_bdev_ocf_set_promotion(struct rpc_bdev_ocf_set_promotion *r) +{ + free(r->cache_name); + free(r->policy); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_set_promotion_decoders[] = { + {"cache_name", offsetof(struct rpc_bdev_ocf_set_promotion, cache_name), spdk_json_decode_string}, + {"policy", offsetof(struct rpc_bdev_ocf_set_promotion, policy), spdk_json_decode_string, true}, + {"nhit_insertion_threshold", offsetof(struct rpc_bdev_ocf_set_promotion, nhit_insertion_threshold), spdk_json_decode_int32, true}, + {"nhit_trigger_threshold", offsetof(struct rpc_bdev_ocf_set_promotion, nhit_trigger_threshold), spdk_json_decode_int32, true}, +}; + +static void +rpc_bdev_ocf_set_promotion_cb(const char *bdev_name, void *cb_arg, int error) +{ + struct spdk_jsonrpc_request *request = cb_arg; + + if (error) { + spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); +} + +static void +rpc_bdev_ocf_set_promotion(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_bdev_ocf_set_promotion req = {}; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_set_promotion_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_set_promotion_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + goto cleanup; + } + + vbdev_ocf_set_promotion(req.cache_name, req.policy, req.nhit_insertion_threshold, + req.nhit_trigger_threshold, rpc_bdev_ocf_set_promotion_cb, request); cleanup: - free_rpc_bdev_ocf_get_bdevs(&req); + free_rpc_bdev_ocf_set_promotion(&req); } -SPDK_RPC_REGISTER("bdev_ocf_get_bdevs", rpc_bdev_ocf_get_bdevs, SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER("bdev_ocf_set_promotion", rpc_bdev_ocf_set_promotion, SPDK_RPC_RUNTIME) + +struct rpc_bdev_ocf_set_cleaning { + char *cache_name; + char *policy; + int32_t acp_wake_up_time; + int32_t acp_flush_max_buffers; + int32_t alru_wake_up_time; + int32_t alru_flush_max_buffers; + int32_t alru_staleness_time; + int32_t alru_activity_threshold; + int32_t alru_max_dirty_ratio; +}; static void -rpc_bdev_ocf_set_cachemode(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +free_rpc_bdev_ocf_set_cleaning(struct rpc_bdev_ocf_set_cleaning *r) { + free(r->cache_name); + free(r->policy); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_set_cleaning_decoders[] = { + {"cache_name", offsetof(struct rpc_bdev_ocf_set_cleaning, cache_name), spdk_json_decode_string}, + {"policy", offsetof(struct rpc_bdev_ocf_set_cleaning, policy), spdk_json_decode_string, true}, + {"acp_wake_up_time", offsetof(struct rpc_bdev_ocf_set_cleaning, acp_wake_up_time), spdk_json_decode_int32, true}, + {"acp_flush_max_buffers", offsetof(struct rpc_bdev_ocf_set_cleaning, acp_flush_max_buffers), spdk_json_decode_int32, true}, + {"alru_wake_up_time", offsetof(struct rpc_bdev_ocf_set_cleaning, alru_wake_up_time), spdk_json_decode_int32, true}, + {"alru_flush_max_buffers", offsetof(struct rpc_bdev_ocf_set_cleaning, alru_flush_max_buffers), spdk_json_decode_int32, true}, + {"alru_staleness_time", offsetof(struct rpc_bdev_ocf_set_cleaning, alru_staleness_time), spdk_json_decode_int32, true}, + {"alru_activity_threshold", offsetof(struct rpc_bdev_ocf_set_cleaning, alru_activity_threshold), spdk_json_decode_int32, true}, + {"alru_max_dirty_ratio", offsetof(struct rpc_bdev_ocf_set_cleaning, alru_max_dirty_ratio), spdk_json_decode_int32, true}, +}; + +static void +rpc_bdev_ocf_set_cleaning_cb(const char *bdev_name, void *cb_arg, int error) +{ + struct spdk_jsonrpc_request *request = cb_arg; + + if (error) { + spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); } -SPDK_RPC_REGISTER("bdev_ocf_set_cachemode", rpc_bdev_ocf_set_cachemode, SPDK_RPC_RUNTIME) static void rpc_bdev_ocf_set_cleaning(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) { + struct rpc_bdev_ocf_set_cleaning req = {}; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_set_cleaning_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_set_cleaning_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + goto cleanup; + } + + vbdev_ocf_set_cleaning(req.cache_name, req.policy, req.acp_wake_up_time, + req.acp_flush_max_buffers, req.alru_wake_up_time, + req.alru_flush_max_buffers, req.alru_staleness_time, + req.alru_activity_threshold, req.alru_max_dirty_ratio, + rpc_bdev_ocf_set_cleaning_cb, request); + +cleanup: + free_rpc_bdev_ocf_set_cleaning(&req); } SPDK_RPC_REGISTER("bdev_ocf_set_cleaning", rpc_bdev_ocf_set_cleaning, SPDK_RPC_RUNTIME) +struct rpc_bdev_ocf_set_seqcutoff { + char *bdev_name; + char *policy; + int32_t threshold; + int32_t promotion_count; + int32_t promote_on_threshold; +}; + +static void +free_rpc_bdev_ocf_set_seqcutoff(struct rpc_bdev_ocf_set_seqcutoff *r) +{ + free(r->bdev_name); + free(r->policy); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_set_seqcutoff_decoders[] = { + {"bdev_name", offsetof(struct rpc_bdev_ocf_set_seqcutoff, bdev_name), spdk_json_decode_string}, + {"policy", offsetof(struct rpc_bdev_ocf_set_seqcutoff, policy), spdk_json_decode_string, true}, + {"threshold", offsetof(struct rpc_bdev_ocf_set_seqcutoff, threshold), spdk_json_decode_int32, true}, + {"promotion_count", offsetof(struct rpc_bdev_ocf_set_seqcutoff, promotion_count), spdk_json_decode_int32, true}, + {"promote_on_threshold", offsetof(struct rpc_bdev_ocf_set_seqcutoff, promote_on_threshold), spdk_json_decode_int32, true}, +}; + +static void +rpc_bdev_ocf_set_seqcutoff_cb(const char *bdev_name, void *cb_arg, int error) +{ + struct spdk_jsonrpc_request *request = cb_arg; + + if (error) { + spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); +} + static void rpc_bdev_ocf_set_seqcutoff(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) { + struct rpc_bdev_ocf_set_seqcutoff req = {}; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_set_seqcutoff_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_set_seqcutoff_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + goto cleanup; + } + + vbdev_ocf_set_seqcutoff(req.bdev_name, req.policy, req.threshold, req.promotion_count, + req.promote_on_threshold, rpc_bdev_ocf_set_seqcutoff_cb, request); + +cleanup: + free_rpc_bdev_ocf_set_seqcutoff(&req); } SPDK_RPC_REGISTER("bdev_ocf_set_seqcutoff", rpc_bdev_ocf_set_seqcutoff, SPDK_RPC_RUNTIME) +struct rpc_bdev_ocf_flush_start { + char *bdev_name; +}; + +static void +free_rpc_bdev_ocf_flush_start(struct rpc_bdev_ocf_flush_start *r) +{ + free(r->bdev_name); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_flush_start_decoders[] = { + {"bdev_name", offsetof(struct rpc_bdev_ocf_flush_start, bdev_name), spdk_json_decode_string}, +}; + +static void +rpc_bdev_ocf_flush_start_cb(const char *bdev_name, void *cb_arg, int error) +{ + struct spdk_jsonrpc_request *request = cb_arg; + + if (error) { + spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); +} + +static void +rpc_bdev_ocf_flush_start(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_bdev_ocf_flush_start req = {}; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_flush_start_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_flush_start_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + goto cleanup; + } + + vbdev_ocf_flush_start(req.bdev_name, rpc_bdev_ocf_flush_start_cb, request); + +cleanup: + free_rpc_bdev_ocf_flush_start(&req); +} +SPDK_RPC_REGISTER("bdev_ocf_flush_start", rpc_bdev_ocf_flush_start, SPDK_RPC_RUNTIME) + static void rpc_bdev_ocf_get_stats(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) { @@ -407,16 +629,53 @@ rpc_bdev_ocf_reset_stats(struct spdk_jsonrpc_request *request, const struct spdk } SPDK_RPC_REGISTER("bdev_ocf_reset_stats", rpc_bdev_ocf_reset_stats, SPDK_RPC_RUNTIME) +struct rpc_bdev_ocf_get_bdevs { + char *name; +}; + static void -rpc_bdev_ocf_flush_start(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +free_rpc_bdev_ocf_get_bdevs(struct rpc_bdev_ocf_get_bdevs *r) { + free(r->name); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_get_bdevs_decoders[] = { + {"name", offsetof(struct rpc_bdev_ocf_get_bdevs, name), spdk_json_decode_string, true}, +}; + +static void +rpc_bdev_ocf_get_bdevs_cb(void *cb_arg1, void *cb_arg2) +{ + struct spdk_json_write_ctx *w = cb_arg1; + struct spdk_jsonrpc_request *request = cb_arg2; + + spdk_json_write_object_end(w); + spdk_jsonrpc_end_result(request, w); } -SPDK_RPC_REGISTER("bdev_ocf_flush_start", rpc_bdev_ocf_flush_start, SPDK_RPC_RUNTIME) static void -rpc_bdev_ocf_flush_status(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +rpc_bdev_ocf_get_bdevs(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) { + struct rpc_bdev_ocf_get_bdevs req = {}; + struct spdk_json_write_ctx *w; + + if (params && spdk_json_decode_object(params, rpc_bdev_ocf_get_bdevs_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_get_bdevs_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + goto cleanup; + } + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_object_begin(w); + + vbdev_ocf_get_bdevs(params ? req.name : NULL, rpc_bdev_ocf_get_bdevs_cb, w, request); + +cleanup: + free_rpc_bdev_ocf_get_bdevs(&req); } -SPDK_RPC_REGISTER("bdev_ocf_flush_status", rpc_bdev_ocf_flush_status, SPDK_RPC_RUNTIME) +SPDK_RPC_REGISTER("bdev_ocf_get_bdevs", rpc_bdev_ocf_get_bdevs, SPDK_RPC_RUNTIME) SPDK_LOG_REGISTER_COMPONENT(vbdev_ocf_rpc) diff --git a/python/spdk/cli/bdev.py b/python/spdk/cli/bdev.py index f9e7dba51be..c9f4848ac9a 100644 --- a/python/spdk/cli/bdev.py +++ b/python/spdk/cli/bdev.py @@ -117,7 +117,7 @@ def bdev_ocf_start_cache(args): help='choose between {wt|wb|wa|wo|wi|pt}; default wt (Write-Through)', choices=['wt', 'wb', 'wa', 'wo', 'wi', 'pt']) p.add_argument('-l', '--cache-line-size', - help='choose between {4|8|16|32|64}; default 4 [KiB]', + help='choose between {4|8|16|32|64} [KiB]; default 4', type=int, choices=[4, 8, 16, 32, 64]) p.add_argument('-n', '--no-load', action='store_true', help='do not load previous cache instance from metadata and force starting a new one instead (WARNING: all cache metadata will be discarded!)') @@ -164,12 +164,6 @@ def bdev_ocf_remove_core(args): p.add_argument('cache_name', help='name of the cache vbdev to remove core from') p.set_defaults(func=bdev_ocf_remove_core) - def bdev_ocf_get_bdevs(args): - print_dict(args.client.bdev_ocf_get_bdevs(name=args.name)) - p = subparsers.add_parser('bdev_ocf_get_bdevs', help='Get info about OCF devices') - p.add_argument('name', nargs='?', help='optional name of specific OCF vbdev; shows all by default') - p.set_defaults(func=bdev_ocf_get_bdevs) - def bdev_ocf_set_cachemode(args): args.client.bdev_ocf_set_cachemode( cache_name=args.cache_name, @@ -181,72 +175,109 @@ def bdev_ocf_set_cachemode(args): choices=['wt', 'wb', 'wa', 'wo', 'wi', 'pt']) p.set_defaults(func=bdev_ocf_set_cachemode) + def bdev_ocf_set_promotion(args): + args.client.bdev_ocf_set_promotion( + cache_name=args.cache_name, + policy=args.policy, + nhit_insertion_threshold=args.nhit_insertion_threshold, + nhit_trigger_threshold=args.nhit_trigger_threshold) + p = subparsers.add_parser('bdev_ocf_set_promotion', help='Set promotion parameters for OCF cache device') + p.add_argument('cache_name', help='name of the cache vbdev') + p.add_argument('-p', '--policy', + help='promotion policy (choose between {always|nhit})', + choices=['always', 'nhit'], + default='none') + p.add_argument('-i', '--nhit-insertion-threshold', + help='number of requests for given core line after which NHIT policy allows insertion into cache (range <2-1000>; default 3)', + type=int, + default=-1) + p.add_argument('-t', '--nhit-trigger-threshold', + help='cache occupancy value over which NHIT promotion is active (range <0-100> [%]; default 80)', + type=int, + default=-1) + p.set_defaults(func=bdev_ocf_set_promotion) + def bdev_ocf_set_cleaning(args): args.client.bdev_ocf_set_cleaning( cache_name=args.cache_name, policy=args.policy, - acp_wake_up=args.acp_wake_up, + acp_wake_up_time=args.acp_wake_up_time, acp_flush_max_buffers=args.acp_flush_max_buffers, - alru_wake_up=args.alru_wake_up, + alru_wake_up_time=args.alru_wake_up_time, alru_flush_max_buffers=args.alru_flush_max_buffers, alru_staleness_time=args.alru_staleness_time, - alru_activity_threshold=args.alru_activity_threshold) + alru_activity_threshold=args.alru_activity_threshold, + alru_max_dirty_ratio=args.alru_max_dirty_ratio) p = subparsers.add_parser('bdev_ocf_set_cleaning', help='Set cleaning parameters for OCF cache device') p.add_argument('cache_name', help='name of the cache vbdev') p.add_argument('-p', '--policy', - help='cleaning policy; choose between {acp|alru|nop}', - choices=['acp', 'alru', 'nop']) - p.add_argument('-u', '--acp-wake-up', - help='time between ACP cleaning thread iterations: <0-10000> [ms]; default 10', - type=int) + help='cleaning policy (choose between {acp|alru|nop})', + choices=['acp', 'alru', 'nop'], + default='none') + p.add_argument('-u', '--acp-wake-up-time', + help='time between ACP cleaning thread iterations (range <0-10000> [ms]; default 10)', + type=int, + default=-1) p.add_argument('-m', '--acp-flush-max-buffers', - help='number of dirty cache lines to be flushed in a single ACP cleaning thread iteration: <1-10000>; default 128', - type=int) - p.add_argument('-v', '--alru-wake-up', - help='cleaning thread sleep time after an idle wake up: <0-3600> [s]; default 20', - type=int) + help='number of dirty cache lines to be flushed in a single ACP cleaning thread iteration (range <1-10000>; default 128)', + type=int, + default=-1) + p.add_argument('-v', '--alru-wake-up-time', + help='cleaning thread sleep time after an idle wake up (range <0-3600> [s]; default 20)', + type=int, + default=-1) p.add_argument('-n', '--alru-flush-max-buffers', - help='number of dirty cache lines to be flushed in one cleaning cycle: <1-10000>; default 100', - type=int) + help='number of dirty cache lines to be flushed in one cleaning cycle (range <1-10000>; default 100)', + type=int, + default=-1) p.add_argument('-s', '--alru-staleness-time', - help='time that has to pass from the last write operation before a dirty cache line can be scheduled to be flushed: <1-3600> [s]; default 120', - type=int) + help='time that has to pass from the last write operation before a dirty cache line can be scheduled to be flushed (range <1-3600> [s]; default 120)', + type=int, + default=-1) p.add_argument('-t', '--alru-activity-threshold', - help='cache idle time before flushing thread can start <0-1000000> [ms]; default 10000', - type=int) + help='cache idle time before flushing thread can start (range <0-1000000> [ms]; default 10000)', + type=int, + default=-1) + p.add_argument('-d', '--alru-max-dirty-ratio', + help='maximum percentage of cache occupancy at which cleaning is triggered immediately (range <0-100> [%]; default 100)', + type=int, + default=-1) p.set_defaults(func=bdev_ocf_set_cleaning) def bdev_ocf_set_seqcutoff(args): args.client.bdev_ocf_set_seqcutoff( - core_name=args.core_name, + bdev_name=args.bdev_name, policy=args.policy, threshold=args.threshold, - promotion_count=args.promotion_count) - p = subparsers.add_parser('bdev_ocf_set_seqcutoff', help='Set sequential cut-off parameters for OCF core device') - p.add_argument('core_name', help='name of the core vbdev') + promotion_count=args.promotion_count, + promote_on_threshold=args.promote_on_threshold) + p = subparsers.add_parser('bdev_ocf_set_seqcutoff', help='Set sequential cut-off parameters for OCF core device or all cores in given cache') + p.add_argument('bdev_name', help='name of OCF vbdev') p.add_argument('-p', '--policy', - help='sequential cut-off policy; choose between {always|full|never}', - choices=['always', 'full', 'never']) + help='sequential cut-off policy (choose between {always|full|never})', + choices=['always', 'full', 'never'], + default='none') p.add_argument('-t', '--threshold', - help='activation threshold [KiB]', - type=int) - p.add_argument('-c', '--promotion_count', - help='request count threshold for cutting off the sequential stream', - type=int) + help='activation threshold (range <1-4194181> [KiB]; default 1)', + type=int, + default=-1) + p.add_argument('-c', '--promotion-count', + help='request count threshold for cutting off the sequential stream (range <1-65535>; default 8)', + type=int, + default=-1) + p.add_argument('-o', '--promote-on-threshold', + help='whether to promote core sequential cut-off stream to global structures when threshold is reached (choose between {0|1}; default 0)', + choices=[0, 1], + type=int, + default=-1) p.set_defaults(func=bdev_ocf_set_seqcutoff) def bdev_ocf_flush_start(args): - args.client.bdev_ocf_flush_start(name=args.name) - p = subparsers.add_parser('bdev_ocf_flush_start', help='Flush all dirty data from the OCF cache device to core devices') - p.add_argument('name', help='name of the OCF vbdev to flush') + args.client.bdev_ocf_flush_start(bdev_name=args.bdev_name) + p = subparsers.add_parser('bdev_ocf_flush_start', help='Flush all dirty data on the given OCF device (from cache to all of its cores or to particular core only). Note that this call only starts the flushing process which will be running in background and may take some time depending on the underlying device size and speed. You can check flushing status using the bdev_ocf_get_bdevs call.') + p.add_argument('bdev_name', help='name of OCF vbdev to flush') p.set_defaults(func=bdev_ocf_flush_start) - def bdev_ocf_flush_status(args): - print_json(args.client.bdev_ocf_flush_status(name=args.name)) - p = subparsers.add_parser('bdev_ocf_flush_status', help='Get flush status of OCF vbdev') - p.add_argument('name', help='name of the OCF vbdev') - p.set_defaults(func=bdev_ocf_flush_status) - def bdev_ocf_get_stats(args): print_dict(args.client.bdev_ocf_get_stats(name=args.name)) p = subparsers.add_parser('bdev_ocf_get_stats', help='Get statistics of chosen OCF block device') @@ -259,6 +290,12 @@ def bdev_ocf_reset_stats(args): p.add_argument('name', help='Name of OCF bdev') p.set_defaults(func=bdev_ocf_reset_stats) + def bdev_ocf_get_bdevs(args): + print_dict(args.client.bdev_ocf_get_bdevs(name=args.name)) + p = subparsers.add_parser('bdev_ocf_get_bdevs', help='Get info about OCF devices') + p.add_argument('name', nargs='?', help='optional name of specific OCF vbdev; shows all by default') + p.set_defaults(func=bdev_ocf_get_bdevs) + def bdev_malloc_create(args): num_blocks = (args.total_size * 1024 * 1024) // args.block_size print_json(args.client.bdev_malloc_create( From 6ff1a714e59dcc295d0e1e6c4d4ab9f301597232 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Fri, 11 Jul 2025 14:20:50 +0200 Subject: [PATCH 12/34] Configuration change tests Change-Id: Ia6aa94a2f352d1f9178958dd34e7e069ea2bc3d9 Signed-off-by: Rafal Stefanowski --- test/ocf/common.sh | 9 + test/ocf/management/config_change.sh | 490 +++++++++++++++++++++++++++ test/ocf/management/run.sh | 1 + 3 files changed, 500 insertions(+) create mode 100755 test/ocf/management/config_change.sh diff --git a/test/ocf/common.sh b/test/ocf/common.sh index cec6c4bf623..47c59a16fc4 100644 --- a/test/ocf/common.sh +++ b/test/ocf/common.sh @@ -189,6 +189,15 @@ __check_cores_waitlist_detached() { $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[].base_attached | not' } +# Generate random number between range. +random_number() { + # $1: lower range (inclusive) + # $2: upper range (inclusive) + # stdout: random number between range + + echo $(($1 + $RANDOM % ($2-$1+1))) +} + # Convert an array of items to comma separated list of items. array_to_comma_list() { # $1: name of array variable diff --git a/test/ocf/management/config_change.sh b/test/ocf/management/config_change.sh new file mode 100755 index 00000000000..bb93fc8b1cd --- /dev/null +++ b/test/ocf/management/config_change.sh @@ -0,0 +1,490 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f $curdir/../../..) +source "$rootdir/test/ocf/common.sh" + +cache_modes=("wt" "wb" "wa" "wo" "wi" "pt") + +promotion_policies=("always" "nhit") +promotion_nhit_insertion_threshold_range=(2 1000) +promotion_nhit_trigger_threshold_range=(0 100) + +cleaning_policies=("alru" "acp" "nop") +cleaning_acp_wake_up_time_range=(0 10000) +cleaning_acp_flush_max_buffers_range=(1 10000) +cleaning_alru_wake_up_time_range=(0 3600) +cleaning_alru_flush_max_buffers_range=(1 10000) +cleaning_alru_staleness_time_range=(1 3600) +cleaning_alru_activity_threshold_range=(0 1000000) +cleaning_alru_max_dirty_ratio_range=(0 100) + +seqcutoff_policies=("always" "full" "never") +seqcutoff_policy_default="full" +seqcutoff_threshold_range=(1 4194181) +seqcutoff_threshold_default=1 +seqcutoff_promotion_count_range=(1 65535) +seqcutoff_promotion_count_default=8 +seqcutoff_promote_on_threshold_range=(0 1) +seqcutoff_promote_on_threshold_default=false + +set_cache_mode() { + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local cache_mode=$1 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_cachemode Ocf_cache$i $cache_mode + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_promotion_always_params() { + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_promotion Ocf_cache$i \ + --policy $policy + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_promotion_nhit_params() { + if [ $# -ne 3 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + local insertion_threshold=$2 + local trigger_threshold=$3 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_promotion Ocf_cache$i \ + --policy $policy \ + --nhit-insertion-threshold $insertion_threshold \ + --nhit-trigger-threshold $trigger_threshold + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_cleaning_alru_params() { + if [ $# -ne 6 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + local wake_up_time=$2 + local flush_max_buffers=$3 + local staleness_time=$4 + local activity_threshold=$5 + local max_dirty_ratio=$6 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_cleaning Ocf_cache$i \ + --policy $policy \ + --alru-wake-up-time $wake_up_time \ + --alru-flush-max-buffers $flush_max_buffers \ + --alru-staleness-time $staleness_time \ + --alru-activity-threshold $activity_threshold \ + --alru-max-dirty-ratio $max_dirty_ratio + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_cleaning_acp_params() { + if [ $# -ne 3 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + local wake_up_time=$2 + local flush_max_buffers=$3 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_cleaning Ocf_cache$i \ + --policy $policy \ + --acp-wake-up-time $wake_up_time \ + --acp-flush-max-buffers $flush_max_buffers + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_cleaning_nop_params() { + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_cleaning Ocf_cache$i \ + --policy $policy + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_seqcutoff_params() { + if [ $# -ne 4 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + local threshold=$2 + local promotion_count=$3 + local promote_on_threshold=$4 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_seqcutoff Ocf_core$i-1 \ + --policy $policy \ + --threshold $threshold \ + --promotion-count $promotion_count \ + --promote-on-threshold $promote_on_threshold + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_seqcutoff_params_all() { + if [ $# -ne 4 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + local threshold=$2 + local promotion_count=$3 + local promote_on_threshold=$4 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_seqcutoff Ocf_cache$i \ + --policy $policy \ + --threshold $threshold \ + --promotion-count $promotion_count \ + --promote-on-threshold $promote_on_threshold + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +__check_cache_mode() { + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local cache_mode=$1 + + $rpc_py bdev_ocf_get_bdevs | jq -e --arg cache_mode $cache_mode \ + '.caches[].cache_mode == $cache_mode' +} + +__check_promotion_always_params() { + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --arg policy $policy \ + '.caches[].promotion.policy == $policy' +} + +__check_promotion_nhit_params() { + if [ $# -ne 3 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + local insertion_threshold=$2 + local trigger_threshold=$3 + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --arg policy $policy \ + --argjson insertion_threshold $insertion_threshold \ + --argjson trigger_threshold $trigger_threshold \ + '.caches[].promotion | + (.policy == $policy) and + (.insertion_threshold == $insertion_threshold) and + (.trigger_threshold == $trigger_threshold)' +} + +__check_cleaning_alru_params() { + if [ $# -ne 6 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + local wake_up_time=$2 + local flush_max_buffers=$3 + local staleness_time=$4 + local activity_threshold=$5 + local max_dirty_ratio=$6 + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --arg policy $policy \ + --argjson wake_up_time $wake_up_time \ + --argjson flush_max_buffers $flush_max_buffers \ + --argjson staleness_time $staleness_time \ + --argjson activity_threshold $activity_threshold \ + --argjson max_dirty_ratio $max_dirty_ratio \ + '.caches[].cleaning | + (.policy == $policy) and + (.wake_up_time == $wake_up_time) and + (.flush_max_buffers == $flush_max_buffers) and + (.staleness_time == $staleness_time) and + (.activity_threshold == $activity_threshold) and + (.max_dirty_ratio == $max_dirty_ratio)' +} + +__check_cleaning_acp_params() { + if [ $# -ne 3 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + local wake_up_time=$2 + local flush_max_buffers=$3 + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --arg policy $policy \ + --argjson wake_up_time $wake_up_time \ + --argjson flush_max_buffers $flush_max_buffers \ + '.caches[].cleaning | + (.policy == $policy) and + (.wake_up_time == $wake_up_time) and + (.flush_max_buffers == $flush_max_buffers)' +} + +__check_cleaning_nop_params() { + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --arg policy $policy \ + '.caches[].cleaning.policy == $policy' +} + +__check_seqcutoff_params() { + if [ $# -ne 4 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + local threshold=$2 + local promotion_count=$3 + if [ $4 -eq 0 ]; then + local promote_on_threshold=false + elif [ $4 -eq 1 ]; then + local promote_on_threshold=true + fi + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --arg policy $policy \ + --argjson threshold $threshold \ + --argjson promotion_count $promotion_count \ + --argjson promote_on_threshold $promote_on_threshold \ + '.caches[].cores[0].seq_cutoff | + (.policy == $policy) and + (.threshold == $threshold) and + (.promotion_count == $promotion_count) and + (.promote_on_threshold == $promote_on_threshold)' + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --arg policy $seqcutoff_policy_default \ + --argjson threshold $seqcutoff_threshold_default \ + --argjson promotion_count $seqcutoff_promotion_count_default \ + --argjson promote_on_threshold $seqcutoff_promote_on_threshold_default \ + '.caches[].cores[1].seq_cutoff | + (.policy == $policy) and + (.threshold == $threshold) and + (.promotion_count == $promotion_count) and + (.promote_on_threshold == $promote_on_threshold)' +} + +__check_seqcutoff_params_all() { + if [ $# -ne 4 ]; then + echo >&2 "invalid number of arguments" + exit 1; + fi + + local policy=$1 + local threshold=$2 + local promotion_count=$3 + if [ $4 -eq 0 ]; then + local promote_on_threshold=false + elif [ $4 -eq 1 ]; then + local promote_on_threshold=true + fi + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --arg policy $policy \ + --argjson threshold $threshold \ + --argjson promotion_count $promotion_count \ + --argjson promote_on_threshold $promote_on_threshold \ + '.caches[].cores[].seq_cutoff | + (.policy == $policy) and + (.threshold == $threshold) and + (.promotion_count == $promotion_count) and + (.promote_on_threshold == $promote_on_threshold)' +} + +# cache modes: + +for add_cores in false true; do +for create_caches in false true; do +for stop_caches in false true; do + start_spdk + start_caches + if [ $create_caches = true ]; then + create_caches + __check_caches_attached + if [ $add_cores = true ]; then + create_cores + add_cores + __check_cores_attached + fi + else + __check_caches_detached + fi + for cache_mode in "${cache_modes[@]}"; do + set_cache_mode $cache_mode + __check_cache_mode $cache_mode + done + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done +done + +# promotion params: + +for promotion_policy in "${promotion_policies[@]}"; do +for add_cores in false true; do +for stop_caches in false true; do + start_spdk + create_caches + start_caches + __check_caches_attached + if [ $add_cores = true ]; then + create_cores + add_cores + __check_cores_attached + fi + if [ $promotion_policy = always ]; then + set_promotion_always_params $promotion_policy + __check_promotion_always_params $promotion_policy + elif [ $promotion_policy = nhit ]; then + insertion_threshold=$(random_number ${promotion_nhit_insertion_threshold_range[0]} ${promotion_nhit_insertion_threshold_range[1]}) + trigger_threshold=$(random_number ${promotion_nhit_trigger_threshold_range[0]} ${promotion_nhit_trigger_threshold_range[1]}) + set_promotion_nhit_params $promotion_policy $insertion_threshold $trigger_threshold + __check_promotion_nhit_params $promotion_policy $insertion_threshold $trigger_threshold + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done +done + +# cleaning params: + +for cleaning_policy in "${cleaning_policies[@]}"; do +for add_cores in false true; do +for stop_caches in false true; do + start_spdk + create_caches + start_caches + __check_caches_attached + if [ $add_cores = true ]; then + create_cores + add_cores + __check_cores_attached + fi + if [ $cleaning_policy = alru ]; then + wake_up_time=$(random_number ${cleaning_alru_wake_up_time_range[0]} ${cleaning_alru_wake_up_time_range[1]}) + flush_max_buffers=$(random_number ${cleaning_alru_flush_max_buffers_range[0]} ${cleaning_alru_flush_max_buffers_range[1]}) + staleness_time=$(random_number ${cleaning_alru_staleness_time_range[0]} ${cleaning_alru_staleness_time_range[1]}) + activity_threshold=$(random_number ${cleaning_alru_activity_threshold_range[0]} ${cleaning_alru_activity_threshold_range[1]}) + max_dirty_ratio=$(random_number ${cleaning_alru_max_dirty_ratio_range[0]} ${cleaning_alru_max_dirty_ratio_range[1]}) + set_cleaning_alru_params $cleaning_policy $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio + __check_cleaning_alru_params $cleaning_policy $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio + elif [ $cleaning_policy = acp ]; then + wake_up_time=$(random_number ${cleaning_acp_wake_up_time_range[0]} ${cleaning_acp_wake_up_time_range[1]}) + flush_max_buffers=$(random_number ${cleaning_acp_flush_max_buffers_range[0]} ${cleaning_acp_flush_max_buffers_range[1]}) + set_cleaning_acp_params $cleaning_policy $wake_up_time $flush_max_buffers + __check_cleaning_acp_params $cleaning_policy $wake_up_time $flush_max_buffers + elif [ $cleaning_policy = nop ]; then + set_cleaning_nop_params $cleaning_policy + __check_cleaning_nop_params $cleaning_policy + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done +done + +# sequential cut-off params: + +for device in core cache; do +for seqcutoff_policy in "${seqcutoff_policies[@]}"; do +for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + threshold=$(random_number ${seqcutoff_threshold_range[0]} ${seqcutoff_threshold_range[1]}) + promotion_count=$(random_number ${seqcutoff_promotion_count_range[0]} ${seqcutoff_promotion_count_range[1]}) + promote_on_threshold=$(random_number ${seqcutoff_promote_on_threshold_range[0]} ${seqcutoff_promote_on_threshold_range[1]}) + if [ $device = core ]; then + set_seqcutoff_params $seqcutoff_policy $threshold $promotion_count $promote_on_threshold + __check_seqcutoff_params $seqcutoff_policy $threshold $promotion_count $promote_on_threshold + elif [ $device = cache ]; then + set_seqcutoff_params_all $seqcutoff_policy $threshold $promotion_count $promote_on_threshold + __check_seqcutoff_params_all $seqcutoff_policy $threshold $promotion_count $promote_on_threshold + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk +done +done +done diff --git a/test/ocf/management/run.sh b/test/ocf/management/run.sh index d58cd724aa6..220ed6be73d 100755 --- a/test/ocf/management/run.sh +++ b/test/ocf/management/run.sh @@ -14,3 +14,4 @@ source $rootdir/test/common/autotest_common.sh run_test "start-stop" "$curdir/start-stop/run.sh" run_test "detach-attach" "$curdir/detach-attach.sh" run_test "hotremove" "$curdir/hotremove.sh" +run_test "config_change" "$curdir/config_change.sh" From 3abb6cde0e1bad1d353dcf1d3fcfbb9a14b55123 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Thu, 24 Jul 2025 10:08:49 +0200 Subject: [PATCH 13/34] Various improvements Change-Id: Ib36ae38a03d1bca9b2595a78b7cc162ab787ebe8 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/ctx.h | 1 - module/bdev/ocf/vbdev_ocf.c | 1102 +++++++++++++---------- module/bdev/ocf/vbdev_ocf.h | 10 +- module/bdev/ocf/vbdev_ocf_cache.c | 55 +- module/bdev/ocf/vbdev_ocf_cache.h | 45 + module/bdev/ocf/vbdev_ocf_core.c | 186 ++-- module/bdev/ocf/vbdev_ocf_core.h | 38 +- module/bdev/ocf/vbdev_ocf_rpc.c | 65 +- python/spdk/cli/bdev.py | 19 +- test/ocf/common.sh | 2 +- test/ocf/management/config_change.sh | 6 +- test/ocf/management/start-stop/basic.sh | 6 +- 12 files changed, 874 insertions(+), 661 deletions(-) diff --git a/module/bdev/ocf/ctx.h b/module/bdev/ocf/ctx.h index fd9a175f4ff..8bfa24e8521 100644 --- a/module/bdev/ocf/ctx.h +++ b/module/bdev/ocf/ctx.h @@ -15,7 +15,6 @@ extern ocf_ctx_t vbdev_ocf_ctx; #define SPDK_OBJECT 1 -// why? #define VBDEV_OCF_QUEUE_RUN_MAX 32 int vbdev_ocf_ctx_init(void); diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index f4ba90bafd6..9380e502d1d 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -34,7 +34,6 @@ struct spdk_bdev_module ocf_if = { .examine_disk = vbdev_ocf_module_examine_disk, .async_fini_start = true, }; - SPDK_BDEV_MODULE_REGISTER(ocf, &ocf_if) static int vbdev_ocf_fn_destruct(void *ctx); @@ -55,68 +54,135 @@ struct spdk_bdev_fn_table vbdev_ocf_fn_table = { .reset_device_stat = NULL, // todo ? }; -// rm -#define __bdev_to_io_dev(bdev) (((char *)bdev) + 1) - static int -_bdev_exists_core_visit(ocf_core_t core, void *cb_arg) +_bdev_exists_cache_visitor(ocf_cache_t cache, void *ctx) { - const char *name = cb_arg; - struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); - - if (!core_ctx) { - /* Skip this core. If there is no context, it means that this core - * was added from metadata during cache load and it's just an empty shell. */ - return 0; - } + char *name = ctx; + ocf_core_t core; + int rc; - if (!strcmp(name, ocf_core_get_name(core))) { + rc = ocf_core_get_by_name(cache, name, OCF_CORE_NAME_SIZE, &core); + /* Check if found core has context (priv) attached as well. Only then + * it counts as a regular core and not just added during cache load. */ + if (!rc && ocf_core_get_priv(core)) { return -EEXIST; + } else if (rc && rc != -OCF_ERR_CORE_NOT_EXIST) { + SPDK_ERRLOG("OCF: failed to get core: %s\n", spdk_strerror(-rc)); } return 0; } -static int -_bdev_exists_cache_visit(ocf_cache_t cache, void *cb_arg) -{ - char *name = cb_arg; - - if (!strcmp(name, ocf_cache_get_name(cache))) { - return -EEXIST; - } - - return ocf_core_visit(cache, _bdev_exists_core_visit, name, false); -} - static bool vbdev_ocf_bdev_exists(const char *name) { - struct vbdev_ocf_core *core_ctx; + ocf_cache_t cache; int rc; - SPDK_DEBUGLOG(vbdev_ocf, "OCF: looking for '%s' in existing bdev names\n", name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': looking for it in existing bdev names\n", name); - vbdev_ocf_foreach_core_in_waitlist(core_ctx) { - if (!strcmp(name, vbdev_ocf_core_get_name(core_ctx))) { - return true; - } + if (vbdev_ocf_core_waitlist_get_by_name(name)) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': found in core wait list\n", name); + + return true; + } + + rc = ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, name, OCF_CACHE_NAME_SIZE, &cache); + if (!rc) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': found cache\n", name); + + /* If cache was found, do not increase its refcount. */ + ocf_mngt_cache_put(cache); + + return true; + } else if (rc && rc != -OCF_ERR_CACHE_NOT_EXIST) { + SPDK_ERRLOG("OCF: failed to get cache: %s\n", spdk_strerror(-rc)); } - rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _bdev_exists_cache_visit, (char *)name); + rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _bdev_exists_cache_visitor, (char *)name); if (rc == -EEXIST) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': found core\n", name); + return true; } else if (rc) { SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); } if (spdk_bdev_get_by_name(name)) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': found in SPDK bdev layer\n", name); + return true; } return false; } +static int +_bdev_resolve_cache_visitor(ocf_cache_t cache, void *ctx) +{ + struct vbdev_ocf_mngt_ctx *mngt_ctx = ctx; + int rc; + + rc = ocf_core_get_by_name(cache, mngt_ctx->bdev_name, OCF_CORE_NAME_SIZE, &mngt_ctx->core); + if (rc && rc != -OCF_ERR_CORE_NOT_EXIST) { + return rc; + } else if (!rc) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': found core\n", mngt_ctx->bdev_name); + + return -EEXIST; + } + + return 0; +} + +/* Takes name of bdev and saves pointer to either cache or + * core of that name inside given management context. */ +static int +vbdev_ocf_bdev_resolve(struct vbdev_ocf_mngt_ctx *mngt_ctx) +{ + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': looking for cache or core of that name\n", + mngt_ctx->bdev_name); + + rc = ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, mngt_ctx->bdev_name, OCF_CACHE_NAME_SIZE, + &mngt_ctx->cache); + if (rc && rc != -OCF_ERR_CACHE_NOT_EXIST) { + return rc; + } else if (!rc) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': found cache\n", mngt_ctx->bdev_name); + + /* If cache was found, do not increase its refcount. */ + ocf_mngt_cache_put(mngt_ctx->cache); + return 0; + } + + rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _bdev_resolve_cache_visitor, mngt_ctx); + assert(!(mngt_ctx->cache && mngt_ctx->core)); + if (rc && rc != -EEXIST) { + return rc; + } else if (!rc || (!mngt_ctx->cache && !mngt_ctx->core)) { + return -ENXIO; + } + + return 0; +} + +static void +vbdev_ocf_mem_calculate(ocf_cache_t cache) +{ + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + uint64_t mem_needed, volume_size; + + volume_size = spdk_bdev_get_block_size(cache_ctx->base.bdev) * + spdk_bdev_get_num_blocks(cache_ctx->base.bdev); + mem_needed = ocf_mngt_get_ram_needed(cache, volume_size); + + SPDK_NOTICELOG("Needed memory to start cache in this configuration " + "(device size: %"PRIu64", cache line size: %"PRIu64"): %"PRIu64"\n", + volume_size, cache_ctx->cache_cfg.cache_line_size, mem_needed); +} + static int vbdev_ocf_module_init(void) { @@ -125,13 +191,13 @@ vbdev_ocf_module_init(void) SPDK_DEBUGLOG(vbdev_ocf, "OCF: starting module\n"); if ((rc = vbdev_ocf_ctx_init())) { - SPDK_ERRLOG("OCF: failed to initialize context: %d\n", rc); + SPDK_ERRLOG("OCF: failed to initialize context: %s\n", spdk_strerror(-rc)); return rc; } if ((rc = vbdev_ocf_volume_init())) { vbdev_ocf_ctx_cleanup(); - SPDK_ERRLOG("OCF: failed to register volume: %d\n", rc); + SPDK_ERRLOG("OCF: failed to register volume: %s\n", spdk_strerror(-rc)); return rc; } @@ -141,7 +207,7 @@ vbdev_ocf_module_init(void) static void _cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *cache_stop_ctx = cb_arg; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; ocf_queue_t cache_mngt_q = ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", @@ -154,8 +220,8 @@ _cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) SPDK_NOTICELOG("OCF cache '%s': stopped\n", ocf_cache_get_name(cache)); } - /* In module fini (no cache stop context) do the cleanup despite the error. */ - if (!error || !cache_stop_ctx) { + /* In module fini (no management context) do the cleanup despite the error. */ + if (!error || !mngt_ctx) { if (vbdev_ocf_cache_is_base_attached(cache)) { vbdev_ocf_cache_base_detach(cache); } else { @@ -168,14 +234,14 @@ _cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_mngt_cache_unlock(cache); - if (cache_stop_ctx) { + if (mngt_ctx) { SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop\n", ocf_cache_get_name(cache)); - cache_stop_ctx->rpc_cb_fn(NULL, cache_stop_ctx->rpc_cb_arg, error); // cache name - free(cache_stop_ctx); + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); + free(mngt_ctx); } else if (!ocf_mngt_cache_get_count(vbdev_ocf_ctx)) { - /* In module fini (no cache stop context) call spdk_bdev_module_fini_start_done() + /* In module fini (no management context) call spdk_bdev_module_fini_start_done() * if there are no caches left to stop. */ spdk_bdev_module_fini_start_done(); } @@ -203,7 +269,6 @@ _cache_stop_lock_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_cache_get_name(cache), error); } - // no need to manually flush ? if (ocf_mngt_cache_is_dirty(cache)) { ocf_mngt_cache_flush(cache, _cache_stop_flush_cb, cb_arg); } else { @@ -217,13 +282,13 @@ _cache_stop_core_unregister_cb(void *cb_arg, int error) ocf_core_t core = cb_arg; ocf_cache_t cache = ocf_core_get_cache(core); struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); - struct vbdev_ocf_mngt_ctx *cache_stop_ctx = core_ctx->mngt_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx = core_ctx->mngt_ctx; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", ocf_core_get_name(core)); if (error) { - SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev: %s\n", + SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during cache stop: %s\n", ocf_core_get_name(core), spdk_strerror(-error)); } @@ -233,14 +298,14 @@ _cache_stop_core_unregister_cb(void *cb_arg, int error) if (ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { /* All cores in this cache were already unregistered * and detached, so proceed with stopping the cache. */ - ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, cache_stop_ctx); + ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, mngt_ctx); } } static int -_cache_stop_core_visit(ocf_core_t core, void *cb_arg) +_cache_stop_core_visitor(ocf_core_t core, void *cb_arg) { - struct vbdev_ocf_mngt_ctx *cache_stop_ctx = cb_arg; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; ocf_cache_t cache = ocf_core_get_cache(core); struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); int rc = 0; @@ -261,7 +326,7 @@ _cache_stop_core_visit(ocf_core_t core, void *cb_arg) return 0; } - core_ctx->mngt_ctx = cache_stop_ctx; + core_ctx->mngt_ctx = mngt_ctx; if ((rc = vbdev_ocf_core_unregister(core_ctx, _cache_stop_core_unregister_cb, core))) { SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev: %s\n", @@ -274,7 +339,7 @@ _cache_stop_core_visit(ocf_core_t core, void *cb_arg) } static int -_module_fini_cache_visit(ocf_cache_t cache, void *cb_arg) +_module_fini_cache_visitor(ocf_cache_t cache, void *cb_arg) { int i, rc = 0; @@ -295,7 +360,7 @@ _module_fini_cache_visit(ocf_cache_t cache, void *cb_arg) ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, NULL); } - if ((rc = ocf_core_visit(cache, _cache_stop_core_visit, NULL, false))) { + if ((rc = ocf_core_visit(cache, _cache_stop_core_visitor, NULL, false))) { SPDK_ERRLOG("OCF cache '%s': failed to iterate over core bdevs: %s\n", ocf_cache_get_name(cache), spdk_strerror(-rc)); ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, NULL); @@ -324,7 +389,7 @@ vbdev_ocf_module_fini_start(void) return; } - if ((rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _module_fini_cache_visit, NULL))) { + if ((rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _module_fini_cache_visitor, NULL))) { SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); spdk_bdev_module_fini_start_done(); return; @@ -339,7 +404,7 @@ vbdev_ocf_module_fini(void) SPDK_DEBUGLOG(vbdev_ocf, "OCF: finishing module stop\n"); while ((core_ctx = STAILQ_FIRST(&g_vbdev_ocf_core_waitlist))) { - STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); + vbdev_ocf_core_waitlist_remove(core_ctx); vbdev_ocf_core_destroy(core_ctx); } @@ -354,7 +419,7 @@ vbdev_ocf_module_get_ctx_size(void) } static int -_examine_config_core_visit(ocf_core_t core, void *cb_arg) +_examine_config_core_visitor(ocf_core_t core, void *cb_arg) { struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); char *bdev_name = cb_arg; @@ -391,14 +456,14 @@ _examine_config_core_visit(ocf_core_t core, void *cb_arg) } static int -_examine_config_cache_visit(ocf_cache_t cache, void *cb_arg) +_examine_config_cache_visitor(ocf_cache_t cache, void *cb_arg) { struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); char *bdev_name = cb_arg; int rc = 0; if (strcmp(bdev_name, cache_ctx->base.name)) { - return ocf_core_visit(cache, _examine_config_core_visit, bdev_name, false); + return ocf_core_visit(cache, _examine_config_core_visitor, bdev_name, false); } SPDK_NOTICELOG("OCF cache '%s': base bdev '%s' found\n", @@ -430,7 +495,7 @@ vbdev_ocf_module_examine_config(struct spdk_bdev *bdev) char *bdev_name = (char *)spdk_bdev_get_name(bdev); int rc; - SPDK_DEBUGLOG(vbdev_ocf, "OCF: looking for vbdevs waiting for '%s'\n", bdev_name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': looking for vbdevs waiting for it\n", bdev_name); vbdev_ocf_foreach_core_in_waitlist(core_ctx) { if (strcmp(bdev_name, core_ctx->base.name)) { @@ -443,15 +508,15 @@ vbdev_ocf_module_examine_config(struct spdk_bdev *bdev) assert(!vbdev_ocf_core_is_base_attached(core_ctx)); if ((rc = vbdev_ocf_core_base_attach(core_ctx, bdev_name))) { - SPDK_ERRLOG("OCF core '%s': failed to attach base bdev '%s'\n", - vbdev_ocf_core_get_name(core_ctx), bdev_name); + SPDK_ERRLOG("OCF core '%s': failed to attach base bdev '%s': %s\n", + vbdev_ocf_core_get_name(core_ctx), bdev_name, spdk_strerror(-rc)); } spdk_bdev_module_examine_done(&ocf_if); return; } - rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _examine_config_cache_visit, bdev_name); + rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _examine_config_cache_visitor, bdev_name); if (rc && rc != -EEXIST) { SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); } @@ -477,7 +542,6 @@ static void _core_add_examine_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int error) { struct vbdev_ocf_core *core_ctx = cb_arg; - struct vbdev_ocf_core *core_ctx_waitlist; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", @@ -496,9 +560,10 @@ _core_add_examine_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int e ocf_core_set_priv(core, core_ctx); if ((rc = vbdev_ocf_core_register(core))) { - SPDK_ERRLOG("OCF core '%s': failed to register vbdev\n", ocf_core_get_name(core)); + SPDK_ERRLOG("OCF core '%s': failed to register vbdev: %s\n", + ocf_core_get_name(core), spdk_strerror(-rc)); ocf_mngt_cache_remove_core(core, _core_add_examine_err_cb, cache); - vbdev_ocf_core_base_detach(core_ctx); // move to callback (pass core_ctx ?) + vbdev_ocf_core_base_detach(core_ctx); // move to callback (pass core_ctx) ? return; } @@ -506,12 +571,8 @@ _core_add_examine_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int e ocf_core_get_name(core), ocf_cache_get_name(cache)); /* If core was taken from waitlist, remove it from there. */ - vbdev_ocf_foreach_core_in_waitlist(core_ctx_waitlist) { - if (strcmp(vbdev_ocf_core_get_name(core_ctx), vbdev_ocf_core_get_name(core_ctx_waitlist))) { - continue; - } - - STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); + if (vbdev_ocf_core_waitlist_get_by_name(vbdev_ocf_core_get_name(core_ctx))) { + vbdev_ocf_core_waitlist_remove(core_ctx); } ocf_mngt_cache_unlock(cache); @@ -542,7 +603,7 @@ _core_add_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) static void _cache_attach_examine_attach_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *examine_attach_ctx = cb_arg; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing device attach\n", ocf_cache_get_name(cache)); @@ -550,26 +611,32 @@ _cache_attach_examine_attach_cb(ocf_cache_t cache, void *cb_arg, int error) /* At this point volume was either moved to ocf_cache_t struct * or is no longer needed due to some errors, so we need to deallocate it. */ vbdev_ocf_cache_config_volume_destroy(cache); + ocf_mngt_cache_unlock(cache); if (error) { SPDK_ERRLOG("OCF cache '%s': failed to attach OCF cache device (OCF error: %d)\n", ocf_cache_get_name(cache), error); + + if (error == -OCF_ERR_NO_MEM) { + SPDK_ERRLOG("Not enough memory to handle cache device of this size. Try to increase hugepage memory size, increase cache line size or use smaller cache device.\n"); + vbdev_ocf_mem_calculate(cache); + } + vbdev_ocf_cache_base_detach(cache); } else { SPDK_NOTICELOG("OCF cache '%s': device attached\n", ocf_cache_get_name(cache)); - } - ocf_mngt_cache_unlock(cache); - free(examine_attach_ctx); - spdk_bdev_module_examine_done(&ocf_if); // move after adding cores from waitlist ? + vbdev_ocf_core_add_from_waitlist(cache); + } - vbdev_ocf_core_add_from_waitlist(cache); + spdk_bdev_module_examine_done(&ocf_if); + free(mngt_ctx); } static void _cache_attach_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *examine_attach_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': attaching OCF cache device\n", @@ -581,25 +648,25 @@ _cache_attach_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) goto err_lock; } - examine_attach_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); - if (!examine_attach_ctx) { + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for examine attach context: %s\n", ocf_cache_get_name(cache), spdk_strerror(-ENOMEM)); goto err_alloc; } - examine_attach_ctx->cache = cache; - examine_attach_ctx->u.att_cb_fn = _cache_attach_examine_attach_cb; + mngt_ctx->cache = cache; + mngt_ctx->u.att_cb_fn = _cache_attach_examine_attach_cb; - if ((rc = vbdev_ocf_cache_volume_attach(cache, examine_attach_ctx))) { - SPDK_ERRLOG("OCF cache '%s': failed to attach volume (OCF error: %d)\n", - ocf_cache_get_name(cache), rc); + if ((rc = vbdev_ocf_cache_volume_attach(cache, mngt_ctx))) { + SPDK_ERRLOG("OCF cache '%s': failed to attach volume: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-rc)); goto err_attach; } return; err_attach: - free(examine_attach_ctx); + free(mngt_ctx); err_alloc: ocf_mngt_cache_unlock(cache); err_lock: @@ -609,7 +676,7 @@ _cache_attach_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } static int -_examine_disk_core_visit(ocf_core_t core, void *cb_arg) +_examine_disk_core_visitor(ocf_core_t core, void *cb_arg) { ocf_cache_t cache = ocf_core_get_cache(core); struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); @@ -633,13 +700,13 @@ _examine_disk_core_visit(ocf_core_t core, void *cb_arg) } static int -_examine_disk_cache_visit(ocf_cache_t cache, void *cb_arg) +_examine_disk_cache_visitor(ocf_cache_t cache, void *cb_arg) { struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); char *bdev_name = cb_arg; if (strcmp(bdev_name, cache_ctx->base.name)) { - return ocf_core_visit(cache, _examine_disk_core_visit, bdev_name, false); + return ocf_core_visit(cache, _examine_disk_core_visitor, bdev_name, false); } assert(!ocf_cache_is_device_attached(cache)); @@ -685,7 +752,7 @@ vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev) return; } - rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _examine_disk_cache_visit, bdev_name); + rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _examine_disk_cache_visitor, bdev_name); if (rc && rc != -EEXIST) { SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); } else if (!rc) { @@ -706,7 +773,6 @@ _destruct_core_detach_cb(ocf_core_t core, void *cb_arg, int error) if (error) { SPDK_ERRLOG("OCF vbdev '%s': failed to remove OCF core device (OCF error: %d)\n", ocf_core_get_name(core), error); - //spdk_bdev_destruct_done(&core->ocf_vbdev, error); } ocf_mngt_cache_unlock(cache); @@ -722,11 +788,9 @@ _destruct_core_flush_cb(ocf_core_t core, void *cb_arg, int error) if (error) { SPDK_ERRLOG("OCF vbdev '%s': failed to flush OCF core device (OCF error: %d)\n", ocf_core_get_name(core), error); - //ocf_mngt_cache_unlock(cache); - //spdk_bdev_destruct_done(&core_ctx->ocf_vbdev, error); - //return; } + /* Detach core instead of removing it, so it stays in the cache metadata. */ ocf_mngt_cache_detach_core(core, _destruct_core_detach_cb, cb_arg); } @@ -741,13 +805,12 @@ _destruct_cache_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (error) { SPDK_ERRLOG("OCF vbdev '%s': failed to acquire OCF cache lock (OCF error: %d)\n", ocf_core_get_name(core), error); - //spdk_bdev_destruct_done(&core_ctx->ocf_vbdev, error); - //return; } if (ocf_mngt_core_is_dirty(core)) { ocf_mngt_core_flush(core, _destruct_core_flush_cb, cache); } else { + /* Detach core instead of removing it, so it stays in the cache metadata. */ ocf_mngt_cache_detach_core(core, _destruct_core_detach_cb, cache); } } @@ -787,8 +850,8 @@ _vbdev_ocf_submit_io_cb(ocf_io_t io, void *priv1, void *priv2, int error) if (error == -OCF_ERR_NO_MEM) { spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); } else if (error) { - SPDK_ERRLOG("OCF vbdev '%s': failed to complete OCF IO\n", - spdk_bdev_get_name(bdev_io->bdev)); + SPDK_ERRLOG("OCF vbdev '%s': failed to complete OCF IO: %s\n", + spdk_bdev_get_name(bdev_io->bdev), spdk_strerror(-error)); spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); } else { spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); @@ -936,8 +999,7 @@ _cache_start_rpc_err_cb(ocf_cache_t cache, void *cb_arg, int error) static void _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *cache_start_ctx = cb_arg; - struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing start\n", ocf_cache_get_name(cache)); @@ -946,36 +1008,26 @@ _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) vbdev_ocf_cache_config_volume_destroy(cache); if (error) { - SPDK_ERRLOG("OCF cache '%s': failed to attach OCF cache device\n", - ocf_cache_get_name(cache)); + SPDK_ERRLOG("OCF cache '%s': failed to attach OCF cache device (OCF error: %d)\n", + ocf_cache_get_name(cache), error); if (error == -OCF_ERR_NO_MEM) { - uint64_t mem_needed, volume_size; - - volume_size = cache_ctx->base.bdev->blockcnt * cache_ctx->base.bdev->blocklen; - mem_needed = ocf_mngt_get_ram_needed(cache, volume_size); - SPDK_ERRLOG("Not enough memory. Try to increase hugepage memory size or cache line size.\n"); - SPDK_NOTICELOG("Needed memory to start cache in this configuration " - "(device size: %"PRIu64", cache line size: %"PRIu64"): %"PRIu64"\n", - volume_size, cache_ctx->cache_cfg.cache_line_size, mem_needed); + SPDK_ERRLOG("Not enough memory to handle cache device of this size. Try to increase hugepage memory size, increase cache line size or use smaller cache device.\n"); + vbdev_ocf_mem_calculate(cache); } vbdev_ocf_cache_base_detach(cache); - ocf_queue_put(cache_ctx->cache_mngt_q); // needed ? + ocf_queue_put(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); // needed ? ocf_mngt_cache_stop(cache, _cache_start_rpc_err_cb, NULL); - cache_start_ctx->rpc_cb_fn(NULL, cache_start_ctx->rpc_cb_arg, error); // cache name - free(cache_start_ctx); + } else { + SPDK_NOTICELOG("OCF cache '%s': started\n", ocf_cache_get_name(cache)); - return; + ocf_mngt_cache_unlock(cache); + vbdev_ocf_core_add_from_waitlist(cache); } - - SPDK_NOTICELOG("OCF cache '%s': started\n", ocf_cache_get_name(cache)); - ocf_mngt_cache_unlock(cache); - cache_start_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_start_ctx->rpc_cb_arg, 0); - free(cache_start_ctx); - - vbdev_ocf_core_add_from_waitlist(cache); + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); + free(mngt_ctx); } /* RPC entry point. */ @@ -985,66 +1037,73 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { ocf_cache_t cache; - struct vbdev_ocf_mngt_ctx *cache_start_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating start\n", cache_name); if (vbdev_ocf_bdev_exists(cache_name)) { - SPDK_ERRLOG("OCF: bdev '%s' already exists\n", cache_name); + SPDK_ERRLOG("OCF '%s': bdev already exists\n", cache_name); rc = -EEXIST; goto err_exist; } if ((rc = vbdev_ocf_cache_create(&cache, cache_name, cache_mode, cache_line_size, no_load))) { - SPDK_ERRLOG("OCF cache '%s': failed to create cache\n", cache_name); + SPDK_ERRLOG("OCF cache '%s': failed to create cache: %s\n", + cache_name, spdk_strerror(-rc)); goto err_create; } if ((rc = vbdev_ocf_cache_mngt_queue_create(cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to create management queue\n", cache_name); + SPDK_ERRLOG("OCF cache '%s': failed to create management queue: %s\n", + cache_name, spdk_strerror(-rc)); goto err_queue; } + /* Check if base device for this cache is already present. */ if ((rc = vbdev_ocf_cache_base_attach(cache, base_name))) { if (rc == -ENODEV) { + /* If not, just leave started cache without the device and exit. */ SPDK_NOTICELOG("OCF cache '%s': start deferred - waiting for base bdev '%s'\n", cache_name, base_name); ocf_mngt_cache_unlock(cache); rpc_cb_fn(cache_name, rpc_cb_arg, -ENODEV); return; } - SPDK_ERRLOG("OCF cache '%s': failed to attach base bdev '%s'\n", cache_name, base_name); + SPDK_ERRLOG("OCF cache '%s': failed to attach base bdev '%s': %s\n", + cache_name, base_name, spdk_strerror(-rc)); goto err_base; } if ((rc = vbdev_ocf_cache_config_volume_create(cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to create config volume\n", cache_name); + SPDK_ERRLOG("OCF cache '%s': failed to create config volume: %s\n", + cache_name, spdk_strerror(-rc)); goto err_volume; } - cache_start_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); - if (!cache_start_ctx) { + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache start context\n", cache_name); rc = -ENOMEM; goto err_alloc; } - cache_start_ctx->rpc_cb_fn = rpc_cb_fn; - cache_start_ctx->rpc_cb_arg = rpc_cb_arg; - cache_start_ctx->cache = cache; - cache_start_ctx->u.att_cb_fn = _cache_start_rpc_attach_cb; + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->cache = cache; + mngt_ctx->u.att_cb_fn = _cache_start_rpc_attach_cb; - if ((rc = vbdev_ocf_cache_volume_attach(cache, cache_start_ctx))) { - SPDK_ERRLOG("OCF cache '%s': failed to attach volume\n", cache_name); + if ((rc = vbdev_ocf_cache_volume_attach(cache, mngt_ctx))) { + SPDK_ERRLOG("OCF cache '%s': failed to attach volume: %s\n", + cache_name, spdk_strerror(-rc)); goto err_attach; } return; err_attach: - free(cache_start_ctx); + free(mngt_ctx); err_alloc: vbdev_ocf_cache_config_volume_destroy(cache); err_volume: @@ -1063,7 +1122,7 @@ void vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { ocf_cache_t cache; - struct vbdev_ocf_mngt_ctx *cache_stop_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; int i, rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop\n", cache_name); @@ -1074,27 +1133,22 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo goto err_cache; } - // DONE ? - // TODO: send hotremove to each core opener first - check how spdk_bdev_unregister does that - // take from bdev_unregister_unsafe() - // or stop cache after unregister ? - - cache_stop_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); - if (!cache_stop_ctx) { + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache stop context\n", cache_name); rc = -ENOMEM; goto err_alloc; } - cache_stop_ctx->cache = cache; - cache_stop_ctx->rpc_cb_fn = rpc_cb_fn; - cache_stop_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->cache = cache; /* Increment cache refcount to not destroy cache structs before destroying all cores. */ for (i = ocf_cache_get_core_count(cache); i > 0; i--) { if ((rc = ocf_mngt_cache_get(cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to increment refcount\n", - ocf_cache_get_name(cache)); + SPDK_ERRLOG("OCF cache '%s': failed to increment refcount (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); for (i = ocf_cache_get_core_count(cache) - i; i > 0; i--) { ocf_mngt_cache_put(cache); } @@ -1106,12 +1160,12 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { /* If there are no cores or all of them are detached, * then cache stop can be triggered already. */ - ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, cache_stop_ctx); + ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, mngt_ctx); } - if ((rc = ocf_core_visit(cache, _cache_stop_core_visit, cache_stop_ctx, false))) { - SPDK_ERRLOG("OCF cache '%s': failed to iterate over core bdevs\n", - ocf_cache_get_name(cache)); + if ((rc = ocf_core_visit(cache, _cache_stop_core_visitor, mngt_ctx, false))) { + SPDK_ERRLOG("OCF cache '%s': failed to iterate over core bdevs: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-rc)); // ocf_mngt_cache_put() ? how many times ? goto err_visit; } @@ -1120,7 +1174,7 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo err_visit: err_get: - free(cache_stop_ctx); + free(mngt_ctx); err_alloc: ocf_mngt_cache_put(cache); err_cache: @@ -1130,7 +1184,7 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo static void _cache_detach_rpc_detach_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *cache_detach_ctx = cb_arg; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing device detach\n", ocf_cache_get_name(cache)); @@ -1146,34 +1200,34 @@ _cache_detach_rpc_detach_cb(ocf_cache_t cache, void *cb_arg, int error) /* Increment queue refcount to prevent destroying management queue after cache device detach. */ ocf_queue_get(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); ocf_mngt_cache_unlock(cache); ocf_mngt_cache_put(cache); - cache_detach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_detach_ctx->rpc_cb_arg, error); - free(cache_detach_ctx); + free(mngt_ctx); } static void _cache_detach_rpc_flush_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *cache_detach_ctx = cb_arg; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; if (error) { SPDK_ERRLOG("OCF cache '%s': failed to flush OCF cache (OCF error: %d)\n", ocf_cache_get_name(cache), error); + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); ocf_mngt_cache_unlock(cache); ocf_mngt_cache_put(cache); - cache_detach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_detach_ctx->rpc_cb_arg, error); - free(cache_detach_ctx); + free(mngt_ctx); return; } - ocf_mngt_cache_detach(cache, _cache_detach_rpc_detach_cb, cache_detach_ctx); + ocf_mngt_cache_detach(cache, _cache_detach_rpc_detach_cb, mngt_ctx); } static void _cache_detach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *cache_detach_ctx = cb_arg; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': detaching OCF cache device\n", ocf_cache_get_name(cache)); @@ -1181,17 +1235,16 @@ _cache_detach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (error) { SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", ocf_cache_get_name(cache), error); + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); ocf_mngt_cache_put(cache); - cache_detach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_detach_ctx->rpc_cb_arg, error); - free(cache_detach_ctx); + free(mngt_ctx); return; } - // no need to check dirty ? if (ocf_mngt_cache_is_dirty(cache)) { - ocf_mngt_cache_flush(cache, _cache_detach_rpc_flush_cb, cache_detach_ctx); + ocf_mngt_cache_flush(cache, _cache_detach_rpc_flush_cb, mngt_ctx); } else { - ocf_mngt_cache_detach(cache, _cache_detach_rpc_detach_cb, cache_detach_ctx); + ocf_mngt_cache_detach(cache, _cache_detach_rpc_detach_cb, mngt_ctx); } } @@ -1200,7 +1253,7 @@ void vbdev_ocf_cache_detach(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { ocf_cache_t cache; - struct vbdev_ocf_mngt_ctx *cache_detach_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating device detach\n", cache_name); @@ -1217,17 +1270,17 @@ vbdev_ocf_cache_detach(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, goto err_state; } - cache_detach_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); - if (!cache_detach_ctx) { + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache detach context\n", cache_name); rc = -ENOMEM; goto err_alloc; } - cache_detach_ctx->rpc_cb_fn = rpc_cb_fn; - cache_detach_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; - ocf_mngt_cache_lock(cache, _cache_detach_rpc_lock_cb, cache_detach_ctx); + ocf_mngt_cache_lock(cache, _cache_detach_rpc_lock_cb, mngt_ctx); return; @@ -1241,7 +1294,7 @@ vbdev_ocf_cache_detach(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, static void _cache_attach_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *cache_attach_ctx = cb_arg; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing device attach\n", ocf_cache_get_name(cache)); @@ -1249,27 +1302,33 @@ _cache_attach_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) /* At this point volume was either moved to ocf_cache_t struct * or is no longer needed due to some errors, so we need to deallocate it. */ vbdev_ocf_cache_config_volume_destroy(cache); + ocf_mngt_cache_unlock(cache); + ocf_mngt_cache_put(cache); if (error) { SPDK_ERRLOG("OCF cache '%s': failed to attach OCF cache device (OCF error: %d)\n", ocf_cache_get_name(cache), error); + + if (error == -OCF_ERR_NO_MEM) { + SPDK_ERRLOG("Not enough memory to handle cache device of this size. Try to increase hugepage memory size, increase cache line size or use smaller cache device.\n"); + vbdev_ocf_mem_calculate(cache); + } + vbdev_ocf_cache_base_detach(cache); } else { SPDK_NOTICELOG("OCF cache '%s': device attached\n", ocf_cache_get_name(cache)); - } - ocf_mngt_cache_unlock(cache); - ocf_mngt_cache_put(cache); - cache_attach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_attach_ctx->rpc_cb_arg, error); - free(cache_attach_ctx); + vbdev_ocf_core_add_from_waitlist(cache); + } - vbdev_ocf_core_add_from_waitlist(cache); + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); + free(mngt_ctx); } static void _cache_attach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *cache_attach_ctx = cb_arg; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': attaching OCF cache device\n", @@ -1278,16 +1337,16 @@ _cache_attach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (error) { SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", ocf_cache_get_name(cache), error); + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); vbdev_ocf_cache_config_volume_destroy(cache); vbdev_ocf_cache_base_detach(cache); ocf_mngt_cache_put(cache); - cache_attach_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_attach_ctx->rpc_cb_arg, error); - free(cache_attach_ctx); + free(mngt_ctx); return; } ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, _cache_attach_rpc_attach_cb, - cache_attach_ctx); + mngt_ctx); } /* RPC entry point. */ @@ -1297,12 +1356,12 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force { ocf_cache_t cache; struct vbdev_ocf_cache *cache_ctx; - struct vbdev_ocf_mngt_ctx *cache_attach_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating device attach\n", cache_name); - // ocf_mngt_cache_put() right away to merge cache_start/attach callbacks ? + // ocf_mngt_cache_put() right away to merge cache_start/attach callbacks ? (or don't start locked) if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; @@ -1320,31 +1379,33 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force if ((rc = vbdev_ocf_cache_base_attach(cache, base_name))) { if (rc == -ENODEV) { - SPDK_NOTICELOG("OCF cache '%s': start deferred - waiting for base bdev '%s'\n", + SPDK_NOTICELOG("OCF cache '%s': attach deferred - waiting for base bdev '%s'\n", cache_name, base_name); rpc_cb_fn(cache_name, rpc_cb_arg, -ENODEV); return; } - SPDK_ERRLOG("OCF cache '%s': failed to attach base bdev '%s'\n", cache_name, base_name); + SPDK_ERRLOG("OCF cache '%s': failed to attach base bdev '%s': %s\n", + cache_name, base_name, spdk_strerror(-rc)); goto err_base; } if ((rc = vbdev_ocf_cache_config_volume_create(cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to create config volume\n", cache_name); + SPDK_ERRLOG("OCF cache '%s': failed to create config volume: %s\n", + cache_name, spdk_strerror(-rc)); goto err_volume; } - cache_attach_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); - if (!cache_attach_ctx) { + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache attach context\n", cache_name); rc = -ENOMEM; goto err_alloc; } - cache_attach_ctx->rpc_cb_fn = rpc_cb_fn; - cache_attach_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; - ocf_mngt_cache_lock(cache, _cache_attach_rpc_lock_cb, cache_attach_ctx); + ocf_mngt_cache_lock(cache, _cache_attach_rpc_lock_cb, mngt_ctx); return; @@ -1362,28 +1423,28 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force static void _core_add_rpc_err_cb(void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *core_add_ctx = cb_arg; - struct vbdev_ocf_core *core_ctx = core_add_ctx->u.core_ctx; - ocf_cache_t cache = core_add_ctx->cache; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + struct vbdev_ocf_core *core_ctx = mngt_ctx->u.core_ctx; + ocf_cache_t cache = mngt_ctx->cache; if (error) { SPDK_ERRLOG("OCF core '%s': failed to remove OCF core device (OCF error: %d)\n", vbdev_ocf_core_get_name(core_ctx), error); } + mngt_ctx->rpc_cb_fn(vbdev_ocf_core_get_name(core_ctx), mngt_ctx->rpc_cb_arg, error); ocf_mngt_cache_unlock(cache); ocf_mngt_cache_put(cache); vbdev_ocf_core_base_detach(core_ctx); vbdev_ocf_core_destroy(core_ctx); - core_add_ctx->rpc_cb_fn(NULL, core_add_ctx->rpc_cb_arg, error); - free(core_add_ctx); + free(mngt_ctx); } static void _core_add_rpc_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *core_add_ctx = cb_arg; - struct vbdev_ocf_core *core_ctx = core_add_ctx->u.core_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + struct vbdev_ocf_core *core_ctx = mngt_ctx->u.core_ctx; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add of OCF core\n", @@ -1391,55 +1452,56 @@ _core_add_rpc_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int error SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing add\n", vbdev_ocf_core_get_name(core_ctx)); if (error) { - SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s'\n", - vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); + SPDK_ERRLOG("OCF core '%s': failed to add core to OCF cache '%s' (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache), error); + mngt_ctx->rpc_cb_fn(vbdev_ocf_core_get_name(core_ctx), mngt_ctx->rpc_cb_arg, error); ocf_mngt_cache_unlock(cache); ocf_mngt_cache_put(cache); vbdev_ocf_core_base_detach(core_ctx); vbdev_ocf_core_destroy(core_ctx); - core_add_ctx->rpc_cb_fn(NULL, core_add_ctx->rpc_cb_arg, error); - free(core_add_ctx); + free(mngt_ctx); return; } ocf_core_set_priv(core, core_ctx); if ((rc = vbdev_ocf_core_register(core))) { - SPDK_ERRLOG("OCF core '%s': failed to register vbdev\n", ocf_core_get_name(core)); - ocf_mngt_cache_remove_core(core, _core_add_rpc_err_cb, core_add_ctx); + SPDK_ERRLOG("OCF core '%s': failed to register vbdev: %s\n", + ocf_core_get_name(core), spdk_strerror(-rc)); + ocf_mngt_cache_remove_core(core, _core_add_rpc_err_cb, mngt_ctx); return; } SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n", ocf_core_get_name(core), ocf_cache_get_name(cache)); + mngt_ctx->rpc_cb_fn(ocf_core_get_name(core), mngt_ctx->rpc_cb_arg, rc); ocf_mngt_cache_unlock(cache); ocf_mngt_cache_put(cache); - core_add_ctx->rpc_cb_fn(ocf_core_get_name(core), core_add_ctx->rpc_cb_arg, rc); - free(core_add_ctx); + free(mngt_ctx); } static void _core_add_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *core_add_ctx = cb_arg; - struct vbdev_ocf_core *core_ctx = core_add_ctx->u.core_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + struct vbdev_ocf_core *core_ctx = mngt_ctx->u.core_ctx; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add of OCF core\n", vbdev_ocf_core_get_name(core_ctx)); if (error) { - SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock\n", - vbdev_ocf_core_get_name(core_ctx)); + SPDK_ERRLOG("OCF core '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), error); + mngt_ctx->rpc_cb_fn(vbdev_ocf_core_get_name(core_ctx), mngt_ctx->rpc_cb_arg, error); ocf_mngt_cache_put(cache); vbdev_ocf_core_base_detach(core_ctx); vbdev_ocf_core_destroy(core_ctx); - core_add_ctx->rpc_cb_fn(NULL, core_add_ctx->rpc_cb_arg, error); // core name - free(core_add_ctx); + free(mngt_ctx); return; } - ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _core_add_rpc_add_cb, core_add_ctx); + ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _core_add_rpc_add_cb, mngt_ctx); } /* RPC entry point. */ @@ -1450,20 +1512,21 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac ocf_cache_t cache; struct vbdev_ocf_cache *cache_ctx; struct vbdev_ocf_core *core_ctx; - struct vbdev_ocf_mngt_ctx *core_add_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; uint32_t cache_block_size, core_block_size; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add\n", core_name); if (vbdev_ocf_bdev_exists(core_name)) { - SPDK_ERRLOG("OCF: bdev '%s' already exists\n", core_name); + SPDK_ERRLOG("OCF '%s': bdev already exists\n", core_name); rc = -EEXIST; goto err_exist; } if ((rc = vbdev_ocf_core_create(&core_ctx, core_name, cache_name))) { - SPDK_ERRLOG("OCF core '%s': failed to create core\n", core_name); + SPDK_ERRLOG("OCF core '%s': failed to create core: %s\n", + core_name, spdk_strerror(-rc)); goto err_create; } @@ -1473,11 +1536,12 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac /* If not, just put core context on the temporary core wait list and exit. */ SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for base bdev '%s'\n", core_name, base_name); - STAILQ_INSERT_TAIL(&g_vbdev_ocf_core_waitlist, core_ctx, waitlist_entry); + vbdev_ocf_core_waitlist_add(core_ctx); rpc_cb_fn(core_name, rpc_cb_arg, -ENODEV); return; } - SPDK_ERRLOG("OCF core '%s': failed to attach base bdev '%s'\n", core_name, base_name); + SPDK_ERRLOG("OCF core '%s': failed to attach base bdev '%s': %s\n", + core_name, base_name, spdk_strerror(-rc)); goto err_base; } @@ -1486,7 +1550,7 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac /* If not, just put core context on the temporary core wait list and exit. */ SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache '%s'\n", core_name, cache_name); - STAILQ_INSERT_TAIL(&g_vbdev_ocf_core_waitlist, core_ctx, waitlist_entry); + vbdev_ocf_core_waitlist_add(core_ctx); rpc_cb_fn(core_name, rpc_cb_arg, -ENODEV); return; } @@ -1497,16 +1561,17 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac * We need to have cache device attached to know if cache was loaded or attached * and then set 'try_add' in core config accordingly. */ if (!ocf_cache_is_device_attached(cache)) { + /* If not, just put core context on the temporary core wait list and exit. */ SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache device '%s'\n", core_name, cache_ctx->base.name); ocf_mngt_cache_put(cache); - STAILQ_INSERT_TAIL(&g_vbdev_ocf_core_waitlist, core_ctx, waitlist_entry); + vbdev_ocf_core_waitlist_add(core_ctx); rpc_cb_fn(core_name, rpc_cb_arg, -ENODEV); return; } - cache_block_size = cache_ctx->base.bdev->blocklen; - core_block_size = core_ctx->base.bdev->blocklen; + cache_block_size = spdk_bdev_get_block_size(cache_ctx->base.bdev); + core_block_size = spdk_bdev_get_block_size(core_ctx->base.bdev); if (cache_block_size > core_block_size) { SPDK_ERRLOG("OCF core '%s': block size (%d) is less than cache '%s' block size (%d)\n", core_name, core_block_size, cache_name, cache_block_size); @@ -1514,26 +1579,23 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac goto err_bsize; } - // DONE - // find a better way to check if cache was loaded or attached - //core_ctx->core_cfg.try_add = cache_ctx->cache_att_cfg.force ? false : true; core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(core_name); // open (and then close) cache-load-added core volumes (with 'base' to save it in priv) ? - core_add_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); - if (!core_add_ctx) { + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { SPDK_ERRLOG("OCF core '%s': failed to allocate memory for core add context\n", core_name); rc = -ENOMEM; goto err_alloc; } - core_add_ctx->rpc_cb_fn = rpc_cb_fn; - core_add_ctx->rpc_cb_arg = rpc_cb_arg; - core_add_ctx->cache = cache; - core_add_ctx->u.core_ctx = core_ctx; + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->cache = cache; + mngt_ctx->u.core_ctx = core_ctx; - ocf_mngt_cache_lock(cache, _core_add_rpc_lock_cb, core_add_ctx); + ocf_mngt_cache_lock(cache, _core_add_rpc_lock_cb, mngt_ctx); return; @@ -1551,9 +1613,9 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac static void _core_remove_rpc_remove_cb(void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *core_rm_ctx = cb_arg; - struct vbdev_ocf_core *core_ctx = core_rm_ctx->u.core_ctx; - ocf_cache_t cache = core_rm_ctx->cache; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + struct vbdev_ocf_core *core_ctx = mngt_ctx->u.core_ctx; + ocf_cache_t cache = mngt_ctx->cache; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing remove of OCF core\n", vbdev_ocf_core_get_name(core_ctx)); @@ -1568,18 +1630,17 @@ _core_remove_rpc_remove_cb(void *cb_arg, int error) vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); } + mngt_ctx->rpc_cb_fn(vbdev_ocf_core_get_name(core_ctx), mngt_ctx->rpc_cb_arg, error); ocf_mngt_cache_unlock(cache); - ocf_mngt_cache_put(cache); vbdev_ocf_core_destroy(core_ctx); - core_rm_ctx->rpc_cb_fn(NULL, core_rm_ctx->rpc_cb_arg, error); // core name - free(core_rm_ctx); + free(mngt_ctx); } static void _core_remove_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *core_rm_ctx = cb_arg; - ocf_core_t core = core_rm_ctx->core; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + ocf_core_t core = mngt_ctx->core; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating remove of OCF core\n", ocf_core_get_name(core)); @@ -1589,46 +1650,43 @@ _core_remove_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_core_get_name(core), error); } - ocf_mngt_cache_remove_core(core, _core_remove_rpc_remove_cb, core_rm_ctx); + /* Do not check core's dirtiness as it was already detached during destruct phase. */ + ocf_mngt_cache_remove_core(core, _core_remove_rpc_remove_cb, mngt_ctx); } static void _core_remove_rpc_unregister_cb(void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *core_rm_ctx = cb_arg; - ocf_core_t core = core_rm_ctx->core; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + ocf_core_t core = mngt_ctx->core; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing unregister of OCF vbdev\n", ocf_core_get_name(core)); if (error) { - SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during core removal\n", - ocf_core_get_name(core)); + SPDK_ERRLOG("OCF core '%s': failed to unregister OCF vbdev during core removal: %s\n", + ocf_core_get_name(core), spdk_strerror(-error)); } - ocf_mngt_cache_lock(ocf_core_get_cache(core), _core_remove_rpc_lock_cb, core_rm_ctx); + ocf_mngt_cache_lock(ocf_core_get_cache(core), _core_remove_rpc_lock_cb, mngt_ctx); } /* RPC entry point. */ void -vbdev_ocf_core_remove(const char *core_name, const char *cache_name, - vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) +vbdev_ocf_core_remove(const char *core_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { ocf_cache_t cache; ocf_core_t core; struct vbdev_ocf_core *core_ctx; - struct vbdev_ocf_mngt_ctx *core_rm_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating removal\n", core_name); /* If core was not added yet due to lack of base or cache device, * just free its structs (and detach its base if exists) and exit. */ - core_ctx = vbdev_ocf_core_waitlist_get_by_name(core_name); - if (core_ctx) { - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': removing from wait list\n", core_name); - - STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); + if ((core_ctx = vbdev_ocf_core_waitlist_get_by_name(core_name))) { + vbdev_ocf_core_waitlist_remove(core_ctx); if (vbdev_ocf_core_is_base_attached(core_ctx)) { vbdev_ocf_core_base_detach(core_ctx); } @@ -1637,63 +1695,59 @@ vbdev_ocf_core_remove(const char *core_name, const char *cache_name, return; } - if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { - SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); - rc = -ENXIO; - goto err_cache; + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { + SPDK_ERRLOG("OCF core '%s': failed to allocate memory for core remove context\n", + core_name); + rc = -ENOMEM; + goto err_alloc; } + mngt_ctx->bdev_name = core_name; - if ((rc = ocf_core_get_by_name(cache, core_name, OCF_CORE_NAME_SIZE, &core))) { - if (rc == -OCF_ERR_CORE_NOT_EXIST) { - SPDK_ERRLOG("OCF core '%s': not exist within cache '%s'\n", core_name, cache_name); - rc = -ENXIO; - } - goto err_core; + if ((rc = vbdev_ocf_bdev_resolve(mngt_ctx))) { + SPDK_ERRLOG("OCF core '%s': failed to find core of that name: %s\n", + core_name, spdk_strerror(-rc)); + goto err_resolve; } + core = mngt_ctx->core; - core_ctx = ocf_core_get_priv(core); - if (!core_ctx) { - /* Skip this core. If there is no context, it means that this core - * was added from metadata during cache load and it's just an empty shell. */ - SPDK_ERRLOG("OCF core '%s': not exist within cache '%s'\n", core_name, cache_name); + /* Check if given core exists and have context assigned. + * If there is no context, it means that this core was added + * from metadata during cache load and it's just an empty shell. */ + if (!core || !(core_ctx = ocf_core_get_priv(core))) { + SPDK_ERRLOG("OCF core '%s': not exist\n", core_name); rc = -ENXIO; - goto err_ctx; + goto err_exist; } + cache = ocf_core_get_cache(core); - core_rm_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); - if (!core_rm_ctx) { - SPDK_ERRLOG("OCF core '%s': failed to allocate memory for core remove context\n", core_name); - rc = -ENOMEM; - goto err_alloc; - } - core_rm_ctx->rpc_cb_fn = rpc_cb_fn; - core_rm_ctx->rpc_cb_arg = rpc_cb_arg; - core_rm_ctx->cache = cache; - core_rm_ctx->core = core; - core_rm_ctx->u.core_ctx = core_ctx; + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->cache = cache; + mngt_ctx->u.core_ctx = core_ctx; if (!vbdev_ocf_core_is_base_attached(core_ctx)) { SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': removing detached (no unregister)\n", core_name); - ocf_mngt_cache_lock(cache, _core_remove_rpc_lock_cb, core_rm_ctx); + ocf_mngt_cache_lock(cache, _core_remove_rpc_lock_cb, mngt_ctx); return; } - if ((rc = vbdev_ocf_core_unregister(core_ctx, _core_remove_rpc_unregister_cb, core_rm_ctx))) { - SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during core removal\n", - ocf_core_get_name(core)); + /* Unregister (and detach) core first before removing it + * to send hotremove signal to all opened descriptors. */ + if ((rc = vbdev_ocf_core_unregister(core_ctx, _core_remove_rpc_unregister_cb, mngt_ctx))) { + SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during core removal: %s\n", + core_name, spdk_strerror(-rc)); goto err_unregister; } return; err_unregister: - free(core_rm_ctx); +err_exist: +err_resolve: + free(mngt_ctx); err_alloc: -err_ctx: -err_core: - ocf_mngt_cache_put(cache); -err_cache: rpc_cb_fn(core_name, rpc_cb_arg, rc); } @@ -1721,7 +1775,8 @@ _cache_save_cb(ocf_cache_t cache, void *cb_arg, int error) static void _cache_mode_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *cache_mode_ctx = cb_arg; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + ocf_cache_mode_t cache_mode = mngt_ctx->u.cache_mode; int rc; if ((rc = error)) { @@ -1730,22 +1785,25 @@ _cache_mode_lock_cb(ocf_cache_t cache, void *cb_arg, int error) goto err; } - if ((rc = ocf_mngt_cache_set_mode(cache, cache_mode_ctx->u.cache_mode))) { + if ((rc = ocf_mngt_cache_set_mode(cache, cache_mode))) { SPDK_ERRLOG("OCF cache '%s': failed to change cache mode to '%s' (OCF error: %d)\n", ocf_cache_get_name(cache), - vbdev_ocf_cachemode_get_name(cache_mode_ctx->u.cache_mode), rc); + vbdev_ocf_cachemode_get_name(cache_mode), rc); ocf_mngt_cache_unlock(cache); goto err; } - ocf_mngt_cache_save(cache, _cache_save_cb, cache_mode_ctx); + SPDK_NOTICELOG("OCF cache '%s': cache mode set to '%s'\n", + ocf_cache_get_name(cache), vbdev_ocf_cachemode_get_name(cache_mode)); + + ocf_mngt_cache_save(cache, _cache_save_cb, mngt_ctx); return; err: - cache_mode_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cache_mode_ctx->rpc_cb_arg, rc); + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, rc); ocf_mngt_cache_put(cache); - free(cache_mode_ctx); + free(mngt_ctx); } /* RPC entry point. */ @@ -1754,7 +1812,7 @@ vbdev_ocf_set_cachemode(const char *cache_name, const char *cache_mode, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { ocf_cache_t cache; - struct vbdev_ocf_mngt_ctx *cache_mode_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': setting new cache mode '%s'\n", cache_name, cache_mode); @@ -1765,18 +1823,18 @@ vbdev_ocf_set_cachemode(const char *cache_name, const char *cache_mode, goto err_cache; } - cache_mode_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); - if (!cache_mode_ctx) { + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cache mode change context\n", cache_name); rc = -ENOMEM; goto err_alloc; } - cache_mode_ctx->u.cache_mode = vbdev_ocf_cachemode_get_by_name(cache_mode); - cache_mode_ctx->rpc_cb_fn = rpc_cb_fn; - cache_mode_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->u.cache_mode = vbdev_ocf_cachemode_get_by_name(cache_mode); - ocf_mngt_cache_lock(cache, _cache_mode_lock_cb, cache_mode_ctx); + ocf_mngt_cache_lock(cache, _cache_mode_lock_cb, mngt_ctx); return; @@ -1825,6 +1883,8 @@ _promotion_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } } + SPDK_NOTICELOG("OCF cache '%s': promotion params set\n", ocf_cache_get_name(cache)); + ocf_mngt_cache_save(cache, _cache_save_cb, mngt_ctx); return; @@ -1881,26 +1941,28 @@ vbdev_ocf_set_promotion(const char *cache_name, const char *policy, static void _cleaning_policy_cb(void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *cleaning_ctx = cb_arg; - ocf_cache_t cache = cleaning_ctx->cache; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + ocf_cache_t cache = mngt_ctx->cache; if (error) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning policy (OCF error: %d)\n", ocf_cache_get_name(cache), error); + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); ocf_mngt_cache_unlock(cache); - cleaning_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cleaning_ctx->rpc_cb_arg, error); ocf_mngt_cache_put(cache); - free(cleaning_ctx); + free(mngt_ctx); return; } - ocf_mngt_cache_save(cache, _cache_save_cb, cleaning_ctx); + SPDK_NOTICELOG("OCF cache '%s': cleaning params set\n", ocf_cache_get_name(cache)); + + ocf_mngt_cache_save(cache, _cache_save_cb, mngt_ctx); } static void _cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *cleaning_ctx = cb_arg; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; int rc; if ((rc = error)) { @@ -1909,75 +1971,77 @@ _cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) goto err_lock; } - if (cleaning_ctx->u.cleaning.acp_wake_up_time >= 0) { + if (mngt_ctx->u.cleaning.acp_wake_up_time >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_acp, ocf_acp_wake_up_time, - cleaning_ctx->u.cleaning.acp_wake_up_time))) { + mngt_ctx->u.cleaning.acp_wake_up_time))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning acp_wake_up_time param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; } } - if (cleaning_ctx->u.cleaning.acp_flush_max_buffers >= 0) { + if (mngt_ctx->u.cleaning.acp_flush_max_buffers >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_acp, ocf_acp_flush_max_buffers, - cleaning_ctx->u.cleaning.acp_flush_max_buffers))) { + mngt_ctx->u.cleaning.acp_flush_max_buffers))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning acp_flush_max_buffers param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; } } - if (cleaning_ctx->u.cleaning.alru_wake_up_time >= 0) { + if (mngt_ctx->u.cleaning.alru_wake_up_time >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_wake_up_time, - cleaning_ctx->u.cleaning.alru_wake_up_time))) { + mngt_ctx->u.cleaning.alru_wake_up_time))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_wake_up_time param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; } } - if (cleaning_ctx->u.cleaning.alru_flush_max_buffers >= 0) { + if (mngt_ctx->u.cleaning.alru_flush_max_buffers >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_flush_max_buffers, - cleaning_ctx->u.cleaning.alru_flush_max_buffers))) { + mngt_ctx->u.cleaning.alru_flush_max_buffers))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_flush_max_buffers param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; } } - if (cleaning_ctx->u.cleaning.alru_staleness_time >= 0) { + if (mngt_ctx->u.cleaning.alru_staleness_time >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_stale_buffer_time, - cleaning_ctx->u.cleaning.alru_staleness_time))) { + mngt_ctx->u.cleaning.alru_staleness_time))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_staleness_time param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; } } - if (cleaning_ctx->u.cleaning.alru_activity_threshold >= 0) { + if (mngt_ctx->u.cleaning.alru_activity_threshold >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_activity_threshold, - cleaning_ctx->u.cleaning.alru_activity_threshold))) { + mngt_ctx->u.cleaning.alru_activity_threshold))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_activity_threshold param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; } } - if (cleaning_ctx->u.cleaning.alru_max_dirty_ratio >= 0) { + if (mngt_ctx->u.cleaning.alru_max_dirty_ratio >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_max_dirty_ratio, - cleaning_ctx->u.cleaning.alru_max_dirty_ratio))) { + mngt_ctx->u.cleaning.alru_max_dirty_ratio))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_max_dirty_ratio param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; } } - if (cleaning_ctx->u.cleaning.policy >= ocf_cleaning_nop && - cleaning_ctx->u.cleaning.policy < ocf_cleaning_max) { - ocf_mngt_cache_cleaning_set_policy(cache, cleaning_ctx->u.cleaning.policy, - _cleaning_policy_cb, cleaning_ctx); + if (mngt_ctx->u.cleaning.policy >= ocf_cleaning_nop && + mngt_ctx->u.cleaning.policy < ocf_cleaning_max) { + ocf_mngt_cache_cleaning_set_policy(cache, mngt_ctx->u.cleaning.policy, + _cleaning_policy_cb, mngt_ctx); } else { - ocf_mngt_cache_save(cache, _cache_save_cb, cleaning_ctx); + SPDK_NOTICELOG("OCF cache '%s': cleaning params set\n", ocf_cache_get_name(cache)); + + ocf_mngt_cache_save(cache, _cache_save_cb, mngt_ctx); } return; @@ -1985,9 +2049,9 @@ _cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) err_param: ocf_mngt_cache_unlock(cache); err_lock: - cleaning_ctx->rpc_cb_fn(ocf_cache_get_name(cache), cleaning_ctx->rpc_cb_arg, rc); + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, rc); ocf_mngt_cache_put(cache); - free(cleaning_ctx); + free(mngt_ctx); } /* RPC entry point. */ @@ -1999,7 +2063,7 @@ vbdev_ocf_set_cleaning(const char *cache_name, const char *policy, int32_t acp_w vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { ocf_cache_t cache; - struct vbdev_ocf_mngt_ctx *cleaning_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': setting cleaning params\n", cache_name); @@ -2010,26 +2074,26 @@ vbdev_ocf_set_cleaning(const char *cache_name, const char *policy, int32_t acp_w goto err_cache; } - cleaning_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); - if (!cleaning_ctx) { + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { SPDK_ERRLOG("OCF cache '%s': failed to allocate memory for cleaning set context\n", cache_name); rc = -ENOMEM; goto err_alloc; } - cleaning_ctx->u.cleaning.policy = vbdev_ocf_cleaning_policy_get_by_name(policy); - cleaning_ctx->u.cleaning.acp_wake_up_time = acp_wake_up_time; - cleaning_ctx->u.cleaning.acp_flush_max_buffers = acp_flush_max_buffers; - cleaning_ctx->u.cleaning.alru_wake_up_time = alru_wake_up_time; - cleaning_ctx->u.cleaning.alru_flush_max_buffers = alru_flush_max_buffers; - cleaning_ctx->u.cleaning.alru_staleness_time = alru_staleness_time; - cleaning_ctx->u.cleaning.alru_activity_threshold = alru_activity_threshold; - cleaning_ctx->u.cleaning.alru_max_dirty_ratio = alru_max_dirty_ratio; - cleaning_ctx->cache = cache; - cleaning_ctx->rpc_cb_fn = rpc_cb_fn; - cleaning_ctx->rpc_cb_arg = rpc_cb_arg; - - ocf_mngt_cache_lock(cache, _cleaning_lock_cb, cleaning_ctx); + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->cache = cache; + mngt_ctx->u.cleaning.policy = vbdev_ocf_cleaning_policy_get_by_name(policy); + mngt_ctx->u.cleaning.acp_wake_up_time = acp_wake_up_time; + mngt_ctx->u.cleaning.acp_flush_max_buffers = acp_flush_max_buffers; + mngt_ctx->u.cleaning.alru_wake_up_time = alru_wake_up_time; + mngt_ctx->u.cleaning.alru_flush_max_buffers = alru_flush_max_buffers; + mngt_ctx->u.cleaning.alru_staleness_time = alru_staleness_time; + mngt_ctx->u.cleaning.alru_activity_threshold = alru_activity_threshold; + mngt_ctx->u.cleaning.alru_max_dirty_ratio = alru_max_dirty_ratio; + + ocf_mngt_cache_lock(cache, _cleaning_lock_cb, mngt_ctx); return; @@ -2043,6 +2107,7 @@ static void _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + ocf_core_t core = mngt_ctx->core; int rc; if ((rc = error)) { @@ -2051,54 +2116,51 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) goto err_lock; } - if (mngt_ctx->core) { + if (core) { SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': setting sequential cut-off on core device\n", - mngt_ctx->bdev_name); + ocf_core_get_name(core)); if (mngt_ctx->u.seqcutoff.policy >= ocf_seq_cutoff_policy_always && mngt_ctx->u.seqcutoff.policy < ocf_seq_cutoff_policy_max) { - if ((rc = ocf_mngt_core_set_seq_cutoff_policy(mngt_ctx->core, - mngt_ctx->u.seqcutoff.policy))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_policy(core, mngt_ctx->u.seqcutoff.policy))) { SPDK_ERRLOG("OCF core '%s': failed to set sequential cut-off policy (OCF error: %d)\n", - ocf_core_get_name(mngt_ctx->core), rc); + ocf_core_get_name(core), rc); goto err_param; } } if (mngt_ctx->u.seqcutoff.threshold >= 0) { - if ((rc = ocf_mngt_core_set_seq_cutoff_threshold(mngt_ctx->core, - mngt_ctx->u.seqcutoff.threshold * KiB))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_threshold(core, mngt_ctx->u.seqcutoff.threshold * KiB))) { SPDK_ERRLOG("OCF core '%s': failed to set sequential cut-off threshold (OCF error: %d)\n", - ocf_core_get_name(mngt_ctx->core), rc); + ocf_core_get_name(core), rc); goto err_param; } } if (mngt_ctx->u.seqcutoff.promotion_count >= 0) { - if ((rc = ocf_mngt_core_set_seq_cutoff_promotion_count(mngt_ctx->core, - mngt_ctx->u.seqcutoff.promotion_count))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promotion_count(core, mngt_ctx->u.seqcutoff.promotion_count))) { SPDK_ERRLOG("OCF core '%s': failed to set sequential cut-off promotion_count (OCF error: %d)\n", - ocf_core_get_name(mngt_ctx->core), rc); + ocf_core_get_name(core), rc); goto err_param; } } if (mngt_ctx->u.seqcutoff.promote_on_threshold >= 0) { - if ((rc = ocf_mngt_core_set_seq_cutoff_promote_on_threshold(mngt_ctx->core, - mngt_ctx->u.seqcutoff.promote_on_threshold))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promote_on_threshold(core, mngt_ctx->u.seqcutoff.promote_on_threshold))) { SPDK_ERRLOG("OCF core '%s': failed to set sequential cut-off promote_on_threshold (OCF error: %d)\n", - ocf_core_get_name(mngt_ctx->core), rc); + ocf_core_get_name(core), rc); goto err_param; } } + + SPDK_NOTICELOG("OCF core '%s': sequential cut-off params set\n", ocf_core_get_name(core)); } else { SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': setting sequential cut-off on all cores in cache device\n", - mngt_ctx->bdev_name); + ocf_cache_get_name(cache)); if (mngt_ctx->u.seqcutoff.policy >= ocf_seq_cutoff_policy_always && mngt_ctx->u.seqcutoff.policy < ocf_seq_cutoff_policy_max) { - if ((rc = ocf_mngt_core_set_seq_cutoff_policy_all(cache, - mngt_ctx->u.seqcutoff.policy))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_policy_all(cache, mngt_ctx->u.seqcutoff.policy))) { SPDK_ERRLOG("OCF cache '%s': failed to set sequential cut-off policy (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2106,8 +2168,7 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } if (mngt_ctx->u.seqcutoff.threshold >= 0) { - if ((rc = ocf_mngt_core_set_seq_cutoff_threshold_all(cache, - mngt_ctx->u.seqcutoff.threshold * KiB))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_threshold_all(cache, mngt_ctx->u.seqcutoff.threshold * KiB))) { SPDK_ERRLOG("OCF cache '%s': failed to set sequential cut-off threshold (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2115,8 +2176,7 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } if (mngt_ctx->u.seqcutoff.promotion_count >= 0) { - if ((rc = ocf_mngt_core_set_seq_cutoff_promotion_count_all(cache, - mngt_ctx->u.seqcutoff.promotion_count))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promotion_count_all(cache, mngt_ctx->u.seqcutoff.promotion_count))) { SPDK_ERRLOG("OCF cache '%s': failed to set sequential cut-off promotion_count (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2124,13 +2184,14 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } if (mngt_ctx->u.seqcutoff.promote_on_threshold >= 0) { - if ((rc = ocf_mngt_core_set_seq_cutoff_promote_on_threshold_all(cache, - mngt_ctx->u.seqcutoff.promote_on_threshold))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promote_on_threshold_all(cache, mngt_ctx->u.seqcutoff.promote_on_threshold))) { SPDK_ERRLOG("OCF cache '%s': failed to set sequential cut-off promote_on_threshold (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; } } + + SPDK_NOTICELOG("OCF cache '%s': sequential cut-off params set\n", ocf_cache_get_name(cache)); } /* For compatibility with global _cache_save_cb(). */ @@ -2147,28 +2208,6 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) free(mngt_ctx); } -static int -_seqcutoff_cache_visitor(ocf_cache_t cache, void *ctx) -{ - ocf_core_t core; - struct vbdev_ocf_mngt_ctx *mngt_ctx = ctx; - int rc; - - if (!strcmp(mngt_ctx->bdev_name, ocf_cache_get_name(cache))) { - ocf_mngt_cache_lock(cache, _seqcutoff_lock_cb, mngt_ctx); - return -EEXIST; - } - - rc = ocf_core_get_by_name(cache, mngt_ctx->bdev_name, OCF_CORE_NAME_SIZE, &core); - if (!rc) { - mngt_ctx->core = core; - ocf_mngt_cache_lock(cache, _seqcutoff_lock_cb, mngt_ctx); - return -EEXIST; - } - - return rc == -OCF_ERR_CORE_NOT_EXIST ? 0 : rc; -} - /* RPC entry point. */ void vbdev_ocf_set_seqcutoff(const char *bdev_name, const char *policy, int32_t threshold, @@ -2187,24 +2226,27 @@ vbdev_ocf_set_seqcutoff(const char *bdev_name, const char *policy, int32_t thres rpc_cb_fn(bdev_name, rpc_cb_arg, -ENOMEM); return; } + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; mngt_ctx->bdev_name = bdev_name; - /* Will be used later if given bdev is a core device. */ + /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ + mngt_ctx->cache = NULL; mngt_ctx->core = NULL; mngt_ctx->u.seqcutoff.policy = vbdev_ocf_seqcutoff_policy_get_by_name(policy); mngt_ctx->u.seqcutoff.threshold = threshold; mngt_ctx->u.seqcutoff.promotion_count = promotion_count; mngt_ctx->u.seqcutoff.promote_on_threshold = promote_on_threshold; - mngt_ctx->rpc_cb_fn = rpc_cb_fn; - mngt_ctx->rpc_cb_arg = rpc_cb_arg; - rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _seqcutoff_cache_visitor, mngt_ctx); - if (rc && rc != -EEXIST) { - SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + if ((rc = vbdev_ocf_bdev_resolve(mngt_ctx))) { + SPDK_ERRLOG("OCF '%s': failed to find cache or core of that name: %s\n", + bdev_name, spdk_strerror(-rc)); rpc_cb_fn(bdev_name, rpc_cb_arg, rc); - } else if (!rc) { - SPDK_ERRLOG("OCF '%s': not exist\n", bdev_name); - rpc_cb_fn(bdev_name, rpc_cb_arg, -ENXIO); + free(mngt_ctx); + return; } + + ocf_mngt_cache_lock(mngt_ctx->cache ? : ocf_core_get_cache(mngt_ctx->core), + _seqcutoff_lock_cb, mngt_ctx); } static void @@ -2215,6 +2257,8 @@ _flush_cache_cb(ocf_cache_t cache, void *cb_arg, int error) SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing flush operation\n", ocf_cache_get_name(cache)); + SPDK_NOTICELOG("OCF cache '%s': flushed\n", ocf_cache_get_name(cache)); + ocf_mngt_cache_read_unlock(cache); cache_ctx->flush.error = error; @@ -2229,6 +2273,8 @@ _flush_core_cb(ocf_core_t core, void *cb_arg, int error) SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': finishing flush operation\n", ocf_core_get_name(core)); + SPDK_NOTICELOG("OCF core '%s': flushed\n", ocf_core_get_name(core)); + ocf_mngt_cache_read_unlock(ocf_core_get_cache(core)); core_ctx->flush.error = error; @@ -2238,7 +2284,8 @@ _flush_core_cb(ocf_core_t core, void *cb_arg, int error) static void _flush_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { - struct vbdev_ocf_mngt_ctx *flush_ctx = cb_arg; + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + ocf_core_t core = mngt_ctx->core; struct vbdev_ocf_cache *cache_ctx; struct vbdev_ocf_core *core_ctx; int rc; @@ -2249,14 +2296,14 @@ _flush_lock_cb(ocf_cache_t cache, void *cb_arg, int error) goto end; } - if (flush_ctx->core) { - SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': flushing core device\n", flush_ctx->bdev_name); + if (core) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': flushing...\n", ocf_core_get_name(core)); - core_ctx = ocf_core_get_priv(flush_ctx->core); + core_ctx = ocf_core_get_priv(core); core_ctx->flush.in_progress = true; - ocf_mngt_core_flush(flush_ctx->core, _flush_core_cb, NULL); + ocf_mngt_core_flush(core, _flush_core_cb, NULL); } else { - SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': flushing cache device\n", flush_ctx->bdev_name); + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': flushing...\n", ocf_cache_get_name(cache)); cache_ctx = ocf_cache_get_priv(cache); cache_ctx->flush.in_progress = true; @@ -2266,65 +2313,46 @@ _flush_lock_cb(ocf_cache_t cache, void *cb_arg, int error) end: /* Flushing process may take some time to finish, so call * RPC callback now and leave flush running in background. */ - flush_ctx->rpc_cb_fn(flush_ctx->bdev_name, flush_ctx->rpc_cb_arg, rc); - free(flush_ctx); -} - -static int -_flush_cache_visitor(ocf_cache_t cache, void *ctx) -{ - ocf_core_t core; - struct vbdev_ocf_mngt_ctx *flush_ctx = ctx; - int rc; - - if (!strcmp(flush_ctx->bdev_name, ocf_cache_get_name(cache))) { - ocf_mngt_cache_read_lock(cache, _flush_lock_cb, flush_ctx); - return -EEXIST; - } - - rc = ocf_core_get_by_name(cache, flush_ctx->bdev_name, OCF_CORE_NAME_SIZE, &core); - if (!rc) { - flush_ctx->core = core; - ocf_mngt_cache_read_lock(cache, _flush_lock_cb, flush_ctx); - return -EEXIST; - } - - return rc == -OCF_ERR_CORE_NOT_EXIST ? 0 : rc; + mngt_ctx->rpc_cb_fn(mngt_ctx->bdev_name, mngt_ctx->rpc_cb_arg, rc); + free(mngt_ctx); } /* RPC entry point. */ void vbdev_ocf_flush_start(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { - struct vbdev_ocf_mngt_ctx *flush_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': initiating flush operation\n", bdev_name); - flush_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); - if (!flush_ctx) { + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { SPDK_ERRLOG("OCF '%s': failed to allocate memory for flush context\n", bdev_name); rpc_cb_fn(bdev_name, rpc_cb_arg, -ENOMEM); return; } - flush_ctx->bdev_name = bdev_name; - /* Will be used later if given bdev is a core device. */ - flush_ctx->core = NULL; - flush_ctx->rpc_cb_fn = rpc_cb_fn; - flush_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->bdev_name = bdev_name; + /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ + mngt_ctx->cache = NULL; + mngt_ctx->core = NULL; - rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _flush_cache_visitor, flush_ctx); - if (rc && rc != -EEXIST) { - SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + if ((rc = vbdev_ocf_bdev_resolve(mngt_ctx))) { + SPDK_ERRLOG("OCF '%s': failed to find cache or core of that name: %s\n", + bdev_name, spdk_strerror(-rc)); rpc_cb_fn(bdev_name, rpc_cb_arg, rc); - } else if (!rc) { - SPDK_ERRLOG("OCF '%s': not exist\n", bdev_name); - rpc_cb_fn(bdev_name, rpc_cb_arg, -ENXIO); + free(mngt_ctx); + return; } + + ocf_mngt_cache_read_lock(mngt_ctx->cache ? : ocf_core_get_cache(mngt_ctx->core), + _flush_lock_cb, mngt_ctx); } static int -_dump_promotion_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) +dump_promotion_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) { ocf_promotion_t promotion_policy; uint32_t param_val; @@ -2358,7 +2386,7 @@ _dump_promotion_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) } static int -_dump_cleaning_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) +dump_cleaning_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) { ocf_cleaning_t cleaning_policy; uint32_t param_val; @@ -2422,7 +2450,7 @@ _dump_cleaning_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) } static int -_dump_seqcutoff_info(struct spdk_json_write_ctx *w, ocf_core_t core) +dump_seqcutoff_info(struct spdk_json_write_ctx *w, ocf_core_t core) { ocf_seq_cutoff_policy seqcutoff_policy; uint32_t param_val_int; @@ -2441,7 +2469,7 @@ _dump_seqcutoff_info(struct spdk_json_write_ctx *w, ocf_core_t core) if ((rc = ocf_mngt_core_get_seq_cutoff_threshold(core, ¶m_val_int))) { return rc; } - spdk_json_write_named_uint32(w, "threshold", param_val_int / KiB); + spdk_json_write_named_uint32(w, "threshold", param_val_int); if ((rc = ocf_mngt_core_get_seq_cutoff_promotion_count(core, ¶m_val_int))) { return rc; @@ -2456,25 +2484,46 @@ _dump_seqcutoff_info(struct spdk_json_write_ctx *w, ocf_core_t core) return 0; } -// handle errors ? static int -_get_bdevs_core_visit(ocf_core_t core, void *cb_arg) +dump_core_waitlist_info(struct spdk_json_write_ctx *w, struct vbdev_ocf_core *core_ctx) +{ + spdk_json_write_named_string(w, "name", vbdev_ocf_core_get_name(core_ctx)); + spdk_json_write_named_string(w, "cache_name", core_ctx->cache_name); + spdk_json_write_named_string(w, "base_name", core_ctx->base.name); + spdk_json_write_named_bool(w, "base_attached", vbdev_ocf_core_is_base_attached(core_ctx)); + spdk_json_write_named_uint64(w, "size", vbdev_ocf_core_is_base_attached(core_ctx) ? + spdk_bdev_get_block_size(core_ctx->base.bdev) * + spdk_bdev_get_num_blocks(core_ctx->base.bdev) : 0); + spdk_json_write_named_uint32(w, "block_size", vbdev_ocf_core_is_base_attached(core_ctx) ? + spdk_bdev_get_block_size(core_ctx->base.bdev) : 0); + + return 0; +} + +static int +dump_core_info(struct spdk_json_write_ctx *w, ocf_core_t core) { - struct spdk_json_write_ctx *w = cb_arg; struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); int rc; - spdk_json_write_object_begin(w); spdk_json_write_named_string(w, "name", ocf_core_get_name(core)); + spdk_json_write_named_string(w, "cache_name", ocf_cache_get_name(ocf_core_get_cache(core))); spdk_json_write_named_string(w, "base_name", core_ctx ? core_ctx->base.name : ""); spdk_json_write_named_bool(w, "base_attached", core_ctx ? vbdev_ocf_core_is_base_attached(core_ctx) : false); + spdk_json_write_named_uint64(w, "size", + (core_ctx && vbdev_ocf_core_is_base_attached(core_ctx)) ? + spdk_bdev_get_block_size(core_ctx->base.bdev) * + spdk_bdev_get_num_blocks(core_ctx->base.bdev) : 0); + spdk_json_write_named_uint32(w, "block_size", + (core_ctx && vbdev_ocf_core_is_base_attached(core_ctx)) ? + spdk_bdev_get_block_size(core_ctx->base.bdev) : 0); spdk_json_write_named_bool(w, "loading", !core_ctx); spdk_json_write_named_object_begin(w, "seq_cutoff"); - if ((rc = _dump_seqcutoff_info(w, core))) { - SPDK_ERRLOG("OCF core '%s': failed to get sequential cut-off params info (OCF error: %d)\n", - ocf_core_get_name(core), rc); + if ((rc = dump_seqcutoff_info(w, core))) { + SPDK_ERRLOG("OCF core '%s': failed to get sequential cut-off params info: %s\n", + ocf_core_get_name(core), spdk_strerror(-rc)); } spdk_json_write_object_end(w); @@ -2483,39 +2532,38 @@ _get_bdevs_core_visit(ocf_core_t core, void *cb_arg) spdk_json_write_named_int32(w, "error", core_ctx ? core_ctx->flush.error : 0); spdk_json_write_object_end(w); - spdk_json_write_object_end(w); - - return 0; + return rc; } -// handle errors ? static int -_get_bdevs_cache_visit(ocf_cache_t cache, void *cb_arg) +dump_cache_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) { - struct spdk_json_write_ctx *w = cb_arg; struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); int rc; - spdk_json_write_object_begin(w); spdk_json_write_named_string(w, "name", ocf_cache_get_name(cache)); spdk_json_write_named_string(w, "base_name", cache_ctx->base.name); spdk_json_write_named_bool(w, "base_attached", ocf_cache_is_device_attached(cache)); + spdk_json_write_named_uint64(w, "size", ocf_cache_is_device_attached(cache) ? + spdk_bdev_get_block_size(cache_ctx->base.bdev) * + spdk_bdev_get_num_blocks(cache_ctx->base.bdev) : 0); + spdk_json_write_named_uint32(w, "block_size", ocf_cache_is_device_attached(cache) ? + spdk_bdev_get_block_size(cache_ctx->base.bdev) : 0); + spdk_json_write_named_uint32(w, "cache_line_size", ocf_cache_get_line_size(cache)); spdk_json_write_named_string(w, "cache_mode", vbdev_ocf_cachemode_get_name(ocf_cache_get_mode(cache))); - spdk_json_write_named_uint32(w, "cache_line_size", - ocf_cache_get_line_size(cache) / KiB); spdk_json_write_named_object_begin(w, "promotion"); - if ((rc = _dump_promotion_info(w, cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to get promotion params info (OCF error: %d)\n", - ocf_cache_get_name(cache), rc); + if ((rc = dump_promotion_info(w, cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to get promotion params info: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-rc)); } spdk_json_write_object_end(w); spdk_json_write_named_object_begin(w, "cleaning"); - if ((rc = _dump_cleaning_info(w, cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to get cleaning params info (OCF error: %d)\n", - ocf_cache_get_name(cache), rc); + if ((rc = dump_cleaning_info(w, cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to get cleaning params info: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-rc)); } spdk_json_write_object_end(w); @@ -2525,8 +2573,41 @@ _get_bdevs_cache_visit(ocf_cache_t cache, void *cb_arg) spdk_json_write_object_end(w); spdk_json_write_named_uint16(w, "cores_count", ocf_cache_get_core_count(cache)); + + return rc; +} + +static int +_get_bdevs_core_visitor(ocf_core_t core, void *cb_arg) +{ + struct spdk_json_write_ctx *w = cb_arg; + int rc; + + spdk_json_write_object_begin(w); + if ((rc = dump_core_info(w, core))) { + SPDK_ERRLOG("OCF core '%s': failed to get core info: %s\n", + ocf_core_get_name(core), spdk_strerror(-rc)); + } + spdk_json_write_object_end(w); + + return rc; +} + +static int +_get_bdevs_cache_visitor(ocf_cache_t cache, void *cb_arg) +{ + struct spdk_json_write_ctx *w = cb_arg; + int rc; + + spdk_json_write_object_begin(w); + + if ((rc = dump_cache_info(w, cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to get cache info: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-rc)); + } + spdk_json_write_named_array_begin(w, "cores"); - rc = ocf_core_visit(cache, _get_bdevs_core_visit, w, false); + rc = ocf_core_visit(cache, _get_bdevs_core_visitor, w, false); spdk_json_write_array_end(w); spdk_json_write_object_end(w); @@ -2536,31 +2617,78 @@ _get_bdevs_cache_visit(ocf_cache_t cache, void *cb_arg) /* RPC entry point. */ void -vbdev_ocf_get_bdevs(const char *name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2) +vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2) { struct spdk_json_write_ctx *w = rpc_cb_arg1; struct vbdev_ocf_core *core_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; int rc; - // TODO: dump info about 'name' bdev only + SPDK_DEBUGLOG(vbdev_ocf, "OCF: getting info about vbdevs\n"); - spdk_json_write_named_array_begin(w, "cores_waitlist"); - vbdev_ocf_foreach_core_in_waitlist(core_ctx) { - spdk_json_write_object_begin(w); - spdk_json_write_named_string(w, "name", vbdev_ocf_core_get_name(core_ctx)); - spdk_json_write_named_string(w, "cache_name", core_ctx->cache_name); - spdk_json_write_named_string(w, "base_name", core_ctx->base.name); - spdk_json_write_named_bool(w, "base_attached", vbdev_ocf_core_is_base_attached(core_ctx)); - spdk_json_write_object_end(w); + if (!bdev_name) { + spdk_json_write_named_array_begin(w, "cores_waitlist"); + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + spdk_json_write_object_begin(w); + if ((rc = dump_core_waitlist_info(w, core_ctx))) { + SPDK_ERRLOG("OCF core '%s': failed to get wait list core info: %s\n", + vbdev_ocf_core_get_name(core_ctx), spdk_strerror(-rc)); + } + spdk_json_write_object_end(w); + } + spdk_json_write_array_end(w); + + spdk_json_write_named_array_begin(w, "caches"); + if ((rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _get_bdevs_cache_visitor, w))) { + SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + } + spdk_json_write_array_end(w); + + goto end; } - spdk_json_write_array_end(w); - spdk_json_write_named_array_begin(w, "caches"); - if ((rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _get_bdevs_cache_visit, w))) { - SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + if ((core_ctx = vbdev_ocf_core_waitlist_get_by_name(bdev_name))) { + if ((rc = dump_core_waitlist_info(w, core_ctx))) { + SPDK_ERRLOG("OCF core '%s': failed to get wait list core info: %s\n", + vbdev_ocf_core_get_name(core_ctx), spdk_strerror(-rc)); + } + + goto end; + } + + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { + SPDK_ERRLOG("OCF '%s': failed to allocate memory for getting bdevs info context\n", + bdev_name); + goto end; + } + mngt_ctx->bdev_name = bdev_name; + /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ + mngt_ctx->cache = NULL; + mngt_ctx->core = NULL; + + if ((rc = vbdev_ocf_bdev_resolve(mngt_ctx))) { + SPDK_ERRLOG("OCF '%s': failed to find cache or core of that name: %s\n", + bdev_name, spdk_strerror(-rc)); + free(mngt_ctx); + goto end; } - spdk_json_write_array_end(w); + if (mngt_ctx->core) { + if ((rc = dump_core_info(w, mngt_ctx->core))) { + SPDK_ERRLOG("OCF core '%s': failed to get core info: %s\n", + ocf_core_get_name(mngt_ctx->core), spdk_strerror(-rc)); + } + } else { + if ((rc = dump_cache_info(w, mngt_ctx->cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to get cache info: %s\n", + ocf_cache_get_name(mngt_ctx->cache), spdk_strerror(-rc)); + } + } + + free(mngt_ctx); + +end: rpc_cb_fn(rpc_cb_arg1, rpc_cb_arg2); } diff --git a/module/bdev/ocf/vbdev_ocf.h b/module/bdev/ocf/vbdev_ocf.h index 93177047bf8..5ec4c6ef061 100644 --- a/module/bdev/ocf/vbdev_ocf.h +++ b/module/bdev/ocf/vbdev_ocf.h @@ -9,10 +9,15 @@ #include "vbdev_ocf_cache.h" #include "vbdev_ocf_core.h" +/* Callback for the RPC layer used in all management operations. */ typedef void (*vbdev_ocf_rpc_mngt_cb)(const char *bdev_name, void *cb_arg, int error); + +/* Callback for the RPC layer used in bdev_ocf_get_bdevs call. */ typedef void (*vbdev_ocf_get_bdevs_cb)(void *cb_arg1, void *cb_arg2); -/* RPC entry points. */ +/* + * RPC entry points: + */ void vbdev_ocf_cache_start(const char *cache_name, const char *base_name, @@ -43,7 +48,6 @@ void vbdev_ocf_core_add(const char *core_name, void *rpc_cb_arg); void vbdev_ocf_core_remove(const char *core_name, - const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg); @@ -70,7 +74,7 @@ void vbdev_ocf_flush_start(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg); -void vbdev_ocf_get_bdevs(const char *name, +void vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2); diff --git a/module/bdev/ocf/vbdev_ocf_cache.c b/module/bdev/ocf/vbdev_ocf_cache.c index 58eef02fcf1..5b2dafa2119 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.c +++ b/module/bdev/ocf/vbdev_ocf_cache.c @@ -7,6 +7,14 @@ #include "ctx.h" #include "utils.h" +bool +vbdev_ocf_cache_is_base_attached(ocf_cache_t cache) +{ + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + + return cache_ctx->base.attached; +} + int vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cache_mode, const uint8_t cache_line_size, bool no_load) @@ -111,6 +119,7 @@ vbdev_ocf_cache_hotremove(struct spdk_bdev *bdev, void *event_ctx) assert(bdev == ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->base.bdev); assert(ocf_cache_is_device_attached(cache)); + assert(vbdev_ocf_cache_is_base_attached(cache)); ocf_mngt_cache_lock(cache, _cache_hotremove_lock_cb, NULL); } @@ -167,9 +176,6 @@ vbdev_ocf_cache_base_attach(ocf_cache_t cache, const char *base_name) base->is_cache = true; base->attached = true; - // why not ? what is it then ?!? - //assert(__bdev_to_io_dev(base->bdev) == base->bdev->ctxt); - return rc; } @@ -196,8 +202,8 @@ vbdev_ocf_cache_config_volume_create(ocf_cache_t cache) ocf_mngt_cache_attach_config_set_default(cache_att_cfg); cache_att_cfg->cache_line_size = cache_ctx->cache_cfg.cache_line_size; cache_att_cfg->open_cores = false; - cache_att_cfg->discard_on_start = false; // needed ? - cache_att_cfg->device.perform_test = false; // needed ? + cache_att_cfg->discard_on_start = false; + cache_att_cfg->device.perform_test = false; // for ocf_volume_open() in ocf_mngt_cache_attach/load() cache_att_cfg->device.volume_params = &cache_ctx->base; cache_att_cfg->force = cache_ctx->no_load; @@ -229,39 +235,39 @@ vbdev_ocf_cache_config_volume_destroy(ocf_cache_t cache) static void _volume_attach_metadata_probe_cb(void *priv, int error, struct ocf_metadata_probe_status *status) { - struct vbdev_ocf_mngt_ctx *ctx = priv; - ocf_cache_t cache = ctx->cache; + struct vbdev_ocf_mngt_ctx *mngt_ctx = priv; + ocf_cache_t cache = mngt_ctx->cache; struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); ocf_volume_close(cache_ctx->cache_att_cfg.device.volume); if (error && error != -OCF_ERR_NO_METADATA) { SPDK_ERRLOG("OCF cache '%s': failed to probe metadata\n", ocf_cache_get_name(cache)); - ctx->u.att_cb_fn(cache, ctx, error); + mngt_ctx->u.att_cb_fn(cache, mngt_ctx, error); } if (error == -OCF_ERR_NO_METADATA) { SPDK_NOTICELOG("OCF cache '%s': metadata not found - starting new cache instance\n", ocf_cache_get_name(cache)); - ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, ctx->u.att_cb_fn, ctx); + ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, mngt_ctx->u.att_cb_fn, mngt_ctx); } else { SPDK_NOTICELOG("OCF cache '%s': metadata found - loading previous cache instance\n", ocf_cache_get_name(cache)); SPDK_NOTICELOG("(start cache with 'no-load' flag to create new cache instead of loading config from metadata)\n"); // check status for cache_name/mode/line_size/dirty ? - ocf_mngt_cache_load(cache, &cache_ctx->cache_att_cfg, ctx->u.att_cb_fn, ctx); + ocf_mngt_cache_load(cache, &cache_ctx->cache_att_cfg, mngt_ctx->u.att_cb_fn, mngt_ctx); } } int -vbdev_ocf_cache_volume_attach(ocf_cache_t cache, struct vbdev_ocf_mngt_ctx *ctx) +vbdev_ocf_cache_volume_attach(ocf_cache_t cache, struct vbdev_ocf_mngt_ctx *mngt_ctx) { struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); int rc; if (cache_ctx->no_load) { SPDK_NOTICELOG("'no-load' flag specified - starting new cache without looking for metadata\n"); - ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, ctx->u.att_cb_fn, ctx); + ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, mngt_ctx->u.att_cb_fn, mngt_ctx); return 0; } @@ -271,7 +277,7 @@ vbdev_ocf_cache_volume_attach(ocf_cache_t cache, struct vbdev_ocf_mngt_ctx *ctx) } ocf_metadata_probe(vbdev_ocf_ctx, cache_ctx->cache_att_cfg.device.volume, - _volume_attach_metadata_probe_cb, ctx); + _volume_attach_metadata_probe_cb, mngt_ctx); return 0; } @@ -349,26 +355,3 @@ vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache) return rc; } - -bool -vbdev_ocf_cache_is_base_attached(ocf_cache_t cache) -{ - struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); - - return cache_ctx->base.attached; -} - -//bool -//vbdev_ocf_any_cache_started(void) -//{ -// /* OCF context is created with refcount set to 1 and any started cache -// * will increment it further. So, if context refcount equals 1, it means -// * that it's just created without any started caches. */ -// //return (ocf_ctx_get_refcnt(vbdev_ocf_ctx) > 1) ? true : false; -// -// int refcnt; -// -// refcnt = ocf_ctx_get_refcnt(vbdev_ocf_ctx); -// printf("*** CTX refcnt: %d\n", refcnt); -// return (refcnt > 1) ? true : false; -//} diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h index c0136c33513..4bba0b88242 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.h +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -17,14 +17,19 @@ extern struct spdk_bdev_module ocf_if; struct vbdev_ocf_cache { /* Base bdev data. */ struct vbdev_ocf_base base; + /* Cache start config. */ struct ocf_mngt_cache_config cache_cfg; + /* Cache device attach config. */ struct ocf_mngt_cache_attach_config cache_att_cfg; + /* Cache management queue. */ ocf_queue_t cache_mngt_q; + /* Indicator for cache start to ignore cache metadata found on device. */ bool no_load; + /* Status of cache flush operation. */ struct { bool in_progress; @@ -32,24 +37,45 @@ struct vbdev_ocf_cache { } flush; }; +/* Callback for the RPC layer used in all management operations. */ typedef void (*vbdev_ocf_rpc_mngt_cb)(const char *bdev_name, void *cb_arg, int error); /* Temporary context for management operations. */ struct vbdev_ocf_mngt_ctx { + /* RPC callback. */ vbdev_ocf_rpc_mngt_cb rpc_cb_fn; + + /* RPC context. */ void * rpc_cb_arg; + + /* Name of bdev. */ const char * bdev_name; + + /* OCF cache handle. */ ocf_cache_t cache; + + /* OCF core handle. */ ocf_core_t core; + union { + /* Callback for cache attach operation that needs to be passed through metadata probe + * while determining if cache needs to be loaded or just started as new. */ ocf_mngt_cache_attach_end_t att_cb_fn; + + /* Core context (priv). */ struct vbdev_ocf_core * core_ctx; + + /* OCF cache mode. */ ocf_cache_mode_t cache_mode; + + /* Promotion parameters. */ struct { ocf_promotion_t policy; int32_t nhit_insertion_threshold; int32_t nhit_trigger_threshold; } promotion; + + /* Cleaning parameters. */ struct { ocf_cleaning_t policy; int32_t acp_wake_up_time; @@ -60,6 +86,8 @@ struct vbdev_ocf_mngt_ctx { int32_t alru_activity_threshold; int32_t alru_max_dirty_ratio; } cleaning; + + /* Sequential cut-off parameters. */ struct { ocf_seq_cutoff_policy policy; int32_t threshold; @@ -73,30 +101,47 @@ struct vbdev_ocf_mngt_ctx { struct vbdev_ocf_cache_mngt_queue_ctx { /* Registered poller. */ struct spdk_poller * poller; + /* Thread on which poller was registered. */ struct spdk_thread * thread; + /* Currently kept only for its name used in debug log. */ ocf_cache_t cache; // rm ? }; +/* + * Helpers: + */ + /* Check if cache base is already attached. */ bool vbdev_ocf_cache_is_base_attached(ocf_cache_t cache); +/* + * Management: + */ + /* Create cache context, fill config and start OCF cache. */ int vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cache_mode, const uint8_t cache_line_size, bool no_load); + /* Free cache context. */ void vbdev_ocf_cache_destroy(ocf_cache_t cache); + /* Open, claim and create IO channel for base bdev. */ int vbdev_ocf_cache_base_attach(ocf_cache_t cache, const char *base_name); + /* Close base bdev and IO channel. */ void vbdev_ocf_cache_base_detach(ocf_cache_t cache); + /* Create OCF cache volume and fill device config. */ int vbdev_ocf_cache_config_volume_create(ocf_cache_t cache); + /* Destroy OCF cache volume from config. */ void vbdev_ocf_cache_config_volume_destroy(ocf_cache_t cache); + /* Attach or load cache device. */ int vbdev_ocf_cache_volume_attach(ocf_cache_t cache, struct vbdev_ocf_mngt_ctx *ctx); + /* Create cache management queue poller. */ int vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache); diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c index 93fba991a30..66b629f10b6 100644 --- a/module/bdev/ocf/vbdev_ocf_core.c +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -3,7 +3,7 @@ * All rights reserved. */ -#include "spdk/string.h" // rm ? +#include "spdk/string.h" #include "vbdev_ocf_core.h" #include "vbdev_ocf_cache.h" @@ -12,6 +12,88 @@ struct vbdev_ocf_core_waitlist_head g_vbdev_ocf_core_waitlist = STAILQ_HEAD_INITIALIZER(g_vbdev_ocf_core_waitlist); +void +vbdev_ocf_core_waitlist_add(struct vbdev_ocf_core *core_ctx) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': adding to wait list\n", + vbdev_ocf_core_get_name(core_ctx)); + + STAILQ_INSERT_TAIL(&g_vbdev_ocf_core_waitlist, core_ctx, waitlist_entry); +} + +void +vbdev_ocf_core_waitlist_remove(struct vbdev_ocf_core *core_ctx) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': removing from wait list\n", + vbdev_ocf_core_get_name(core_ctx)); + + STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); +} + +struct vbdev_ocf_core * +vbdev_ocf_core_waitlist_get_by_name(const char *core_name) +{ + struct vbdev_ocf_core *core_ctx; + + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + if (!strcmp(core_name, vbdev_ocf_core_get_name(core_ctx))) { + return core_ctx; + } + } + + return NULL; +} + +char * +vbdev_ocf_core_get_name(struct vbdev_ocf_core *core_ctx) +{ + return core_ctx->core_cfg.name; +} + +bool +vbdev_ocf_core_is_base_attached(struct vbdev_ocf_core *core_ctx) +{ + return core_ctx->base.attached; +} + +static int +_core_is_loaded_cache_visitor(ocf_cache_t cache, void *cb_arg) +{ + const char *core_name = cb_arg; + ocf_core_t core; + int rc; + + rc = ocf_core_get_by_name(cache, core_name, OCF_CORE_NAME_SIZE, &core); + if (!rc && !ocf_core_get_priv(core)) { + /* Core context is assigned only after manual core add (either right + * away if all devices are present, after corresponding cache start, + * or base bdev appearance). + * If there is no context, it means that this core was just added from + * metadata during cache load and that's what we're looking for here. */ + + return -EEXIST; + } else if (rc && rc != -OCF_ERR_CORE_NOT_EXIST) { + SPDK_ERRLOG("OCF: failed to get core: %s\n", spdk_strerror(-rc)); + } + + return 0; +} + +bool +vbdev_ocf_core_is_loaded(const char *core_name) +{ + int rc; + + rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _core_is_loaded_cache_visitor, (char *)core_name); + if (rc == -EEXIST) { + return true; + } else if (rc) { + SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + } + + return false; +} + int vbdev_ocf_core_create(struct vbdev_ocf_core **out, const char *core_name, const char *cache_name) { @@ -59,30 +141,26 @@ static void vbdev_ocf_core_hotremove(struct spdk_bdev *bdev, void *event_ctx) { struct vbdev_ocf_core *core_ctx = event_ctx; - struct vbdev_ocf_core *core_ctx_waitlist; int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating hot removal of base bdev '%s'\n", vbdev_ocf_core_get_name(core_ctx), spdk_bdev_get_name(bdev)); assert(bdev == core_ctx->base.bdev); + assert(vbdev_ocf_core_is_base_attached(core_ctx)); - vbdev_ocf_foreach_core_in_waitlist(core_ctx_waitlist) { - if (core_ctx == core_ctx_waitlist) { - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': hot removing from wait list\n", - vbdev_ocf_core_get_name(core_ctx)); - // always true ? - if (vbdev_ocf_core_is_base_attached(core_ctx)) { - vbdev_ocf_core_base_detach(core_ctx); - return; - } - } + if (vbdev_ocf_core_waitlist_get_by_name(vbdev_ocf_core_get_name(core_ctx))) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': hot removing from wait list\n", + vbdev_ocf_core_get_name(core_ctx)); + + vbdev_ocf_core_base_detach(core_ctx); + return; } if ((rc = vbdev_ocf_core_unregister(core_ctx, NULL, NULL))) { SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev during core hot removal: %s\n", vbdev_ocf_core_get_name(core_ctx), spdk_strerror(-rc)); - // detach base despite the error ? + /* Base bdev is already removed so detach it despite the error. */ vbdev_ocf_core_base_detach(core_ctx); } } @@ -142,9 +220,6 @@ vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core_ctx, const char *base_nam // for ocf_volume_open() in ocf_mngt_cache_add_core() core_cfg->volume_params = base; - // why not ? what is it then ?!? - //assert(__bdev_to_io_dev(base->bdev) == base->bdev->ctxt); - return rc; } @@ -301,9 +376,9 @@ vbdev_ocf_core_register(ocf_core_t core) ocf_vbdev->write_cache = base->bdev->write_cache; ocf_vbdev->blocklen = base->bdev->blocklen; ocf_vbdev->blockcnt = base->bdev->blockcnt; - // ? - //ocf_vbdev->required_alignment = base->bdev->required_alignment; - //ocf_vbdev->optimal_io_boundary = base->bdev->optimal_io_boundary; + // cache_line_size align ? + ocf_vbdev->required_alignment = base->bdev->required_alignment; + ocf_vbdev->optimal_io_boundary = base->bdev->optimal_io_boundary; // generate UUID based on namespace UUID + base bdev UUID (take from old module?) ocf_vbdev->fn_table = &vbdev_ocf_fn_table; ocf_vbdev->module = &ocf_if; @@ -374,7 +449,7 @@ _core_add_from_waitlist_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, ocf_core_get_name(core), ocf_cache_get_name(cache)); ocf_mngt_cache_unlock(cache); - STAILQ_REMOVE(&g_vbdev_ocf_core_waitlist, core_ctx, vbdev_ocf_core, waitlist_entry); + vbdev_ocf_core_waitlist_remove(core_ctx); } static void @@ -397,8 +472,9 @@ _core_add_from_waitlist_lock_cb(ocf_cache_t cache, void *cb_arg, int error) void vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache) { + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); struct vbdev_ocf_core *core_ctx; - uint32_t cache_block_size = ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->base.bdev->blocklen; + uint32_t cache_block_size = spdk_bdev_get_block_size(cache_ctx->base.bdev); uint32_t core_block_size; vbdev_ocf_foreach_core_in_waitlist(core_ctx) { @@ -410,7 +486,7 @@ vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache) SPDK_NOTICELOG("OCF core '%s': adding from waitlist to cache '%s'\n", vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); - core_block_size = core_ctx->base.bdev->blocklen; + core_block_size = spdk_bdev_get_block_size(core_ctx->base.bdev); if (cache_block_size > core_block_size) { SPDK_ERRLOG("OCF core '%s': failed to add to cache '%s': cache block size (%d) is greater than core block size (%d)\n", vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache), @@ -423,69 +499,3 @@ vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache) ocf_mngt_cache_lock(cache, _core_add_from_waitlist_lock_cb, core_ctx); } } - -char * -vbdev_ocf_core_get_name(struct vbdev_ocf_core *core_ctx) -{ - return core_ctx->core_cfg.name; -} - -struct vbdev_ocf_core * -vbdev_ocf_core_waitlist_get_by_name(const char *core_name) -{ - struct vbdev_ocf_core *core_ctx; - - vbdev_ocf_foreach_core_in_waitlist(core_ctx) { - // if (core_ctx && ...) ? - if (!strcmp(core_name, vbdev_ocf_core_get_name(core_ctx))) { - return core_ctx; - } - } - - return NULL; -} - -bool -vbdev_ocf_core_is_base_attached(struct vbdev_ocf_core *core_ctx) -{ - return core_ctx->base.attached; -} - -static int -_core_is_loaded_core_visit(ocf_core_t core, void *cb_arg) -{ - const char *core_name = cb_arg; - struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); - - if (!strcmp(core_name, ocf_core_get_name(core)) && !core_ctx) { - /* Core context is assigned only after manual core add (either right away - * if all devices are present, after corresponding cache start, or base bdev - * appearance). - * If there is no context, it means that this core was added from metadata - * during cache load. */ - return -EEXIST; - } - - return 0; -} - -static int -_core_is_loaded_cache_visit(ocf_cache_t cache, void *cb_arg) -{ - return ocf_core_visit(cache, _core_is_loaded_core_visit, cb_arg, false); -} - -bool -vbdev_ocf_core_is_loaded(const char *core_name) -{ - int rc; - - rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _core_is_loaded_cache_visit, (char *)core_name); - if (rc == -EEXIST) { - return true; - } else if (rc) { - SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); - } - - return false; -} diff --git a/module/bdev/ocf/vbdev_ocf_core.h b/module/bdev/ocf/vbdev_ocf_core.h index 37c131d3685..89bd1480377 100644 --- a/module/bdev/ocf/vbdev_ocf_core.h +++ b/module/bdev/ocf/vbdev_ocf_core.h @@ -20,6 +20,7 @@ extern struct spdk_bdev_fn_table vbdev_ocf_fn_table; * the cache yet due to lack of cache or base device. */ extern STAILQ_HEAD(vbdev_ocf_core_waitlist_head, vbdev_ocf_core) g_vbdev_ocf_core_waitlist; +/* Core wait list iterator. */ #define vbdev_ocf_foreach_core_in_waitlist(core_ctx) \ STAILQ_FOREACH(core_ctx, &g_vbdev_ocf_core_waitlist, waitlist_entry) @@ -27,14 +28,19 @@ extern STAILQ_HEAD(vbdev_ocf_core_waitlist_head, vbdev_ocf_core) g_vbdev_ocf_cor struct vbdev_ocf_core { /* Exposed OCF vbdev; the one that is registered in bdev layer for usage. */ struct spdk_bdev ocf_vbdev; + /* Base bdev data. */ struct vbdev_ocf_base base; + /* Core add config. */ struct ocf_mngt_core_config core_cfg; + /* Name of cache that this core is assigned to. */ char cache_name[OCF_CACHE_NAME_SIZE]; + /* Context for cache stop management operation. */ struct vbdev_ocf_mngt_ctx * mngt_ctx; + /* Status of core flush operation. */ struct { bool in_progress; @@ -48,39 +54,67 @@ struct vbdev_ocf_core { struct vbdev_ocf_core_io_channel_ctx { /* OCF queue. */ ocf_queue_t queue; + /* Cache base bdev IO channel. */ struct spdk_io_channel * cache_ch; + /* Core base bdev IO channel. */ struct spdk_io_channel * core_ch; + /* Registered poller. */ struct spdk_poller * poller; + /* Thread on which poller was registered and channels were opened. */ struct spdk_thread * thread; + /* Currently kept only for its name used in debug log. */ ocf_core_t core; // rm ? }; -/* Return core name string from given core context. */ -char *vbdev_ocf_core_get_name(struct vbdev_ocf_core *core_ctx); +/* + * Helpers: + */ + +/* Add core context to wait list. */ +void vbdev_ocf_core_waitlist_add(struct vbdev_ocf_core *core_ctx); + +/* Remove core context from wait list. */ +void vbdev_ocf_core_waitlist_remove(struct vbdev_ocf_core *core_ctx); + /* Get core from wait list by its name. */ struct vbdev_ocf_core *vbdev_ocf_core_waitlist_get_by_name(const char *core_name); + +/* Return core name string from given core context. */ +char *vbdev_ocf_core_get_name(struct vbdev_ocf_core *core_ctx); + /* Check if core base is already attached. */ bool vbdev_ocf_core_is_base_attached(struct vbdev_ocf_core *core_ctx); + /* Check if core was added to cache from metadata during cache load. */ bool vbdev_ocf_core_is_loaded(const char *core_name); +/* + * Management: + */ + /* Create core context and fill config. */ int vbdev_ocf_core_create(struct vbdev_ocf_core **out, const char *core_name, const char *cache_name); + /* Free core context. */ void vbdev_ocf_core_destroy(struct vbdev_ocf_core *core_ctx); + /* Open, claim and create IO channel for base bdev. */ int vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core_ctx, const char *base_name); + /* Close base bdev and IO channel. */ void vbdev_ocf_core_base_detach(struct vbdev_ocf_core *core_ctx); + /* Register core in SPDK bdev layer for usage. */ int vbdev_ocf_core_register(ocf_core_t core); + /* Unregister core from SPDK bdev layer. */ int vbdev_ocf_core_unregister(struct vbdev_ocf_core *core_ctx, spdk_bdev_unregister_cb cb_fn, void *cb_arg); + /* Add all cores from wait list assigned to given cache. */ void vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache); diff --git a/module/bdev/ocf/vbdev_ocf_rpc.c b/module/bdev/ocf/vbdev_ocf_rpc.c index e6d4f0ecd3c..854dd72fc58 100644 --- a/module/bdev/ocf/vbdev_ocf_rpc.c +++ b/module/bdev/ocf/vbdev_ocf_rpc.c @@ -38,9 +38,9 @@ rpc_bdev_ocf_start_cache_cb(const char *bdev_name, void *cb_arg, int error) struct spdk_json_write_ctx *w; if (error && error != -ENODEV) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Could not start OCF cache: %s", - spdk_strerror(-error)); + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to start OCF cache '%s': %s", + bdev_name, spdk_strerror(-error)); return; } @@ -91,7 +91,9 @@ rpc_bdev_ocf_stop_cache_cb(const char *bdev_name, void *cb_arg, int error) struct spdk_jsonrpc_request *request = cb_arg; if (error) { - spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to stop OCF cache '%s': %s", + bdev_name, spdk_strerror(-error)); return; } @@ -139,9 +141,9 @@ rpc_bdev_ocf_detach_cache_cb(const char *bdev_name, void *cb_arg, int error) struct spdk_jsonrpc_request *request = cb_arg; if (error) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Could not detach OCF cache: %s", - spdk_strerror(-error)); + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to detach device from OCF cache '%s': %s", + bdev_name, spdk_strerror(-error)); return; } @@ -194,9 +196,9 @@ rpc_bdev_ocf_attach_cache_cb(const char *bdev_name, void *cb_arg, int error) struct spdk_jsonrpc_request *request = cb_arg; if (error && error != -ENODEV) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Could not attach OCF cache: %s", - spdk_strerror(-error)); + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to attach device to OCF cache '%s': %s", + bdev_name, spdk_strerror(-error)); return; } @@ -252,9 +254,9 @@ rpc_bdev_ocf_add_core_cb(const char *bdev_name, void *cb_arg, int error) struct spdk_json_write_ctx *w; if (error && error != -ENODEV) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Could not add core to OCF cache: %s", - spdk_strerror(-error)); + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to add core '%s' to OCF cache: %s", + bdev_name, spdk_strerror(-error)); return; } @@ -286,19 +288,16 @@ SPDK_RPC_REGISTER("bdev_ocf_add_core", rpc_bdev_ocf_add_core, SPDK_RPC_RUNTIME) struct rpc_bdev_ocf_remove_core { char *core_name; - char *cache_name; }; static void free_rpc_bdev_ocf_remove_core(struct rpc_bdev_ocf_remove_core *r) { free(r->core_name); - free(r->cache_name); } static const struct spdk_json_object_decoder rpc_bdev_ocf_remove_core_decoders[] = { {"core_name", offsetof(struct rpc_bdev_ocf_remove_core, core_name), spdk_json_decode_string}, - {"cache_name", offsetof(struct rpc_bdev_ocf_remove_core, cache_name), spdk_json_decode_string}, }; static void @@ -307,7 +306,9 @@ rpc_bdev_ocf_remove_core_cb(const char *bdev_name, void *cb_arg, int error) struct spdk_jsonrpc_request *request = cb_arg; if (error) { - spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to remove core '%s' from OCF cache: %s", + bdev_name, spdk_strerror(-error)); return; } @@ -328,7 +329,7 @@ rpc_bdev_ocf_remove_core(struct spdk_jsonrpc_request *request, const struct spdk goto cleanup; } - vbdev_ocf_core_remove(req.core_name, req.cache_name, rpc_bdev_ocf_remove_core_cb, request); + vbdev_ocf_core_remove(req.core_name, rpc_bdev_ocf_remove_core_cb, request); cleanup: free_rpc_bdev_ocf_remove_core(&req); @@ -358,7 +359,9 @@ rpc_bdev_ocf_set_cachemode_cb(const char *bdev_name, void *cb_arg, int error) struct spdk_jsonrpc_request *request = cb_arg; if (error) { - spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to change cache mode on OCF cache '%s': %s", + bdev_name, spdk_strerror(-error)); return; } @@ -413,7 +416,9 @@ rpc_bdev_ocf_set_promotion_cb(const char *bdev_name, void *cb_arg, int error) struct spdk_jsonrpc_request *request = cb_arg; if (error) { - spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to set promotion params on OCF cache '%s': %s", + bdev_name, spdk_strerror(-error)); return; } @@ -479,7 +484,9 @@ rpc_bdev_ocf_set_cleaning_cb(const char *bdev_name, void *cb_arg, int error) struct spdk_jsonrpc_request *request = cb_arg; if (error) { - spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to set cleaning params on OCF cache '%s': %s", + bdev_name, spdk_strerror(-error)); return; } @@ -540,7 +547,9 @@ rpc_bdev_ocf_set_seqcutoff_cb(const char *bdev_name, void *cb_arg, int error) struct spdk_jsonrpc_request *request = cb_arg; if (error) { - spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to set sequential cut-off params on OCF cache or core '%s': %s", + bdev_name, spdk_strerror(-error)); return; } @@ -589,7 +598,9 @@ rpc_bdev_ocf_flush_start_cb(const char *bdev_name, void *cb_arg, int error) struct spdk_jsonrpc_request *request = cb_arg; if (error) { - spdk_jsonrpc_send_error_response(request, error, spdk_strerror(-error)); + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to start flushing OCF cache or core '%s': %s", + bdev_name, spdk_strerror(-error)); return; } @@ -630,17 +641,17 @@ rpc_bdev_ocf_reset_stats(struct spdk_jsonrpc_request *request, const struct spdk SPDK_RPC_REGISTER("bdev_ocf_reset_stats", rpc_bdev_ocf_reset_stats, SPDK_RPC_RUNTIME) struct rpc_bdev_ocf_get_bdevs { - char *name; + char *bdev_name; }; static void free_rpc_bdev_ocf_get_bdevs(struct rpc_bdev_ocf_get_bdevs *r) { - free(r->name); + free(r->bdev_name); } static const struct spdk_json_object_decoder rpc_bdev_ocf_get_bdevs_decoders[] = { - {"name", offsetof(struct rpc_bdev_ocf_get_bdevs, name), spdk_json_decode_string, true}, + {"bdev_name", offsetof(struct rpc_bdev_ocf_get_bdevs, bdev_name), spdk_json_decode_string, true}, }; static void @@ -671,7 +682,7 @@ rpc_bdev_ocf_get_bdevs(struct spdk_jsonrpc_request *request, const struct spdk_j w = spdk_jsonrpc_begin_result(request); spdk_json_write_object_begin(w); - vbdev_ocf_get_bdevs(params ? req.name : NULL, rpc_bdev_ocf_get_bdevs_cb, w, request); + vbdev_ocf_get_bdevs(params ? req.bdev_name : NULL, rpc_bdev_ocf_get_bdevs_cb, w, request); cleanup: free_rpc_bdev_ocf_get_bdevs(&req); diff --git a/python/spdk/cli/bdev.py b/python/spdk/cli/bdev.py index c9f4848ac9a..cfb91eb5b0c 100644 --- a/python/spdk/cli/bdev.py +++ b/python/spdk/cli/bdev.py @@ -151,17 +151,16 @@ def bdev_ocf_add_core(args): core_name=args.core_name, base_name=args.base_name, cache_name=args.cache_name)) - p = subparsers.add_parser('bdev_ocf_add_core', help='Add core device to OCF cache') + p = subparsers.add_parser('bdev_ocf_add_core', help='Add core (backend device) to OCF cache') p.add_argument('core_name', help='name for the new OCF core vbdev') p.add_argument('base_name', help='name of the base bdev to use as core') p.add_argument('cache_name', help='name of already started OCF cache vbdev') p.set_defaults(func=bdev_ocf_add_core) def bdev_ocf_remove_core(args): - args.client.bdev_ocf_remove_core(core_name=args.core_name, cache_name=args.cache_name) - p = subparsers.add_parser('bdev_ocf_remove_core', help='Remove core device from OCF cache') + args.client.bdev_ocf_remove_core(core_name=args.core_name) + p = subparsers.add_parser('bdev_ocf_remove_core', help='Remove core (backend device) from OCF cache') p.add_argument('core_name', help='name of the core vbdev to remove from OCF cache instance') - p.add_argument('cache_name', help='name of the cache vbdev to remove core from') p.set_defaults(func=bdev_ocf_remove_core) def bdev_ocf_set_cachemode(args): @@ -181,7 +180,7 @@ def bdev_ocf_set_promotion(args): policy=args.policy, nhit_insertion_threshold=args.nhit_insertion_threshold, nhit_trigger_threshold=args.nhit_trigger_threshold) - p = subparsers.add_parser('bdev_ocf_set_promotion', help='Set promotion parameters for OCF cache device') + p = subparsers.add_parser('bdev_ocf_set_promotion', help='Set promotion parameters for OCF cache') p.add_argument('cache_name', help='name of the cache vbdev') p.add_argument('-p', '--policy', help='promotion policy (choose between {always|nhit})', @@ -208,7 +207,7 @@ def bdev_ocf_set_cleaning(args): alru_staleness_time=args.alru_staleness_time, alru_activity_threshold=args.alru_activity_threshold, alru_max_dirty_ratio=args.alru_max_dirty_ratio) - p = subparsers.add_parser('bdev_ocf_set_cleaning', help='Set cleaning parameters for OCF cache device') + p = subparsers.add_parser('bdev_ocf_set_cleaning', help='Set cleaning parameters for OCF cache') p.add_argument('cache_name', help='name of the cache vbdev') p.add_argument('-p', '--policy', help='cleaning policy (choose between {acp|alru|nop})', @@ -251,7 +250,7 @@ def bdev_ocf_set_seqcutoff(args): threshold=args.threshold, promotion_count=args.promotion_count, promote_on_threshold=args.promote_on_threshold) - p = subparsers.add_parser('bdev_ocf_set_seqcutoff', help='Set sequential cut-off parameters for OCF core device or all cores in given cache') + p = subparsers.add_parser('bdev_ocf_set_seqcutoff', help='Set sequential cut-off parameters for OCF core or all cores in given cache') p.add_argument('bdev_name', help='name of OCF vbdev') p.add_argument('-p', '--policy', help='sequential cut-off policy (choose between {always|full|never})', @@ -291,9 +290,9 @@ def bdev_ocf_reset_stats(args): p.set_defaults(func=bdev_ocf_reset_stats) def bdev_ocf_get_bdevs(args): - print_dict(args.client.bdev_ocf_get_bdevs(name=args.name)) - p = subparsers.add_parser('bdev_ocf_get_bdevs', help='Get info about OCF devices') - p.add_argument('name', nargs='?', help='optional name of specific OCF vbdev; shows all by default') + print_dict(args.client.bdev_ocf_get_bdevs(bdev_name=args.bdev_name)) + p = subparsers.add_parser('bdev_ocf_get_bdevs', help='Get info about OCF vbdevs') + p.add_argument('bdev_name', nargs='?', help='optional name of specific OCF vbdev (shows all by default)') p.set_defaults(func=bdev_ocf_get_bdevs) def bdev_malloc_create(args): diff --git a/test/ocf/common.sh b/test/ocf/common.sh index 47c59a16fc4..a4bb46475ba 100644 --- a/test/ocf/common.sh +++ b/test/ocf/common.sh @@ -129,7 +129,7 @@ add_cores() { remove_cores() { for i in {1..3}; do for j in {1..3}; do - $rpc_py bdev_ocf_remove_core Ocf_core$i-$j Ocf_cache$i + $rpc_py bdev_ocf_remove_core Ocf_core$i-$j done done $rpc_py bdev_ocf_get_bdevs | jq -e '.' diff --git a/test/ocf/management/config_change.sh b/test/ocf/management/config_change.sh index bb93fc8b1cd..30901823e4e 100755 --- a/test/ocf/management/config_change.sh +++ b/test/ocf/management/config_change.sh @@ -312,7 +312,7 @@ __check_seqcutoff_params() { --argjson promote_on_threshold $promote_on_threshold \ '.caches[].cores[0].seq_cutoff | (.policy == $policy) and - (.threshold == $threshold) and + (.threshold == $threshold * 1024) and (.promotion_count == $promotion_count) and (.promote_on_threshold == $promote_on_threshold)' @@ -323,7 +323,7 @@ __check_seqcutoff_params() { --argjson promote_on_threshold $seqcutoff_promote_on_threshold_default \ '.caches[].cores[1].seq_cutoff | (.policy == $policy) and - (.threshold == $threshold) and + (.threshold == $threshold * 1024) and (.promotion_count == $promotion_count) and (.promote_on_threshold == $promote_on_threshold)' } @@ -350,7 +350,7 @@ __check_seqcutoff_params_all() { --argjson promote_on_threshold $promote_on_threshold \ '.caches[].cores[].seq_cutoff | (.policy == $policy) and - (.threshold == $threshold) and + (.threshold == $threshold * 1024) and (.promotion_count == $promotion_count) and (.promote_on_threshold == $promote_on_threshold)' } diff --git a/test/ocf/management/start-stop/basic.sh b/test/ocf/management/start-stop/basic.sh index 46c2cb1aa10..1d6072522e0 100755 --- a/test/ocf/management/start-stop/basic.sh +++ b/test/ocf/management/start-stop/basic.sh @@ -130,7 +130,7 @@ for stop_caches in false true; do __check_caches_attached __check_cores_attached for i in {1..3}; do - $rpc_py bdev_ocf_remove_core Ocf_core$i-1 Ocf_cache$i + $rpc_py bdev_ocf_remove_core Ocf_core$i-1 done $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 2' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 2' @@ -153,8 +153,8 @@ for stop_caches in false true; do __check_caches_attached __check_cores_attached for i in {1..3}; do - $rpc_py bdev_ocf_remove_core Ocf_core$i-2 Ocf_cache$i - $rpc_py bdev_ocf_remove_core Ocf_core$i-3 Ocf_cache$i + $rpc_py bdev_ocf_remove_core Ocf_core$i-2 + $rpc_py bdev_ocf_remove_core Ocf_core$i-3 done $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 1' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 1' From 936d755706acdffa1418e1a84e37bfba27e5b861 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Thu, 24 Jul 2025 18:33:36 +0200 Subject: [PATCH 14/34] Implement statistics Change-Id: I70e3d6470dc3f0c7a6a493472746ccdc3bd3c6e7 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/stats.c | 61 ++++++-------- module/bdev/ocf/stats.h | 9 ++- module/bdev/ocf/vbdev_ocf.c | 138 ++++++++++++++++++++++++++++++++ module/bdev/ocf/vbdev_ocf.h | 9 +++ module/bdev/ocf/vbdev_ocf_rpc.c | 87 ++++++++++++++++++++ python/spdk/cli/bdev.py | 12 +-- 6 files changed, 272 insertions(+), 44 deletions(-) diff --git a/module/bdev/ocf/stats.c b/module/bdev/ocf/stats.c index f27036c269e..aa21fa04a06 100644 --- a/module/bdev/ocf/stats.c +++ b/module/bdev/ocf/stats.c @@ -1,38 +1,33 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2018 Intel Corporation. + * Copyright (C) 2025 Huawei Technologies * All rights reserved. */ -#include "ctx.h" #include "stats.h" int -vbdev_ocf_stats_get(ocf_cache_t cache, char *core_name, struct vbdev_ocf_stats *stats) +vbdev_ocf_stats_cache_get(ocf_cache_t cache, struct vbdev_ocf_stats *stats) { - int status; - ocf_core_t core; - - status = ocf_core_get_by_name(cache, core_name, strlen(core_name), &core); - if (status) { - return status; - } - - return ocf_stats_collect_core(core, &stats->usage, &stats->reqs, &stats->blocks, &stats->errors); + return ocf_stats_collect_cache(cache, &stats->usage, &stats->req, &stats->blocks, &stats->errors); } int -vbdev_ocf_stats_reset(ocf_cache_t cache, char *core_name) +vbdev_ocf_stats_core_get(ocf_core_t core, struct vbdev_ocf_stats *stats) { - int status; - ocf_core_t core; + return ocf_stats_collect_core(core, &stats->usage, &stats->req, &stats->blocks, &stats->errors); +} - status = ocf_core_get_by_name(cache, core_name, strlen(core_name), &core); - if (status) { - return status; - } +int +vbdev_ocf_stats_cache_reset(ocf_cache_t cache) +{ + return ocf_core_stats_initialize_all(cache); +} +int +vbdev_ocf_stats_core_reset(ocf_core_t core) +{ ocf_core_stats_initialize(core); - return 0; } @@ -47,8 +42,6 @@ vbdev_ocf_stats_reset(ocf_cache_t cache, char *core_name) void vbdev_ocf_stats_write_json(struct spdk_json_write_ctx *w, struct vbdev_ocf_stats *stats) { - spdk_json_write_object_begin(w); - spdk_json_write_named_object_begin(w, "usage"); WJSON_STAT(w, stats, usage, occupancy, "4KiB blocks"); WJSON_STAT(w, stats, usage, free, "4KiB blocks"); @@ -57,18 +50,18 @@ vbdev_ocf_stats_write_json(struct spdk_json_write_ctx *w, struct vbdev_ocf_stats spdk_json_write_object_end(w); spdk_json_write_named_object_begin(w, "requests"); - WJSON_STAT(w, stats, reqs, rd_hits, "Requests"); - WJSON_STAT(w, stats, reqs, rd_partial_misses, "Requests"); - WJSON_STAT(w, stats, reqs, rd_full_misses, "Requests"); - WJSON_STAT(w, stats, reqs, rd_total, "Requests"); - WJSON_STAT(w, stats, reqs, wr_hits, "Requests"); - WJSON_STAT(w, stats, reqs, wr_partial_misses, "Requests"); - WJSON_STAT(w, stats, reqs, wr_full_misses, "Requests"); - WJSON_STAT(w, stats, reqs, wr_total, "Requests"); - WJSON_STAT(w, stats, reqs, rd_pt, "Requests"); - WJSON_STAT(w, stats, reqs, wr_pt, "Requests"); - WJSON_STAT(w, stats, reqs, serviced, "Requests"); - WJSON_STAT(w, stats, reqs, total, "Requests"); + WJSON_STAT(w, stats, req, rd_hits, "Requests"); + WJSON_STAT(w, stats, req, rd_partial_misses, "Requests"); + WJSON_STAT(w, stats, req, rd_full_misses, "Requests"); + WJSON_STAT(w, stats, req, rd_total, "Requests"); + WJSON_STAT(w, stats, req, wr_hits, "Requests"); + WJSON_STAT(w, stats, req, wr_partial_misses, "Requests"); + WJSON_STAT(w, stats, req, wr_full_misses, "Requests"); + WJSON_STAT(w, stats, req, wr_total, "Requests"); + WJSON_STAT(w, stats, req, rd_pt, "Requests"); + WJSON_STAT(w, stats, req, wr_pt, "Requests"); + WJSON_STAT(w, stats, req, serviced, "Requests"); + WJSON_STAT(w, stats, req, total, "Requests"); spdk_json_write_object_end(w); spdk_json_write_named_object_begin(w, "blocks"); @@ -92,6 +85,4 @@ vbdev_ocf_stats_write_json(struct spdk_json_write_ctx *w, struct vbdev_ocf_stats WJSON_STAT(w, stats, errors, cache_volume_total, "Requests"); WJSON_STAT(w, stats, errors, total, "Requests"); spdk_json_write_object_end(w); - - spdk_json_write_object_end(w); } diff --git a/module/bdev/ocf/stats.h b/module/bdev/ocf/stats.h index b48a30c0194..0ceb641560f 100644 --- a/module/bdev/ocf/stats.h +++ b/module/bdev/ocf/stats.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2018 Intel Corporation. + * Copyright (C) 2025 Huawei Technologies * All rights reserved. */ @@ -11,13 +12,15 @@ struct vbdev_ocf_stats { struct ocf_stats_usage usage; - struct ocf_stats_requests reqs; + struct ocf_stats_requests req; struct ocf_stats_blocks blocks; struct ocf_stats_errors errors; }; -int vbdev_ocf_stats_get(ocf_cache_t cache, char *core_name, struct vbdev_ocf_stats *stats); -int vbdev_ocf_stats_reset(ocf_cache_t cache, char *core_name); +int vbdev_ocf_stats_cache_get(ocf_cache_t cache, struct vbdev_ocf_stats *stats); +int vbdev_ocf_stats_core_get(ocf_core_t core, struct vbdev_ocf_stats *stats); +int vbdev_ocf_stats_cache_reset(ocf_cache_t cache); +int vbdev_ocf_stats_core_reset(ocf_core_t core); void vbdev_ocf_stats_write_json(struct spdk_json_write_ctx *w, struct vbdev_ocf_stats *stats); diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 9380e502d1d..415ceb63e82 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -11,6 +11,7 @@ #include "vbdev_ocf.h" #include "ctx.h" #include "data.h" +#include "stats.h" #include "utils.h" #include "volume.h" @@ -2351,6 +2352,142 @@ vbdev_ocf_flush_start(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo _flush_lock_cb, mngt_ctx); } +// TODO: read lock cache +/* RPC entry point. */ +void +vbdev_ocf_get_stats(const char *bdev_name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, + void *rpc_cb_arg1, void *rpc_cb_arg2) +{ + struct spdk_json_write_ctx *w = rpc_cb_arg1; + ocf_cache_t cache; + ocf_core_t core; + struct vbdev_ocf_mngt_ctx *mngt_ctx; + struct vbdev_ocf_stats stats; + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': getting statistics\n", bdev_name); + + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { + SPDK_ERRLOG("OCF '%s': failed to allocate memory for getting statistics context\n", + bdev_name); + goto end; + } + mngt_ctx->bdev_name = bdev_name; + /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ + mngt_ctx->cache = NULL; + mngt_ctx->core = NULL; + + if ((rc = vbdev_ocf_bdev_resolve(mngt_ctx))) { + SPDK_ERRLOG("OCF '%s': failed to find cache or core of that name: %s\n", + bdev_name, spdk_strerror(-rc)); + free(mngt_ctx); + goto end; + } + cache = mngt_ctx->cache; + core = mngt_ctx->core; + free(mngt_ctx); + + if (core) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': collecting statistics\n", ocf_core_get_name(core)); + + if ((rc = vbdev_ocf_stats_core_get(core, &stats))) { + SPDK_ERRLOG("OCF core '%s': failed to collect statistics (OCF error: %d)\n", + ocf_core_get_name(core), rc); + goto end; + } + } else { + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': collecting statistics\n", ocf_cache_get_name(cache)); + + if ((rc = vbdev_ocf_stats_cache_get(cache, &stats))) { + SPDK_ERRLOG("OCF cache '%s': failed to collect statistics (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto end; + } + } + + vbdev_ocf_stats_write_json(w, &stats); + +end: + rpc_cb_fn(rpc_cb_arg1, rpc_cb_arg2); +} + +static void +_reset_stats_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + ocf_core_t core = mngt_ctx->core; + int rc; + + if ((rc = error)) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + goto end; + } + + if (core) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': stats reset\n", ocf_core_get_name(core)); + + if ((rc = vbdev_ocf_stats_core_reset(core))) { + SPDK_ERRLOG("OCF core '%s': failed to reset statistics (OCF error: %d)\n", + ocf_core_get_name(core), rc); + } + } else { + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': stats reset\n", ocf_cache_get_name(cache)); + + if ((rc = vbdev_ocf_stats_cache_reset(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to reset statistics (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + } + } + + ocf_mngt_cache_unlock(cache); + +end: + mngt_ctx->rpc_cb_fn(mngt_ctx->bdev_name, mngt_ctx->rpc_cb_arg, rc); + free(mngt_ctx); +} + +/* RPC entry point. */ +void +vbdev_ocf_reset_stats(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) +{ + struct vbdev_ocf_mngt_ctx *mngt_ctx; + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': resetting statistics\n", bdev_name); + + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); + if (!mngt_ctx) { + SPDK_ERRLOG("OCF '%s': failed to allocate memory for resetting statistics context\n", + bdev_name); + rc = -ENOMEM; + goto err_alloc; + } + mngt_ctx->rpc_cb_fn = rpc_cb_fn; + mngt_ctx->rpc_cb_arg = rpc_cb_arg; + mngt_ctx->bdev_name = bdev_name; + /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ + mngt_ctx->cache = NULL; + mngt_ctx->core = NULL; + + if ((rc = vbdev_ocf_bdev_resolve(mngt_ctx))) { + SPDK_ERRLOG("OCF '%s': failed to find cache or core of that name: %s\n", + bdev_name, spdk_strerror(-rc)); + goto err_resolve; + } + + ocf_mngt_cache_lock(mngt_ctx->cache ? : ocf_core_get_cache(mngt_ctx->core), + _reset_stats_lock_cb, mngt_ctx); + + return; + +err_resolve: + free(mngt_ctx); +err_alloc: + rpc_cb_fn(bdev_name, rpc_cb_arg, rc); +} + static int dump_promotion_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) { @@ -2615,6 +2752,7 @@ _get_bdevs_cache_visitor(ocf_cache_t cache, void *cb_arg) return rc; } +// TODO: read lock cache ? /* RPC entry point. */ void vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2) diff --git a/module/bdev/ocf/vbdev_ocf.h b/module/bdev/ocf/vbdev_ocf.h index 5ec4c6ef061..a9afd1c1466 100644 --- a/module/bdev/ocf/vbdev_ocf.h +++ b/module/bdev/ocf/vbdev_ocf.h @@ -74,6 +74,15 @@ void vbdev_ocf_flush_start(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg); +void vbdev_ocf_get_stats(const char *bdev_name, + vbdev_ocf_get_bdevs_cb rpc_cb_fn, + void *rpc_cb_arg1, + void *rpc_cb_arg2); + +void vbdev_ocf_reset_stats(const char *bdev_name, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, + void *rpc_cb_arg); + void vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rpc_cb_arg1, diff --git a/module/bdev/ocf/vbdev_ocf_rpc.c b/module/bdev/ocf/vbdev_ocf_rpc.c index 854dd72fc58..8f968405ae3 100644 --- a/module/bdev/ocf/vbdev_ocf_rpc.c +++ b/module/bdev/ocf/vbdev_ocf_rpc.c @@ -628,15 +628,102 @@ rpc_bdev_ocf_flush_start(struct spdk_jsonrpc_request *request, const struct spdk } SPDK_RPC_REGISTER("bdev_ocf_flush_start", rpc_bdev_ocf_flush_start, SPDK_RPC_RUNTIME) +struct rpc_bdev_ocf_get_stats { + char *bdev_name; +}; + +static void +free_rpc_bdev_ocf_get_stats(struct rpc_bdev_ocf_get_stats *r) +{ + free(r->bdev_name); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_get_stats_decoders[] = { + {"bdev_name", offsetof(struct rpc_bdev_ocf_get_stats, bdev_name), spdk_json_decode_string}, +}; + +static void +rpc_bdev_ocf_get_stats_cb(void *cb_arg1, void *cb_arg2) +{ + struct spdk_json_write_ctx *w = cb_arg1; + struct spdk_jsonrpc_request *request = cb_arg2; + + spdk_json_write_object_end(w); + spdk_jsonrpc_end_result(request, w); +} + static void rpc_bdev_ocf_get_stats(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) { + struct rpc_bdev_ocf_get_stats req = {}; + struct spdk_json_write_ctx *w; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_get_stats_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_get_stats_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + goto cleanup; + } + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_object_begin(w); + + vbdev_ocf_get_stats(req.bdev_name, rpc_bdev_ocf_get_stats_cb, w, request); + +cleanup: + free_rpc_bdev_ocf_get_stats(&req); } SPDK_RPC_REGISTER("bdev_ocf_get_stats", rpc_bdev_ocf_get_stats, SPDK_RPC_RUNTIME) +struct rpc_bdev_ocf_reset_stats { + char *bdev_name; +}; + +static void +free_rpc_bdev_ocf_reset_stats(struct rpc_bdev_ocf_reset_stats *r) +{ + free(r->bdev_name); +} + +static const struct spdk_json_object_decoder rpc_bdev_ocf_reset_stats_decoders[] = { + {"bdev_name", offsetof(struct rpc_bdev_ocf_reset_stats, bdev_name), spdk_json_decode_string}, +}; + +static void +rpc_bdev_ocf_reset_stats_cb(const char *bdev_name, void *cb_arg, int error) +{ + struct spdk_jsonrpc_request *request = cb_arg; + + if (error) { + spdk_jsonrpc_send_error_response_fmt(request, error, + "Failed to reset statistics on OCF cache or core '%s': %s", + bdev_name, spdk_strerror(-error)); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); +} + static void rpc_bdev_ocf_reset_stats(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) { + struct rpc_bdev_ocf_reset_stats req = {}; + + if (spdk_json_decode_object(params, rpc_bdev_ocf_reset_stats_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_reset_stats_decoders), + &req)) { + SPDK_DEBUGLOG(vbdev_ocf_rpc, "spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + goto cleanup; + } + + vbdev_ocf_reset_stats(req.bdev_name, rpc_bdev_ocf_reset_stats_cb, request); + +cleanup: + free_rpc_bdev_ocf_reset_stats(&req); } SPDK_RPC_REGISTER("bdev_ocf_reset_stats", rpc_bdev_ocf_reset_stats, SPDK_RPC_RUNTIME) diff --git a/python/spdk/cli/bdev.py b/python/spdk/cli/bdev.py index cfb91eb5b0c..7045e8e53d6 100644 --- a/python/spdk/cli/bdev.py +++ b/python/spdk/cli/bdev.py @@ -278,15 +278,15 @@ def bdev_ocf_flush_start(args): p.set_defaults(func=bdev_ocf_flush_start) def bdev_ocf_get_stats(args): - print_dict(args.client.bdev_ocf_get_stats(name=args.name)) - p = subparsers.add_parser('bdev_ocf_get_stats', help='Get statistics of chosen OCF block device') - p.add_argument('name', help='Name of OCF bdev') + print_dict(args.client.bdev_ocf_get_stats(bdev_name=args.bdev_name)) + p = subparsers.add_parser('bdev_ocf_get_stats', help='Get statistics of OCF vbdev') + p.add_argument('bdev_name', help='name of OCF vbdev') p.set_defaults(func=bdev_ocf_get_stats) def bdev_ocf_reset_stats(args): - print_dict(args.client.bdev_ocf_reset_stats(name=args.name)) - p = subparsers.add_parser('bdev_ocf_reset_stats', help='Reset statistics of chosen OCF block device') - p.add_argument('name', help='Name of OCF bdev') + args.client.bdev_ocf_reset_stats(bdev_name=args.bdev_name) + p = subparsers.add_parser('bdev_ocf_reset_stats', help='Reset statistics of OCF vbdev') + p.add_argument('bdev_name', help='name of OCF vbdev') p.set_defaults(func=bdev_ocf_reset_stats) def bdev_ocf_get_bdevs(args): From 974fbf320d89853d51b064e1acbae0016547fb5b Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Fri, 12 Sep 2025 17:00:50 +0200 Subject: [PATCH 15/34] Multiple fixes - update cache IO channels after device detach/attach - fix management queue refcount - fix IO channels double free - handle spdk_thread_send_msg() errors - do not allow using OCF core as base bdev for another OCF core - remove redundant checks - remove cluttering logs Change-Id: I34f2aceb434d5752c787ae6071cce3930bc44575 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/ctx.c | 1 + module/bdev/ocf/vbdev_ocf.c | 112 ++++++++++----------- module/bdev/ocf/vbdev_ocf_cache.c | 36 +++++-- module/bdev/ocf/vbdev_ocf_cache.h | 3 + module/bdev/ocf/vbdev_ocf_core.c | 161 +++++++++++++++++++++++++++--- module/bdev/ocf/vbdev_ocf_core.h | 6 ++ module/bdev/ocf/vbdev_ocf_rpc.c | 4 +- module/bdev/ocf/volume.c | 18 +++- 8 files changed, 253 insertions(+), 88 deletions(-) diff --git a/module/bdev/ocf/ctx.c b/module/bdev/ocf/ctx.c index d5dec5badef..deb6b3dacb3 100644 --- a/module/bdev/ocf/ctx.c +++ b/module/bdev/ocf/ctx.c @@ -342,6 +342,7 @@ vbdev_ocf_ctx_cleaner_init(ocf_cleaner_t c) } priv->mngt_queue = vbdev_ocf_cache->cache_mngt_q; + ocf_queue_get(priv->mngt_queue); ocf_cleaner_set_cmpl(c, cleaner_cmpl); ocf_cleaner_set_priv(c, priv); diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 415ceb63e82..9e19b4d315b 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -209,7 +209,6 @@ static void _cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; - ocf_queue_t cache_mngt_q = ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing stop of OCF cache\n", ocf_cache_get_name(cache)); @@ -225,11 +224,8 @@ _cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) if (!error || !mngt_ctx) { if (vbdev_ocf_cache_is_base_attached(cache)) { vbdev_ocf_cache_base_detach(cache); - } else { - /* If device was not attached to cache, then - * cache stop won't put its management queue. */ - ocf_queue_put(cache_mngt_q); } + vbdev_ocf_cache_mngt_queue_put(cache); vbdev_ocf_cache_destroy(cache); } @@ -293,7 +289,6 @@ _cache_stop_core_unregister_cb(void *cb_arg, int error) ocf_core_get_name(core), spdk_strerror(-error)); } - ocf_mngt_cache_put(cache); vbdev_ocf_core_destroy(core_ctx); if (ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { @@ -307,7 +302,6 @@ static int _cache_stop_core_visitor(ocf_core_t core, void *cb_arg) { struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; - ocf_cache_t cache = ocf_core_get_cache(core); struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); int rc = 0; @@ -316,13 +310,11 @@ _cache_stop_core_visitor(ocf_core_t core, void *cb_arg) if (!core_ctx) { /* Skip this core. If there is no context, it means that this core * was added from metadata during cache load and it's just an empty shell. */ - ocf_mngt_cache_put(cache); return 0; } /* If core is detached it's already unregistered, so just free its data and exit. */ if (!vbdev_ocf_core_is_base_attached(core_ctx)) { - ocf_mngt_cache_put(cache); vbdev_ocf_core_destroy(core_ctx); return 0; } @@ -332,7 +324,6 @@ _cache_stop_core_visitor(ocf_core_t core, void *cb_arg) if ((rc = vbdev_ocf_core_unregister(core_ctx, _cache_stop_core_unregister_cb, core))) { SPDK_ERRLOG("OCF core '%s': failed to start unregistering OCF vbdev: %s\n", ocf_core_get_name(core), spdk_strerror(-rc)); - ocf_mngt_cache_put(cache); return rc; } @@ -342,18 +333,10 @@ _cache_stop_core_visitor(ocf_core_t core, void *cb_arg) static int _module_fini_cache_visitor(ocf_cache_t cache, void *cb_arg) { - int i, rc = 0; + int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': module stop visit\n", ocf_cache_get_name(cache)); - /* Increment cache refcount to not destroy cache structs before destroying all cores. */ - for (i = ocf_cache_get_core_count(cache); i > 0; i--) { - if ((rc = ocf_mngt_cache_get(cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to increment refcount: %s\n", - ocf_cache_get_name(cache), spdk_strerror(-rc)); - } - } - if (!ocf_cache_get_core_count(cache) || ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { /* If there are no cores or all of them are detached, @@ -439,6 +422,12 @@ _examine_config_core_visitor(ocf_core_t core, void *cb_arg) SPDK_NOTICELOG("OCF core '%s': base bdev '%s' found\n", ocf_core_get_name(core), bdev_name); + if (!strcmp(spdk_bdev_get_product_name(spdk_bdev_get_by_name(bdev_name)), "OCF_disk")) { + SPDK_ERRLOG("OCF core '%s': base bdev '%s' is already an OCF core\n", + ocf_core_get_name(core), bdev_name); + return -ENOTSUP; + } + assert(!vbdev_ocf_core_is_base_attached(core_ctx)); if ((rc = vbdev_ocf_core_base_attach(core_ctx, bdev_name))) { @@ -506,11 +495,20 @@ vbdev_ocf_module_examine_config(struct spdk_bdev *bdev) SPDK_NOTICELOG("OCF core '%s': base bdev '%s' found\n", vbdev_ocf_core_get_name(core_ctx), bdev_name); + if (!strcmp(spdk_bdev_get_product_name(bdev), "OCF_disk")) { + SPDK_ERRLOG("OCF core '%s': base bdev '%s' is already an OCF core\n", + vbdev_ocf_core_get_name(core_ctx), bdev_name); + spdk_bdev_module_examine_done(&ocf_if); + return; + } + assert(!vbdev_ocf_core_is_base_attached(core_ctx)); if ((rc = vbdev_ocf_core_base_attach(core_ctx, bdev_name))) { SPDK_ERRLOG("OCF core '%s': failed to attach base bdev '%s': %s\n", vbdev_ocf_core_get_name(core_ctx), bdev_name, spdk_strerror(-rc)); + spdk_bdev_module_examine_done(&ocf_if); + return; } spdk_bdev_module_examine_done(&ocf_if); @@ -627,7 +625,13 @@ _cache_attach_examine_attach_cb(ocf_cache_t cache, void *cb_arg, int error) } else { SPDK_NOTICELOG("OCF cache '%s': device attached\n", ocf_cache_get_name(cache)); - vbdev_ocf_core_add_from_waitlist(cache); + /* Update cache IO channel after new device attach. */ + if ((error = vbdev_ocf_core_create_cache_channel(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to create channel for newly attached cache: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-error)); + } else { + vbdev_ocf_core_add_from_waitlist(cache); + } } spdk_bdev_module_examine_done(&ocf_if); @@ -843,9 +847,6 @@ _vbdev_ocf_submit_io_cb(ocf_io_t io, void *priv1, void *priv2, int error) { struct spdk_bdev_io *bdev_io = priv1; - SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': finishing submit of IO request\n", - spdk_bdev_get_name(bdev_io->bdev)); - ocf_io_put(io); if (error == -OCF_ERR_NO_MEM) { @@ -870,13 +871,8 @@ vbdev_ocf_submit_io(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, ui struct vbdev_ocf_core_io_channel_ctx *ch_ctx = spdk_io_channel_get_ctx(ch); ocf_io_t io = NULL; - // impossible to be true ? - if (!core) { - SPDK_ERRLOG("OCF vbdev '%s': failed to submit IO - no OCF core device\n", - spdk_bdev_get_name(bdev_io->bdev)); - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); - return; - } + /* OCF core should be added before vbdev register and removed after vbdev unregister. */ + assert(core); io = ocf_volume_new_io(ocf_core_get_front_volume(core), ch_ctx->queue, offset, len, dir, 0, flags); @@ -917,9 +913,6 @@ vbdev_ocf_fn_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bde uint64_t offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen; uint32_t len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen; - SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': initiating submit of IO request\n", - spdk_bdev_get_name(bdev_io->bdev)); - switch (bdev_io->type) { case SPDK_BDEV_IO_TYPE_READ: // align buffer for write as well ? (comment in old vbdev_ocf.c) @@ -948,9 +941,6 @@ vbdev_ocf_fn_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) ocf_core_t core = ctx; struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); - SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': checking if IO type '%s' is supported\n", - spdk_bdev_get_name(&core_ctx->ocf_vbdev), spdk_bdev_get_io_type_name(io_type)); - switch (io_type) { case SPDK_BDEV_IO_TYPE_READ: case SPDK_BDEV_IO_TYPE_WRITE: @@ -994,7 +984,6 @@ _cache_start_rpc_err_cb(ocf_cache_t cache, void *cb_arg, int error) vbdev_ocf_cache_destroy(cache); ocf_mngt_cache_unlock(cache); // use in error path only ? - //ocf_mngt_cache_put(cache); // no need ? (check refcnt) } static void @@ -1018,7 +1007,7 @@ _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) } vbdev_ocf_cache_base_detach(cache); - ocf_queue_put(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); // needed ? + vbdev_ocf_cache_mngt_queue_put(cache); ocf_mngt_cache_stop(cache, _cache_start_rpc_err_cb, NULL); } else { SPDK_NOTICELOG("OCF cache '%s': started\n", ocf_cache_get_name(cache)); @@ -1110,7 +1099,7 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, err_volume: vbdev_ocf_cache_base_detach(cache); err_base: - ocf_queue_put(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); + vbdev_ocf_cache_mngt_queue_put(cache); err_queue: ocf_mngt_cache_stop(cache, _cache_start_rpc_err_cb, NULL); err_create: @@ -1124,7 +1113,7 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo { ocf_cache_t cache; struct vbdev_ocf_mngt_ctx *mngt_ctx; - int i, rc; + int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop\n", cache_name); @@ -1145,18 +1134,6 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo mngt_ctx->rpc_cb_arg = rpc_cb_arg; mngt_ctx->cache = cache; - /* Increment cache refcount to not destroy cache structs before destroying all cores. */ - for (i = ocf_cache_get_core_count(cache); i > 0; i--) { - if ((rc = ocf_mngt_cache_get(cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to increment refcount (OCF error: %d)\n", - ocf_cache_get_name(cache), rc); - for (i = ocf_cache_get_core_count(cache) - i; i > 0; i--) { - ocf_mngt_cache_put(cache); - } - goto err_get; - } - } - if (!ocf_cache_get_core_count(cache) || ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { /* If there are no cores or all of them are detached, @@ -1167,14 +1144,12 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo if ((rc = ocf_core_visit(cache, _cache_stop_core_visitor, mngt_ctx, false))) { SPDK_ERRLOG("OCF cache '%s': failed to iterate over core bdevs: %s\n", ocf_cache_get_name(cache), spdk_strerror(-rc)); - // ocf_mngt_cache_put() ? how many times ? goto err_visit; } return; err_visit: -err_get: free(mngt_ctx); err_alloc: ocf_mngt_cache_put(cache); @@ -1195,12 +1170,16 @@ _cache_detach_rpc_detach_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_cache_get_name(cache), error); } else { vbdev_ocf_cache_base_detach(cache); + + /* Update cache IO channel after device detach. */ + if ((error = vbdev_ocf_core_destroy_cache_channel(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to destroy channel for detached cache: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-error)); + } + SPDK_NOTICELOG("OCF cache '%s': device detached\n", ocf_cache_get_name(cache)); } - /* Increment queue refcount to prevent destroying management queue after cache device detach. */ - ocf_queue_get(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); - mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); ocf_mngt_cache_unlock(cache); ocf_mngt_cache_put(cache); @@ -1319,7 +1298,13 @@ _cache_attach_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) } else { SPDK_NOTICELOG("OCF cache '%s': device attached\n", ocf_cache_get_name(cache)); - vbdev_ocf_core_add_from_waitlist(cache); + /* Update cache IO channel after new device attach. */ + if ((error = vbdev_ocf_core_create_cache_channel(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to create channel for newly attached cache: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-error)); + } else { + vbdev_ocf_core_add_from_waitlist(cache); + } } mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); @@ -1543,7 +1528,13 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac } SPDK_ERRLOG("OCF core '%s': failed to attach base bdev '%s': %s\n", core_name, base_name, spdk_strerror(-rc)); - goto err_base; + goto err_no_base; + } + + if (!strcmp(spdk_bdev_get_product_name(spdk_bdev_get_by_name(base_name)), "OCF_disk")) { + SPDK_ERRLOG("OCF core '%s': base bdev '%s' is already an OCF core\n", core_name, base_name); + rc = -ENOTSUP; + goto err_ocf_base; } /* Second, check if OCF cache for this core is already started. */ @@ -1603,8 +1594,9 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac err_alloc: err_bsize: ocf_mngt_cache_put(cache); +err_ocf_base: vbdev_ocf_core_base_detach(core_ctx); -err_base: +err_no_base: vbdev_ocf_core_destroy(core_ctx); err_create: err_exist: diff --git a/module/bdev/ocf/vbdev_ocf_cache.c b/module/bdev/ocf/vbdev_ocf_cache.c index 5b2dafa2119..a38bee67981 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.c +++ b/module/bdev/ocf/vbdev_ocf_cache.c @@ -3,6 +3,8 @@ * All rights reserved. */ +#include "spdk/string.h" + #include "vbdev_ocf_cache.h" #include "ctx.h" #include "utils.h" @@ -71,6 +73,8 @@ vbdev_ocf_cache_destroy(ocf_cache_t cache) free(cache_ctx); } +int vbdev_ocf_core_destroy_cache_channel(ocf_cache_t cache); + static void _cache_hotremove_detach_cb(ocf_cache_t cache, void *cb_arg, int error) { @@ -91,8 +95,12 @@ _cache_hotremove_detach_cb(ocf_cache_t cache, void *cb_arg, int error) SPDK_NOTICELOG("OCF cache '%s': device detached\n", ocf_cache_get_name(cache)); vbdev_ocf_cache_base_detach(cache); - /* Increment queue refcount to prevent destroying management queue after cache device detach. */ - ocf_queue_get(((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->cache_mngt_q); + + /* Update cache IO channel after device detach. */ + if ((error = vbdev_ocf_core_destroy_cache_channel(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to destroy channel for detached cache: %s\n", + ocf_cache_get_name(cache), spdk_strerror(-error)); + } } static void @@ -254,7 +262,6 @@ _volume_attach_metadata_probe_cb(void *priv, int error, struct ocf_metadata_prob SPDK_NOTICELOG("OCF cache '%s': metadata found - loading previous cache instance\n", ocf_cache_get_name(cache)); SPDK_NOTICELOG("(start cache with 'no-load' flag to create new cache instead of loading config from metadata)\n"); - // check status for cache_name/mode/line_size/dirty ? ocf_mngt_cache_load(cache, &cache_ctx->cache_att_cfg, mngt_ctx->u.att_cb_fn, mngt_ctx); } } @@ -295,12 +302,20 @@ static void vbdev_ocf_cache_mngt_queue_stop(ocf_queue_t queue) { struct vbdev_ocf_cache_mngt_queue_ctx *mngt_q_ctx = ocf_queue_get_priv(queue); + int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': destroying OCF management queue\n", ocf_cache_get_name(mngt_q_ctx->cache)); if (mngt_q_ctx->thread && mngt_q_ctx->thread != spdk_get_thread()) { - spdk_thread_send_msg(mngt_q_ctx->thread, _cache_mngt_queue_stop, mngt_q_ctx); + if ((rc = spdk_thread_send_msg(mngt_q_ctx->thread, _cache_mngt_queue_stop, mngt_q_ctx))) { + SPDK_ERRLOG("OCF cache '%s': failed to send message to thread (name: %s, id: %ld): %s\n", + ocf_cache_get_name(mngt_q_ctx->cache), + spdk_thread_get_name(mngt_q_ctx->thread), + spdk_thread_get_id(mngt_q_ctx->thread), + spdk_strerror(-rc)); + assert(false); + } } else { _cache_mngt_queue_stop(mngt_q_ctx); } @@ -333,6 +348,8 @@ vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache) ocf_cache_get_name(cache)); return -ENOMEM; } + mngt_q_ctx->cache = cache; // keep? (only for DEBUGLOG) + mngt_q_ctx->thread = spdk_get_thread(); if ((rc = ocf_queue_create_mngt(cache, &cache_ctx->cache_mngt_q, &cache_mngt_queue_ops))) { SPDK_ERRLOG("OCF cache '%s': failed to create OCF management queue\n", @@ -350,8 +367,13 @@ vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache) return -ENOMEM; } - mngt_q_ctx->cache = cache; // keep? (only for DEBUGLOG) - mngt_q_ctx->thread = spdk_get_thread(); - return rc; } + +void +vbdev_ocf_cache_mngt_queue_put(ocf_cache_t cache) +{ + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + + vbdev_ocf_queue_put(cache_ctx->cache_mngt_q); +} diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h index 4bba0b88242..d3ffe3bef2a 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.h +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -145,4 +145,7 @@ int vbdev_ocf_cache_volume_attach(ocf_cache_t cache, struct vbdev_ocf_mngt_ctx * /* Create cache management queue poller. */ int vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache); +/* Decrement refcount on cache management queue. */ +void vbdev_ocf_cache_mngt_queue_put(ocf_cache_t cache); + #endif diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c index 66b629f10b6..1fbdc813e15 100644 --- a/module/bdev/ocf/vbdev_ocf_core.c +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -240,10 +240,16 @@ _core_io_queue_stop(void *ctx) struct vbdev_ocf_core_io_channel_ctx *ch_ctx = ctx; spdk_poller_unregister(&ch_ctx->poller); + if (ch_ctx->cache_ch) { spdk_put_io_channel(ch_ctx->cache_ch); } - spdk_put_io_channel(ch_ctx->core_ch); + + /* Core channel may not exist only on error path. */ + if (likely(ch_ctx->core_ch)) { + spdk_put_io_channel(ch_ctx->core_ch); + } + free(ch_ctx); } @@ -251,12 +257,21 @@ static void vbdev_ocf_core_io_queue_stop(ocf_queue_t queue) { struct vbdev_ocf_core_io_channel_ctx *ch_ctx = ocf_queue_get_priv(queue); + struct spdk_bdev *ocf_vbdev = &((struct vbdev_ocf_core *)ocf_core_get_priv(ch_ctx->core))->ocf_vbdev; + int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': deallocating external IO channel context\n", - spdk_bdev_get_name(&((struct vbdev_ocf_core *)ocf_core_get_priv(ch_ctx->core))->ocf_vbdev)); + spdk_bdev_get_name(ocf_vbdev)); if (ch_ctx->thread && ch_ctx->thread != spdk_get_thread()) { - spdk_thread_send_msg(ch_ctx->thread, _core_io_queue_stop, ch_ctx); + if ((rc = spdk_thread_send_msg(ch_ctx->thread, _core_io_queue_stop, ch_ctx))) { + SPDK_ERRLOG("OCF vbdev '%s': failed to send message to thread (name: %s, id: %ld): %s\n", + spdk_bdev_get_name(ocf_vbdev), + spdk_thread_get_name(ch_ctx->thread), + spdk_thread_get_id(ch_ctx->thread), + spdk_strerror(-rc)); + assert(false); + } } else { _core_io_queue_stop(ch_ctx); } @@ -300,6 +315,8 @@ _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) vbdev_name); return -ENOMEM; } + ch_ctx->core = core; // keep? (only for DEBUGLOG) + ch_ctx->thread = spdk_get_thread(); if ((rc = ocf_queue_create(cache, &ch_ctx->queue, &core_io_queue_ops))) { SPDK_ERRLOG("OCF vbdev '%s': failed to create OCF queue\n", vbdev_name); @@ -307,6 +324,10 @@ _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) return rc; } ocf_queue_set_priv(ch_ctx->queue, ch_ctx); + /* Save queue pointer in buffer provided by the IO channel callback. + * Only this will be needed in channel destroy callback to decrement + * the refcount. The rest is freed in queue stop callback. */ + ch_destroy_ctx->queue = ch_ctx->queue; if (!ocf_cache_is_detached(cache)) { ch_ctx->cache_ch = spdk_bdev_get_io_channel(cache_base->desc); @@ -322,7 +343,8 @@ _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) if (!ch_ctx->core_ch) { SPDK_ERRLOG("OCF vbdev '%s': failed to create IO channel for base bdev '%s'\n", vbdev_name, spdk_bdev_get_name(core_base->bdev)); - spdk_put_io_channel(ch_ctx->cache_ch); + /* Do not spdk_put_io_channel() here as it + * will be done during stop of OCF queue. */ ocf_queue_put(ch_ctx->queue); return -ENOMEM; } @@ -330,20 +352,12 @@ _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) ch_ctx->poller = SPDK_POLLER_REGISTER(vbdev_ocf_queue_poller, ch_ctx->queue, 0); if (!ch_ctx->poller) { SPDK_ERRLOG("OCF vbdev '%s': failed to create IO queue poller\n", vbdev_name); - spdk_put_io_channel(ch_ctx->core_ch); - spdk_put_io_channel(ch_ctx->cache_ch); + /* Do not spdk_put_io_channel() here as it + * will be done during stop of OCF queue. */ ocf_queue_put(ch_ctx->queue); return -ENOMEM; } - ch_ctx->core = core; // keep? (only for DEBUGLOG) - ch_ctx->thread = spdk_get_thread(); - - /* Save queue pointer in buffer provided by the IO channel callback. - * Only this will be needed in channel destroy callback to decrement - * the refcount. The rest is freed in queue stop callback. */ - ch_destroy_ctx->queue = ch_ctx->queue; - return rc; } @@ -499,3 +513,122 @@ vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache) ocf_mngt_cache_lock(cache, _core_add_from_waitlist_lock_cb, core_ctx); } } + +static void +_create_cache_ch_cb(struct spdk_io_channel_iter *i, int error) +{ + ocf_core_t core = spdk_io_channel_iter_get_io_device(i); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to create cache IO channels: %s\n", + ocf_core_get_name(core), spdk_strerror(-error)); + return; + } + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': all cache IO channels created\n", + ocf_core_get_name(core)); +} + +static void +_create_cache_ch_fn(struct spdk_io_channel_iter *i) +{ + ocf_cache_t cache = spdk_io_channel_iter_get_ctx(i); + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + struct vbdev_ocf_core_io_channel_ctx *ch_ctx, *queue_ch_ctx; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s' on channel %p: creating cache IO channel\n", + ocf_core_get_name(spdk_io_channel_iter_get_io_device(i)), + spdk_io_channel_iter_get_channel(i)); + + /* The actual IO channel context is saved in queue priv. + * The one saved in channel context buffer is used only + * to properly destroy the channel. + * (See the comment in _vbdev_ocf_ch_create_cb() in vbdev_ocf_core.c + * for more details.) */ + ch_ctx = spdk_io_channel_get_ctx(spdk_io_channel_iter_get_channel(i)); + queue_ch_ctx = ocf_queue_get_priv(ch_ctx->queue); + + /* Cache device IO channel should not exist at this point. + * It should be cleaned after cache device detach (either + * manual or hot removed) or not initialized after starting + * cache without device present. */ + assert(!queue_ch_ctx->cache_ch); + + queue_ch_ctx->cache_ch = spdk_bdev_get_io_channel(cache_ctx->base.desc); + + spdk_for_each_channel_continue(i, 0); +} + +static int +_create_cache_ch_core_visitor(ocf_core_t core, void *ctx) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': creating all cache IO channels\n", + ocf_core_get_name(core)); + + spdk_for_each_channel(core, _create_cache_ch_fn, ctx, _create_cache_ch_cb); + + return 0; +} + +int +vbdev_ocf_core_create_cache_channel(ocf_cache_t cache) +{ + return ocf_core_visit(cache, _create_cache_ch_core_visitor, cache, true); +} + +static void +_destroy_cache_ch_cb(struct spdk_io_channel_iter *i, int error) +{ + ocf_core_t core = spdk_io_channel_iter_get_io_device(i); + + if (error) { + SPDK_ERRLOG("OCF core '%s': failed to destroy cache IO channels: %s\n", + ocf_core_get_name(core), spdk_strerror(-error)); + return; + } + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': all cache IO channels destroyed\n", + ocf_core_get_name(core)); +} + +static void +_destroy_cache_ch_fn(struct spdk_io_channel_iter *i) +{ + struct vbdev_ocf_core_io_channel_ctx *ch_ctx, *queue_ch_ctx; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s' on channel %p: destroying cache IO channel\n", + ocf_core_get_name(spdk_io_channel_iter_get_io_device(i)), + spdk_io_channel_iter_get_channel(i)); + + /* The actual IO channel context is saved in queue priv. + * The one saved in channel context buffer is used only + * to properly destroy the channel. + * (See the comment in _vbdev_ocf_ch_create_cb() in vbdev_ocf_core.c + * for more details.) */ + ch_ctx = spdk_io_channel_get_ctx(spdk_io_channel_iter_get_channel(i)); + queue_ch_ctx = ocf_queue_get_priv(ch_ctx->queue); + + assert(queue_ch_ctx->cache_ch); + + spdk_put_io_channel(queue_ch_ctx->cache_ch); + queue_ch_ctx->cache_ch = NULL; + + spdk_for_each_channel_continue(i, 0); +} + +static int +_destroy_cache_ch_core_visitor(ocf_core_t core, void *ctx) +{ + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': destroying all cache IO channels\n", + ocf_core_get_name(core)); + + spdk_for_each_channel(core, _destroy_cache_ch_fn, ctx, _destroy_cache_ch_cb); + + return 0; +} + +int +vbdev_ocf_core_destroy_cache_channel(ocf_cache_t cache) +{ + return ocf_core_visit(cache, _destroy_cache_ch_core_visitor, NULL, true); +} diff --git a/module/bdev/ocf/vbdev_ocf_core.h b/module/bdev/ocf/vbdev_ocf_core.h index 89bd1480377..b31540ffe8d 100644 --- a/module/bdev/ocf/vbdev_ocf_core.h +++ b/module/bdev/ocf/vbdev_ocf_core.h @@ -118,4 +118,10 @@ int vbdev_ocf_core_unregister(struct vbdev_ocf_core *core_ctx, spdk_bdev_unregis /* Add all cores from wait list assigned to given cache. */ void vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache); +/* Create new cache IO channel on all opened channels context for each opened core. */ +int vbdev_ocf_core_create_cache_channel(ocf_cache_t cache); + +/* Destroy cache IO channel on all opened channels context for each opened core. */ +int vbdev_ocf_core_destroy_cache_channel(ocf_cache_t cache); + #endif diff --git a/module/bdev/ocf/vbdev_ocf_rpc.c b/module/bdev/ocf/vbdev_ocf_rpc.c index 8f968405ae3..66f764e2653 100644 --- a/module/bdev/ocf/vbdev_ocf_rpc.c +++ b/module/bdev/ocf/vbdev_ocf_rpc.c @@ -39,8 +39,8 @@ rpc_bdev_ocf_start_cache_cb(const char *bdev_name, void *cb_arg, int error) if (error && error != -ENODEV) { spdk_jsonrpc_send_error_response_fmt(request, error, - "Failed to start OCF cache '%s': %s", - bdev_name, spdk_strerror(-error)); + "Failed to start OCF cache: %s", + spdk_strerror(-error)); return; } diff --git a/module/bdev/ocf/volume.c b/module/bdev/ocf/volume.c index e09f824fdfd..75ee71de3ea 100644 --- a/module/bdev/ocf/volume.c +++ b/module/bdev/ocf/volume.c @@ -8,8 +8,9 @@ #include "spdk/bdev_module.h" #include "spdk/env.h" -#include "spdk/thread.h" #include "spdk/log.h" +#include "spdk/string.h" +#include "spdk/thread.h" #include "vbdev_ocf_core.h" #include "data.h" @@ -212,10 +213,8 @@ vbdev_forward_flush(ocf_volume_t volume, ocf_forward_token_t token) uint64_t bytes = base->bdev->blockcnt * base->bdev->blocklen; int status; - // workaround (?); check for better solution - /* If base device doesn't support flush just warn about it and exit. */ + /* If base device doesn't support flush just ignore it and exit. */ if (unlikely(!spdk_bdev_io_type_supported(base->bdev, SPDK_BDEV_IO_TYPE_FLUSH))) { - SPDK_WARNLOG("Base bdev '%s': attempt to flush device that doesn't support it\n", base->bdev->name); ocf_forward_end(token, 0); return; } @@ -356,8 +355,17 @@ _base_detach(void *ctx) void vbdev_ocf_base_detach(struct vbdev_ocf_base *base) { + int rc; + if (base->thread && base->thread != spdk_get_thread()) { - spdk_thread_send_msg(base->thread, _base_detach, base); + if ((rc = spdk_thread_send_msg(base->thread, _base_detach, base))) { + SPDK_ERRLOG("OCF '%s': failed to send message to thread (name: %s, id: %ld): %s\n", + base->name, + spdk_thread_get_name(base->thread), + spdk_thread_get_id(base->thread), + spdk_strerror(-rc)); + assert(false); + } } else { _base_detach(base); } From e2307bc87ceb89c2ad1f97266f2ebb9a4af0d8ac Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Mon, 15 Sep 2025 16:19:42 +0200 Subject: [PATCH 16/34] Implement info ops + tests - current module config - generic bdev info - generic bdev stats - OCF stats Change-Id: If63774a399902caaa50c09ef2e89a9837e5db95a Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 140 ++++++++- test/ocf/management/info_dump.sh | 57 ++++ .../management/info_dump_driver_specific.json | 60 ++++ test/ocf/management/info_dump_get_bdevs.json | 282 ++++++++++++++++++ .../ocf/management/info_dump_save_config.json | 123 ++++++++ test/ocf/management/run.sh | 1 + 6 files changed, 657 insertions(+), 6 deletions(-) create mode 100755 test/ocf/management/info_dump.sh create mode 100644 test/ocf/management/info_dump_driver_specific.json create mode 100644 test/ocf/management/info_dump_get_bdevs.json create mode 100644 test/ocf/management/info_dump_save_config.json diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 9e19b4d315b..f08b283c0dc 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -24,16 +24,18 @@ static void vbdev_ocf_module_fini(void); static int vbdev_ocf_module_get_ctx_size(void); static void vbdev_ocf_module_examine_config(struct spdk_bdev *bdev); static void vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev); +static int vbdev_ocf_module_config_json(struct spdk_json_write_ctx *w); struct spdk_bdev_module ocf_if = { .name = "OCF", .module_init = vbdev_ocf_module_init, .fini_start = vbdev_ocf_module_fini_start, + .async_fini_start = true, .module_fini = vbdev_ocf_module_fini, .get_ctx_size = vbdev_ocf_module_get_ctx_size, .examine_config = vbdev_ocf_module_examine_config, .examine_disk = vbdev_ocf_module_examine_disk, - .async_fini_start = true, + .config_json = vbdev_ocf_module_config_json, }; SPDK_BDEV_MODULE_REGISTER(ocf, &ocf_if) @@ -42,7 +44,8 @@ static void vbdev_ocf_fn_submit_request(struct spdk_io_channel *ch, struct spdk_ static bool vbdev_ocf_fn_io_type_supported(void *ctx, enum spdk_bdev_io_type); static struct spdk_io_channel *vbdev_ocf_fn_get_io_channel(void *ctx); static int vbdev_ocf_fn_dump_info_json(void *ctx, struct spdk_json_write_ctx *w); -static void vbdev_ocf_fn_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w); +static void vbdev_ocf_fn_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w); +static void vbdev_ocf_fn_reset_device_stat(void *ctx); struct spdk_bdev_fn_table vbdev_ocf_fn_table = { .destruct = vbdev_ocf_fn_destruct, @@ -50,9 +53,8 @@ struct spdk_bdev_fn_table vbdev_ocf_fn_table = { .io_type_supported = vbdev_ocf_fn_io_type_supported, .get_io_channel = vbdev_ocf_fn_get_io_channel, .dump_info_json = vbdev_ocf_fn_dump_info_json, - .write_config_json = vbdev_ocf_fn_write_config_json, - .dump_device_stat_json = NULL, // todo ? - .reset_device_stat = NULL, // todo ? + .dump_device_stat_json = vbdev_ocf_fn_dump_device_stat_json, + .reset_device_stat = vbdev_ocf_fn_reset_device_stat, }; static int @@ -766,6 +768,91 @@ vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev) } } +static void +dump_core_config(struct spdk_json_write_ctx *w, struct vbdev_ocf_core *core_ctx) +{ + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "bdev_ocf_add_core"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "core_name", vbdev_ocf_core_get_name(core_ctx)); + spdk_json_write_named_string(w, "base_name", core_ctx->base.name); + spdk_json_write_named_string(w, "cache_name", core_ctx->cache_name); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); +} + +static void +dump_cache_config(struct spdk_json_write_ctx *w, ocf_cache_t cache) +{ + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "bdev_ocf_start_cache"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "cache_name", ocf_cache_get_name(cache)); + spdk_json_write_named_string(w, "base_name", cache_ctx->base.name); + spdk_json_write_named_string(w, "cache_mode", + vbdev_ocf_cachemode_get_name(ocf_cache_get_mode(cache))); + spdk_json_write_named_uint32(w, "cache_line_size", ocf_cache_get_line_size(cache)); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); +} + +static int +_module_config_json_core_visitor(ocf_core_t core, void *ctx) +{ + struct spdk_json_write_ctx *w = ctx; + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': module config visit\n", ocf_core_get_name(core)); + + if (!core_ctx) { + /* Skip this core. If there is no context, it means that this core + * was added from metadata during cache load and not manually by RPC call. */ + return 0; + } + + dump_core_config(w, core_ctx); + + return 0; +} + +static int +_module_config_json_cache_visitor(ocf_cache_t cache, void *ctx) +{ + struct spdk_json_write_ctx *w = ctx; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': module config visit\n", ocf_cache_get_name(cache)); + + dump_cache_config(w, cache); + + return ocf_core_visit(cache, _module_config_json_core_visitor, w, false); +} + +static int +vbdev_ocf_module_config_json(struct spdk_json_write_ctx *w) +{ + struct vbdev_ocf_core *core_ctx; + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF: generating current module configuration\n"); + + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { + dump_core_config(w, core_ctx); + } + + if ((rc = ocf_mngt_cache_visit(vbdev_ocf_ctx, _module_config_json_cache_visitor, w))) { + SPDK_ERRLOG("OCF: failed to iterate over bdevs: %s\n", spdk_strerror(-rc)); + return rc; + } + + return 0; +} + static void _destruct_core_detach_cb(ocf_core_t core, void *cb_arg, int error) { @@ -966,11 +1053,52 @@ vbdev_ocf_fn_get_io_channel(void *ctx) // ctx == ocf_vbdev.ctxt static int vbdev_ocf_fn_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) { + ocf_core_t core = ctx; + ocf_cache_t cache = ocf_core_get_cache(core); + struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); + + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': dumping driver specific info\n", ocf_core_get_name(core)); + + spdk_json_write_named_object_begin(w, "ocf"); + spdk_json_write_named_string(w, "name", ocf_core_get_name(core)); + spdk_json_write_named_string(w, "base_name", core_ctx ? core_ctx->base.name : ""); + + spdk_json_write_named_object_begin(w, "cache"); + spdk_json_write_named_string(w, "name", ocf_cache_get_name(cache)); + spdk_json_write_named_string(w, "base_name", cache_ctx->base.name); + spdk_json_write_named_string(w, "cache_mode", + vbdev_ocf_cachemode_get_name(ocf_cache_get_mode(cache))); + spdk_json_write_named_uint32(w, "cache_line_size", ocf_cache_get_line_size(cache)); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); + return 0; } static void -vbdev_ocf_fn_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) +vbdev_ocf_fn_dump_device_stat_json(void *ctx, struct spdk_json_write_ctx *w) +{ + ocf_core_t core = ctx; + struct vbdev_ocf_stats stats; + int rc; + + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': collecting statistics\n", ocf_core_get_name(core)); + + if ((rc = vbdev_ocf_stats_core_get(core, &stats))) { + SPDK_ERRLOG("OCF core '%s': failed to collect statistics (OCF error: %d)\n", + ocf_core_get_name(core), rc); + return; + } + + vbdev_ocf_stats_write_json(w, &stats); +} + +/* Do not define this function to not reset OCF stats when resetting exposed bdev's stats. + * Let the user reset OCF stats independently by calling bdev_ocf_reset_stats RPC. */ +static void +vbdev_ocf_fn_reset_device_stat(void *ctx) { } diff --git a/test/ocf/management/info_dump.sh b/test/ocf/management/info_dump.sh new file mode 100755 index 00000000000..ad22f440319 --- /dev/null +++ b/test/ocf/management/info_dump.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f "$curdir/../../..") +source "$rootdir/test/ocf/common.sh" + +start_spdk +create_caches +$rpc_py bdev_ocf_start_cache Ocf_cache1 Cache_dev1 --no-load +$rpc_py bdev_ocf_start_cache Ocf_cache2 Cache_dev2 --no-load +$rpc_py bdev_ocf_start_cache Ocf_cache3 Cache_dev3 --no-load --cache-mode pt --cache-line-size 64 +create_cores +add_cores + +# First cache detached. +$rpc_py bdev_malloc_delete Cache_dev1 +# First core in each cache detached. +for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 +done +# First core in wait list detached. +for i in {2..3}; do + $rpc_py bdev_malloc_create -b Core_dev_waitlist$i 200 512 +done +for i in {1..3}; do + $rpc_py bdev_ocf_add_core Ocf_core_waitlist$i Core_dev_waitlist$i Ocf_cache_none +done +$rpc_py bdev_ocf_set_cachemode Ocf_cache2 wb + +$rpc_py bdev_ocf_get_bdevs | jq -e . + +# Test OCF statistics: +$rpc_py bdev_ocf_get_stats Ocf_cache1 | jq -e \ + '([.requests[], .blocks[], .errors[] | .count] | any(. != 0)) and + ([.requests[], .blocks[], .errors[] | .percentage] | any(. != "0.0"))' +$rpc_py bdev_ocf_reset_stats Ocf_cache1 +$rpc_py bdev_ocf_get_stats Ocf_cache1 | jq -e \ + '.requests[], .blocks[], .errors[] | .count == 0 and .percentage == "0.0"' + +# Test OCF get bdevs: +diff <($rpc_py bdev_ocf_get_bdevs | jq -e .) <(jq -e . "$curdir/info_dump_get_bdevs.json") + +# Test general get bdevs driver specific info: +diff <($rpc_py bdev_get_bdevs | jq -e '.[].driver_specific.ocf | select(. != null)') \ + <(jq -e . "$curdir/info_dump_driver_specific.json") + +# Test current config dump: +diff <($rpc_py save_subsystem_config -n bdev | jq -e '.config[] | select(.method|test("bdev_ocf_"))') \ + <(jq -e . "$curdir/info_dump_save_config.json") + +stop_spdk diff --git a/test/ocf/management/info_dump_driver_specific.json b/test/ocf/management/info_dump_driver_specific.json new file mode 100644 index 00000000000..42d45cffb9b --- /dev/null +++ b/test/ocf/management/info_dump_driver_specific.json @@ -0,0 +1,60 @@ +{ + "name": "Ocf_core1-2", + "base_name": "Core_dev1-2", + "cache": { + "name": "Ocf_cache1", + "base_name": "Cache_dev1", + "cache_mode": "wt", + "cache_line_size": 4096 + } +} +{ + "name": "Ocf_core1-3", + "base_name": "Core_dev1-3", + "cache": { + "name": "Ocf_cache1", + "base_name": "Cache_dev1", + "cache_mode": "wt", + "cache_line_size": 4096 + } +} +{ + "name": "Ocf_core2-2", + "base_name": "Core_dev2-2", + "cache": { + "name": "Ocf_cache2", + "base_name": "Cache_dev2", + "cache_mode": "wb", + "cache_line_size": 4096 + } +} +{ + "name": "Ocf_core2-3", + "base_name": "Core_dev2-3", + "cache": { + "name": "Ocf_cache2", + "base_name": "Cache_dev2", + "cache_mode": "wb", + "cache_line_size": 4096 + } +} +{ + "name": "Ocf_core3-2", + "base_name": "Core_dev3-2", + "cache": { + "name": "Ocf_cache3", + "base_name": "Cache_dev3", + "cache_mode": "pt", + "cache_line_size": 65536 + } +} +{ + "name": "Ocf_core3-3", + "base_name": "Core_dev3-3", + "cache": { + "name": "Ocf_cache3", + "base_name": "Cache_dev3", + "cache_mode": "pt", + "cache_line_size": 65536 + } +} diff --git a/test/ocf/management/info_dump_get_bdevs.json b/test/ocf/management/info_dump_get_bdevs.json new file mode 100644 index 00000000000..0a2a4ef3669 --- /dev/null +++ b/test/ocf/management/info_dump_get_bdevs.json @@ -0,0 +1,282 @@ +{ + "cores_waitlist": [ + { + "name": "Ocf_core_waitlist1", + "cache_name": "Ocf_cache_none", + "base_name": "Core_dev_waitlist1", + "base_attached": false, + "size": 0, + "block_size": 0 + }, + { + "name": "Ocf_core_waitlist2", + "cache_name": "Ocf_cache_none", + "base_name": "Core_dev_waitlist2", + "base_attached": true, + "size": 209715200, + "block_size": 512 + }, + { + "name": "Ocf_core_waitlist3", + "cache_name": "Ocf_cache_none", + "base_name": "Core_dev_waitlist3", + "base_attached": true, + "size": 209715200, + "block_size": 512 + } + ], + "caches": [ + { + "name": "Ocf_cache1", + "base_name": "Cache_dev1", + "base_attached": false, + "size": 0, + "block_size": 0, + "cache_line_size": 4096, + "cache_mode": "wt", + "promotion": { + "policy": "always" + }, + "cleaning": { + "policy": "alru", + "wake_up_time": 20, + "flush_max_buffers": 100, + "staleness_time": 120, + "activity_threshold": 10000, + "max_dirty_ratio": 100 + }, + "flush": { + "in_progress": false, + "error": 0 + }, + "cores_count": 3, + "cores": [ + { + "name": "Ocf_core1-1", + "cache_name": "Ocf_cache1", + "base_name": "Core_dev1-1", + "base_attached": false, + "size": 0, + "block_size": 0, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + }, + { + "name": "Ocf_core1-2", + "cache_name": "Ocf_cache1", + "base_name": "Core_dev1-2", + "base_attached": true, + "size": 209715200, + "block_size": 512, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + }, + { + "name": "Ocf_core1-3", + "cache_name": "Ocf_cache1", + "base_name": "Core_dev1-3", + "base_attached": true, + "size": 209715200, + "block_size": 512, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + } + ] + }, + { + "name": "Ocf_cache2", + "base_name": "Cache_dev2", + "base_attached": true, + "size": 104857600, + "block_size": 512, + "cache_line_size": 4096, + "cache_mode": "wb", + "promotion": { + "policy": "always" + }, + "cleaning": { + "policy": "alru", + "wake_up_time": 20, + "flush_max_buffers": 100, + "staleness_time": 120, + "activity_threshold": 10000, + "max_dirty_ratio": 100 + }, + "flush": { + "in_progress": false, + "error": 0 + }, + "cores_count": 3, + "cores": [ + { + "name": "Ocf_core2-1", + "cache_name": "Ocf_cache2", + "base_name": "Core_dev2-1", + "base_attached": false, + "size": 0, + "block_size": 0, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + }, + { + "name": "Ocf_core2-2", + "cache_name": "Ocf_cache2", + "base_name": "Core_dev2-2", + "base_attached": true, + "size": 209715200, + "block_size": 512, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + }, + { + "name": "Ocf_core2-3", + "cache_name": "Ocf_cache2", + "base_name": "Core_dev2-3", + "base_attached": true, + "size": 209715200, + "block_size": 512, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + } + ] + }, + { + "name": "Ocf_cache3", + "base_name": "Cache_dev3", + "base_attached": true, + "size": 104857600, + "block_size": 512, + "cache_line_size": 65536, + "cache_mode": "pt", + "promotion": { + "policy": "always" + }, + "cleaning": { + "policy": "alru", + "wake_up_time": 20, + "flush_max_buffers": 100, + "staleness_time": 120, + "activity_threshold": 10000, + "max_dirty_ratio": 100 + }, + "flush": { + "in_progress": false, + "error": 0 + }, + "cores_count": 3, + "cores": [ + { + "name": "Ocf_core3-1", + "cache_name": "Ocf_cache3", + "base_name": "Core_dev3-1", + "base_attached": false, + "size": 0, + "block_size": 0, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + }, + { + "name": "Ocf_core3-2", + "cache_name": "Ocf_cache3", + "base_name": "Core_dev3-2", + "base_attached": true, + "size": 209715200, + "block_size": 512, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + }, + { + "name": "Ocf_core3-3", + "cache_name": "Ocf_cache3", + "base_name": "Core_dev3-3", + "base_attached": true, + "size": 209715200, + "block_size": 512, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + } + ] + } + ] +} diff --git a/test/ocf/management/info_dump_save_config.json b/test/ocf/management/info_dump_save_config.json new file mode 100644 index 00000000000..8083ad721f8 --- /dev/null +++ b/test/ocf/management/info_dump_save_config.json @@ -0,0 +1,123 @@ +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core_waitlist1", + "base_name": "Core_dev_waitlist1", + "cache_name": "Ocf_cache_none" + } +} +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core_waitlist2", + "base_name": "Core_dev_waitlist2", + "cache_name": "Ocf_cache_none" + } +} +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core_waitlist3", + "base_name": "Core_dev_waitlist3", + "cache_name": "Ocf_cache_none" + } +} +{ + "method": "bdev_ocf_start_cache", + "params": { + "cache_name": "Ocf_cache1", + "base_name": "Cache_dev1", + "cache_mode": "wt", + "cache_line_size": 4096 + } +} +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core1-1", + "base_name": "Core_dev1-1", + "cache_name": "Ocf_cache1" + } +} +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core1-2", + "base_name": "Core_dev1-2", + "cache_name": "Ocf_cache1" + } +} +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core1-3", + "base_name": "Core_dev1-3", + "cache_name": "Ocf_cache1" + } +} +{ + "method": "bdev_ocf_start_cache", + "params": { + "cache_name": "Ocf_cache2", + "base_name": "Cache_dev2", + "cache_mode": "wb", + "cache_line_size": 4096 + } +} +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core2-1", + "base_name": "Core_dev2-1", + "cache_name": "Ocf_cache2" + } +} +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core2-2", + "base_name": "Core_dev2-2", + "cache_name": "Ocf_cache2" + } +} +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core2-3", + "base_name": "Core_dev2-3", + "cache_name": "Ocf_cache2" + } +} +{ + "method": "bdev_ocf_start_cache", + "params": { + "cache_name": "Ocf_cache3", + "base_name": "Cache_dev3", + "cache_mode": "pt", + "cache_line_size": 65536 + } +} +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core3-1", + "base_name": "Core_dev3-1", + "cache_name": "Ocf_cache3" + } +} +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core3-2", + "base_name": "Core_dev3-2", + "cache_name": "Ocf_cache3" + } +} +{ + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core3-3", + "base_name": "Core_dev3-3", + "cache_name": "Ocf_cache3" + } +} diff --git a/test/ocf/management/run.sh b/test/ocf/management/run.sh index 220ed6be73d..337e1b38adc 100755 --- a/test/ocf/management/run.sh +++ b/test/ocf/management/run.sh @@ -12,6 +12,7 @@ rootdir=$(readlink -f $curdir/../../..) source $rootdir/test/common/autotest_common.sh run_test "start-stop" "$curdir/start-stop/run.sh" +run_test "info_dump" "$curdir/info_dump.sh" run_test "detach-attach" "$curdir/detach-attach.sh" run_test "hotremove" "$curdir/hotremove.sh" run_test "config_change" "$curdir/config_change.sh" From 1e808cfbde42a0882e227a72bc1636d283f61154 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Thu, 18 Sep 2025 15:07:29 +0200 Subject: [PATCH 17/34] Address all of the remaining in-code comments Change-Id: I106829e59cdf145ed62998c6b101d4efd7389821 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/ctx.c | 3 +- module/bdev/ocf/vbdev_ocf.c | 262 ++++++++++--------- module/bdev/ocf/vbdev_ocf.h | 10 +- module/bdev/ocf/vbdev_ocf_cache.c | 19 +- module/bdev/ocf/vbdev_ocf_cache.h | 13 +- module/bdev/ocf/vbdev_ocf_core.c | 47 +++- module/bdev/ocf/vbdev_ocf_core.h | 2 +- module/bdev/ocf/vbdev_ocf_rpc.c | 4 +- module/bdev/ocf/volume.c | 9 +- test/ocf/management/info_dump_get_bdevs.json | 20 +- 10 files changed, 214 insertions(+), 175 deletions(-) diff --git a/module/bdev/ocf/ctx.c b/module/bdev/ocf/ctx.c index deb6b3dacb3..37b21cf3372 100644 --- a/module/bdev/ocf/ctx.c +++ b/module/bdev/ocf/ctx.c @@ -362,7 +362,8 @@ vbdev_ocf_ctx_cleaner_stop(ocf_cleaner_t c) } } -// this one should only set next_run to "now" and poller registration should be move to _init with saved thread +/* TODO?: this one should only set next_run to "now" and poller + * registration should be move to _init with saved thread */ static void vbdev_ocf_ctx_cleaner_kick(ocf_cleaner_t cleaner) { diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index f08b283c0dc..ac2dc41c24b 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -216,7 +216,7 @@ _cache_stop_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_cache_get_name(cache)); if (error) { - SPDK_ERRLOG("OCF cache '%s': failed to stop OCF cache (OCF error: %d)\n", + SPDK_ERRLOG("OCF cache '%s': failed to properly stop OCF cache (OCF error: %d)\n", ocf_cache_get_name(cache), error); } else { SPDK_NOTICELOG("OCF cache '%s': stopped\n", ocf_cache_get_name(cache)); @@ -528,14 +528,26 @@ vbdev_ocf_module_examine_config(struct spdk_bdev *bdev) static void _core_add_examine_err_cb(void *cb_arg, int error) { - ocf_cache_t cache = cb_arg; + struct vbdev_ocf_core *core_ctx = cb_arg; + ocf_cache_t cache; + int rc; if (error) { - SPDK_ERRLOG("OCF core: failed to remove OCF core device (OCF error: %d)\n", error); + SPDK_ERRLOG("OCF core '%s': failed to remove OCF core device (OCF error: %d)\n", + vbdev_ocf_core_get_name(core_ctx), error); + } + + if ((rc = ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, core_ctx->cache_name, + OCF_CACHE_NAME_SIZE, &cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to find cache of that name (OCF error: %d)\n", + core_ctx->cache_name, rc); + assert(false); } + ocf_mngt_cache_put(cache); ocf_mngt_cache_unlock(cache); ocf_mngt_cache_put(cache); + vbdev_ocf_core_base_detach(core_ctx); spdk_bdev_module_examine_done(&ocf_if); } @@ -563,8 +575,7 @@ _core_add_examine_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int e if ((rc = vbdev_ocf_core_register(core))) { SPDK_ERRLOG("OCF core '%s': failed to register vbdev: %s\n", ocf_core_get_name(core), spdk_strerror(-rc)); - ocf_mngt_cache_remove_core(core, _core_add_examine_err_cb, cache); - vbdev_ocf_core_base_detach(core_ctx); // move to callback (pass core_ctx) ? + ocf_mngt_cache_remove_core(core, _core_add_examine_err_cb, core_ctx); return; } @@ -602,15 +613,15 @@ _core_add_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } static void -_cache_attach_examine_attach_cb(ocf_cache_t cache, void *cb_arg, int error) +_cache_attach_cb(ocf_cache_t cache, void *cb_arg, int error) { struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing device attach\n", ocf_cache_get_name(cache)); - /* At this point volume was either moved to ocf_cache_t struct - * or is no longer needed due to some errors, so we need to deallocate it. */ + /* At this point volume was either moved to ocf_cache_t struct or is no longer + * needed due to some errors, so we need to deallocate it either way. */ vbdev_ocf_cache_config_volume_destroy(cache); ocf_mngt_cache_unlock(cache); @@ -636,7 +647,11 @@ _cache_attach_examine_attach_cb(ocf_cache_t cache, void *cb_arg, int error) } } - spdk_bdev_module_examine_done(&ocf_if); + if (mngt_ctx->rpc_cb_fn) { + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); + } else { + spdk_bdev_module_examine_done(&ocf_if); + } free(mngt_ctx); } @@ -662,7 +677,7 @@ _cache_attach_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) goto err_alloc; } mngt_ctx->cache = cache; - mngt_ctx->u.att_cb_fn = _cache_attach_examine_attach_cb; + mngt_ctx->u.att_cb_fn = _cache_attach_cb; if ((rc = vbdev_ocf_cache_volume_attach(cache, mngt_ctx))) { SPDK_ERRLOG("OCF cache '%s': failed to attach volume: %s\n", @@ -748,7 +763,7 @@ vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev) SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache device '%s'\n", vbdev_ocf_core_get_name(core_ctx), ((struct vbdev_ocf_cache *)ocf_cache_get_priv(cache))->base.name); - ocf_mngt_cache_put(cache); // ? + ocf_mngt_cache_put(cache); spdk_bdev_module_examine_done(&ocf_if); return; } @@ -963,7 +978,7 @@ vbdev_ocf_submit_io(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, ui io = ocf_volume_new_io(ocf_core_get_front_volume(core), ch_ctx->queue, offset, len, dir, 0, flags); - if (!io) { + if (spdk_unlikely(!io)) { spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_NOMEM); return; } @@ -983,7 +998,7 @@ _io_read_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bo uint64_t offset = bdev_io->u.bdev.offset_blocks * bdev_io->bdev->blocklen; uint32_t len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen; - if (!success) { + if (spdk_unlikely(!success)) { SPDK_ERRLOG("OCF vbdev '%s': failed to allocate IO buffer - size of the " "buffer to allocate might be greater than the permitted maximum\n", spdk_bdev_get_name(bdev_io->bdev)); @@ -1002,8 +1017,6 @@ vbdev_ocf_fn_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bde switch (bdev_io->type) { case SPDK_BDEV_IO_TYPE_READ: - // align buffer for write as well ? (comment in old vbdev_ocf.c) - // from doc: This function *must* be called from the thread issuing bdev_io. spdk_bdev_io_get_buf(bdev_io, _io_read_get_buf_cb, len); break; case SPDK_BDEV_IO_TYPE_WRITE: @@ -1040,7 +1053,7 @@ vbdev_ocf_fn_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) } static struct spdk_io_channel * -vbdev_ocf_fn_get_io_channel(void *ctx) // ctx == ocf_vbdev.ctxt +vbdev_ocf_fn_get_io_channel(void *ctx) { ocf_core_t core = ctx; @@ -1111,7 +1124,7 @@ _cache_start_rpc_err_cb(ocf_cache_t cache, void *cb_arg, int error) } vbdev_ocf_cache_destroy(cache); - ocf_mngt_cache_unlock(cache); // use in error path only ? + ocf_mngt_cache_unlock(cache); } static void @@ -1121,8 +1134,8 @@ _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing start\n", ocf_cache_get_name(cache)); - /* At this point volume was either moved to ocf_cache_t struct - * or is no longer needed due to some errors, so we need to deallocate it. */ + /* At this point volume was either moved to ocf_cache_t struct or is no longer + * needed due to some errors, so we need to deallocate it either way. */ vbdev_ocf_cache_config_volume_destroy(cache); if (error) { @@ -1151,7 +1164,7 @@ _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) /* RPC entry point. */ void vbdev_ocf_cache_start(const char *cache_name, const char *base_name, - const char *cache_mode, const uint8_t cache_line_size, bool no_load, + const char *cache_mode, const uint32_t cache_line_size, bool no_load, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { ocf_cache_t cache; @@ -1183,6 +1196,7 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, if ((rc = vbdev_ocf_cache_base_attach(cache, base_name))) { if (rc == -ENODEV) { /* If not, just leave started cache without the device and exit. */ + /* It will be attached later at the examine stage when the device appears. */ SPDK_NOTICELOG("OCF cache '%s': start deferred - waiting for base bdev '%s'\n", cache_name, base_name); ocf_mngt_cache_unlock(cache); @@ -1374,7 +1388,7 @@ vbdev_ocf_cache_detach(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, if (!ocf_cache_is_device_attached(cache)) { SPDK_ERRLOG("OCF cache '%s': device already detached\n", cache_name); - rc = -EALREADY; // better errno ? + rc = -EALREADY; goto err_state; } @@ -1399,46 +1413,6 @@ vbdev_ocf_cache_detach(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, rpc_cb_fn(cache_name, rpc_cb_arg, rc); } -static void -_cache_attach_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) -{ - struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; - - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing device attach\n", - ocf_cache_get_name(cache)); - - /* At this point volume was either moved to ocf_cache_t struct - * or is no longer needed due to some errors, so we need to deallocate it. */ - vbdev_ocf_cache_config_volume_destroy(cache); - ocf_mngt_cache_unlock(cache); - ocf_mngt_cache_put(cache); - - if (error) { - SPDK_ERRLOG("OCF cache '%s': failed to attach OCF cache device (OCF error: %d)\n", - ocf_cache_get_name(cache), error); - - if (error == -OCF_ERR_NO_MEM) { - SPDK_ERRLOG("Not enough memory to handle cache device of this size. Try to increase hugepage memory size, increase cache line size or use smaller cache device.\n"); - vbdev_ocf_mem_calculate(cache); - } - - vbdev_ocf_cache_base_detach(cache); - } else { - SPDK_NOTICELOG("OCF cache '%s': device attached\n", ocf_cache_get_name(cache)); - - /* Update cache IO channel after new device attach. */ - if ((error = vbdev_ocf_core_create_cache_channel(cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to create channel for newly attached cache: %s\n", - ocf_cache_get_name(cache), spdk_strerror(-error)); - } else { - vbdev_ocf_core_add_from_waitlist(cache); - } - } - - mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); - free(mngt_ctx); -} - static void _cache_attach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { @@ -1454,12 +1428,11 @@ _cache_attach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); vbdev_ocf_cache_config_volume_destroy(cache); vbdev_ocf_cache_base_detach(cache); - ocf_mngt_cache_put(cache); free(mngt_ctx); return; } - ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, _cache_attach_rpc_attach_cb, + ocf_mngt_cache_attach(cache, &cache_ctx->cache_att_cfg, _cache_attach_cb, mngt_ctx); } @@ -1475,7 +1448,6 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating device attach\n", cache_name); - // ocf_mngt_cache_put() right away to merge cache_start/attach callbacks ? (or don't start locked) if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; @@ -1484,15 +1456,18 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force if (!ocf_cache_is_detached(cache)) { SPDK_ERRLOG("OCF cache '%s': device already attached\n", cache_name); - rc = -EEXIST; // better errno ? + rc = -EEXIST; goto err_state; } cache_ctx = ocf_cache_get_priv(cache); cache_ctx->no_load = force; + /* Check if base device to attach to this cache is already present. */ if ((rc = vbdev_ocf_cache_base_attach(cache, base_name))) { if (rc == -ENODEV) { + /* If not, just leave it here and exit. It will be attached + * later at the examine stage when the device appears. */ SPDK_NOTICELOG("OCF cache '%s': attach deferred - waiting for base bdev '%s'\n", cache_name, base_name); rpc_cb_fn(cache_name, rpc_cb_arg, -ENODEV); @@ -1519,6 +1494,7 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force mngt_ctx->rpc_cb_fn = rpc_cb_fn; mngt_ctx->rpc_cb_arg = rpc_cb_arg; + ocf_mngt_cache_put(cache); ocf_mngt_cache_lock(cache, _cache_attach_rpc_lock_cb, mngt_ctx); return; @@ -1648,6 +1624,7 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac if ((rc = vbdev_ocf_core_base_attach(core_ctx, base_name))) { if (rc == -ENODEV) { /* If not, just put core context on the temporary core wait list and exit. */ + /* It will be attached later at the examine stage when the device appears. */ SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for base bdev '%s'\n", core_name, base_name); vbdev_ocf_core_waitlist_add(core_ctx); @@ -1668,6 +1645,7 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac /* Second, check if OCF cache for this core is already started. */ if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { /* If not, just put core context on the temporary core wait list and exit. */ + /* Core will be automatically added later when this cache finally starts. */ SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache '%s'\n", core_name, cache_name); vbdev_ocf_core_waitlist_add(core_ctx); @@ -1680,8 +1658,9 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac /* And finally, check if OCF cache device is already attached. * We need to have cache device attached to know if cache was loaded or attached * and then set 'try_add' in core config accordingly. */ - if (!ocf_cache_is_device_attached(cache)) { + if (!ocf_cache_is_device_attached(cache) || !vbdev_ocf_cache_is_base_attached(cache)) { /* If not, just put core context on the temporary core wait list and exit. */ + /* Core will be automatically added later when device for this cache gets attached. */ SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache device '%s'\n", core_name, cache_ctx->base.name); ocf_mngt_cache_put(cache); @@ -1701,8 +1680,6 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(core_name); - // open (and then close) cache-load-added core volumes (with 'base' to save it in priv) ? - mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!mngt_ctx) { SPDK_ERRLOG("OCF core '%s': failed to allocate memory for core add context\n", @@ -2432,8 +2409,9 @@ _flush_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } end: - /* Flushing process may take some time to finish, so call - * RPC callback now and leave flush running in background. */ + /* Flushing process may take some time to finish, so call RPC callback now and + * leave flush running in background. Current status of flushing is dumped in + * the bdev_ocf_get_bdevs RPC call output. */ mngt_ctx->rpc_cb_fn(mngt_ctx->bdev_name, mngt_ctx->rpc_cb_arg, rc); free(mngt_ctx); } @@ -2472,17 +2450,54 @@ vbdev_ocf_flush_start(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo _flush_lock_cb, mngt_ctx); } -// TODO: read lock cache +static void +_get_stats_lock_cb(ocf_cache_t cache, void *cb_arg, int error) +{ + struct vbdev_ocf_mngt_ctx *mngt_ctx = cb_arg; + struct spdk_json_write_ctx *w = mngt_ctx->u.rpc_dump.rpc_cb_arg; + ocf_core_t core = mngt_ctx->core; + struct vbdev_ocf_stats stats; + int rc; + + if ((rc = error)) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + goto end; + } + + if (core) { + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': collecting statistics\n", ocf_core_get_name(core)); + + if ((rc = vbdev_ocf_stats_core_get(core, &stats))) { + SPDK_ERRLOG("OCF core '%s': failed to collect statistics (OCF error: %d)\n", + ocf_core_get_name(core), rc); + } + } else { + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': collecting statistics\n", ocf_cache_get_name(cache)); + + if ((rc = vbdev_ocf_stats_cache_get(cache, &stats))) { + SPDK_ERRLOG("OCF cache '%s': failed to collect statistics (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + } + } + + if (!rc) { + vbdev_ocf_stats_write_json(w, &stats); + } + + ocf_mngt_cache_read_unlock(cache); + +end: + mngt_ctx->u.rpc_dump.rpc_cb_fn(mngt_ctx->u.rpc_dump.rpc_cb_arg, mngt_ctx->rpc_cb_arg); + free(mngt_ctx); +} + /* RPC entry point. */ void -vbdev_ocf_get_stats(const char *bdev_name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, +vbdev_ocf_get_stats(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2) { - struct spdk_json_write_ctx *w = rpc_cb_arg1; - ocf_cache_t cache; - ocf_core_t core; struct vbdev_ocf_mngt_ctx *mngt_ctx; - struct vbdev_ocf_stats stats; int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': getting statistics\n", bdev_name); @@ -2491,8 +2506,11 @@ vbdev_ocf_get_stats(const char *bdev_name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, if (!mngt_ctx) { SPDK_ERRLOG("OCF '%s': failed to allocate memory for getting statistics context\n", bdev_name); - goto end; + goto err_alloc; } + mngt_ctx->u.rpc_dump.rpc_cb_fn = rpc_cb_fn; + mngt_ctx->u.rpc_dump.rpc_cb_arg = rpc_cb_arg1; + mngt_ctx->rpc_cb_arg = rpc_cb_arg2; mngt_ctx->bdev_name = bdev_name; /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ mngt_ctx->cache = NULL; @@ -2501,34 +2519,17 @@ vbdev_ocf_get_stats(const char *bdev_name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, if ((rc = vbdev_ocf_bdev_resolve(mngt_ctx))) { SPDK_ERRLOG("OCF '%s': failed to find cache or core of that name: %s\n", bdev_name, spdk_strerror(-rc)); - free(mngt_ctx); - goto end; + goto err_resolve; } - cache = mngt_ctx->cache; - core = mngt_ctx->core; - free(mngt_ctx); - - if (core) { - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': collecting statistics\n", ocf_core_get_name(core)); - if ((rc = vbdev_ocf_stats_core_get(core, &stats))) { - SPDK_ERRLOG("OCF core '%s': failed to collect statistics (OCF error: %d)\n", - ocf_core_get_name(core), rc); - goto end; - } - } else { - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': collecting statistics\n", ocf_cache_get_name(cache)); - - if ((rc = vbdev_ocf_stats_cache_get(cache, &stats))) { - SPDK_ERRLOG("OCF cache '%s': failed to collect statistics (OCF error: %d)\n", - ocf_cache_get_name(cache), rc); - goto end; - } - } + ocf_mngt_cache_read_lock(mngt_ctx->cache ? : ocf_core_get_cache(mngt_ctx->core), + _get_stats_lock_cb, mngt_ctx); - vbdev_ocf_stats_write_json(w, &stats); + return; -end: +err_resolve: + free(mngt_ctx); +err_alloc: rpc_cb_fn(rpc_cb_arg1, rpc_cb_arg2); } @@ -2546,18 +2547,18 @@ _reset_stats_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } if (core) { - SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': stats reset\n", ocf_core_get_name(core)); + SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': resetting statistics\n", ocf_core_get_name(core)); if ((rc = vbdev_ocf_stats_core_reset(core))) { - SPDK_ERRLOG("OCF core '%s': failed to reset statistics (OCF error: %d)\n", - ocf_core_get_name(core), rc); + SPDK_ERRLOG("OCF core '%s': failed to reset statistics (OCF error: %d)\n", + ocf_core_get_name(core), rc); } } else { - SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': stats reset\n", ocf_cache_get_name(cache)); + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': resetting statistics\n", ocf_cache_get_name(cache)); if ((rc = vbdev_ocf_stats_cache_reset(cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to reset statistics (OCF error: %d)\n", - ocf_cache_get_name(cache), rc); + SPDK_ERRLOG("OCF cache '%s': failed to reset statistics (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); } } @@ -2748,11 +2749,14 @@ dump_core_waitlist_info(struct spdk_json_write_ctx *w, struct vbdev_ocf_core *co spdk_json_write_named_string(w, "cache_name", core_ctx->cache_name); spdk_json_write_named_string(w, "base_name", core_ctx->base.name); spdk_json_write_named_bool(w, "base_attached", vbdev_ocf_core_is_base_attached(core_ctx)); - spdk_json_write_named_uint64(w, "size", vbdev_ocf_core_is_base_attached(core_ctx) ? - spdk_bdev_get_block_size(core_ctx->base.bdev) * - spdk_bdev_get_num_blocks(core_ctx->base.bdev) : 0); - spdk_json_write_named_uint32(w, "block_size", vbdev_ocf_core_is_base_attached(core_ctx) ? - spdk_bdev_get_block_size(core_ctx->base.bdev) : 0); + if (vbdev_ocf_core_is_base_attached(core_ctx)) { + spdk_json_write_named_uint64(w, "size", spdk_bdev_get_block_size(core_ctx->base.bdev) * + spdk_bdev_get_num_blocks(core_ctx->base.bdev)); + spdk_json_write_named_uint32(w, "block_size", spdk_bdev_get_block_size(core_ctx->base.bdev)); + } else { + spdk_json_write_named_null(w, "size"); + spdk_json_write_named_null(w, "block_size"); + } return 0; } @@ -2768,13 +2772,14 @@ dump_core_info(struct spdk_json_write_ctx *w, ocf_core_t core) spdk_json_write_named_string(w, "base_name", core_ctx ? core_ctx->base.name : ""); spdk_json_write_named_bool(w, "base_attached", core_ctx ? vbdev_ocf_core_is_base_attached(core_ctx) : false); - spdk_json_write_named_uint64(w, "size", - (core_ctx && vbdev_ocf_core_is_base_attached(core_ctx)) ? - spdk_bdev_get_block_size(core_ctx->base.bdev) * - spdk_bdev_get_num_blocks(core_ctx->base.bdev) : 0); - spdk_json_write_named_uint32(w, "block_size", - (core_ctx && vbdev_ocf_core_is_base_attached(core_ctx)) ? - spdk_bdev_get_block_size(core_ctx->base.bdev) : 0); + if (core_ctx && vbdev_ocf_core_is_base_attached(core_ctx)) { + spdk_json_write_named_uint64(w, "size", spdk_bdev_get_block_size(core_ctx->base.bdev) * + spdk_bdev_get_num_blocks(core_ctx->base.bdev)); + spdk_json_write_named_uint32(w, "block_size", spdk_bdev_get_block_size(core_ctx->base.bdev)); + } else { + spdk_json_write_named_null(w, "size"); + spdk_json_write_named_null(w, "block_size"); + } spdk_json_write_named_bool(w, "loading", !core_ctx); spdk_json_write_named_object_begin(w, "seq_cutoff"); @@ -2800,12 +2805,16 @@ dump_cache_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) spdk_json_write_named_string(w, "name", ocf_cache_get_name(cache)); spdk_json_write_named_string(w, "base_name", cache_ctx->base.name); - spdk_json_write_named_bool(w, "base_attached", ocf_cache_is_device_attached(cache)); - spdk_json_write_named_uint64(w, "size", ocf_cache_is_device_attached(cache) ? - spdk_bdev_get_block_size(cache_ctx->base.bdev) * - spdk_bdev_get_num_blocks(cache_ctx->base.bdev) : 0); - spdk_json_write_named_uint32(w, "block_size", ocf_cache_is_device_attached(cache) ? - spdk_bdev_get_block_size(cache_ctx->base.bdev) : 0); + spdk_json_write_named_bool(w, "base_attached", ocf_cache_is_device_attached(cache) && + vbdev_ocf_cache_is_base_attached(cache)); + if (vbdev_ocf_cache_is_base_attached(cache)) { + spdk_json_write_named_uint64(w, "size", spdk_bdev_get_block_size(cache_ctx->base.bdev) * + spdk_bdev_get_num_blocks(cache_ctx->base.bdev)); + spdk_json_write_named_uint32(w, "block_size", spdk_bdev_get_block_size(cache_ctx->base.bdev)); + } else { + spdk_json_write_named_null(w, "size"); + spdk_json_write_named_null(w, "block_size"); + } spdk_json_write_named_uint32(w, "cache_line_size", ocf_cache_get_line_size(cache)); spdk_json_write_named_string(w, "cache_mode", vbdev_ocf_cachemode_get_name(ocf_cache_get_mode(cache))); @@ -2872,10 +2881,9 @@ _get_bdevs_cache_visitor(ocf_cache_t cache, void *cb_arg) return rc; } -// TODO: read lock cache ? /* RPC entry point. */ void -vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_get_bdevs_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2) +vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2) { struct spdk_json_write_ctx *w = rpc_cb_arg1; struct vbdev_ocf_core *core_ctx; diff --git a/module/bdev/ocf/vbdev_ocf.h b/module/bdev/ocf/vbdev_ocf.h index a9afd1c1466..8c429b2bf1d 100644 --- a/module/bdev/ocf/vbdev_ocf.h +++ b/module/bdev/ocf/vbdev_ocf.h @@ -12,8 +12,8 @@ /* Callback for the RPC layer used in all management operations. */ typedef void (*vbdev_ocf_rpc_mngt_cb)(const char *bdev_name, void *cb_arg, int error); -/* Callback for the RPC layer used in bdev_ocf_get_bdevs call. */ -typedef void (*vbdev_ocf_get_bdevs_cb)(void *cb_arg1, void *cb_arg2); +/* Callback for the RPC layer used in info dumping calls. */ +typedef void (*vbdev_ocf_rpc_dump_cb)(void *cb_arg1, void *cb_arg2); /* * RPC entry points: @@ -22,7 +22,7 @@ typedef void (*vbdev_ocf_get_bdevs_cb)(void *cb_arg1, void *cb_arg2); void vbdev_ocf_cache_start(const char *cache_name, const char *base_name, const char *cache_mode, - const uint8_t cache_line_size, + const uint32_t cache_line_size, bool no_load, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg); @@ -75,7 +75,7 @@ void vbdev_ocf_flush_start(const char *bdev_name, void *rpc_cb_arg); void vbdev_ocf_get_stats(const char *bdev_name, - vbdev_ocf_get_bdevs_cb rpc_cb_fn, + vbdev_ocf_rpc_dump_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2); @@ -84,7 +84,7 @@ void vbdev_ocf_reset_stats(const char *bdev_name, void *rpc_cb_arg); void vbdev_ocf_get_bdevs(const char *bdev_name, - vbdev_ocf_get_bdevs_cb rpc_cb_fn, + vbdev_ocf_rpc_dump_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2); diff --git a/module/bdev/ocf/vbdev_ocf_cache.c b/module/bdev/ocf/vbdev_ocf_cache.c index a38bee67981..afa9402d176 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.c +++ b/module/bdev/ocf/vbdev_ocf_cache.c @@ -19,7 +19,7 @@ vbdev_ocf_cache_is_base_attached(ocf_cache_t cache) int vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cache_mode, - const uint8_t cache_line_size, bool no_load) + const uint32_t cache_line_size, bool no_load) { ocf_cache_t cache; struct ocf_mngt_cache_config *cache_cfg; @@ -41,13 +41,13 @@ vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cac cache_cfg = &cache_ctx->cache_cfg; ocf_mngt_cache_config_set_default(cache_cfg); - strncpy(cache_cfg->name, cache_name, OCF_CACHE_NAME_SIZE); + strlcpy(cache_cfg->name, cache_name, OCF_CACHE_NAME_SIZE); if (cache_mode) { cache_cfg->cache_mode = vbdev_ocf_cachemode_get_by_name(cache_mode); } if (cache_line_size) { - cache_cfg->cache_line_size = cache_line_size * KiB; + cache_cfg->cache_line_size = cache_line_size; } cache_cfg->locked = true; @@ -81,19 +81,15 @@ _cache_hotremove_detach_cb(ocf_cache_t cache, void *cb_arg, int error) SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': finishing hot removal\n", ocf_cache_get_name(cache)); - // how to detach base despite the error (detached core) and not cause use-after-free on module fini? - //vbdev_ocf_cache_base_detach(cache); - ocf_mngt_cache_unlock(cache); if (error) { SPDK_ERRLOG("OCF cache '%s': failed to detach OCF cache device (OCF error: %d)\n", ocf_cache_get_name(cache), error); - return; + } else { + SPDK_NOTICELOG("OCF cache '%s': device detached\n", ocf_cache_get_name(cache)); } - SPDK_NOTICELOG("OCF cache '%s': device detached\n", ocf_cache_get_name(cache)); - vbdev_ocf_cache_base_detach(cache); /* Update cache IO channel after device detach. */ @@ -157,7 +153,7 @@ vbdev_ocf_cache_base_attach(ocf_cache_t cache, const char *base_name) SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': attaching base bdev '%s'\n", ocf_cache_get_name(cache), base_name); - strncpy(base->name, base_name, OCF_CACHE_NAME_SIZE); + strlcpy(base->name, base_name, OCF_CACHE_NAME_SIZE); if ((rc = spdk_bdev_open_ext(base_name, true, _vbdev_ocf_cache_event_cb, cache, &base->desc))) { return rc; @@ -212,7 +208,6 @@ vbdev_ocf_cache_config_volume_create(ocf_cache_t cache) cache_att_cfg->open_cores = false; cache_att_cfg->discard_on_start = false; cache_att_cfg->device.perform_test = false; - // for ocf_volume_open() in ocf_mngt_cache_attach/load() cache_att_cfg->device.volume_params = &cache_ctx->base; cache_att_cfg->force = cache_ctx->no_load; @@ -348,7 +343,7 @@ vbdev_ocf_cache_mngt_queue_create(ocf_cache_t cache) ocf_cache_get_name(cache)); return -ENOMEM; } - mngt_q_ctx->cache = cache; // keep? (only for DEBUGLOG) + mngt_q_ctx->cache = cache; mngt_q_ctx->thread = spdk_get_thread(); if ((rc = ocf_queue_create_mngt(cache, &cache_ctx->cache_mngt_q, &cache_mngt_queue_ops))) { diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h index d3ffe3bef2a..0503f5423b4 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.h +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -37,8 +37,9 @@ struct vbdev_ocf_cache { } flush; }; -/* Callback for the RPC layer used in all management operations. */ +/* Callbacks for the RPC layer. */ typedef void (*vbdev_ocf_rpc_mngt_cb)(const char *bdev_name, void *cb_arg, int error); +typedef void (*vbdev_ocf_rpc_dump_cb)(void *cb_arg1, void *cb_arg2); /* Temporary context for management operations. */ struct vbdev_ocf_mngt_ctx { @@ -68,6 +69,12 @@ struct vbdev_ocf_mngt_ctx { /* OCF cache mode. */ ocf_cache_mode_t cache_mode; + /* Callback and context for RPCs that dump info. */ + struct { + vbdev_ocf_rpc_dump_cb rpc_cb_fn; + void * rpc_cb_arg; + } rpc_dump; + /* Promotion parameters. */ struct { ocf_promotion_t policy; @@ -106,7 +113,7 @@ struct vbdev_ocf_cache_mngt_queue_ctx { struct spdk_thread * thread; /* Currently kept only for its name used in debug log. */ - ocf_cache_t cache; // rm ? + ocf_cache_t cache; }; /* @@ -122,7 +129,7 @@ bool vbdev_ocf_cache_is_base_attached(ocf_cache_t cache); /* Create cache context, fill config and start OCF cache. */ int vbdev_ocf_cache_create(ocf_cache_t *out, const char *cache_name, const char *cache_mode, - const uint8_t cache_line_size, bool no_load); + const uint32_t cache_line_size, bool no_load); /* Free cache context. */ void vbdev_ocf_cache_destroy(ocf_cache_t cache); diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c index 1fbdc813e15..08c20acd93c 100644 --- a/module/bdev/ocf/vbdev_ocf_core.c +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -114,8 +114,8 @@ vbdev_ocf_core_create(struct vbdev_ocf_core **out, const char *core_name, const core_cfg = &core_ctx->core_cfg; ocf_mngt_core_config_set_default(core_cfg); - strncpy(core_cfg->name, core_name, OCF_CORE_NAME_SIZE); - strncpy(core_ctx->cache_name, cache_name, OCF_CACHE_NAME_SIZE); + strlcpy(core_cfg->name, core_name, OCF_CORE_NAME_SIZE); + strlcpy(core_ctx->cache_name, cache_name, OCF_CACHE_NAME_SIZE); if ((rc = ocf_uuid_set_str(&core_cfg->uuid, core_cfg->name))) { SPDK_ERRLOG("OCF core '%s': failed to set OCF volume uuid\n", core_name); @@ -190,7 +190,7 @@ vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core_ctx, const char *base_nam SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': attaching base bdev '%s'\n", vbdev_ocf_core_get_name(core_ctx), base_name); - strncpy(base->name, base_name, OCF_CORE_NAME_SIZE); + strlcpy(base->name, base_name, OCF_CORE_NAME_SIZE); if ((rc = spdk_bdev_open_ext(base_name, true, _vbdev_ocf_core_event_cb, core_ctx, &base->desc))) { return rc; @@ -217,7 +217,6 @@ vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core_ctx, const char *base_nam base->is_cache = false; base->attached = true; core_cfg->volume_type = SPDK_OBJECT; - // for ocf_volume_open() in ocf_mngt_cache_add_core() core_cfg->volume_params = base; return rc; @@ -246,7 +245,7 @@ _core_io_queue_stop(void *ctx) } /* Core channel may not exist only on error path. */ - if (likely(ch_ctx->core_ch)) { + if (spdk_likely(ch_ctx->core_ch)) { spdk_put_io_channel(ch_ctx->core_ch); } @@ -315,7 +314,7 @@ _vbdev_ocf_ch_create_cb(void *io_device, void *ctx_buf) vbdev_name); return -ENOMEM; } - ch_ctx->core = core; // keep? (only for DEBUGLOG) + ch_ctx->core = core; ch_ctx->thread = spdk_get_thread(); if ((rc = ocf_queue_create(cache, &ch_ctx->queue, &core_io_queue_ops))) { @@ -373,6 +372,20 @@ _vbdev_ocf_ch_destroy_cb(void *io_device, void *ctx_buf) ocf_queue_put(ch_destroy_ctx->queue); } +static void +_core_register(void *ctx) +{ + struct spdk_bdev *ocf_vbdev = ctx; + ocf_core_t core = ocf_vbdev->ctxt; + int rc; + + if ((rc = spdk_bdev_register(ocf_vbdev))) { + SPDK_ERRLOG("OCF vbdev '%s': failed to register SPDK bdev through message to app thread: %s\n", + spdk_bdev_get_name(ocf_vbdev), spdk_strerror(-rc)); + spdk_io_device_unregister(core, NULL); + } +} + int vbdev_ocf_core_register(ocf_core_t core) { @@ -390,7 +403,6 @@ vbdev_ocf_core_register(ocf_core_t core) ocf_vbdev->write_cache = base->bdev->write_cache; ocf_vbdev->blocklen = base->bdev->blocklen; ocf_vbdev->blockcnt = base->bdev->blockcnt; - // cache_line_size align ? ocf_vbdev->required_alignment = base->bdev->required_alignment; ocf_vbdev->optimal_io_boundary = base->bdev->optimal_io_boundary; // generate UUID based on namespace UUID + base bdev UUID (take from old module?) @@ -402,11 +414,22 @@ vbdev_ocf_core_register(ocf_core_t core) SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': io_device created at %p\n", spdk_bdev_get_name(ocf_vbdev), core); - if ((rc = spdk_bdev_register(ocf_vbdev))) { // needs to be called from SPDK app thread - SPDK_ERRLOG("OCF vbdev '%s': failed to register SPDK bdev\n", - spdk_bdev_get_name(ocf_vbdev)); - spdk_io_device_unregister(core, NULL); - return rc; + if (!spdk_thread_is_app_thread(NULL)) { + if ((rc = spdk_thread_send_msg(spdk_thread_get_app_thread(), _core_register, ocf_vbdev))) { + SPDK_ERRLOG("OCF vbdev '%s': failed to send message to thread (name: %s, id: %ld): %s\n", + spdk_bdev_get_name(ocf_vbdev), + spdk_thread_get_name(spdk_thread_get_app_thread()), + spdk_thread_get_id(spdk_thread_get_app_thread()), + spdk_strerror(-rc)); + assert(false); + } + } else { + if ((rc = spdk_bdev_register(ocf_vbdev))) { + SPDK_ERRLOG("OCF vbdev '%s': failed to register SPDK bdev\n", + spdk_bdev_get_name(ocf_vbdev)); + spdk_io_device_unregister(core, NULL); + return rc; + } } return rc; diff --git a/module/bdev/ocf/vbdev_ocf_core.h b/module/bdev/ocf/vbdev_ocf_core.h index b31540ffe8d..da28ec25441 100644 --- a/module/bdev/ocf/vbdev_ocf_core.h +++ b/module/bdev/ocf/vbdev_ocf_core.h @@ -68,7 +68,7 @@ struct vbdev_ocf_core_io_channel_ctx { struct spdk_thread * thread; /* Currently kept only for its name used in debug log. */ - ocf_core_t core; // rm ? + ocf_core_t core; }; /* diff --git a/module/bdev/ocf/vbdev_ocf_rpc.c b/module/bdev/ocf/vbdev_ocf_rpc.c index 66f764e2653..e90144307c1 100644 --- a/module/bdev/ocf/vbdev_ocf_rpc.c +++ b/module/bdev/ocf/vbdev_ocf_rpc.c @@ -11,7 +11,7 @@ struct rpc_bdev_ocf_start_cache { char *cache_name; char *base_name; char *cache_mode; - uint8_t cache_line_size; + uint32_t cache_line_size; bool no_load; }; @@ -27,7 +27,7 @@ static const struct spdk_json_object_decoder rpc_bdev_ocf_start_cache_decoders[] {"cache_name", offsetof(struct rpc_bdev_ocf_start_cache, cache_name), spdk_json_decode_string}, {"base_name", offsetof(struct rpc_bdev_ocf_start_cache, base_name), spdk_json_decode_string}, {"cache_mode", offsetof(struct rpc_bdev_ocf_start_cache, cache_mode), spdk_json_decode_string, true}, - {"cache_line_size", offsetof(struct rpc_bdev_ocf_start_cache, cache_line_size), spdk_json_decode_uint8, true}, + {"cache_line_size", offsetof(struct rpc_bdev_ocf_start_cache, cache_line_size), spdk_json_decode_uint32, true}, {"no_load", offsetof(struct rpc_bdev_ocf_start_cache, no_load), spdk_json_decode_bool, true}, }; diff --git a/module/bdev/ocf/volume.c b/module/bdev/ocf/volume.c index 75ee71de3ea..b7f27faa0e8 100644 --- a/module/bdev/ocf/volume.c +++ b/module/bdev/ocf/volume.c @@ -17,13 +17,12 @@ #include "volume.h" #include "ctx.h" +/* TODO?: refactor (like ocf_core_volume in ocf_core.c ?) */ static int vbdev_ocf_volume_open(ocf_volume_t volume, void *opts) { struct vbdev_ocf_base **priv = ocf_volume_get_priv(volume); - // refactor (like ocf_core_volume in ocf_core.c ?) - if (opts) { *priv = opts; } @@ -152,6 +151,12 @@ vbdev_forward_io(ocf_volume_t volume, ocf_forward_token_t token, int iovcnt, skip, status = -1; struct iovec *iovs; + if (!base->attached) { + SPDK_ERRLOG("Base bdev '%s' not attached\n", base->name); + ocf_forward_end(token, -ENXIO); + return; + } + ch = vbdev_forward_get_channel(volume, token); if (unlikely(ch == NULL)) { ocf_forward_end(token, -EFAULT); diff --git a/test/ocf/management/info_dump_get_bdevs.json b/test/ocf/management/info_dump_get_bdevs.json index 0a2a4ef3669..d7698f4be89 100644 --- a/test/ocf/management/info_dump_get_bdevs.json +++ b/test/ocf/management/info_dump_get_bdevs.json @@ -5,8 +5,8 @@ "cache_name": "Ocf_cache_none", "base_name": "Core_dev_waitlist1", "base_attached": false, - "size": 0, - "block_size": 0 + "size": null, + "block_size": null }, { "name": "Ocf_core_waitlist2", @@ -30,8 +30,8 @@ "name": "Ocf_cache1", "base_name": "Cache_dev1", "base_attached": false, - "size": 0, - "block_size": 0, + "size": null, + "block_size": null, "cache_line_size": 4096, "cache_mode": "wt", "promotion": { @@ -56,8 +56,8 @@ "cache_name": "Ocf_cache1", "base_name": "Core_dev1-1", "base_attached": false, - "size": 0, - "block_size": 0, + "size": null, + "block_size": null, "loading": false, "seq_cutoff": { "policy": "full", @@ -140,8 +140,8 @@ "cache_name": "Ocf_cache2", "base_name": "Core_dev2-1", "base_attached": false, - "size": 0, - "block_size": 0, + "size": null, + "block_size": null, "loading": false, "seq_cutoff": { "policy": "full", @@ -224,8 +224,8 @@ "cache_name": "Ocf_cache3", "base_name": "Core_dev3-1", "base_attached": false, - "size": 0, - "block_size": 0, + "size": null, + "block_size": null, "loading": false, "seq_cutoff": { "policy": "full", From 7d846475fcb667d9056d3c787fa49c06372dc396 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Fri, 19 Sep 2025 12:02:33 +0200 Subject: [PATCH 18/34] Fix formatting Change-Id: Ib43ab22b3fa457e624deae912b3c587bdf6b0648 Signed-off-by: Rafal Stefanowski --- lib/env_ocf/ocf_env_refcnt.c | 37 +- lib/env_ocf/ocf_env_refcnt.h | 9 +- module/bdev/ocf/ctx.c | 2 +- module/bdev/ocf/vbdev_ocf.c | 69 +- module/bdev/ocf/vbdev_ocf_cache.c | 4 +- module/bdev/ocf/vbdev_ocf_cache.h | 20 +- module/bdev/ocf/vbdev_ocf_core.c | 14 +- module/bdev/ocf/vbdev_ocf_core.h | 20 +- module/bdev/ocf/volume.h | 14 +- python/spdk/cli/bdev.py | 25 +- test/ocf/common.sh | 4 +- test/ocf/management/config_change.sh | 258 +++---- test/ocf/management/detach-attach.sh | 212 +++--- test/ocf/management/hotremove.sh | 686 +++++++++---------- test/ocf/management/start-stop/examine.sh | 40 +- test/ocf/management/start-stop/incomplete.sh | 104 +-- test/ocf/management/start-stop/load.sh | 72 +- 17 files changed, 811 insertions(+), 779 deletions(-) diff --git a/lib/env_ocf/ocf_env_refcnt.c b/lib/env_ocf/ocf_env_refcnt.c index 69045e6ee32..4c2c3c6dabb 100644 --- a/lib/env_ocf/ocf_env_refcnt.c +++ b/lib/env_ocf/ocf_env_refcnt.c @@ -6,7 +6,8 @@ #include "ocf_env_refcnt.h" -int env_refcnt_init(struct env_refcnt *rc, const char *name, size_t name_len) +int +env_refcnt_init(struct env_refcnt *rc, const char *name, size_t name_len) { env_atomic_set(&rc->counter, 0); env_atomic_set(&rc->freeze, 0); @@ -16,43 +17,50 @@ int env_refcnt_init(struct env_refcnt *rc, const char *name, size_t name_len) return 0; } -void env_refcnt_deinit(struct env_refcnt *rc) +void +env_refcnt_deinit(struct env_refcnt *rc) { } -void env_refcnt_dec(struct env_refcnt *rc) +void +env_refcnt_dec(struct env_refcnt *rc) { int val = env_atomic_dec_return(&rc->counter); ENV_BUG_ON(val < 0); - if (!val && env_atomic_cmpxchg(&rc->callback, 1, 0)) + if (!val && env_atomic_cmpxchg(&rc->callback, 1, 0)) { rc->cb(rc->priv); + } } -bool env_refcnt_inc(struct env_refcnt *rc) +bool +env_refcnt_inc(struct env_refcnt *rc) { int val; if (!env_atomic_read(&rc->freeze)) { val = env_atomic_inc_return(&rc->counter); - if (!env_atomic_read(&rc->freeze)) + if (!env_atomic_read(&rc->freeze)) { return !!val; - else + } else { env_refcnt_dec(rc); + } } return 0; } -void env_refcnt_freeze(struct env_refcnt *rc) +void +env_refcnt_freeze(struct env_refcnt *rc) { env_atomic_inc(&rc->freeze); } -void env_refcnt_register_zero_cb(struct env_refcnt *rc, env_refcnt_cb_t cb, - void *priv) +void +env_refcnt_register_zero_cb(struct env_refcnt *rc, env_refcnt_cb_t cb, + void *priv) { ENV_BUG_ON(!env_atomic_read(&rc->freeze)); ENV_BUG_ON(env_atomic_read(&rc->callback)); @@ -64,18 +72,21 @@ void env_refcnt_register_zero_cb(struct env_refcnt *rc, env_refcnt_cb_t cb, env_refcnt_dec(rc); } -void env_refcnt_unfreeze(struct env_refcnt *rc) +void +env_refcnt_unfreeze(struct env_refcnt *rc) { int val = env_atomic_dec_return(&rc->freeze); ENV_BUG_ON(val < 0); } -bool env_refcnt_frozen(struct env_refcnt *rc) +bool +env_refcnt_frozen(struct env_refcnt *rc) { return !!env_atomic_read(&rc->freeze); } -bool env_refcnt_zeroed(struct env_refcnt *rc) +bool +env_refcnt_zeroed(struct env_refcnt *rc) { return (env_atomic_read(&rc->counter) == 0); } diff --git a/lib/env_ocf/ocf_env_refcnt.h b/lib/env_ocf/ocf_env_refcnt.h index 0395ed6e47b..e4ea113ff62 100644 --- a/lib/env_ocf/ocf_env_refcnt.h +++ b/lib/env_ocf/ocf_env_refcnt.h @@ -11,8 +11,7 @@ typedef void (*env_refcnt_cb_t)(void *priv); -struct env_refcnt -{ +struct env_refcnt { env_atomic counter __attribute__((aligned(64))); env_atomic freeze; env_atomic callback; @@ -35,7 +34,7 @@ void env_refcnt_dec(struct env_refcnt *rc); /* Disallow incrementing of underlying counter - attempts to increment counter * will be failing until env_refcnt_unfreeze is calleed. * It's ok to call freeze multiple times, in which case counter is frozen - * until all freeze calls are offset by a corresponding unfreeze.*/ + * until all freeze calls are offset by a corresponding unfreeze. */ void env_refcnt_freeze(struct env_refcnt *rc); /* Cancel the effect of single env_refcnt_freeze call */ @@ -49,6 +48,6 @@ bool env_refcnt_zeroed(struct env_refcnt *rc); * Must be called after counter is frozen. * Cannot be called until previously regsitered callback had fired. */ void env_refcnt_register_zero_cb(struct env_refcnt *rc, env_refcnt_cb_t cb, - void *priv); + void *priv); -#endif // __OCF_ENV_REFCNT_H__ +#endif /* __OCF_ENV_REFCNT_H__ */ diff --git a/module/bdev/ocf/ctx.c b/module/bdev/ocf/ctx.c index 37b21cf3372..35ac9465866 100644 --- a/module/bdev/ocf/ctx.c +++ b/module/bdev/ocf/ctx.c @@ -363,7 +363,7 @@ vbdev_ocf_ctx_cleaner_stop(ocf_cleaner_t c) } /* TODO?: this one should only set next_run to "now" and poller - * registration should be move to _init with saved thread */ + * registration should be move to _init with saved thread */ static void vbdev_ocf_ctx_cleaner_kick(ocf_cleaner_t cleaner) { diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index ac2dc41c24b..37bcc172e75 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -340,7 +340,7 @@ _module_fini_cache_visitor(ocf_cache_t cache, void *cb_arg) SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': module stop visit\n", ocf_cache_get_name(cache)); if (!ocf_cache_get_core_count(cache) || - ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { + ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { /* If there are no cores or all of them are detached, * then cache stop can be triggered already. */ ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, NULL); @@ -975,7 +975,7 @@ vbdev_ocf_submit_io(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, ui /* OCF core should be added before vbdev register and removed after vbdev unregister. */ assert(core); - + io = ocf_volume_new_io(ocf_core_get_front_volume(core), ch_ctx->queue, offset, len, dir, 0, flags); if (spdk_unlikely(!io)) { @@ -1156,7 +1156,7 @@ _cache_start_rpc_attach_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_mngt_cache_unlock(cache); vbdev_ocf_core_add_from_waitlist(cache); } - + mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); free(mngt_ctx); } @@ -1277,7 +1277,7 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo mngt_ctx->cache = cache; if (!ocf_cache_get_core_count(cache) || - ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { + ocf_cache_get_core_count(cache) == ocf_cache_get_core_inactive_count(cache)) { /* If there are no cores or all of them are detached, * then cache stop can be triggered already. */ ocf_mngt_cache_lock(cache, _cache_stop_lock_cb, mngt_ctx); @@ -1964,8 +1964,9 @@ _promotion_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } if (mngt_ctx->u.promotion.nhit_insertion_threshold >= 0) { - if ((rc = ocf_mngt_cache_promotion_set_param(cache, ocf_promotion_nhit, ocf_nhit_insertion_threshold, - mngt_ctx->u.promotion.nhit_insertion_threshold))) { + if ((rc = ocf_mngt_cache_promotion_set_param(cache, ocf_promotion_nhit, + ocf_nhit_insertion_threshold, + mngt_ctx->u.promotion.nhit_insertion_threshold))) { SPDK_ERRLOG("OCF cache '%s': failed to set promotion nhit_insertion_threshold param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -1974,7 +1975,7 @@ _promotion_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (mngt_ctx->u.promotion.nhit_trigger_threshold >= 0) { if ((rc = ocf_mngt_cache_promotion_set_param(cache, ocf_promotion_nhit, ocf_nhit_trigger_threshold, - mngt_ctx->u.promotion.nhit_trigger_threshold))) { + mngt_ctx->u.promotion.nhit_trigger_threshold))) { SPDK_ERRLOG("OCF cache '%s': failed to set promotion nhit_trigger_threshold param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2071,7 +2072,7 @@ _cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (mngt_ctx->u.cleaning.acp_wake_up_time >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_acp, ocf_acp_wake_up_time, - mngt_ctx->u.cleaning.acp_wake_up_time))) { + mngt_ctx->u.cleaning.acp_wake_up_time))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning acp_wake_up_time param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2080,7 +2081,7 @@ _cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (mngt_ctx->u.cleaning.acp_flush_max_buffers >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_acp, ocf_acp_flush_max_buffers, - mngt_ctx->u.cleaning.acp_flush_max_buffers))) { + mngt_ctx->u.cleaning.acp_flush_max_buffers))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning acp_flush_max_buffers param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2089,7 +2090,7 @@ _cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (mngt_ctx->u.cleaning.alru_wake_up_time >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_wake_up_time, - mngt_ctx->u.cleaning.alru_wake_up_time))) { + mngt_ctx->u.cleaning.alru_wake_up_time))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_wake_up_time param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2098,7 +2099,7 @@ _cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (mngt_ctx->u.cleaning.alru_flush_max_buffers >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_flush_max_buffers, - mngt_ctx->u.cleaning.alru_flush_max_buffers))) { + mngt_ctx->u.cleaning.alru_flush_max_buffers))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_flush_max_buffers param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2107,7 +2108,7 @@ _cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (mngt_ctx->u.cleaning.alru_staleness_time >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_stale_buffer_time, - mngt_ctx->u.cleaning.alru_staleness_time))) { + mngt_ctx->u.cleaning.alru_staleness_time))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_staleness_time param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2116,7 +2117,7 @@ _cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (mngt_ctx->u.cleaning.alru_activity_threshold >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_activity_threshold, - mngt_ctx->u.cleaning.alru_activity_threshold))) { + mngt_ctx->u.cleaning.alru_activity_threshold))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_activity_threshold param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2125,7 +2126,7 @@ _cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (mngt_ctx->u.cleaning.alru_max_dirty_ratio >= 0) { if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_max_dirty_ratio, - mngt_ctx->u.cleaning.alru_max_dirty_ratio))) { + mngt_ctx->u.cleaning.alru_max_dirty_ratio))) { SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_max_dirty_ratio param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2236,7 +2237,8 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } if (mngt_ctx->u.seqcutoff.promotion_count >= 0) { - if ((rc = ocf_mngt_core_set_seq_cutoff_promotion_count(core, mngt_ctx->u.seqcutoff.promotion_count))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promotion_count(core, + mngt_ctx->u.seqcutoff.promotion_count))) { SPDK_ERRLOG("OCF core '%s': failed to set sequential cut-off promotion_count (OCF error: %d)\n", ocf_core_get_name(core), rc); goto err_param; @@ -2244,7 +2246,8 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } if (mngt_ctx->u.seqcutoff.promote_on_threshold >= 0) { - if ((rc = ocf_mngt_core_set_seq_cutoff_promote_on_threshold(core, mngt_ctx->u.seqcutoff.promote_on_threshold))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promote_on_threshold(core, + mngt_ctx->u.seqcutoff.promote_on_threshold))) { SPDK_ERRLOG("OCF core '%s': failed to set sequential cut-off promote_on_threshold (OCF error: %d)\n", ocf_core_get_name(core), rc); goto err_param; @@ -2266,7 +2269,8 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } if (mngt_ctx->u.seqcutoff.threshold >= 0) { - if ((rc = ocf_mngt_core_set_seq_cutoff_threshold_all(cache, mngt_ctx->u.seqcutoff.threshold * KiB))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_threshold_all(cache, + mngt_ctx->u.seqcutoff.threshold * KiB))) { SPDK_ERRLOG("OCF cache '%s': failed to set sequential cut-off threshold (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2274,7 +2278,8 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } if (mngt_ctx->u.seqcutoff.promotion_count >= 0) { - if ((rc = ocf_mngt_core_set_seq_cutoff_promotion_count_all(cache, mngt_ctx->u.seqcutoff.promotion_count))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promotion_count_all(cache, + mngt_ctx->u.seqcutoff.promotion_count))) { SPDK_ERRLOG("OCF cache '%s': failed to set sequential cut-off promotion_count (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2282,7 +2287,8 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } if (mngt_ctx->u.seqcutoff.promote_on_threshold >= 0) { - if ((rc = ocf_mngt_core_set_seq_cutoff_promote_on_threshold_all(cache, mngt_ctx->u.seqcutoff.promote_on_threshold))) { + if ((rc = ocf_mngt_core_set_seq_cutoff_promote_on_threshold_all(cache, + mngt_ctx->u.seqcutoff.promote_on_threshold))) { SPDK_ERRLOG("OCF cache '%s': failed to set sequential cut-off promote_on_threshold (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; @@ -2309,8 +2315,8 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) /* RPC entry point. */ void vbdev_ocf_set_seqcutoff(const char *bdev_name, const char *policy, int32_t threshold, - int32_t promotion_count, int32_t promote_on_threshold, - vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) + int32_t promotion_count, int32_t promote_on_threshold, + vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { struct vbdev_ocf_mngt_ctx *mngt_ctx; int rc; @@ -2627,13 +2633,13 @@ dump_promotion_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) if (promotion_policy == ocf_promotion_nhit) { if ((rc = ocf_mngt_cache_promotion_get_param(cache, ocf_promotion_nhit, - ocf_nhit_insertion_threshold, ¶m_val))) { + ocf_nhit_insertion_threshold, ¶m_val))) { return rc; } spdk_json_write_named_uint32(w, "insertion_threshold", param_val); if ((rc = ocf_mngt_cache_promotion_get_param(cache, ocf_promotion_nhit, - ocf_nhit_trigger_threshold, ¶m_val))) { + ocf_nhit_trigger_threshold, ¶m_val))) { return rc; } spdk_json_write_named_uint32(w, "trigger_threshold", param_val); @@ -2661,44 +2667,44 @@ dump_cleaning_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) if (cleaning_policy == ocf_cleaning_acp) { if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_acp, - ocf_acp_wake_up_time, ¶m_val))) { + ocf_acp_wake_up_time, ¶m_val))) { return rc; } spdk_json_write_named_uint32(w, "wake_up_time", param_val); if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_acp, - ocf_acp_flush_max_buffers, ¶m_val))) { + ocf_acp_flush_max_buffers, ¶m_val))) { return rc; } spdk_json_write_named_uint32(w, "flush_max_buffers", param_val); } else if (cleaning_policy == ocf_cleaning_alru) { if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, - ocf_alru_wake_up_time, ¶m_val))) { + ocf_alru_wake_up_time, ¶m_val))) { return rc; } spdk_json_write_named_uint32(w, "wake_up_time", param_val); if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, - ocf_alru_flush_max_buffers, ¶m_val))) { + ocf_alru_flush_max_buffers, ¶m_val))) { return rc; } spdk_json_write_named_uint32(w, "flush_max_buffers", param_val); if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, - ocf_alru_stale_buffer_time, ¶m_val))) { + ocf_alru_stale_buffer_time, ¶m_val))) { return rc; } spdk_json_write_named_uint32(w, "staleness_time", param_val); if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, - ocf_alru_activity_threshold, ¶m_val))) { + ocf_alru_activity_threshold, ¶m_val))) { return rc; } spdk_json_write_named_uint32(w, "activity_threshold", param_val); if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, - ocf_alru_max_dirty_ratio, ¶m_val))) { + ocf_alru_max_dirty_ratio, ¶m_val))) { return rc; } spdk_json_write_named_uint32(w, "max_dirty_ratio", param_val); @@ -2883,7 +2889,8 @@ _get_bdevs_cache_visitor(ocf_cache_t cache, void *cb_arg) /* RPC entry point. */ void -vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, void *rpc_cb_arg1, void *rpc_cb_arg2) +vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, void *rpc_cb_arg1, + void *rpc_cb_arg2) { struct spdk_json_write_ctx *w = rpc_cb_arg1; struct vbdev_ocf_core *core_ctx; diff --git a/module/bdev/ocf/vbdev_ocf_cache.c b/module/bdev/ocf/vbdev_ocf_cache.c index afa9402d176..f6753d4b2be 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.c +++ b/module/bdev/ocf/vbdev_ocf_cache.c @@ -160,7 +160,7 @@ vbdev_ocf_cache_base_attach(ocf_cache_t cache, const char *base_name) } if ((rc = spdk_bdev_module_claim_bdev_desc(base->desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, - NULL, &ocf_if))) { + NULL, &ocf_if))) { SPDK_ERRLOG("OCF cache '%s': failed to claim base bdev '%s'\n", ocf_cache_get_name(cache), base_name); spdk_bdev_close(base->desc); @@ -298,7 +298,7 @@ vbdev_ocf_cache_mngt_queue_stop(ocf_queue_t queue) { struct vbdev_ocf_cache_mngt_queue_ctx *mngt_q_ctx = ocf_queue_get_priv(queue); int rc; - + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': destroying OCF management queue\n", ocf_cache_get_name(mngt_q_ctx->cache)); diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h index 0503f5423b4..90d3887b6bb 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.h +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -44,19 +44,19 @@ typedef void (*vbdev_ocf_rpc_dump_cb)(void *cb_arg1, void *cb_arg2); /* Temporary context for management operations. */ struct vbdev_ocf_mngt_ctx { /* RPC callback. */ - vbdev_ocf_rpc_mngt_cb rpc_cb_fn; + vbdev_ocf_rpc_mngt_cb rpc_cb_fn; /* RPC context. */ - void * rpc_cb_arg; + void *rpc_cb_arg; /* Name of bdev. */ - const char * bdev_name; + const char *bdev_name; /* OCF cache handle. */ - ocf_cache_t cache; + ocf_cache_t cache; /* OCF core handle. */ - ocf_core_t core; + ocf_core_t core; union { /* Callback for cache attach operation that needs to be passed through metadata probe @@ -64,7 +64,7 @@ struct vbdev_ocf_mngt_ctx { ocf_mngt_cache_attach_end_t att_cb_fn; /* Core context (priv). */ - struct vbdev_ocf_core * core_ctx; + struct vbdev_ocf_core *core_ctx; /* OCF cache mode. */ ocf_cache_mode_t cache_mode; @@ -72,7 +72,7 @@ struct vbdev_ocf_mngt_ctx { /* Callback and context for RPCs that dump info. */ struct { vbdev_ocf_rpc_dump_cb rpc_cb_fn; - void * rpc_cb_arg; + void *rpc_cb_arg; } rpc_dump; /* Promotion parameters. */ @@ -107,13 +107,13 @@ struct vbdev_ocf_mngt_ctx { /* Cache management queue context. */ struct vbdev_ocf_cache_mngt_queue_ctx { /* Registered poller. */ - struct spdk_poller * poller; + struct spdk_poller *poller; /* Thread on which poller was registered. */ - struct spdk_thread * thread; + struct spdk_thread *thread; /* Currently kept only for its name used in debug log. */ - ocf_cache_t cache; + ocf_cache_t cache; }; /* diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c index 08c20acd93c..22d0a52f631 100644 --- a/module/bdev/ocf/vbdev_ocf_core.c +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -10,7 +10,7 @@ #include "ctx.h" struct vbdev_ocf_core_waitlist_head g_vbdev_ocf_core_waitlist = - STAILQ_HEAD_INITIALIZER(g_vbdev_ocf_core_waitlist); + STAILQ_HEAD_INITIALIZER(g_vbdev_ocf_core_waitlist); void vbdev_ocf_core_waitlist_add(struct vbdev_ocf_core *core_ctx) @@ -197,7 +197,7 @@ vbdev_ocf_core_base_attach(struct vbdev_ocf_core *core_ctx, const char *base_nam } if ((rc = spdk_bdev_module_claim_bdev_desc(base->desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, - NULL, &ocf_if))) { + NULL, &ocf_if))) { SPDK_ERRLOG("OCF core '%s': failed to claim base bdev '%s'\n", vbdev_ocf_core_get_name(core_ctx), base_name); spdk_bdev_close(base->desc); @@ -256,9 +256,10 @@ static void vbdev_ocf_core_io_queue_stop(ocf_queue_t queue) { struct vbdev_ocf_core_io_channel_ctx *ch_ctx = ocf_queue_get_priv(queue); - struct spdk_bdev *ocf_vbdev = &((struct vbdev_ocf_core *)ocf_core_get_priv(ch_ctx->core))->ocf_vbdev; + struct spdk_bdev *ocf_vbdev = &((struct vbdev_ocf_core *)ocf_core_get_priv( + ch_ctx->core))->ocf_vbdev; int rc; - + SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': deallocating external IO channel context\n", spdk_bdev_get_name(ocf_vbdev)); @@ -436,7 +437,8 @@ vbdev_ocf_core_register(ocf_core_t core) } int -vbdev_ocf_core_unregister(struct vbdev_ocf_core *core_ctx, spdk_bdev_unregister_cb cb_fn, void *cb_arg) +vbdev_ocf_core_unregister(struct vbdev_ocf_core *core_ctx, spdk_bdev_unregister_cb cb_fn, + void *cb_arg) { SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating unregister of OCF vbdev\n", vbdev_ocf_core_get_name(core_ctx)); @@ -516,7 +518,7 @@ vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache) vbdev_ocf_foreach_core_in_waitlist(core_ctx) { if (strcmp(ocf_cache_get_name(cache), core_ctx->cache_name) || - !vbdev_ocf_core_is_base_attached(core_ctx)) { + !vbdev_ocf_core_is_base_attached(core_ctx)) { continue; } diff --git a/module/bdev/ocf/vbdev_ocf_core.h b/module/bdev/ocf/vbdev_ocf_core.h index da28ec25441..b82530124bd 100644 --- a/module/bdev/ocf/vbdev_ocf_core.h +++ b/module/bdev/ocf/vbdev_ocf_core.h @@ -39,7 +39,7 @@ struct vbdev_ocf_core { char cache_name[OCF_CACHE_NAME_SIZE]; /* Context for cache stop management operation. */ - struct vbdev_ocf_mngt_ctx * mngt_ctx; + struct vbdev_ocf_mngt_ctx *mngt_ctx; /* Status of core flush operation. */ struct { @@ -53,22 +53,22 @@ struct vbdev_ocf_core { /* Core IO channel context. */ struct vbdev_ocf_core_io_channel_ctx { /* OCF queue. */ - ocf_queue_t queue; + ocf_queue_t queue; /* Cache base bdev IO channel. */ - struct spdk_io_channel * cache_ch; + struct spdk_io_channel *cache_ch; /* Core base bdev IO channel. */ - struct spdk_io_channel * core_ch; + struct spdk_io_channel *core_ch; /* Registered poller. */ - struct spdk_poller * poller; + struct spdk_poller *poller; /* Thread on which poller was registered and channels were opened. */ - struct spdk_thread * thread; + struct spdk_thread *thread; /* Currently kept only for its name used in debug log. */ - ocf_core_t core; + ocf_core_t core; }; /* @@ -98,7 +98,8 @@ bool vbdev_ocf_core_is_loaded(const char *core_name); */ /* Create core context and fill config. */ -int vbdev_ocf_core_create(struct vbdev_ocf_core **out, const char *core_name, const char *cache_name); +int vbdev_ocf_core_create(struct vbdev_ocf_core **out, const char *core_name, + const char *cache_name); /* Free core context. */ void vbdev_ocf_core_destroy(struct vbdev_ocf_core *core_ctx); @@ -113,7 +114,8 @@ void vbdev_ocf_core_base_detach(struct vbdev_ocf_core *core_ctx); int vbdev_ocf_core_register(ocf_core_t core); /* Unregister core from SPDK bdev layer. */ -int vbdev_ocf_core_unregister(struct vbdev_ocf_core *core_ctx, spdk_bdev_unregister_cb cb_fn, void *cb_arg); +int vbdev_ocf_core_unregister(struct vbdev_ocf_core *core_ctx, spdk_bdev_unregister_cb cb_fn, + void *cb_arg); /* Add all cores from wait list assigned to given cache. */ void vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache); diff --git a/module/bdev/ocf/volume.h b/module/bdev/ocf/volume.h index dda02a6f8e0..210649892bc 100644 --- a/module/bdev/ocf/volume.h +++ b/module/bdev/ocf/volume.h @@ -8,13 +8,13 @@ #define VBDEV_OCF_VOLUME_H struct vbdev_ocf_base { - char name[spdk_max(OCF_CACHE_NAME_SIZE, OCF_CORE_NAME_SIZE)]; - bool is_cache; - bool attached; - struct spdk_bdev * bdev; - struct spdk_bdev_desc * desc; - struct spdk_io_channel * mngt_ch; - struct spdk_thread * thread; + char name[spdk_max(OCF_CACHE_NAME_SIZE, OCF_CORE_NAME_SIZE)]; + bool is_cache; + bool attached; + struct spdk_bdev *bdev; + struct spdk_bdev_desc *desc; + struct spdk_io_channel *mngt_ch; + struct spdk_thread *thread; }; void vbdev_ocf_base_detach(struct vbdev_ocf_base *base); diff --git a/python/spdk/cli/bdev.py b/python/spdk/cli/bdev.py index 7045e8e53d6..d6ac2686911 100644 --- a/python/spdk/cli/bdev.py +++ b/python/spdk/cli/bdev.py @@ -120,7 +120,10 @@ def bdev_ocf_start_cache(args): help='choose between {4|8|16|32|64} [KiB]; default 4', type=int, choices=[4, 8, 16, 32, 64]) - p.add_argument('-n', '--no-load', action='store_true', help='do not load previous cache instance from metadata and force starting a new one instead (WARNING: all cache metadata will be discarded!)') + p.add_argument('-n', '--no-load', + action='store_true', + help='do not load previous cache instance from metadata and force starting a \ + new one instead (WARNING: all cache metadata will be discarded!)') p.set_defaults(func=bdev_ocf_start_cache) def bdev_ocf_stop_cache(args): @@ -170,7 +173,8 @@ def bdev_ocf_set_cachemode(args): p = subparsers.add_parser('bdev_ocf_set_cachemode', help='Set cache mode of OCF cache') p.add_argument('cache_name', help='name of the cache vbdev') p.add_argument('cache_mode', - help='choose between {wt|wb|wa|wo|wi|pt} (Write-Through, Write-Back, Write-Around, Write-Only, Write-Invalidate, Pass-Through)', + help='choose between {wt|wb|wa|wo|wi|pt} (Write-Through, Write-Back, \ + Write-Around, Write-Only, Write-Invalidate, Pass-Through)', choices=['wt', 'wb', 'wa', 'wo', 'wi', 'pt']) p.set_defaults(func=bdev_ocf_set_cachemode) @@ -187,7 +191,8 @@ def bdev_ocf_set_promotion(args): choices=['always', 'nhit'], default='none') p.add_argument('-i', '--nhit-insertion-threshold', - help='number of requests for given core line after which NHIT policy allows insertion into cache (range <2-1000>; default 3)', + help='number of requests for given core line after which NHIT policy \ + allows insertion into cache (range <2-1000>; default 3)', type=int, default=-1) p.add_argument('-t', '--nhit-trigger-threshold', @@ -218,7 +223,8 @@ def bdev_ocf_set_cleaning(args): type=int, default=-1) p.add_argument('-m', '--acp-flush-max-buffers', - help='number of dirty cache lines to be flushed in a single ACP cleaning thread iteration (range <1-10000>; default 128)', + help='number of dirty cache lines to be flushed in a single ACP \ + cleaning thread iteration (range <1-10000>; default 128)', type=int, default=-1) p.add_argument('-v', '--alru-wake-up-time', @@ -230,7 +236,8 @@ def bdev_ocf_set_cleaning(args): type=int, default=-1) p.add_argument('-s', '--alru-staleness-time', - help='time that has to pass from the last write operation before a dirty cache line can be scheduled to be flushed (range <1-3600> [s]; default 120)', + help='time that has to pass from the last write operation before a dirty cache line \ + can be scheduled to be flushed (range <1-3600> [s]; default 120)', type=int, default=-1) p.add_argument('-t', '--alru-activity-threshold', @@ -265,7 +272,8 @@ def bdev_ocf_set_seqcutoff(args): type=int, default=-1) p.add_argument('-o', '--promote-on-threshold', - help='whether to promote core sequential cut-off stream to global structures when threshold is reached (choose between {0|1}; default 0)', + help='whether to promote core sequential cut-off stream to global structures \ + when threshold is reached (choose between {0|1}; default 0)', choices=[0, 1], type=int, default=-1) @@ -273,7 +281,10 @@ def bdev_ocf_set_seqcutoff(args): def bdev_ocf_flush_start(args): args.client.bdev_ocf_flush_start(bdev_name=args.bdev_name) - p = subparsers.add_parser('bdev_ocf_flush_start', help='Flush all dirty data on the given OCF device (from cache to all of its cores or to particular core only). Note that this call only starts the flushing process which will be running in background and may take some time depending on the underlying device size and speed. You can check flushing status using the bdev_ocf_get_bdevs call.') + p = subparsers.add_parser('bdev_ocf_flush_start', help='Flush all dirty data on the given OCF device \ + (from cache to all of its cores or to particular core only). Note that this call only starts \ + the flushing process which will be running in background and may take some time depending on \ + the underlying device size and speed. You can check flushing status using the bdev_ocf_get_bdevs call.') p.add_argument('bdev_name', help='name of OCF vbdev to flush') p.set_defaults(func=bdev_ocf_flush_start) diff --git a/test/ocf/common.sh b/test/ocf/common.sh index a4bb46475ba..3c750ce547c 100644 --- a/test/ocf/common.sh +++ b/test/ocf/common.sh @@ -42,7 +42,7 @@ destroy_caches() { create_caches_persistent() { for i in {1..3}; do - $rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[i-1]}" -d blk Cache_dev$i + $rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[i - 1]}" -d blk Cache_dev$i done # Give a bit more time to settle, because this will be called after @@ -195,7 +195,7 @@ random_number() { # $2: upper range (inclusive) # stdout: random number between range - echo $(($1 + $RANDOM % ($2-$1+1))) + echo $(($1 + RANDOM % ($2 - $1 + 1))) } # Convert an array of items to comma separated list of items. diff --git a/test/ocf/management/config_change.sh b/test/ocf/management/config_change.sh index 30901823e4e..33468a7f24e 100755 --- a/test/ocf/management/config_change.sh +++ b/test/ocf/management/config_change.sh @@ -37,7 +37,7 @@ seqcutoff_promote_on_threshold_default=false set_cache_mode() { if [ $# -ne 1 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local cache_mode=$1 @@ -51,7 +51,7 @@ set_cache_mode() { set_promotion_always_params() { if [ $# -ne 1 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -66,7 +66,7 @@ set_promotion_always_params() { set_promotion_nhit_params() { if [ $# -ne 3 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -85,7 +85,7 @@ set_promotion_nhit_params() { set_cleaning_alru_params() { if [ $# -ne 6 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -110,7 +110,7 @@ set_cleaning_alru_params() { set_cleaning_acp_params() { if [ $# -ne 3 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -129,7 +129,7 @@ set_cleaning_acp_params() { set_cleaning_nop_params() { if [ $# -ne 1 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -144,7 +144,7 @@ set_cleaning_nop_params() { set_seqcutoff_params() { if [ $# -ne 4 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -165,7 +165,7 @@ set_seqcutoff_params() { set_seqcutoff_params_all() { if [ $# -ne 4 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -186,7 +186,7 @@ set_seqcutoff_params_all() { __check_cache_mode() { if [ $# -ne 1 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local cache_mode=$1 @@ -198,7 +198,7 @@ __check_cache_mode() { __check_promotion_always_params() { if [ $# -ne 1 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -211,7 +211,7 @@ __check_promotion_always_params() { __check_promotion_nhit_params() { if [ $# -ne 3 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -231,7 +231,7 @@ __check_promotion_nhit_params() { __check_cleaning_alru_params() { if [ $# -ne 6 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -260,7 +260,7 @@ __check_cleaning_alru_params() { __check_cleaning_acp_params() { if [ $# -ne 3 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -280,7 +280,7 @@ __check_cleaning_acp_params() { __check_cleaning_nop_params() { if [ $# -ne 1 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -293,7 +293,7 @@ __check_cleaning_nop_params() { __check_seqcutoff_params() { if [ $# -ne 4 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -331,7 +331,7 @@ __check_seqcutoff_params() { __check_seqcutoff_params_all() { if [ $# -ne 4 ]; then echo >&2 "invalid number of arguments" - exit 1; + exit 1 fi local policy=$1 @@ -358,133 +358,133 @@ __check_seqcutoff_params_all() { # cache modes: for add_cores in false true; do -for create_caches in false true; do -for stop_caches in false true; do - start_spdk - start_caches - if [ $create_caches = true ]; then - create_caches - __check_caches_attached - if [ $add_cores = true ]; then - create_cores - add_cores - __check_cores_attached - fi - else - __check_caches_detached - fi - for cache_mode in "${cache_modes[@]}"; do - set_cache_mode $cache_mode - __check_cache_mode $cache_mode + for create_caches in false true; do + for stop_caches in false true; do + start_spdk + start_caches + if [ $create_caches = true ]; then + create_caches + __check_caches_attached + if [ $add_cores = true ]; then + create_cores + add_cores + __check_cores_attached + fi + else + __check_caches_detached + fi + for cache_mode in "${cache_modes[@]}"; do + set_cache_mode $cache_mode + __check_cache_mode $cache_mode + done + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done -done done # promotion params: for promotion_policy in "${promotion_policies[@]}"; do -for add_cores in false true; do -for stop_caches in false true; do - start_spdk - create_caches - start_caches - __check_caches_attached - if [ $add_cores = true ]; then - create_cores - add_cores - __check_cores_attached - fi - if [ $promotion_policy = always ]; then - set_promotion_always_params $promotion_policy - __check_promotion_always_params $promotion_policy - elif [ $promotion_policy = nhit ]; then - insertion_threshold=$(random_number ${promotion_nhit_insertion_threshold_range[0]} ${promotion_nhit_insertion_threshold_range[1]}) - trigger_threshold=$(random_number ${promotion_nhit_trigger_threshold_range[0]} ${promotion_nhit_trigger_threshold_range[1]}) - set_promotion_nhit_params $promotion_policy $insertion_threshold $trigger_threshold - __check_promotion_nhit_params $promotion_policy $insertion_threshold $trigger_threshold - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done -done + for add_cores in false true; do + for stop_caches in false true; do + start_spdk + create_caches + start_caches + __check_caches_attached + if [ $add_cores = true ]; then + create_cores + add_cores + __check_cores_attached + fi + if [ $promotion_policy = always ]; then + set_promotion_always_params $promotion_policy + __check_promotion_always_params $promotion_policy + elif [ $promotion_policy = nhit ]; then + insertion_threshold=$(random_number ${promotion_nhit_insertion_threshold_range[0]} ${promotion_nhit_insertion_threshold_range[1]}) + trigger_threshold=$(random_number ${promotion_nhit_trigger_threshold_range[0]} ${promotion_nhit_trigger_threshold_range[1]}) + set_promotion_nhit_params $promotion_policy $insertion_threshold $trigger_threshold + __check_promotion_nhit_params $promotion_policy $insertion_threshold $trigger_threshold + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done + done done # cleaning params: for cleaning_policy in "${cleaning_policies[@]}"; do -for add_cores in false true; do -for stop_caches in false true; do - start_spdk - create_caches - start_caches - __check_caches_attached - if [ $add_cores = true ]; then - create_cores - add_cores - __check_cores_attached - fi - if [ $cleaning_policy = alru ]; then - wake_up_time=$(random_number ${cleaning_alru_wake_up_time_range[0]} ${cleaning_alru_wake_up_time_range[1]}) - flush_max_buffers=$(random_number ${cleaning_alru_flush_max_buffers_range[0]} ${cleaning_alru_flush_max_buffers_range[1]}) - staleness_time=$(random_number ${cleaning_alru_staleness_time_range[0]} ${cleaning_alru_staleness_time_range[1]}) - activity_threshold=$(random_number ${cleaning_alru_activity_threshold_range[0]} ${cleaning_alru_activity_threshold_range[1]}) - max_dirty_ratio=$(random_number ${cleaning_alru_max_dirty_ratio_range[0]} ${cleaning_alru_max_dirty_ratio_range[1]}) - set_cleaning_alru_params $cleaning_policy $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio - __check_cleaning_alru_params $cleaning_policy $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio - elif [ $cleaning_policy = acp ]; then - wake_up_time=$(random_number ${cleaning_acp_wake_up_time_range[0]} ${cleaning_acp_wake_up_time_range[1]}) - flush_max_buffers=$(random_number ${cleaning_acp_flush_max_buffers_range[0]} ${cleaning_acp_flush_max_buffers_range[1]}) - set_cleaning_acp_params $cleaning_policy $wake_up_time $flush_max_buffers - __check_cleaning_acp_params $cleaning_policy $wake_up_time $flush_max_buffers - elif [ $cleaning_policy = nop ]; then - set_cleaning_nop_params $cleaning_policy - __check_cleaning_nop_params $cleaning_policy - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done -done + for add_cores in false true; do + for stop_caches in false true; do + start_spdk + create_caches + start_caches + __check_caches_attached + if [ $add_cores = true ]; then + create_cores + add_cores + __check_cores_attached + fi + if [ $cleaning_policy = alru ]; then + wake_up_time=$(random_number ${cleaning_alru_wake_up_time_range[0]} ${cleaning_alru_wake_up_time_range[1]}) + flush_max_buffers=$(random_number ${cleaning_alru_flush_max_buffers_range[0]} ${cleaning_alru_flush_max_buffers_range[1]}) + staleness_time=$(random_number ${cleaning_alru_staleness_time_range[0]} ${cleaning_alru_staleness_time_range[1]}) + activity_threshold=$(random_number ${cleaning_alru_activity_threshold_range[0]} ${cleaning_alru_activity_threshold_range[1]}) + max_dirty_ratio=$(random_number ${cleaning_alru_max_dirty_ratio_range[0]} ${cleaning_alru_max_dirty_ratio_range[1]}) + set_cleaning_alru_params $cleaning_policy $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio + __check_cleaning_alru_params $cleaning_policy $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio + elif [ $cleaning_policy = acp ]; then + wake_up_time=$(random_number ${cleaning_acp_wake_up_time_range[0]} ${cleaning_acp_wake_up_time_range[1]}) + flush_max_buffers=$(random_number ${cleaning_acp_flush_max_buffers_range[0]} ${cleaning_acp_flush_max_buffers_range[1]}) + set_cleaning_acp_params $cleaning_policy $wake_up_time $flush_max_buffers + __check_cleaning_acp_params $cleaning_policy $wake_up_time $flush_max_buffers + elif [ $cleaning_policy = nop ]; then + set_cleaning_nop_params $cleaning_policy + __check_cleaning_nop_params $cleaning_policy + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done + done done # sequential cut-off params: for device in core cache; do -for seqcutoff_policy in "${seqcutoff_policies[@]}"; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - threshold=$(random_number ${seqcutoff_threshold_range[0]} ${seqcutoff_threshold_range[1]}) - promotion_count=$(random_number ${seqcutoff_promotion_count_range[0]} ${seqcutoff_promotion_count_range[1]}) - promote_on_threshold=$(random_number ${seqcutoff_promote_on_threshold_range[0]} ${seqcutoff_promote_on_threshold_range[1]}) - if [ $device = core ]; then - set_seqcutoff_params $seqcutoff_policy $threshold $promotion_count $promote_on_threshold - __check_seqcutoff_params $seqcutoff_policy $threshold $promotion_count $promote_on_threshold - elif [ $device = cache ]; then - set_seqcutoff_params_all $seqcutoff_policy $threshold $promotion_count $promote_on_threshold - __check_seqcutoff_params_all $seqcutoff_policy $threshold $promotion_count $promote_on_threshold - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done -done + for seqcutoff_policy in "${seqcutoff_policies[@]}"; do + for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + threshold=$(random_number ${seqcutoff_threshold_range[0]} ${seqcutoff_threshold_range[1]}) + promotion_count=$(random_number ${seqcutoff_promotion_count_range[0]} ${seqcutoff_promotion_count_range[1]}) + promote_on_threshold=$(random_number ${seqcutoff_promote_on_threshold_range[0]} ${seqcutoff_promote_on_threshold_range[1]}) + if [ $device = core ]; then + set_seqcutoff_params $seqcutoff_policy $threshold $promotion_count $promote_on_threshold + __check_seqcutoff_params $seqcutoff_policy $threshold $promotion_count $promote_on_threshold + elif [ $device = cache ]; then + set_seqcutoff_params_all $seqcutoff_policy $threshold $promotion_count $promote_on_threshold + __check_seqcutoff_params_all $seqcutoff_policy $threshold $promotion_count $promote_on_threshold + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done + done done diff --git a/test/ocf/management/detach-attach.sh b/test/ocf/management/detach-attach.sh index 977ebbed0bf..592fd974487 100755 --- a/test/ocf/management/detach-attach.sh +++ b/test/ocf/management/detach-attach.sh @@ -13,141 +13,141 @@ source "$rootdir/test/ocf/common.sh" # caches only: for attach_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - start_caches - __check_caches_attached - detach_caches - __check_caches_detached - if [ $attach_caches = true ]; then - attach_caches + for stop_caches in false true; do + start_spdk + create_caches + start_caches __check_caches_attached - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + detach_caches + __check_caches_detached + if [ $attach_caches = true ]; then + attach_caches + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # add cores before detaching caches: for attach_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - detach_caches - __check_caches_detached - __check_cores_attached - if [ $attach_caches = true ]; then - attach_caches + for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores __check_caches_attached __check_cores_attached - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + detach_caches + __check_caches_detached + __check_cores_attached + if [ $attach_caches = true ]; then + attach_caches + __check_caches_attached + __check_cores_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # add cores after detaching caches: for attach_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - __check_caches_attached - detach_caches - __check_caches_detached - add_cores - __check_caches_detached - __check_cores_waitlist_attached - if [ $attach_caches = true ]; then - attach_caches + for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches __check_caches_attached - __check_cores_attached - __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + detach_caches + __check_caches_detached + add_cores + __check_caches_detached + __check_cores_waitlist_attached + if [ $attach_caches = true ]; then + attach_caches + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # add cores before and remove after detaching caches: for attach_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - detach_caches - __check_caches_detached - __check_cores_attached - remove_cores - __check_caches_detached - __check_cores_empty - if [ $attach_caches = true ]; then - attach_caches + for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores __check_caches_attached + __check_cores_attached + detach_caches + __check_caches_detached + __check_cores_attached + remove_cores + __check_caches_detached __check_cores_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + if [ $attach_caches = true ]; then + attach_caches + __check_caches_attached + __check_cores_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # add cores after detaching caches and remove before/without attaching: for attach_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - __check_caches_attached - detach_caches - __check_caches_detached - add_cores - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_attached - remove_cores - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_empty - if [ $attach_caches = true ]; then - attach_caches + for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches __check_caches_attached + detach_caches + __check_caches_detached + add_cores + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_attached + remove_cores + __check_caches_detached __check_cores_empty __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + if [ $attach_caches = true ]; then + attach_caches + __check_caches_attached + __check_cores_empty + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # add cores after detaching caches and remove after attaching: diff --git a/test/ocf/management/hotremove.sh b/test/ocf/management/hotremove.sh index 605892d4c6a..049248a7312 100755 --- a/test/ocf/management/hotremove.sh +++ b/test/ocf/management/hotremove.sh @@ -75,434 +75,434 @@ __check_cores_waitlist_detached_all_but_first() { # cores in waitlist: for remove_cores in false true; do -for create_cores in false true; do - start_spdk - create_cores - add_cores - __check_cores_waitlist_attached - destroy_cores - __check_cores_waitlist_detached - if [ $create_cores = true ]; then + for create_cores in false true; do + start_spdk create_cores + add_cores __check_cores_waitlist_attached - fi - if [ $remove_cores = true ]; then - remove_cores - __check_cores_waitlist_empty - fi - stop_spdk -done + destroy_cores + __check_cores_waitlist_detached + if [ $create_cores = true ]; then + create_cores + __check_cores_waitlist_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + stop_spdk + done done # cores in waitlist (destroying only first): for remove_cores in false true; do -for create_cores in false true; do - start_spdk - create_cores - add_cores - __check_cores_waitlist_attached - for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_cores_waitlist_detached_only_first - if [ $create_cores = true ]; then + for create_cores in false true; do + start_spdk + create_cores + add_cores + __check_cores_waitlist_attached for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 + $rpc_py bdev_malloc_delete Core_dev$i-1 done $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_cores_waitlist_attached - fi - if [ $remove_cores = true ]; then - remove_cores - __check_cores_waitlist_empty - fi - stop_spdk -done + __check_cores_waitlist_detached_only_first + if [ $create_cores = true ]; then + for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_cores_waitlist_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + stop_spdk + done done # cores in waitlist (destroying all but first): for remove_cores in false true; do -for create_cores in false true; do - start_spdk - create_cores - add_cores - __check_cores_waitlist_attached - for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_cores_waitlist_detached_all_but_first - if [ $create_cores = true ]; then + for create_cores in false true; do + start_spdk + create_cores + add_cores + __check_cores_waitlist_attached for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 - $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 done $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_cores_waitlist_attached - fi - if [ $remove_cores = true ]; then - remove_cores - __check_cores_waitlist_empty - fi - stop_spdk -done + __check_cores_waitlist_detached_all_but_first + if [ $create_cores = true ]; then + for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 + $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_cores_waitlist_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + stop_spdk + done done # caches only: for create_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - start_caches - __check_caches_attached - destroy_caches - __check_caches_detached - if [ $create_caches = true ]; then + for stop_caches in false true; do + start_spdk create_caches + start_caches __check_caches_attached - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + destroy_caches + __check_caches_detached + if [ $create_caches = true ]; then + create_caches + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # caches only (destroying only first): for create_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - start_caches - __check_caches_attached - $rpc_py bdev_malloc_delete Cache_dev1 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_caches_detached_only_first - if [ $create_caches = true ]; then - $rpc_py bdev_malloc_create -b Cache_dev1 100 512 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + for stop_caches in false true; do + start_spdk + create_caches + start_caches __check_caches_attached - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + $rpc_py bdev_malloc_delete Cache_dev1 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_detached_only_first + if [ $create_caches = true ]; then + $rpc_py bdev_malloc_create -b Cache_dev1 100 512 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # caches only (destroying all but first): for create_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - start_caches - __check_caches_attached - $rpc_py bdev_malloc_delete Cache_dev2 - $rpc_py bdev_malloc_delete Cache_dev3 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_caches_detached_all_but_first - if [ $create_caches = true ]; then - $rpc_py bdev_malloc_create -b Cache_dev2 100 512 - $rpc_py bdev_malloc_create -b Cache_dev3 100 512 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + for stop_caches in false true; do + start_spdk + create_caches + start_caches __check_caches_attached - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + $rpc_py bdev_malloc_delete Cache_dev2 + $rpc_py bdev_malloc_delete Cache_dev3 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_detached_all_but_first + if [ $create_caches = true ]; then + $rpc_py bdev_malloc_create -b Cache_dev2 100 512 + $rpc_py bdev_malloc_create -b Cache_dev3 100 512 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # hotremove caches: for remove_cores in false true; do -for create_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - destroy_caches - __check_caches_detached - __check_cores_attached - if [ $remove_cores = true ]; then - remove_cores - __check_caches_detached - __check_cores_empty - fi - if [ $create_caches = true ]; then - create_caches - __check_caches_attached - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done -done + for create_caches in false true; do + for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + destroy_caches + __check_caches_detached + __check_cores_attached + if [ $remove_cores = true ]; then + remove_cores + __check_caches_detached + __check_cores_empty + fi + if [ $create_caches = true ]; then + create_caches + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done + done done # hotremove caches (destroying only first): for remove_cores in false true; do -for create_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - $rpc_py bdev_malloc_delete Cache_dev1 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_caches_detached_only_first - __check_cores_attached - if [ $remove_cores = true ]; then - remove_cores - __check_caches_detached_only_first - __check_cores_empty - fi - if [ $create_caches = true ]; then - $rpc_py bdev_malloc_create -b Cache_dev1 100 512 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_caches_attached - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done -done + for create_caches in false true; do + for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + $rpc_py bdev_malloc_delete Cache_dev1 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_detached_only_first + __check_cores_attached + if [ $remove_cores = true ]; then + remove_cores + __check_caches_detached_only_first + __check_cores_empty + fi + if [ $create_caches = true ]; then + $rpc_py bdev_malloc_create -b Cache_dev1 100 512 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done + done done # hotremove caches (destroying all but first): for remove_cores in false true; do -for create_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - $rpc_py bdev_malloc_delete Cache_dev2 - $rpc_py bdev_malloc_delete Cache_dev3 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_caches_detached_all_but_first - __check_cores_attached - if [ $remove_cores = true ]; then - remove_cores - __check_caches_detached_all_but_first - __check_cores_empty - fi - if [ $create_caches = true ]; then - $rpc_py bdev_malloc_create -b Cache_dev2 100 512 - $rpc_py bdev_malloc_create -b Cache_dev3 100 512 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_caches_attached - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done -done + for create_caches in false true; do + for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + $rpc_py bdev_malloc_delete Cache_dev2 + $rpc_py bdev_malloc_delete Cache_dev3 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_detached_all_but_first + __check_cores_attached + if [ $remove_cores = true ]; then + remove_cores + __check_caches_detached_all_but_first + __check_cores_empty + fi + if [ $create_caches = true ]; then + $rpc_py bdev_malloc_create -b Cache_dev2 100 512 + $rpc_py bdev_malloc_create -b Cache_dev3 100 512 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done + done done # hotremove cores: for create_cores in false true; do -for remove_cores in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - destroy_cores - __check_caches_attached - __check_cores_detached - if [ $create_cores = true ]; then - create_cores - __check_caches_attached - __check_cores_attached - fi - if [ $remove_cores = true ]; then - remove_cores - __check_caches_attached - __check_cores_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done -done + for remove_cores in false true; do + for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + destroy_cores + __check_caches_attached + __check_cores_detached + if [ $create_cores = true ]; then + create_cores + __check_caches_attached + __check_cores_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_caches_attached + __check_cores_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done + done done # hotremove cores (destroying only first): for create_cores in false true; do -for remove_cores in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_caches_attached - __check_cores_detached_only_first - if [ $create_cores = true ]; then - for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 + for remove_cores in false true; do + for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-1 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + __check_cores_detached_only_first + if [ $create_cores = true ]; then + for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + __check_cores_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_caches_attached + __check_cores_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_caches_attached - __check_cores_attached - fi - if [ $remove_cores = true ]; then - remove_cores - __check_caches_attached - __check_cores_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done -done + done done # hotremove cores (destroying all but first): for create_cores in false true; do -for remove_cores in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_caches_attached - __check_cores_detached_all_but_first - if [ $create_cores = true ]; then - for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 - $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 + for remove_cores in false true; do + for stop_caches in false true; do + start_spdk + create_caches + create_cores + start_caches + add_cores + __check_caches_attached + __check_cores_attached + for i in {1..3}; do + $rpc_py bdev_malloc_delete Core_dev$i-2 + $rpc_py bdev_malloc_delete Core_dev$i-3 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + __check_cores_detached_all_but_first + if [ $create_cores = true ]; then + for i in {1..3}; do + $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 + $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_attached + __check_cores_attached + fi + if [ $remove_cores = true ]; then + remove_cores + __check_caches_attached + __check_cores_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' - __check_caches_attached - __check_cores_attached - fi - if [ $remove_cores = true ]; then - remove_cores - __check_caches_attached - __check_cores_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done -done + done done # hotremove caches and cores: for create_caches_and_cores in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - destroy_caches - __check_caches_detached - destroy_cores - __check_cores_detached - if [ $create_caches_and_cores = true ]; then + for stop_caches in false true; do + start_spdk create_caches - __check_caches_attached create_cores + start_caches + add_cores + __check_caches_attached __check_cores_attached - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + destroy_caches + __check_caches_detached + destroy_cores + __check_cores_detached + if [ $create_caches_and_cores = true ]; then + create_caches + __check_caches_attached + create_cores + __check_cores_attached + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # hotremove caches and cores, then manually remove cores: for create_caches in false true; do -for stop_caches in false true; do - start_spdk - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - destroy_caches - __check_caches_detached - destroy_cores - __check_cores_detached - remove_cores - __check_caches_detached - __check_cores_empty - if [ $create_caches = true ]; then + for stop_caches in false true; do + start_spdk create_caches + create_cores + start_caches + add_cores __check_caches_attached + __check_cores_attached + destroy_caches + __check_caches_detached + destroy_cores + __check_cores_detached + remove_cores + __check_caches_detached __check_cores_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + if [ $create_caches = true ]; then + create_caches + __check_caches_attached + __check_cores_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # hotremove caches and cores (inverse create after destroy order): diff --git a/test/ocf/management/start-stop/examine.sh b/test/ocf/management/start-stop/examine.sh index 80448cc9b4a..c8ea6a3bf94 100755 --- a/test/ocf/management/start-stop/examine.sh +++ b/test/ocf/management/start-stop/examine.sh @@ -102,26 +102,26 @@ done # found core device, then start cache: for remove_cores in false true; do -for stop_caches in false true; do - start_spdk - add_cores - __check_cores_waitlist_detached - create_cores - __check_cores_waitlist_attached - start_caches - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_attached - if [ $remove_cores = true ]; then - remove_cores - __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + for stop_caches in false true; do + start_spdk + add_cores + __check_cores_waitlist_detached + create_cores + __check_cores_waitlist_attached + start_caches + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_attached + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # found cache and then core device: diff --git a/test/ocf/management/start-stop/incomplete.sh b/test/ocf/management/start-stop/incomplete.sh index 525af974dda..02250f378f0 100755 --- a/test/ocf/management/start-stop/incomplete.sh +++ b/test/ocf/management/start-stop/incomplete.sh @@ -39,66 +39,66 @@ done # incomplete caches and incomplete cores: for remove_cores in false true; do -for stop_caches in false true; do - start_spdk - start_caches - add_cores - __check_caches_detached - __check_cores_waitlist_detached - if [ $remove_cores = true ]; then - remove_cores - __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + for stop_caches in false true; do + start_spdk + start_caches + add_cores + __check_caches_detached + __check_cores_waitlist_detached + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # complete caches and incomplete cores: for remove_cores in false true; do -for stop_caches in false true; do - start_spdk - create_caches - start_caches - add_cores - __check_caches_attached - __check_cores_empty - __check_cores_waitlist_detached - if [ $remove_cores = true ]; then - remove_cores - __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + for stop_caches in false true; do + start_spdk + create_caches + start_caches + add_cores + __check_caches_attached + __check_cores_empty + __check_cores_waitlist_detached + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # incomplete caches and complete cores: for remove_cores in false true; do -for stop_caches in false true; do - start_spdk - create_cores - start_caches - add_cores - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_attached - if [ $remove_cores = true ]; then - remove_cores - __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + for stop_caches in false true; do + start_spdk + create_cores + start_caches + add_cores + __check_caches_detached + __check_cores_empty + __check_cores_waitlist_attached + if [ $remove_cores = true ]; then + remove_cores + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done diff --git a/test/ocf/management/start-stop/load.sh b/test/ocf/management/start-stop/load.sh index 41959d251e6..da7ca3a08dd 100755 --- a/test/ocf/management/start-stop/load.sh +++ b/test/ocf/management/start-stop/load.sh @@ -53,50 +53,50 @@ create_caches_with_metadata_with_cores_persistent() { # load caches without cores in metadata: for add_cores in false true; do -for stop_caches in false true; do - start_spdk - create_caches_with_metadata - create_cores - start_caches_try_load - __check_caches_attached - __check_cores_empty - __check_cores_waitlist_empty - if [ $add_cores = true ]; then - add_cores - __check_cores_attached + for stop_caches in false true; do + start_spdk + create_caches_with_metadata + create_cores + start_caches_try_load + __check_caches_attached + __check_cores_empty __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + if [ $add_cores = true ]; then + add_cores + __check_cores_attached + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # load caches with cores in metadata: for add_cores in false true; do -for stop_caches in false true; do - start_spdk - create_caches_with_metadata_with_cores - create_cores - start_caches_try_load - __check_caches_attached - __check_cores_loading - __check_cores_waitlist_empty - if [ $add_cores = true ]; then - add_cores + for stop_caches in false true; do + start_spdk + create_caches_with_metadata_with_cores + create_cores + start_caches_try_load __check_caches_attached - __check_cores_attached + __check_cores_loading __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk -done + if [ $add_cores = true ]; then + add_cores + __check_caches_attached + __check_cores_attached + __check_cores_waitlist_empty + fi + if [ $stop_caches = true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk + done done # load caches with cores in metadata (add cores first): From 445e59ede5d2081345e70ed7975306ea82f5617b Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Wed, 24 Sep 2025 11:04:34 +0200 Subject: [PATCH 19/34] Update documentation Change-Id: Id6ddfa4042427715f7927c7d66fb98ee9904881c Signed-off-by: Rafal Stefanowski --- doc/bdev.md | 57 ++- doc/jsonrpc.md.jinja2 | 812 +++++++++++++++++++++++++++------------- python/spdk/cli/bdev.py | 12 +- 3 files changed, 589 insertions(+), 292 deletions(-) diff --git a/doc/bdev.md b/doc/bdev.md index bb5ad1625cf..fd4b08614cd 100644 --- a/doc/bdev.md +++ b/doc/bdev.md @@ -311,32 +311,57 @@ To delete an aio bdev use the bdev_aio_delete command. `rpc.py bdev_aio_delete aio0` -## OCF Virtual bdev {#bdev_config_cas} +## OCF Virtual bdev {#bdev_config_ocf} -OCF virtual bdev module is based on [Open CAS Framework](https://github.com/Open-CAS/ocf) - a -high performance block storage caching meta-library. +OCF virtual bdev module is based on [Open CAS Framework](https://github.com/Open-CAS/ocf) - a high performance block storage caching meta-library.
To enable the module, configure SPDK using `--with-ocf` flag. OCF bdev can be used to enable caching for any underlying bdev. -Below is an example command for creating OCF bdev: +To use OCF caching, first we need to create a caching device: -`rpc.py bdev_ocf_create Cache1 wt Malloc0 Nvme0n1` +`rpc.py bdev_ocf_start_cache Ocf_cache0 Malloc0` -This command will create new OCF bdev `Cache1` having bdev `Malloc0` as caching-device -and `Nvme0n1` as core-device and initial cache mode `Write-Through`. -`Malloc0` will be used as cache for `Nvme0n1`, so data written to `Cache1` will be present -on `Nvme0n1` eventually. -By default, OCF will be configured with cache line size equal 4KiB -and non-volatile metadata will be disabled. +This will start OCF cache **Ocf_cache0** using **Malloc0** as a caching device. **Ocf_cache0** will be used internally by OCF and will not be exposed as an SPDK bdev. -To remove `Cache1`: +Then we can add a storage (core) device to our cache: -`rpc.py bdev_ocf_delete Cache1` +`rpc.py bdev_ocf_add_core Ocf_core0 Nvme0n1 Ocf_cache0` -During removal OCF-cache will be stopped and all cached data will be written to the core device. +This will add **Ocf_core0** to previously created cache instance **Ocf_cache0** using **Nvme0n1** as core device. +**Ocf_core0** on the other hand, will be registered in the SPDK bdev layer and will be available for usage as any other SPDK bdev. +(You can add up to 4096 core devices to a single cache instance.)
+In such configuration **Malloc0** will be used as cache for **Nvme0n1**, +so data written to **Ocf_core0** will be present on **Nvme0n1** eventually, depending on chosen cache mode.
+By default, OCF will be configured with Write-Through cache mode, 4KiB cache line size and loading of previous configuration if OCF metadata is present on the cache device. +You can change those parameters when creating cache with `bdev_ocf_start_cache`. In addition, cache mode can also be changed in runtime using `bdev_ocf_set_cachemode`.
+There are also other caching parameters to tweak (including: promotion, cleaning or sequential stream cut-off), +as well as issuing manual flushing of dirty data or dumping OCF statistics. -Note that OCF has a per-device RAM requirement. More details can be found in the -[OCF documentation](https://open-cas.github.io/guide_system_requirements.html). +To remove particular core device: + +`rpc.py bdev_ocf_remove_core Ocf_core0` + +To stop the whole OCF cache (and remove all of its core devices): + +`rpc.py bdev_ocf_stop_cache Ocf_cache0` + +During core removal or cache stop all cached data will be written (flushed) to the core device first. + +A few things to mention: + +- If cache's base bdev contains OCF metadata from previous runs, cache configuration will be loaded from the metadata, unless the no-load flag is specified. + This configuration includes all the cores that were added to this cache before it was stopped. + Those cores however will be added only as a place holders with a "loading" status until manually added to cache with `bdev_ocf_add_core`. + Also, the cache itself will be in incomplete state until all of its cores are added. This behavior has two reasons. + First and foremost, it prevents data corruption in case an arbitrary bdev is automatically added as a core device just because of a matching name. + And second, it allows for the same JSON bdev config either it's loading or starting a new cache. Such incomplete cache state might also be desired, + e.g. to just inspect which cores were previously added to cache. + +- When the core device is added using `bdev_ocf_add_core` but its base bdev or cache bdev is not present yet (or cache is in detached state - no base bdev), + it is put on a temporary core wait list. Such core will be automatically moved from this wait list to the cache once its base bdev is created and its cache is up and running. + This means that all constructing operations can be issued in any order and the outcome should always be the same. + +- OCF has a per-device RAM requirement. More details can be found in the [OCF documentation](https://open-cas.github.io/guide_system_requirements.html). ## Malloc bdev {#bdev_config_malloc} diff --git a/doc/jsonrpc.md.jinja2 b/doc/jsonrpc.md.jinja2 index deb49ee1fc2..764b22c6615 100644 --- a/doc/jsonrpc.md.jinja2 +++ b/doc/jsonrpc.md.jinja2 @@ -2898,19 +2898,19 @@ Example response: } ~~~ -### bdev_ocf_create {#rpc_bdev_ocf_create} +### bdev_ocf_start_cache {#rpc_bdev_ocf_start_cache} -Construct new OCF bdev. -Command accepts cache mode that is going to be used. -You can find more details about supported cache modes in the [OCF documentation](https://open-cas.github.io/cache_configuration.html#cache-mode) +Start OCF cache instance. + +You can find more details about supported options in the [OCF documentation](https://open-cas.com/cache_configuration.html) #### Parameters -{{ bdev_ocf_create_params }} +{{ bdev_ocf_start_cache_params }} #### Response -Name of newly created bdev. +Name of created OCF cache vbdev. #### Example @@ -2919,14 +2919,14 @@ Example request: ~~~json { "params": { - "name": "ocf0", - "mode": "wt", + "cache_name": "Ocf_cache0", + "base_name": "Malloc0", + "cache_mode": "wb", "cache_line_size": 64, - "cache_bdev_name": "Nvme0n1", - "core_bdev_name": "aio0" + "no_load": true }, "jsonrpc": "2.0", - "method": "bdev_ocf_create", + "method": "bdev_ocf_start_cache", "id": 1 } ~~~ @@ -2937,17 +2937,17 @@ Example response: { "jsonrpc": "2.0", "id": 1, - "result": "ocf0" + "result": "Ocf_cache0" } ~~~ -### bdev_ocf_delete {#rpc_bdev_ocf_delete} +### bdev_ocf_stop_cache {#rpc_bdev_ocf_stop_cache} -Delete the OCF bdev +Stop OCF cache instance. #### Parameters -{{ bdev_ocf_delete_params }} +{{ bdev_ocf_stop_cache_params }} #### Example @@ -2956,10 +2956,10 @@ Example request: ~~~json { "params": { - "name": "ocf0" + "cache_name": "Ocf_cache0" }, "jsonrpc": "2.0", - "method": "bdev_ocf_delete", + "method": "bdev_ocf_stop_cache", "id": 1 } ~~~ @@ -2974,17 +2974,46 @@ Example response: } ~~~ -### bdev_ocf_get_stats {#rpc_bdev_ocf_get_stats} +### bdev_ocf_detach_cache {#rpc_bdev_ocf_detach_cache} -Get statistics of chosen OCF block device. +Detach caching device from OCF cache. #### Parameters -{{ bdev_ocf_get_stats_params }} +{{ bdev_ocf_detach_cache_params }} -#### Response +#### Example -Statistics as json object. +Example request: + +~~~json +{ + "params": { + "cache_name": "Ocf_cache0" + }, + "jsonrpc": "2.0", + "method": "bdev_ocf_detach_cache", + "id": 1 +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +### bdev_ocf_attach_cache {#rpc_bdev_ocf_attach_cache} + +Attach caching device to OCF cache. + +#### Parameters + +{{ bdev_ocf_attach_cache_params }} #### Example @@ -2993,10 +3022,12 @@ Example request: ~~~json { "params": { - "name": "ocf0" + "cache_name": "Ocf_cache0", + "base_name": "Nvme0n1", + "force": true }, "jsonrpc": "2.0", - "method": "bdev_ocf_get_stats", + "method": "bdev_ocf_attach_cache", "id": 1 } ~~~ @@ -3007,198 +3038,21 @@ Example response: { "jsonrpc": "2.0", "id": 1, - "result": [ - { - "usage": { - "clean": { - "count": 76033, - "units": "4KiB blocks", - "percentage": "100.0" - }, - "free": { - "count": 767, - "units": "4KiB blocks", - "percentage": "0.9" - }, - "occupancy": { - "count": 76033, - "units": "4KiB blocks", - "percentage": "99.0" - }, - "dirty": { - "count": 0, - "units": "4KiB blocks", - "percentage": "0.0" - } - } - }, - { - "requests": { - "rd_total": { - "count": 2, - "units": "Requests", - "percentage": "0.0" - }, - "wr_full_misses": { - "count": 76280, - "units": "Requests", - "percentage": "35.6" - }, - "rd_full_misses": { - "count": 1, - "units": "Requests", - "percentage": "0.0" - }, - "rd_partial_misses": { - "count": 0, - "units": "Requests", - "percentage": "0.0" - }, - "wr_total": { - "count": 212416, - "units": "Requests", - "percentage": "99.2" - }, - "wr_pt": { - "count": 1535, - "units": "Requests", - "percentage": "0.7" - }, - "wr_partial_misses": { - "count": 0, - "units": "Requests", - "percentage": "0.0" - }, - "serviced": { - "count": 212418, - "units": "Requests", - "percentage": "99.2" - }, - "rd_pt": { - "count": 0, - "units": "Requests", - "percentage": "0.0" - }, - "total": { - "count": 213953, - "units": "Requests", - "percentage": "100.0" - }, - "rd_hits": { - "count": 1, - "units": "Requests", - "percentage": "0.0" - }, - "wr_hits": { - "count": 136136, - "units": "Requests", - "percentage": "63.6" - } - } - }, - { - "errors": { - "total": { - "count": 0, - "units": "Requests", - "percentage": "0.0" - }, - "cache_obj_total": { - "count": 0, - "units": "Requests", - "percentage": "0.0" - }, - "core_obj_total": { - "count": 0, - "units": "Requests", - "percentage": "0.0" - }, - "cache_obj_rd": { - "count": 0, - "units": "Requests", - "percentage": "0.0" - }, - "core_obj_wr": { - "count": 0, - "units": "Requests", - "percentage": "0.0" - }, - "core_obj_rd": { - "count": 0, - "units": "Requests", - "percentage": "0.0" - }, - "cache_obj_wr": { - "count": 0, - "units": "Requests", - "percentage": "0.0" - } - } - }, - { - "blocks": { - "volume_rd": { - "count": 9, - "units": "4KiB blocks", - "percentage": "0.0" - }, - "volume_wr": { - "count": 213951, - "units": "4KiB blocks", - "percentage": "99.9" - }, - "cache_obj_total": { - "count": 212425, - "units": "4KiB blocks", - "percentage": "100.0" - }, - "core_obj_total": { - "count": 213959, - "units": "4KiB blocks", - "percentage": "100.0" - }, - "cache_obj_rd": { - "count": 1, - "units": "4KiB blocks", - "percentage": "0.0" - }, - "core_obj_wr": { - "count": 213951, - "units": "4KiB blocks", - "percentage": "99.9" - }, - "volume_total": { - "count": 213960, - "units": "4KiB blocks", - "percentage": "100.0" - }, - "core_obj_rd": { - "count": 8, - "units": "4KiB blocks", - "percentage": "0.0" - }, - "cache_obj_wr": { - "count": 212424, - "units": "4KiB blocks", - "percentage": "99.9" - } - } - } - ] + "result": true } ~~~ -### bdev_ocf_reset_stats {#rpc_bdev_ocf_reset_stats} +### bdev_ocf_add_core {#rpc_bdev_ocf_add_core} -Reset statistics of chosen OCF block device. +Add core (backend device) to OCF cache. #### Parameters -{{ bdev_ocf_reset_stats_params }} +{{ bdev_ocf_add_core_params }} #### Response -Completion status of reset statistics operation returned as a boolean. +Name of created OCF core vbdev. #### Example @@ -3207,10 +3061,12 @@ Example request: ~~~json { "params": { - "name": "ocf0" + "core_name": "Ocf_core0", + "base_name": "Malloc0", + "cache_name": "Ocf_cache0" }, "jsonrpc": "2.0", - "method": "bdev_ocf_reset_stats", + "method": "bdev_ocf_add_core", "id": 1 } ~~~ @@ -3221,21 +3077,52 @@ Example response: { "jsonrpc": "2.0", "id": 1, - "result": true + "result": "Ocf_core0" } ~~~ -### bdev_ocf_get_bdevs {#rpc_bdev_ocf_get_bdevs} +### bdev_ocf_remove_core {#rpc_bdev_ocf_remove_core} -Get list of OCF devices including unregistered ones. +Remove core (backend device) from OCF cache. #### Parameters -{{ bdev_ocf_get_bdevs_params }} +{{ bdev_ocf_remove_core_params }} -#### Response +#### Example + +Example request: + +~~~json +{ + "params": { + "core_name": "Ocf_core0" + }, + "jsonrpc": "2.0", + "method": "bdev_ocf_remove_core", + "id": 1 +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +### bdev_ocf_set_cachemode {#rpc_bdev_ocf_set_cachemode} -Array of OCF devices with their current status, along with core and cache bdevs. +Set cache mode of OCF cache. + +You can find more details about supported cache modes in the [OCF documentation](https://open-cas.com/cache_configuration.html#cache-mode) + +#### Parameters + +{{ bdev_ocf_set_cachemode_params }} #### Example @@ -3243,8 +3130,12 @@ Example request: ~~~json { + "params": { + "cache_name": "Ocf_cache0", + "cache_mode": "wb" + }, "jsonrpc": "2.0", - "method": "bdev_ocf_get_bdevs", + "method": "bdev_ocf_set_cachemode", "id": 1 } ~~~ @@ -3255,34 +3146,57 @@ Example response: { "jsonrpc": "2.0", "id": 1, - "result": [ - { - "name": "PartCache", - "started": false, - "cache": { - "name": "Malloc0", - "attached": true - }, - "core": { - "name": "Malloc1", - "attached": false - } - } - ] + "result": true } ~~~ -### bdev_ocf_set_cache_mode {#rpc_bdev_ocf_set_cache_mode} +### bdev_ocf_set_promotion {#rpc_bdev_ocf_set_promotion} -Set new cache mode on OCF bdev. +Set promotion parameters for OCF cache. + +You can find more details about supported promotion parameters in the [OCF documentation](https://open-cas.com/promotion.html) #### Parameters -{{ bdev_ocf_set_cache_mode_params }} +{{ bdev_ocf_set_promotion_params }} -#### Response +#### Example + +Example request: + +~~~json +{ + "params": { + "cache_name": "Ocf_cache0", + "policy": "nhit", + "nhit_insertion_threshold": 100, + "nhit_trigger_threshold": 50 + }, + "jsonrpc": "2.0", + "method": "bdev_ocf_set_promotion", + "id": 1 +} +~~~ -New cache mode name. +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +### bdev_ocf_set_cleaning {#rpc_bdev_ocf_set_cleaning} + +Set cleaning parameters for OCF cache. + +You can find more details about supported cleaning parameters in the [OCF documentation](https://open-cas.com/cleaner.html) + +#### Parameters + +{{ bdev_ocf_set_cleaning_params }} #### Example @@ -3291,11 +3205,13 @@ Example request: ~~~json { "params": { - "name": "ocf0", - "mode": "pt" + "cache_name": "Ocf_cache0", + "policy": "acp", + "acp_wake_up_time": 150, + "acp_flush_max_buffers": 512 }, "jsonrpc": "2.0", - "method": "bdev_ocf_set_cache_mode", + "method": "bdev_ocf_set_cleaning", "id": 1 } ~~~ @@ -3306,14 +3222,15 @@ Example response: { "jsonrpc": "2.0", "id": 1, - "result": "pt" + "result": true } ~~~ ### bdev_ocf_set_seqcutoff {#rpc_bdev_ocf_set_seqcutoff} -Set sequential cutoff parameters on all cores for the given OCF cache device. -A brief description of this functionality can be found in [OpenCAS documentation](https://open-cas.github.io/guide_tool_details.html#seq-cutoff). +Set sequential cut-off parameters for OCF core or all cores in given cache. + +You can find more details about supported sequential cut-off parameters in the [OCF documentation](https://open-cas.com/seq_cutoff.html) #### Parameters @@ -3326,10 +3243,11 @@ Example request: ~~~json { "params": { - "name": "ocf0", - "policy": "full", - "threshold": 4, - "promotion_count": 2 + "bdev_name": "Ocf_core0", + "policy": "always", + "threshold": 4096, + "promotion_count": 65535, + "promote_on_threshold": 1 }, "jsonrpc": "2.0", "method": "bdev_ocf_set_seqcutoff", @@ -3349,16 +3267,10 @@ Example response: ### bdev_ocf_flush_start {#rpc_bdev_ocf_flush_start} -Start flushing OCF cache device. +Flush all dirty data on the given OCF device (from cache to all of its cores or to particular core only). -Automatic flushes of dirty data are managed by OCF cleaning policy settings. -In addition to that, all dirty data is flushed to core device when there is -an attempt to stop caching. -On the other hand, this RPC call gives a possibility to flush dirty data manually -when there is a need for it, e.g. to speed up the shutdown process when data -hasn't been flushed for a long time. -This RPC returns immediately, and flush is then being performed in the -background. To see the status of flushing operation use bdev_ocf_flush_status. +Note that this call only starts the flushing process which will be running in background and may take some time depending on the underlying device size and speed. +You can check flushing status using the bdev_ocf_get_bdevs call. #### Parameters @@ -3371,7 +3283,7 @@ Example request: ~~~json { "params": { - "name": "ocf0" + "bdev_name": "Ocf_core0" }, "jsonrpc": "2.0", "method": "bdev_ocf_flush_start", @@ -3389,26 +3301,223 @@ Example response: } ~~~ -### bdev_ocf_flush_status {#rpc_bdev_ocf_flush_status} +### bdev_ocf_get_stats {#rpc_bdev_ocf_get_stats} -Get flush status of OCF cache device. +Get statistics of OCF vbdev. -Automatic flushes of dirty data are managed by OCF cleaning policy settings. -In addition to that, all dirty data is flushed to core device when there is -an attempt to stop caching. -On the other hand, there is a possibility to flush dirty data manually -when there is a need for it, e.g. to speed up the shutdown process when data -hasn't been flushed for a long time. -This RPC reports if such manual flush is still in progress and if the operation -was successful. To start manual flush use bdev_ocf_flush_start. +Statistics are shown for particular core or all cores in given cache combined depending on which bdev was passed as a parameter. #### Parameters -{{ bdev_ocf_flush_status_params }} +{{ bdev_ocf_get_stats_params }} #### Response -Status of OCF cache device flush. +Statistics in JSON format. + +#### Example + +Example request: + +~~~json +{ + "params": { + "name": "Ocf_cache0" + }, + "jsonrpc": "2.0", + "method": "bdev_ocf_get_stats", + "id": 1 +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "usage": { + "occupancy": { + "count": 55521, + "percentage": "2.14", + "units": "4KiB blocks" + }, + "free": { + "count": 2529759, + "percentage": "97.85", + "units": "4KiB blocks" + }, + "clean": { + "count": 55521, + "percentage": "2.14", + "units": "4KiB blocks" + }, + "dirty": { + "count": 0, + "percentage": "0.0", + "units": "4KiB blocks" + } + }, + "requests": { + "rd_hits": { + "count": 12, + "percentage": "0.2", + "units": "Requests" + }, + "rd_partial_misses": { + "count": 0, + "percentage": "0.0", + "units": "Requests" + }, + "rd_full_misses": { + "count": 27726, + "percentage": "49.83", + "units": "Requests" + }, + "rd_total": { + "count": 27738, + "percentage": "49.85", + "units": "Requests" + }, + "wr_hits": { + "count": 7, + "percentage": "0.1", + "units": "Requests" + }, + "wr_partial_misses": { + "count": 0, + "percentage": "0.0", + "units": "Requests" + }, + "wr_full_misses": { + "count": 27893, + "percentage": "50.13", + "units": "Requests" + }, + "wr_total": { + "count": 27900, + "percentage": "50.14", + "units": "Requests" + }, + "rd_pt": { + "count": 0, + "percentage": "0.0", + "units": "Requests" + }, + "wr_pt": { + "count": 0, + "percentage": "0.0", + "units": "Requests" + }, + "serviced": { + "count": 55638, + "percentage": "100.0", + "units": "Requests" + }, + "total": { + "count": 55638, + "percentage": "100.0", + "units": "Requests" + } + }, + "blocks": { + "core_volume_rd": { + "count": 3474, + "percentage": "49.89", + "units": "4KiB blocks" + }, + "core_volume_wr": { + "count": 3488, + "percentage": "50.10", + "units": "4KiB blocks" + }, + "core_volume_total": { + "count": 6962, + "percentage": "100.0", + "units": "4KiB blocks" + }, + "cache_volume_rd": { + "count": 3, + "percentage": "0.4", + "units": "4KiB blocks" + }, + "cache_volume_wr": { + "count": 6961, + "percentage": "99.95", + "units": "4KiB blocks" + }, + "cache_volume_total": { + "count": 6964, + "percentage": "100.0", + "units": "4KiB blocks" + }, + "volume_rd": { + "count": 3476, + "percentage": "49.91", + "units": "4KiB blocks" + }, + "volume_wr": { + "count": 3488, + "percentage": "50.8", + "units": "4KiB blocks" + }, + "volume_total": { + "count": 6964, + "percentage": "100.0", + "units": "4KiB blocks" + } + }, + "errors": { + "core_volume_rd": { + "count": 0, + "percentage": "0.0", + "units": "Requests" + }, + "core_volume_wr": { + "count": 0, + "percentage": "0.0", + "units": "Requests" + }, + "core_volume_total": { + "count": 0, + "percentage": "0.0", + "units": "Requests" + }, + "cache_volume_rd": { + "count": 0, + "percentage": "0.0", + "units": "Requests" + }, + "cache_volume_wr": { + "count": 0, + "percentage": "0.0", + "units": "Requests" + }, + "cache_volume_total": { + "count": 0, + "percentage": "0.0", + "units": "Requests" + }, + "total": { + "count": 0, + "percentage": "0.0", + "units": "Requests" + } + } + } +} +~~~ + +### bdev_ocf_reset_stats {#rpc_bdev_ocf_reset_stats} + +Reset statistics of OCF vbdev. + +Statistics are reset for particular core or all cores in given cache depending on which bdev was passed as a parameter. + +#### Parameters + +{{ bdev_ocf_reset_stats_params }} #### Example @@ -3417,10 +3526,44 @@ Example request: ~~~json { "params": { - "name": "ocf0" + "name": "Ocf_cache0" }, "jsonrpc": "2.0", - "method": "bdev_ocf_flush_status", + "method": "bdev_ocf_reset_stats", + "id": 1 +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +### bdev_ocf_get_bdevs {#rpc_bdev_ocf_get_bdevs} + +Get detailed info about OCF vbdevs. + +#### Parameters + +{{ bdev_ocf_get_bdevs_params }} + +#### Response + +List of OCF vbdev details in JSON format. + +#### Example + +Example request: + +~~~json +{ + "jsonrpc": "2.0", + "method": "bdev_ocf_get_bdevs", "id": 1 } ~~~ @@ -3432,8 +3575,137 @@ Example response: "jsonrpc": "2.0", "id": 1, "result": { - "in_progress": false, - "status": 0 + "cores_waitlist": [ + { + "name": "Ocf_core_waitlist0", + "cache_name": "Ocf_cache2", + "base_name": "Nvme2n1", + "base_attached": false, + "size": null, + "block_size": null + }, + { + "name": "Ocf_core_waitlist1", + "cache_name": "Ocf_cache2", + "base_name": "Malloc3", + "base_attached": true, + "size": 209715200, + "block_size": 512 + } + ], + "caches": [ + { + "name": "Ocf_cache0", + "base_name": "Malloc0", + "base_attached": true, + "size": 209715200, + "block_size": 512, + "cache_line_size": 4096, + "cache_mode": "wt", + "promotion": { + "policy": "always" + }, + "cleaning": { + "policy": "alru", + "wake_up_time": 20, + "flush_max_buffers": 100, + "staleness_time": 120, + "activity_threshold": 10000, + "max_dirty_ratio": 100 + }, + "flush": { + "in_progress": false, + "error": 0 + }, + "cores_count": 1, + "cores": [ + { + "name": "Ocf_core0-0", + "cache_name": "Ocf_cache0", + "base_name": "Nvme0n1", + "base_attached": true, + "size": 10737418240, + "block_size": 512, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + } + ] + }, + { + "name": "Ocf_cache1", + "base_name": "Malloc1", + "base_attached": true, + "size": 209715200, + "block_size": 512, + "cache_line_size": 65536, + "cache_mode": "wb", + "promotion": { + "policy": "always" + }, + "cleaning": { + "policy": "alru", + "wake_up_time": 20, + "flush_max_buffers": 100, + "staleness_time": 120, + "activity_threshold": 10000, + "max_dirty_ratio": 100 + }, + "flush": { + "in_progress": false, + "error": 0 + }, + "cores_count": 2, + "cores": [ + { + "name": "Ocf_core1-0", + "cache_name": "Ocf_cache1", + "base_name": "Nvme1n1", + "base_attached": true, + "size": 10737418240, + "block_size": 512, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + }, + { + "name": "Ocf_core1-1", + "cache_name": "Ocf_cache1", + "base_name": "Malloc2", + "base_attached": true, + "size": 262144000, + "block_size": 512, + "loading": false, + "seq_cutoff": { + "policy": "full", + "threshold": 1024, + "promotion_count": 8, + "promote_on_threshold": false + }, + "flush": { + "in_progress": false, + "error": 0 + } + } + ] + } + ] } } ~~~ diff --git a/python/spdk/cli/bdev.py b/python/spdk/cli/bdev.py index d6ac2686911..0bcbf37259b 100644 --- a/python/spdk/cli/bdev.py +++ b/python/spdk/cli/bdev.py @@ -157,7 +157,7 @@ def bdev_ocf_add_core(args): p = subparsers.add_parser('bdev_ocf_add_core', help='Add core (backend device) to OCF cache') p.add_argument('core_name', help='name for the new OCF core vbdev') p.add_argument('base_name', help='name of the base bdev to use as core') - p.add_argument('cache_name', help='name of already started OCF cache vbdev') + p.add_argument('cache_name', help='name of the cache vbdev to add this core to') p.set_defaults(func=bdev_ocf_add_core) def bdev_ocf_remove_core(args): @@ -174,7 +174,7 @@ def bdev_ocf_set_cachemode(args): p.add_argument('cache_name', help='name of the cache vbdev') p.add_argument('cache_mode', help='choose between {wt|wb|wa|wo|wi|pt} (Write-Through, Write-Back, \ - Write-Around, Write-Only, Write-Invalidate, Pass-Through)', + Write-Around, Write-Only, Write-Invalidate, Pass-Through); default "wt"', choices=['wt', 'wb', 'wa', 'wo', 'wi', 'pt']) p.set_defaults(func=bdev_ocf_set_cachemode) @@ -187,7 +187,7 @@ def bdev_ocf_set_promotion(args): p = subparsers.add_parser('bdev_ocf_set_promotion', help='Set promotion parameters for OCF cache') p.add_argument('cache_name', help='name of the cache vbdev') p.add_argument('-p', '--policy', - help='promotion policy (choose between {always|nhit})', + help='promotion policy (choose between {always|nhit}; default "always")', choices=['always', 'nhit'], default='none') p.add_argument('-i', '--nhit-insertion-threshold', @@ -215,7 +215,7 @@ def bdev_ocf_set_cleaning(args): p = subparsers.add_parser('bdev_ocf_set_cleaning', help='Set cleaning parameters for OCF cache') p.add_argument('cache_name', help='name of the cache vbdev') p.add_argument('-p', '--policy', - help='cleaning policy (choose between {acp|alru|nop})', + help='cleaning policy (choose between {acp|alru|nop}; default "alru")', choices=['acp', 'alru', 'nop'], default='none') p.add_argument('-u', '--acp-wake-up-time', @@ -260,7 +260,7 @@ def bdev_ocf_set_seqcutoff(args): p = subparsers.add_parser('bdev_ocf_set_seqcutoff', help='Set sequential cut-off parameters for OCF core or all cores in given cache') p.add_argument('bdev_name', help='name of OCF vbdev') p.add_argument('-p', '--policy', - help='sequential cut-off policy (choose between {always|full|never})', + help='sequential cut-off policy (choose between {always|full|never}; default "full")', choices=['always', 'full', 'never'], default='none') p.add_argument('-t', '--threshold', @@ -302,7 +302,7 @@ def bdev_ocf_reset_stats(args): def bdev_ocf_get_bdevs(args): print_dict(args.client.bdev_ocf_get_bdevs(bdev_name=args.bdev_name)) - p = subparsers.add_parser('bdev_ocf_get_bdevs', help='Get info about OCF vbdevs') + p = subparsers.add_parser('bdev_ocf_get_bdevs', help='Get detailed info about OCF vbdevs') p.add_argument('bdev_name', nargs='?', help='optional name of specific OCF vbdev (shows all by default)') p.set_defaults(func=bdev_ocf_get_bdevs) From e1f8effa93b9d4d034dfffb4dda27afb58d1fc6e Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Tue, 30 Sep 2025 15:52:10 +0200 Subject: [PATCH 20/34] Prevent running any calls when module is stopping Change-Id: I7242902219bcb5604df51f8a06feb50e86426541 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 98 +++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 37bcc172e75..57c87e3ca32 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -17,6 +17,7 @@ /* This namespace UUID was generated using uuid_generate() method. */ #define BDEV_OCF_NAMESPACE_UUID "f92b7f49-f6c0-44c8-bd23-3205e8c3b6ad" +bool g_vbdev_ocf_module_is_running = false; static int vbdev_ocf_module_init(void); static void vbdev_ocf_module_fini_start(void); @@ -204,6 +205,8 @@ vbdev_ocf_module_init(void) return rc; } + g_vbdev_ocf_module_is_running = true; + return rc; } @@ -364,6 +367,8 @@ vbdev_ocf_module_fini_start(void) SPDK_DEBUGLOG(vbdev_ocf, "OCF: initiating module stop\n"); + g_vbdev_ocf_module_is_running = false; + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { if (vbdev_ocf_core_is_base_attached(core_ctx)) { vbdev_ocf_core_base_detach(core_ctx); @@ -1173,6 +1178,12 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating start\n", cache_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rc = -EPERM; + goto err_module; + } + if (vbdev_ocf_bdev_exists(cache_name)) { SPDK_ERRLOG("OCF '%s': bdev already exists\n", cache_name); rc = -EEXIST; @@ -1246,6 +1257,7 @@ vbdev_ocf_cache_start(const char *cache_name, const char *base_name, ocf_mngt_cache_stop(cache, _cache_start_rpc_err_cb, NULL); err_create: err_exist: +err_module: rpc_cb_fn(cache_name, rpc_cb_arg, rc); } @@ -1259,6 +1271,12 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating stop\n", cache_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rc = -EPERM; + goto err_module; + } + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; @@ -1296,6 +1314,7 @@ vbdev_ocf_cache_stop(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo err_alloc: ocf_mngt_cache_put(cache); err_cache: +err_module: rpc_cb_fn(cache_name, rpc_cb_arg, rc); } @@ -1380,6 +1399,12 @@ vbdev_ocf_cache_detach(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating device detach\n", cache_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rc = -EPERM; + goto err_module; + } + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; @@ -1410,6 +1435,7 @@ vbdev_ocf_cache_detach(const char *cache_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, err_state: ocf_mngt_cache_put(cache); err_cache: +err_module: rpc_cb_fn(cache_name, rpc_cb_arg, rc); } @@ -1448,6 +1474,12 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': initiating device attach\n", cache_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rc = -EPERM; + goto err_module; + } + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; @@ -1507,6 +1539,7 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force err_state: ocf_mngt_cache_put(cache); err_cache: +err_module: rpc_cb_fn(cache_name, rpc_cb_arg, rc); } @@ -1608,6 +1641,12 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating add\n", core_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rc = -EPERM; + goto err_module; + } + if (vbdev_ocf_bdev_exists(core_name)) { SPDK_ERRLOG("OCF '%s': bdev already exists\n", core_name); rc = -EEXIST; @@ -1705,6 +1744,7 @@ vbdev_ocf_core_add(const char *core_name, const char *base_name, const char *cac vbdev_ocf_core_destroy(core_ctx); err_create: err_exist: +err_module: rpc_cb_fn(core_name, rpc_cb_arg, rc); } @@ -1781,6 +1821,12 @@ vbdev_ocf_core_remove(const char *core_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': initiating removal\n", core_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rc = -EPERM; + goto err_module; + } + /* If core was not added yet due to lack of base or cache device, * just free its structs (and detach its base if exists) and exit. */ if ((core_ctx = vbdev_ocf_core_waitlist_get_by_name(core_name))) { @@ -1846,6 +1892,7 @@ vbdev_ocf_core_remove(const char *core_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo err_resolve: free(mngt_ctx); err_alloc: +err_module: rpc_cb_fn(core_name, rpc_cb_arg, rc); } @@ -1915,6 +1962,12 @@ vbdev_ocf_set_cachemode(const char *cache_name, const char *cache_mode, SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': setting new cache mode '%s'\n", cache_name, cache_mode); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rc = -EPERM; + goto err_module; + } + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; @@ -1939,6 +1992,7 @@ vbdev_ocf_set_cachemode(const char *cache_name, const char *cache_mode, err_alloc: ocf_mngt_cache_put(cache); err_cache: +err_module: rpc_cb_fn(cache_name, rpc_cb_arg, rc); } @@ -2008,6 +2062,12 @@ vbdev_ocf_set_promotion(const char *cache_name, const char *policy, SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': setting promotion params\n", cache_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rc = -EPERM; + goto err_module; + } + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; @@ -2034,6 +2094,7 @@ vbdev_ocf_set_promotion(const char *cache_name, const char *policy, err_alloc: ocf_mngt_cache_put(cache); err_cache: +err_module: rpc_cb_fn(cache_name, rpc_cb_arg, rc); } @@ -2167,6 +2228,12 @@ vbdev_ocf_set_cleaning(const char *cache_name, const char *policy, int32_t acp_w SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': setting cleaning params\n", cache_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rc = -EPERM; + goto err_module; + } + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_ERRLOG("OCF cache '%s': not exist\n", cache_name); rc = -ENXIO; @@ -2199,6 +2266,7 @@ vbdev_ocf_set_cleaning(const char *cache_name, const char *policy, int32_t acp_w err_alloc: ocf_mngt_cache_put(cache); err_cache: +err_module: rpc_cb_fn(cache_name, rpc_cb_arg, rc); } @@ -2323,6 +2391,12 @@ vbdev_ocf_set_seqcutoff(const char *bdev_name, const char *policy, int32_t thres SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': setting sequential cut-off params\n", bdev_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rpc_cb_fn(bdev_name, rpc_cb_arg, -EPERM); + return; + } + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!mngt_ctx) { SPDK_ERRLOG("OCF '%s': failed to allocate memory for sequential cut-off set context\n", @@ -2431,6 +2505,12 @@ vbdev_ocf_flush_start(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': initiating flush operation\n", bdev_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rpc_cb_fn(bdev_name, rpc_cb_arg, -EPERM); + return; + } + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!mngt_ctx) { SPDK_ERRLOG("OCF '%s': failed to allocate memory for flush context\n", bdev_name); @@ -2508,6 +2588,11 @@ vbdev_ocf_get_stats(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': getting statistics\n", bdev_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + goto err_module; + } + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!mngt_ctx) { SPDK_ERRLOG("OCF '%s': failed to allocate memory for getting statistics context\n", @@ -2536,6 +2621,7 @@ vbdev_ocf_get_stats(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, err_resolve: free(mngt_ctx); err_alloc: +err_module: rpc_cb_fn(rpc_cb_arg1, rpc_cb_arg2); } @@ -2584,6 +2670,12 @@ vbdev_ocf_reset_stats(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': resetting statistics\n", bdev_name); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + rc = -EPERM; + goto err_module; + } + mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!mngt_ctx) { SPDK_ERRLOG("OCF '%s': failed to allocate memory for resetting statistics context\n", @@ -2612,6 +2704,7 @@ vbdev_ocf_reset_stats(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo err_resolve: free(mngt_ctx); err_alloc: +err_module: rpc_cb_fn(bdev_name, rpc_cb_arg, rc); } @@ -2899,6 +2992,11 @@ vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, void SPDK_DEBUGLOG(vbdev_ocf, "OCF: getting info about vbdevs\n"); + if (!g_vbdev_ocf_module_is_running) { + SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); + goto end; + } + if (!bdev_name) { spdk_json_write_named_array_begin(w, "cores_waitlist"); vbdev_ocf_foreach_core_in_waitlist(core_ctx) { From d5c058554c501cf56e4bfbb3e07cf143db3797d5 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Thu, 9 Oct 2025 14:11:00 +0200 Subject: [PATCH 21/34] Do not print uninitialized core info Change-Id: Ia187bee78d1e1078eefcb49ebcaf73ffd601f479 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 57c87e3ca32..50f3f741839 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -2866,6 +2866,14 @@ dump_core_info(struct spdk_json_write_ctx *w, ocf_core_t core) struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); int rc; + /* If this core returns no cache it means that it is + * still in the process of adding itself to the cache. + * Do not print any core info until adding is finished. + * Return an empty JSON object instead. */ + if (!ocf_core_get_cache(core)) { + return 0; + } + spdk_json_write_named_string(w, "name", ocf_core_get_name(core)); spdk_json_write_named_string(w, "cache_name", ocf_cache_get_name(ocf_core_get_cache(core))); spdk_json_write_named_string(w, "base_name", core_ctx ? core_ctx->base.name : ""); From 74ea79c5e3d9c9862b5502c5a3c18c0813d60831 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Tue, 30 Sep 2025 16:48:28 +0200 Subject: [PATCH 22/34] Generate UUIDs for registered bdevs Change-Id: I3753edb1643a4465c9768e6a8a13cd4c21cc6443 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 7 +++++-- module/bdev/ocf/vbdev_ocf_core.c | 15 ++++++++++++++- module/bdev/ocf/vbdev_ocf_core.h | 3 +++ test/ocf/management/info_dump.sh | 4 +++- test/ocf/management/info_dump_get_bdevs.json | 9 +++++++++ 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 50f3f741839..ef2f7aafa4c 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -15,8 +15,6 @@ #include "utils.h" #include "volume.h" -/* This namespace UUID was generated using uuid_generate() method. */ -#define BDEV_OCF_NAMESPACE_UUID "f92b7f49-f6c0-44c8-bd23-3205e8c3b6ad" bool g_vbdev_ocf_module_is_running = false; static int vbdev_ocf_module_init(void); @@ -2875,6 +2873,11 @@ dump_core_info(struct spdk_json_write_ctx *w, ocf_core_t core) } spdk_json_write_named_string(w, "name", ocf_core_get_name(core)); + if (core_ctx && !spdk_uuid_is_null(spdk_bdev_get_uuid(&core_ctx->ocf_vbdev))) { + spdk_json_write_named_uuid(w, "uuid", spdk_bdev_get_uuid(&core_ctx->ocf_vbdev)); + } else { + spdk_json_write_named_null(w, "uuid"); + } spdk_json_write_named_string(w, "cache_name", ocf_cache_get_name(ocf_core_get_cache(core))); spdk_json_write_named_string(w, "base_name", core_ctx ? core_ctx->base.name : ""); spdk_json_write_named_bool(w, "base_attached", diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c index 22d0a52f631..1edc85a32a5 100644 --- a/module/bdev/ocf/vbdev_ocf_core.c +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -393,6 +393,7 @@ vbdev_ocf_core_register(ocf_core_t core) struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); struct vbdev_ocf_base *base = &core_ctx->base; struct spdk_bdev *ocf_vbdev = &core_ctx->ocf_vbdev; + struct spdk_uuid ns_uuid; int rc = 0; SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': registering OCF vbdev in SPDK bdev layer\n", @@ -406,10 +407,22 @@ vbdev_ocf_core_register(ocf_core_t core) ocf_vbdev->blockcnt = base->bdev->blockcnt; ocf_vbdev->required_alignment = base->bdev->required_alignment; ocf_vbdev->optimal_io_boundary = base->bdev->optimal_io_boundary; - // generate UUID based on namespace UUID + base bdev UUID (take from old module?) ocf_vbdev->fn_table = &vbdev_ocf_fn_table; ocf_vbdev->module = &ocf_if; + /* Generate UUID based on namespace UUID + base bdev UUID. */ + if ((rc = spdk_uuid_parse(&ns_uuid, BDEV_OCF_NAMESPACE_UUID))) { + SPDK_ERRLOG("OCF vbdev '%s': failed to parse namespace UUID\n", + spdk_bdev_get_name(ocf_vbdev)); + return rc; + } + if ((rc = spdk_uuid_generate_sha1(&ocf_vbdev->uuid, &ns_uuid, + (const char *)&base->bdev->uuid, sizeof(struct spdk_uuid)))) { + SPDK_ERRLOG("OCF vbdev '%s': failed to generate new UUID\n", + spdk_bdev_get_name(ocf_vbdev)); + return rc; + } + spdk_io_device_register(core, _vbdev_ocf_ch_create_cb, _vbdev_ocf_ch_destroy_cb, sizeof(struct vbdev_ocf_core_io_channel_ctx), ocf_core_get_name(core)); SPDK_DEBUGLOG(vbdev_ocf, "OCF vbdev '%s': io_device created at %p\n", diff --git a/module/bdev/ocf/vbdev_ocf_core.h b/module/bdev/ocf/vbdev_ocf_core.h index b82530124bd..b24db73e24f 100644 --- a/module/bdev/ocf/vbdev_ocf_core.h +++ b/module/bdev/ocf/vbdev_ocf_core.h @@ -10,6 +10,9 @@ #include "spdk/bdev_module.h" #include "volume.h" +/* This namespace UUID was generated using uuid_generate() method. */ +#define BDEV_OCF_NAMESPACE_UUID "f92b7f49-f6c0-44c8-bd23-3205e8c3b6ad" + /* OCF module interface. */ extern struct spdk_bdev_module ocf_if; diff --git a/test/ocf/management/info_dump.sh b/test/ocf/management/info_dump.sh index ad22f440319..eedce4bcbf6 100755 --- a/test/ocf/management/info_dump.sh +++ b/test/ocf/management/info_dump.sh @@ -44,7 +44,9 @@ $rpc_py bdev_ocf_get_stats Ocf_cache1 | jq -e \ '.requests[], .blocks[], .errors[] | .count == 0 and .percentage == "0.0"' # Test OCF get bdevs: -diff <($rpc_py bdev_ocf_get_bdevs | jq -e .) <(jq -e . "$curdir/info_dump_get_bdevs.json") +# remove UUID fields before comparison as they are changing each run +diff <($rpc_py bdev_ocf_get_bdevs | sed '/"uuid": /d' | jq -e .) \ + <(sed '/"uuid": /d' "$curdir/info_dump_get_bdevs.json" | jq -e .) # Test general get bdevs driver specific info: diff <($rpc_py bdev_get_bdevs | jq -e '.[].driver_specific.ocf | select(. != null)') \ diff --git a/test/ocf/management/info_dump_get_bdevs.json b/test/ocf/management/info_dump_get_bdevs.json index d7698f4be89..d13a1190d23 100644 --- a/test/ocf/management/info_dump_get_bdevs.json +++ b/test/ocf/management/info_dump_get_bdevs.json @@ -53,6 +53,7 @@ "cores": [ { "name": "Ocf_core1-1", + "uuid": "0be66590-b2b0-58cf-8dde-aa8ed86ee223", "cache_name": "Ocf_cache1", "base_name": "Core_dev1-1", "base_attached": false, @@ -72,6 +73,7 @@ }, { "name": "Ocf_core1-2", + "uuid": "7ae5d5b1-f2b8-56e6-838e-d6c3e4e4a52d", "cache_name": "Ocf_cache1", "base_name": "Core_dev1-2", "base_attached": true, @@ -91,6 +93,7 @@ }, { "name": "Ocf_core1-3", + "uuid": "6d587297-39c4-5e4b-899b-faa488c5a976", "cache_name": "Ocf_cache1", "base_name": "Core_dev1-3", "base_attached": true, @@ -137,6 +140,7 @@ "cores": [ { "name": "Ocf_core2-1", + "uuid": "a3dbab70-2366-5a7b-a22d-a5f27f958fde", "cache_name": "Ocf_cache2", "base_name": "Core_dev2-1", "base_attached": false, @@ -156,6 +160,7 @@ }, { "name": "Ocf_core2-2", + "uuid": "089e4715-908c-5635-8ff7-e45ce42f0b2b", "cache_name": "Ocf_cache2", "base_name": "Core_dev2-2", "base_attached": true, @@ -175,6 +180,7 @@ }, { "name": "Ocf_core2-3", + "uuid": "373d6301-3d56-5261-b0bd-dd959c48aa3e", "cache_name": "Ocf_cache2", "base_name": "Core_dev2-3", "base_attached": true, @@ -221,6 +227,7 @@ "cores": [ { "name": "Ocf_core3-1", + "uuid": "b3c4e428-b6cb-572d-98bf-33d192ba4a62", "cache_name": "Ocf_cache3", "base_name": "Core_dev3-1", "base_attached": false, @@ -240,6 +247,7 @@ }, { "name": "Ocf_core3-2", + "uuid": "7c321a44-f6b1-5af3-9ce3-2088fa529566", "cache_name": "Ocf_cache3", "base_name": "Core_dev3-2", "base_attached": true, @@ -259,6 +267,7 @@ }, { "name": "Ocf_core3-3", + "uuid": "5cfe8234-1fd5-5b73-9026-9214652a4475", "cache_name": "Ocf_cache3", "base_name": "Core_dev3-3", "base_attached": true, From 62630ab63411049a7f7743d2f9d723e27fde0315 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Thu, 9 Oct 2025 13:35:43 +0200 Subject: [PATCH 23/34] Remove old tests Change-Id: I488ec6c802299aba39f38743cad12a429ece066e Signed-off-by: Rafal Stefanowski --- test/ocf/integrity/bdevperf-iotypes.sh | 15 --- test/ocf/integrity/fio-modes.sh | 103 -------------------- test/ocf/integrity/mallocs.conf | 62 ------------ test/ocf/integrity/stats.sh | 19 ---- test/ocf/integrity/test.fio | 40 -------- test/ocf/management/configuration-change.sh | 63 ------------ test/ocf/management/create-destruct.sh | 91 ----------------- test/ocf/management/multicore.sh | 85 ---------------- test/ocf/management/remove.sh | 87 ----------------- 9 files changed, 565 deletions(-) delete mode 100755 test/ocf/integrity/bdevperf-iotypes.sh delete mode 100755 test/ocf/integrity/fio-modes.sh delete mode 100644 test/ocf/integrity/mallocs.conf delete mode 100755 test/ocf/integrity/stats.sh delete mode 100644 test/ocf/integrity/test.fio delete mode 100755 test/ocf/management/configuration-change.sh delete mode 100755 test/ocf/management/create-destruct.sh delete mode 100755 test/ocf/management/multicore.sh delete mode 100755 test/ocf/management/remove.sh diff --git a/test/ocf/integrity/bdevperf-iotypes.sh b/test/ocf/integrity/bdevperf-iotypes.sh deleted file mode 100755 index 4d159c5c36c..00000000000 --- a/test/ocf/integrity/bdevperf-iotypes.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (C) 2019 Intel Corporation -# All rights reserved. -# -curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) -source $rootdir/test/common/autotest_common.sh - -bdevperf=$rootdir/build/examples/bdevperf - -source "$curdir/mallocs.conf" -$bdevperf --json <(gen_malloc_ocf_json) -q 128 -o 4096 -t 4 -w flush -$bdevperf --json <(gen_malloc_ocf_json) -q 128 -o 4096 -t 4 -w unmap -$bdevperf --json <(gen_malloc_ocf_json) -q 128 -o 4096 -t 4 -w write diff --git a/test/ocf/integrity/fio-modes.sh b/test/ocf/integrity/fio-modes.sh deleted file mode 100755 index ca930b7ff2c..00000000000 --- a/test/ocf/integrity/fio-modes.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (C) 2019 Intel Corporation -# All rights reserved. -# -curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) - -source $rootdir/test/ocf/common.sh - -function fio_verify() { - fio_bdev $curdir/test.fio --aux-path=/tmp/ --ioengine=spdk_bdev "$@" -} - -function cleanup() { - rm -f $curdir/modes.conf -} - -# Clear nvme device which we will use in test -clear_nvme - -trap "cleanup; exit 1" SIGINT SIGTERM EXIT - -# Building config is not backtrace worthy ... -xtrace_disable - -config=() ocf_names=() ocf_modes=() - -ocf_names[1]=PT_Nvme ocf_modes[1]=pt -ocf_names[2]=WT_Nvme ocf_modes[2]=wt -ocf_names[3]=WB_Nvme0 ocf_modes[3]=wb -ocf_names[4]=WB_Nvme1 ocf_modes[4]=wb - -mapfile -t config < <("$rootdir/scripts/gen_nvme.sh") - -# Drop anything from last closing ] so we can inject our own config pieces ... -config=("${config[@]::${#config[@]}-2}") -# ... and now convert entire array to a single string item -config=("${config[*]}") - -config+=( - "$( - cat <<- JSON - { - "method": "bdev_split_create", - "params": { - "base_bdev": "Nvme0n1", - "split_count": 8, - "split_size_mb": 101 - } - } - JSON - )" -) - -for ((d = 0, c = 1; d <= ${#ocf_names[@]} + 2; d += 2, c++)); do - config+=( - "$( - cat <<- JSON - { - "method": "bdev_ocf_create", - "params": { - "name": "${ocf_names[c]}", - "mode": "${ocf_modes[c]}", - "cache_bdev_name": "Nvme0n1p$d", - "core_bdev_name": "Nvme0n1p$((d + 1))" - } - } - JSON - )" - ) -done - -config+=( - "$( - cat <<- JSON - { - "method": "bdev_wait_for_examine" - } - JSON - )" -) - -# First ']}' closes our config and bdev subsystem blocks -cat <<- CONFIG > "$curdir/modes.conf" - {"subsystems":[ - $( - IFS="," - printf '%s\n' "${config[*]}" - ) - ]}]} -CONFIG - -# Format the config nicely and dump it to stdout for everyone to marvel at it ... -jq . "$curdir/modes.conf" - -# ... and now back to our regularly scheduled program -xtrace_restore - -fio_verify --filename=PT_Nvme:WT_Nvme:WB_Nvme0:WB_Nvme1 --spdk_json_conf="$curdir/modes.conf" --thread=1 - -trap - SIGINT SIGTERM EXIT -cleanup diff --git a/test/ocf/integrity/mallocs.conf b/test/ocf/integrity/mallocs.conf deleted file mode 100644 index 0e9b7993e22..00000000000 --- a/test/ocf/integrity/mallocs.conf +++ /dev/null @@ -1,62 +0,0 @@ -gen_malloc_ocf_json () { - local size=300 # MB - local block_size=512 - local config - - local malloc malloc_devs=3 - for (( malloc = 0; malloc < malloc_devs; malloc++ )); do - config+=( - "$( - cat <<-JSON - { - "method": "bdev_malloc_create", - "params": { - "name": "Malloc$malloc", - "num_blocks": $(( (size << 20) / block_size )), - "block_size": 512 - } - } - JSON - )" - ) - done - - local ocfs ocf ocf_mode ocf_cache ocf_core - ocfs=(1 2) - ocf_mode[1]=wt ocf_cache[1]=Malloc0 ocf_core[1]=Malloc1 - ocf_mode[2]=pt ocf_cache[2]=Malloc0 ocf_core[2]=Malloc2 - - for ocf in "${ocfs[@]}"; do - config+=( - "$( - cat <<-JSON - { - "method": "bdev_ocf_create", - "params": { - "name": "MalCache$ocf", - "mode": "${ocf_mode[ocf]}", - "cache_bdev_name": "${ocf_cache[ocf]}", - "core_bdev_name": "${ocf_core[ocf]}" - } - } - JSON - )" - ) - done - - jq . <<-JSON - { - "subsystems": [ - { - "subsystem": "bdev", - "config": [ - $(IFS=","; printf '%s\n' "${config[*]}"), - { - "method": "bdev_wait_for_examine" - } - ] - } - ] - } - JSON -} diff --git a/test/ocf/integrity/stats.sh b/test/ocf/integrity/stats.sh deleted file mode 100755 index 85177dde5e5..00000000000 --- a/test/ocf/integrity/stats.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (C) 2019 Intel Corporation -# All rights reserved. -# -curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) -source $rootdir/test/common/autotest_common.sh - -bdevperf=$rootdir/build/examples/bdevperf - -source "$curdir/mallocs.conf" -$bdevperf --json <(gen_malloc_ocf_json) -q 128 -o 4096 -w write -t 120 -r /var/tmp/spdk.sock & -bdev_perf_pid=$! -waitforlisten $bdev_perf_pid -sleep 1 -$rpc_py bdev_ocf_get_stats MalCache1 -kill -9 $bdev_perf_pid -wait $bdev_perf_pid || true diff --git a/test/ocf/integrity/test.fio b/test/ocf/integrity/test.fio deleted file mode 100644 index e45d1b95865..00000000000 --- a/test/ocf/integrity/test.fio +++ /dev/null @@ -1,40 +0,0 @@ -[global] -thread=1 -group_reporting=1 -direct=1 -serialize_overlap=1 -time_based=1 -do_verify=1 -verify=md5 -verify_backlog=1024 -iodepth=128 -bs=4K -runtime=10 -verify_state_save=0 - -size=20% - -[job_1] -offset=0 -rw=randwrite -name=randwrite - -[job_2] -offset=20% -rw=randrw -name=randrw - -[job_3] -offset=40% -rw=write -name=write - -[job_4] -offset=60% -rw=rw -name=rw - -[job_5] -offset=80% -rw=randwrite -name=randwrite diff --git a/test/ocf/management/configuration-change.sh b/test/ocf/management/configuration-change.sh deleted file mode 100755 index b9e517033e8..00000000000 --- a/test/ocf/management/configuration-change.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (C) 2021 Intel Corporation -# All rights reserved. -# -curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) -source $rootdir/test/common/autotest_common.sh - -rpc_py=$rootdir/scripts/rpc.py -cache_line_sizes=(4 8 16 32 64) -cache_modes=(wt wb pt wa wi wo) - -$SPDK_BIN_DIR/iscsi_tgt & -spdk_pid=$! - -waitforlisten $spdk_pid - -# Create OCF cache with different cache line sizes -for cache_line_size in "${cache_line_sizes[@]}"; do - $rpc_py bdev_malloc_create 101 512 -b Malloc0 - $rpc_py bdev_malloc_create 101 512 -b Malloc1 - $rpc_py bdev_ocf_create Cache0 wt Malloc0 Malloc1 --cache-line-size $cache_line_size - - $rpc_py bdev_ocf_get_bdevs | jq -e \ - '.[0] | .started and .cache.attached and .core.attached' - - # Check if cache line size values are reported correctly - $rpc_py bdev_get_bdevs -b Cache0 | jq -e \ - ".[0] | .driver_specific.cache_line_size == $cache_line_size" - $rpc_py save_subsystem_config -n bdev | jq -e \ - ".config | .[] | select(.method == \"bdev_ocf_create\") | .params.cache_line_size == $cache_line_size" - - $rpc_py bdev_ocf_delete Cache0 - $rpc_py bdev_malloc_delete Malloc0 - $rpc_py bdev_malloc_delete Malloc1 -done - -# Prepare OCF cache for dynamic configuration switching -$rpc_py bdev_malloc_create 101 512 -b Malloc0 -$rpc_py bdev_malloc_create 101 512 -b Malloc1 -$rpc_py bdev_ocf_create Cache0 wt Malloc0 Malloc1 - -$rpc_py bdev_ocf_get_bdevs | jq -e \ - '.[0] | .started and .cache.attached and .core.attached' - -# Change cache mode -for cache_mode in "${cache_modes[@]}"; do - $rpc_py bdev_ocf_set_cache_mode Cache0 $cache_mode - - # Check if cache mode values are reported correctly - $rpc_py bdev_get_bdevs -b Cache0 | jq -e \ - ".[0] | .driver_specific.mode == \"$cache_mode\"" - $rpc_py save_subsystem_config -n bdev | jq -e \ - ".config | .[] | select(.method == \"bdev_ocf_create\") | .params.mode == \"$cache_mode\"" -done - -# Change sequential cutoff -$rpc_py bdev_ocf_set_seqcutoff Cache0 -p always -t 64 -$rpc_py bdev_ocf_set_seqcutoff Cache0 -p never -t 16 - -trap - SIGINT SIGTERM EXIT -killprocess $spdk_pid diff --git a/test/ocf/management/create-destruct.sh b/test/ocf/management/create-destruct.sh deleted file mode 100755 index 839b56945dc..00000000000 --- a/test/ocf/management/create-destruct.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (C) 2019 Intel Corporation -# All rights reserved. -# -curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) -source $rootdir/test/common/autotest_common.sh - -rpc_py=$rootdir/scripts/rpc.py - -function bdev_check_claimed() { - if [ "$($rpc_py bdev_get_bdevs -b "$@" | jq '.[0].claimed')" = "true" ]; then - return 0 - else - return 1 - fi -} - -$SPDK_BIN_DIR/iscsi_tgt & -spdk_pid=$! - -trap 'killprocess $spdk_pid; exit 1' SIGINT SIGTERM EXIT - -waitforlisten $spdk_pid - -$rpc_py bdev_malloc_create 101 512 -b Malloc0 -$rpc_py bdev_malloc_create 101 512 -b Malloc1 - -$rpc_py bdev_ocf_create PartCache wt Malloc0 NonExisting - -$rpc_py bdev_ocf_get_bdevs PartCache | jq -e \ - '.[0] | .started == false and .cache.attached and .core.attached == false' - -$rpc_py bdev_ocf_get_bdevs NonExisting | jq -e \ - '.[0] | .name == "PartCache"' - -if ! bdev_check_claimed Malloc0; then - echo >&2 "Base device expected to be claimed now" - exit 1 -fi - -$rpc_py bdev_ocf_delete PartCache -if bdev_check_claimed Malloc0; then - echo >&2 "Base device is not expected to be claimed now" - exit 1 -fi - -$rpc_py bdev_ocf_create FullCache wt Malloc0 Malloc1 - -$rpc_py bdev_ocf_get_bdevs FullCache | jq -e \ - '.[0] | .started and .cache.attached and .core.attached' - -if ! (bdev_check_claimed Malloc0 && bdev_check_claimed Malloc1); then - echo >&2 "Base devices expected to be claimed now" - exit 1 -fi - -$rpc_py bdev_ocf_delete FullCache -if bdev_check_claimed Malloc0 && bdev_check_claimed Malloc1; then - echo >&2 "Base devices are not expected to be claimed now" - exit 1 -fi - -$rpc_py bdev_ocf_create HotCache wt Malloc0 Malloc1 - -if ! (bdev_check_claimed Malloc0 && bdev_check_claimed Malloc1); then - echo >&2 "Base devices expected to be claimed now" - exit 1 -fi - -$rpc_py bdev_malloc_delete Malloc0 - -if bdev_check_claimed Malloc1; then - echo >&2 "Base device is not expected to be claimed now" - exit 1 -fi - -status=$($rpc_py bdev_get_bdevs) -gone=$(echo $status | jq 'map(select(.name == "HotCache")) == []') -if [[ $gone == false ]]; then - echo >&2 "OCF bdev is expected to unregister" - exit 1 -fi - -# check if shutdown of running CAS bdev is ok -$rpc_py bdev_ocf_create PartCache wt NonExisting Malloc1 - -trap - SIGINT SIGTERM EXIT - -killprocess $spdk_pid diff --git a/test/ocf/management/multicore.sh b/test/ocf/management/multicore.sh deleted file mode 100755 index ce1ff5950c2..00000000000 --- a/test/ocf/management/multicore.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (C) 2019 Intel Corporation -# All rights reserved. -# -curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) -source $rootdir/test/common/autotest_common.sh - -rpc_py=$rootdir/scripts/rpc.py - -spdk_pid='?' -function start_spdk() { - $SPDK_BIN_DIR/iscsi_tgt & - spdk_pid=$! - trap 'killprocess $spdk_pid; exit 1' SIGINT SIGTERM EXIT - waitforlisten $spdk_pid -} -function stop_spdk() { - killprocess $spdk_pid - trap - SIGINT SIGTERM EXIT -} - -start_spdk - -# Hotplug case - -$rpc_py bdev_malloc_create 1 512 -b Core0 -$rpc_py bdev_malloc_create 1 512 -b Core1 - -$rpc_py bdev_ocf_create C1 wt Cache Core0 -$rpc_py bdev_ocf_create C2 wt Cache Core1 - -$rpc_py bdev_ocf_get_bdevs | jq -e \ - 'any(select(.started)) == false' - -$rpc_py bdev_malloc_create 101 512 -b Cache - -$rpc_py bdev_ocf_get_bdevs | jq -e \ - 'all(select(.started)) == true' - -#Be sure that we will not fail delete because examine is still in progress -waitforbdev C2 - -# Detaching cores - -$rpc_py bdev_ocf_delete C2 - -$rpc_py bdev_ocf_get_bdevs C1 | jq -e \ - '.[0] | .started' - -$rpc_py bdev_ocf_create C2 wt Cache Core1 - -$rpc_py bdev_ocf_get_bdevs C2 | jq -e \ - '.[0] | .started' - -# Normal shutdown - -stop_spdk - -# Hotremove case -start_spdk - -$rpc_py bdev_malloc_create 101 512 -b Cache -$rpc_py bdev_malloc_create 101 512 -b Malloc -$rpc_py bdev_malloc_create 1 512 -b Core - -$rpc_py bdev_ocf_create C1 wt Cache Malloc -$rpc_py bdev_ocf_create C2 wt Cache Core - -$rpc_py bdev_ocf_get_bdevs Cache | jq \ - 'length == 2' - -$rpc_py bdev_malloc_delete Cache - -$rpc_py bdev_ocf_get_bdevs | jq -e \ - '. == []' - -# Not fully initialized shutdown - -$rpc_py bdev_ocf_create C1 wt Malloc NonExisting -$rpc_py bdev_ocf_create C2 wt Malloc NonExisting -$rpc_py bdev_ocf_create C3 wt Malloc Core - -stop_spdk diff --git a/test/ocf/management/remove.sh b/test/ocf/management/remove.sh deleted file mode 100755 index 17795411b6d..00000000000 --- a/test/ocf/management/remove.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (C) 2019 Intel Corporation -# All rights reserved. -# -curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) -source $rootdir/test/common/autotest_common.sh - -rpc_py=$rootdir/scripts/rpc.py - -rm -f aio* -truncate -s 128M aio0 -truncate -s 128M aio1 - -jq . <<- JSON > "$curdir/config" - { - "subsystems": [ - { - "subsystem": "bdev", - "config": [ - { - "method": "bdev_aio_create", - "params": { - "name": "ai0", - "block_size": 512, - "filename": "./aio0" - } - }, - { - "method": "bdev_aio_create", - "params": { - "name": "aio1", - "block_size": 512, - "filename": "./aio1" - } - }, - { - "method": "bdev_wait_for_examine" - } - ] - } - ] - } -JSON - -"$SPDK_BIN_DIR/iscsi_tgt" --json "$curdir/config" & -spdk_pid=$! - -waitforlisten $spdk_pid - -# Create ocf on persistent storage - -$rpc_py bdev_ocf_create ocfWT wt aio0 aio1 - -# Check that ocfWT was created properly - -$rpc_py bdev_ocf_get_bdevs | jq -r '.[] .name' | grep -qw ocfWT - -# Remove ocfWT, after delete via rpc ocf bdev should not load on next app start - -$rpc_py bdev_ocf_delete ocfWT - -# Check that ocfWT was deleted properly - -[[ -z $("$rpc_py" bdev_ocf_get_bdevs | jq -r '.[] | select(.name == "ocfWT") | .name') ]] - -trap - SIGINT SIGTERM EXIT - -killprocess $spdk_pid - -# Check for ocfWT was deleted permanently -"$SPDK_BIN_DIR/iscsi_tgt" --json "$curdir/config" & -spdk_pid=$! - -trap 'killprocess $spdk_pid; rm -f aio* $curdir/config ocf_bdevs ocf_bdevs_verify; exit 1' SIGINT SIGTERM EXIT - -waitforlisten $spdk_pid - -# Check that ocfWT was not loaded on app start - -(($("$rpc_py" bdev_ocf_get_bdevs | jq 'length') == 0)) - -trap - SIGINT SIGTERM EXIT - -killprocess $spdk_pid -rm -f aio* $curdir/config ocf_bdevs ocf_bdevs_verify From cbbefafbe75cee9fac685eb3ad08181c7c878a4e Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Tue, 7 Oct 2025 17:16:59 +0200 Subject: [PATCH 24/34] Refactor new tests Change-Id: I107349f86aef598b5ec112d1a6df5de20ea4a2c3 Signed-off-by: Rafal Stefanowski --- test/ocf/common.sh | 739 +++++++++++++++++-- test/ocf/management/config_change.sh | 442 ++--------- test/ocf/management/detach-attach.sh | 24 +- test/ocf/management/hotremove.sh | 206 ++---- test/ocf/management/info_dump.sh | 16 +- test/ocf/management/run.sh | 4 +- test/ocf/management/start-stop/basic.sh | 117 ++- test/ocf/management/start-stop/examine.sh | 243 ++++-- test/ocf/management/start-stop/incomplete.sh | 165 +++-- test/ocf/management/start-stop/load.sh | 337 ++++++--- test/ocf/management/start-stop/run.sh | 4 +- test/ocf/ocf.sh | 4 +- 12 files changed, 1426 insertions(+), 875 deletions(-) diff --git a/test/ocf/common.sh b/test/ocf/common.sh index 3c750ce547c..8c793c3a010 100644 --- a/test/ocf/common.sh +++ b/test/ocf/common.sh @@ -7,10 +7,106 @@ source "$rootdir/test/common/autotest_common.sh" rpc_py="$rootdir/scripts/rpc.py" -cache_dev_persistent_addr=(05:00.0 06:00.0 08:00.0) + +cache_line_sizes=(4 8 16 32 64) +cache_modes=("wt" "wb" "wa" "wo" "wi" "pt") +promotion_policies=("always" "nhit") +promotion_nhit_insertion_threshold_range=(2 1000) +promotion_nhit_trigger_threshold_range=(0 100) +cleaning_policies=("alru" "acp" "nop") +cleaning_acp_wake_up_time_range=(0 10000) +cleaning_acp_flush_max_buffers_range=(1 10000) +cleaning_alru_wake_up_time_range=(0 3600) +cleaning_alru_flush_max_buffers_range=(1 10000) +cleaning_alru_staleness_time_range=(1 3600) +cleaning_alru_activity_threshold_range=(0 1000000) +cleaning_alru_max_dirty_ratio_range=(0 100) +seqcutoff_policies=("always" "full" "never") +seqcutoff_policy_default="full" +seqcutoff_threshold_range=(1 4194181) +seqcutoff_threshold_default=1 +seqcutoff_promotion_count_range=(1 65535) +seqcutoff_promotion_count_default=8 +seqcutoff_promote_on_threshold_range=(0 1) +seqcutoff_promote_on_threshold_default=false + +###### Change those depending on persistent storage availability: +persistent_cache_addr=(05:00.0 06:00.0 08:00.0) + +persistent_cache_create() { + # $1: name of bdev to create + # $2: address/path/identifier of specific device + + if [ $# -ne 2 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local name=$1 + local addr=$2 + + $rpc_py bdev_virtio_attach_controller -t pci -a $addr -d blk $name +} + +persistent_cache_destroy() { + # $1: name of bdev to destroy + + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local name=$1 + + $rpc_py bdev_virtio_detach_controller $name +} +###### + +malloc_cache_create() { + # $1: name of bdev to create + + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local name=$1 + + $rpc_py bdev_malloc_create -b $name 100 512 +} + +malloc_cache_destroy() { + # $1: name of bdev to destroy + + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local name=$1 + + $rpc_py bdev_malloc_delete $name +} + +malloc_core_create() { + # $1: name of bdev to create + + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local name=$1 + + $rpc_py bdev_malloc_create -b $name 200 512 +} + +malloc_core_destroy() { + malloc_cache_destroy "$@" +} start_spdk() { - $SPDK_BIN_DIR/spdk_tgt "$@" & + "$SPDK_BIN_DIR/spdk_tgt" -L vbdev_ocf "$@" & spdk_pid=$! trap 'killprocess $spdk_pid; exit 1' SIGINT SIGTERM EXIT waitforlisten $spdk_pid @@ -21,78 +117,237 @@ stop_spdk() { killprocess $spdk_pid } -create_caches() { +ocf_settled() { + # Check if there are no caches currently busy (under a lock). + # Instead of the usual cache info, an empty JSON object is printed for such cache. + if ! ./scripts/rpc.py bdev_ocf_get_bdevs | jq -e \ + '.caches | any(. == {}) | not'; then + + return 1 + fi + + # Check if there are no cores in caches still in the initialization process. + # Such condition can happen when base bdevs with OCF metadata were found for + # started caches and their configuration is loaded from this metadata in background. + if ! ./scripts/rpc.py bdev_ocf_get_bdevs | jq -e \ + 'if (.caches | length == 0) then true + else ([.caches[].cores[]] | any(. == {}) | not) end'; then + + return 1 + fi + + # By using array arithmetic check if there is no attached + # cores in wait list that belong to any attached cache. + # Those cores should be automatically added to their caches, so if there + # are any cores left it means that the adding process did not finish yet. + # Such condition can happen when base bdevs were found for started caches + # (or added cores in wait list) and then attaching caches and moving cores + # from wait list is done in background. + if ! ./scripts/rpc.py bdev_ocf_get_bdevs | jq -e \ + '([.cores_waitlist[] | select(.base_attached).cache_name] | unique) - + (([.cores_waitlist[] | select(.base_attached).cache_name] | unique) - + ([.caches[] | select(.base_attached).name])) == []'; then + + return 1 + fi +} + +create_caches_do() { for i in {1..3}; do - $rpc_py bdev_malloc_create -b Cache_dev$i 100 512 + malloc_cache_create Cache_dev$i done +} + +create_caches() { + create_caches_do - # Give it some time to settle before checking, as there might be - # some cores in waitlist that needs to be moved to started caches. - sleep 3 + # Give it some time to settle before returning, as there might be + # some cores in wait list that need to be moved to attached caches. + waitforcondition ocf_settled $rpc_py bdev_ocf_get_bdevs | jq -e '.' } +create_caches_with_metadata() { + create_caches + start_caches + stop_caches +} + +create_caches_with_metadata_with_cores() { + create_caches + create_cores + start_caches + add_cores + stop_caches + destroy_cores +} + destroy_caches() { for i in {1..3}; do - $rpc_py bdev_malloc_delete Cache_dev$i + malloc_cache_destroy Cache_dev$i done $rpc_py bdev_ocf_get_bdevs | jq -e '.' } -create_caches_persistent() { +create_caches_only_first() { + malloc_cache_create Cache_dev1 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +destroy_caches_only_first() { + malloc_cache_destroy Cache_dev1 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +create_caches_all_but_first() { + malloc_cache_create Cache_dev2 + malloc_cache_create Cache_dev3 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +destroy_caches_all_but_first() { + malloc_cache_destroy Cache_dev2 + malloc_cache_destroy Cache_dev3 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +create_caches_persistent_do() { for i in {1..3}; do - $rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[i - 1]}" -d blk Cache_dev$i + persistent_cache_create Cache_dev$i "${persistent_cache_addr[i - 1]}" done +} + +create_caches_persistent() { + create_caches_persistent_do - # Give a bit more time to settle, because this will be called after - # caches load with attached cores in waitlist and it may take more time - # to move all of them to loaded caches. - sleep 15 + # Give it a bit more time to settle, because this might be called after + # caches load with attached cores in wait list and it may take more time + # to load caches configuration and then move all cores to loaded caches. + waitforcondition ocf_settled 20 $rpc_py bdev_ocf_get_bdevs | jq -e '.' } +create_caches_persistent_with_metadata() { + create_caches_persistent + start_caches + stop_caches +} + +create_caches_persistent_with_metadata_with_cores() { + create_caches_persistent + create_cores + start_caches + add_cores + stop_caches + destroy_cores +} + destroy_caches_persistent() { for i in {1..3}; do - $rpc_py bdev_virtio_detach_controller Cache_dev$i + persistent_cache_destroy Cache_dev$i done $rpc_py bdev_ocf_get_bdevs | jq -e '.' } -create_cores() { +create_cores_do() { for i in {1..3}; do for j in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-$j 200 512 + malloc_core_create Core_dev$i-$j done done +} + +create_cores() { + create_cores_do + + # Give it some time to settle before returning, as there might be some cores + # in wait list that need to be moved to their caches after attaching bases. + waitforcondition ocf_settled + $rpc_py bdev_ocf_get_bdevs | jq -e '.' } destroy_cores() { for i in {1..3}; do for j in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-$j + malloc_core_destroy Core_dev$i-$j done done $rpc_py bdev_ocf_get_bdevs | jq -e '.' } -start_caches() { +create_cores_only_first() { for i in {1..3}; do - $rpc_py bdev_ocf_start_cache Ocf_cache$i Cache_dev$i --no-load + malloc_core_create Core_dev$i-1 done - # executed before cores from waitlist are added - sleep 3 $rpc_py bdev_ocf_get_bdevs | jq -e '.' } -start_caches_try_load() { +destroy_cores_only_first() { + for i in {1..3}; do + malloc_core_destroy Core_dev$i-1 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +create_cores_all_but_first() { + for i in {1..3}; do + malloc_core_create Core_dev$i-2 + malloc_core_create Core_dev$i-3 + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +destroy_cores_all_but_first() { for i in {1..3}; do - $rpc_py bdev_ocf_start_cache Ocf_cache$i Cache_dev$i + malloc_core_destroy Core_dev$i-2 + malloc_core_destroy Core_dev$i-3 done - # executed before cores from waitlist are added - sleep 3 + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +start_caches_do() { + for i in {1..3}; do + $rpc_py bdev_ocf_start_cache Ocf_cache$i Cache_dev$i "$@" + done +} + +start_caches() { + start_caches_do --no-load + + # Give it some time to settle before returning, as there might be + # some cores in wait list that need to be moved to attached caches. + waitforcondition ocf_settled + + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +start_caches_try_load() { + start_caches_do + + # Give it some time to settle before returning, as there might be + # some cores in wait list that need to be moved to attached caches. + waitforcondition ocf_settled + + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +start_caches_with_cache_line_size() { + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local cache_line_size=$1 + + start_caches_do --cache-line-size $cache_line_size --no-load + + # Give it some time to settle before returning, as there might be + # some cores in wait list that need to be moved to attached caches. + waitforcondition ocf_settled + $rpc_py bdev_ocf_get_bdevs | jq -e '.' } @@ -117,12 +372,17 @@ attach_caches() { $rpc_py bdev_ocf_get_bdevs | jq -e '.' } -add_cores() { +add_cores_do() { for i in {1..3}; do for j in {1..3}; do $rpc_py bdev_ocf_add_core Ocf_core$i-$j Core_dev$i-$j Ocf_cache$i done done +} + +add_cores() { + add_cores_do + $rpc_py bdev_ocf_get_bdevs | jq -e '.' } @@ -135,44 +395,228 @@ remove_cores() { $rpc_py bdev_ocf_get_bdevs | jq -e '.' } +set_cache_mode() { + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local cache_mode=$1 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_cachemode Ocf_cache$i $cache_mode + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_promotion_always_params() { + for i in {1..3}; do + $rpc_py bdev_ocf_set_promotion Ocf_cache$i \ + --policy always + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_promotion_nhit_params() { + if [ $# -ne 2 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local insertion_threshold=$1 + local trigger_threshold=$2 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_promotion Ocf_cache$i \ + --policy nhit \ + --nhit-insertion-threshold $insertion_threshold \ + --nhit-trigger-threshold $trigger_threshold + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_cleaning_alru_params() { + if [ $# -ne 5 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local wake_up_time=$1 + local flush_max_buffers=$2 + local staleness_time=$3 + local activity_threshold=$4 + local max_dirty_ratio=$5 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_cleaning Ocf_cache$i \ + --policy alru \ + --alru-wake-up-time $wake_up_time \ + --alru-flush-max-buffers $flush_max_buffers \ + --alru-staleness-time $staleness_time \ + --alru-activity-threshold $activity_threshold \ + --alru-max-dirty-ratio $max_dirty_ratio + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_cleaning_acp_params() { + if [ $# -ne 2 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local wake_up_time=$1 + local flush_max_buffers=$2 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_cleaning Ocf_cache$i \ + --policy acp \ + --acp-wake-up-time $wake_up_time \ + --acp-flush-max-buffers $flush_max_buffers + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_cleaning_nop_params() { + for i in {1..3}; do + $rpc_py bdev_ocf_set_cleaning Ocf_cache$i \ + --policy nop + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_seqcutoff_params() { + if [ $# -ne 4 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local policy=$1 + local threshold=$2 + local promotion_count=$3 + local promote_on_threshold=$4 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_seqcutoff Ocf_core$i-1 \ + --policy $policy \ + --threshold $threshold \ + --promotion-count $promotion_count \ + --promote-on-threshold $promote_on_threshold + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +set_seqcutoff_params_all() { + if [ $# -ne 4 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local policy=$1 + local threshold=$2 + local promotion_count=$3 + local promote_on_threshold=$4 + + for i in {1..3}; do + $rpc_py bdev_ocf_set_seqcutoff Ocf_cache$i \ + --policy $policy \ + --threshold $threshold \ + --promotion-count $promotion_count \ + --promote-on-threshold $promote_on_threshold + done + $rpc_py bdev_ocf_get_bdevs | jq -e '.' +} + +__check_caches_base_claimed() { + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Cache_dev"))] | all(.claimed)' +} + +__check_caches_base_not_claimed() { + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Cache_dev"))] | any(.claimed) | not' +} + __check_caches_empty() { $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 0' } __check_caches_attached() { $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | all(.base_attached)' } __check_caches_detached() { $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | any(.base_attached) | not' +} + +__check_caches_detached_only_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' +} + +__check_caches_detached_all_but_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached | not' +} + +__check_cores_base_claimed() { + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Core_dev"))] | all(.claimed)' +} + +__check_cores_base_not_claimed() { + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Core_dev"))] | any(.claimed) | not' } __check_cores_empty() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 0' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 0' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores | length] | all(. == 0)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores_count] | all(. == 0)' +} + +__check_cores_loading() { + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores | length] | all(. == 3)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores_count] | all(. == 3)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[]] | any(.base_attached) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[]] | all(.loading)' } __check_cores_attached() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores | length] | all(. == 3)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores_count] | all(. == 3)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[]] | all(.base_attached)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[]] | any(.loading) | not' } __check_cores_detached() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores | length] | all(. == 3)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores_count] | all(. == 3)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[]] | any(.base_attached) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[]] | any(.loading) | not' } -__check_cores_loading() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].loading' +__check_cores_detached_only_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores | length] | all(. == 3)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores_count] | all(. == 3)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[0]] | any(.base_attached) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[0]] | any(.loading) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[1]] | all(.base_attached)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[1]] | any(.loading) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[2]] | all(.base_attached)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[2]] | any(.loading) | not' +} + +__check_cores_detached_all_but_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores | length] | all(. == 3)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores_count] | all(. == 3)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[0]] | all(.base_attached)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[0]] | any(.loading) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[1]] | any(.base_attached) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[1]] | any(.loading) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[2]] | any(.base_attached) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[2]] | any(.loading) | not' } __check_cores_waitlist_empty() { @@ -181,12 +625,215 @@ __check_cores_waitlist_empty() { __check_cores_waitlist_attached() { $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 9' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | all(.base_attached)' } __check_cores_waitlist_detached() { $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 9' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | any(.base_attached) | not' +} + +__check_cores_waitlist_detached_only_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 9' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[0].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[1].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[2].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[3].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[4].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[5].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[6].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[7].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[8].base_attached' +} + +__check_cores_waitlist_detached_all_but_first() { + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 9' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[0].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[1].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[2].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[3].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[4].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[5].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[6].base_attached' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[7].base_attached | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[8].base_attached | not' +} + +__check_setup_completed() { + for i in {1..3}; do + for j in {1..3}; do + waitforbdev Core_dev$i-$j 20000 + done + done + + $rpc_py bdev_ocf_get_bdevs | jq -e '.' + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_claimed + __check_cores_waitlist_empty + __check_cores_attached +} + +__check_cache_line_size() { + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local cache_line_size=$1 + + $rpc_py bdev_ocf_get_bdevs | jq -e --argjson cache_line_size $cache_line_size \ + '.caches | all(.cache_line_size == $cache_line_size * 1024)' +} + +__check_cache_mode() { + if [ $# -ne 1 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local cache_mode=$1 + + $rpc_py bdev_ocf_get_bdevs | jq -e --arg cache_mode $cache_mode \ + '.caches | all(.cache_mode == $cache_mode)' +} + +__check_promotion_always_params() { + $rpc_py bdev_ocf_get_bdevs | jq -e \ + '.caches | all(.promotion.policy == "always")' +} + +__check_promotion_nhit_params() { + if [ $# -ne 2 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local insertion_threshold=$1 + local trigger_threshold=$2 + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --argjson insertion_threshold $insertion_threshold \ + --argjson trigger_threshold $trigger_threshold \ + '[.caches[].promotion] | all( + (.policy == "nhit") and + (.insertion_threshold == $insertion_threshold) and + (.trigger_threshold == $trigger_threshold))' +} + +__check_cleaning_alru_params() { + if [ $# -ne 5 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local wake_up_time=$1 + local flush_max_buffers=$2 + local staleness_time=$3 + local activity_threshold=$4 + local max_dirty_ratio=$5 + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --argjson wake_up_time $wake_up_time \ + --argjson flush_max_buffers $flush_max_buffers \ + --argjson staleness_time $staleness_time \ + --argjson activity_threshold $activity_threshold \ + --argjson max_dirty_ratio $max_dirty_ratio \ + '[.caches[].cleaning] | all( + (.policy == "alru") and + (.wake_up_time == $wake_up_time) and + (.flush_max_buffers == $flush_max_buffers) and + (.staleness_time == $staleness_time) and + (.activity_threshold == $activity_threshold) and + (.max_dirty_ratio == $max_dirty_ratio))' +} + +__check_cleaning_acp_params() { + if [ $# -ne 2 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local wake_up_time=$1 + local flush_max_buffers=$2 + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --argjson wake_up_time $wake_up_time \ + --argjson flush_max_buffers $flush_max_buffers \ + '[.caches[].cleaning] | all( + (.policy == "acp") and + (.wake_up_time == $wake_up_time) and + (.flush_max_buffers == $flush_max_buffers))' +} + +__check_cleaning_nop_params() { + $rpc_py bdev_ocf_get_bdevs | jq -e \ + '.caches | all(.cleaning.policy == "nop")' +} + +__check_seqcutoff_params() { + if [ $# -ne 4 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local policy=$1 + local threshold=$2 + local promotion_count=$3 + if [ $4 -eq 0 ]; then + local promote_on_threshold=false + elif [ $4 -eq 1 ]; then + local promote_on_threshold=true + fi + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --arg policy $policy \ + --argjson threshold $threshold \ + --argjson promotion_count $promotion_count \ + --argjson promote_on_threshold $promote_on_threshold \ + '[.caches[].cores[0].seq_cutoff] | all( + (.policy == $policy) and + (.threshold == $threshold * 1024) and + (.promotion_count == $promotion_count) and + (.promote_on_threshold == $promote_on_threshold))' + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --arg policy $seqcutoff_policy_default \ + --argjson threshold $seqcutoff_threshold_default \ + --argjson promotion_count $seqcutoff_promotion_count_default \ + --argjson promote_on_threshold $seqcutoff_promote_on_threshold_default \ + '[.caches[].cores[1].seq_cutoff] | all( + (.policy == $policy) and + (.threshold == $threshold * 1024) and + (.promotion_count == $promotion_count) and + (.promote_on_threshold == $promote_on_threshold))' +} + +__check_seqcutoff_params_all() { + if [ $# -ne 4 ]; then + echo >&2 "invalid number of arguments" + exit 1 + fi + + local policy=$1 + local threshold=$2 + local promotion_count=$3 + if [ $4 -eq 0 ]; then + local promote_on_threshold=false + elif [ $4 -eq 1 ]; then + local promote_on_threshold=true + fi + + $rpc_py bdev_ocf_get_bdevs | jq -e \ + --arg policy $policy \ + --argjson threshold $threshold \ + --argjson promotion_count $promotion_count \ + --argjson promote_on_threshold $promote_on_threshold \ + '[.caches[].cores[].seq_cutoff] | all( + (.policy == $policy) and + (.threshold == $threshold * 1024) and + (.promotion_count == $promotion_count) and + (.promote_on_threshold == $promote_on_threshold))' } # Generate random number between range. diff --git a/test/ocf/management/config_change.sh b/test/ocf/management/config_change.sh index 33468a7f24e..5aeebbeecaa 100755 --- a/test/ocf/management/config_change.sh +++ b/test/ocf/management/config_change.sh @@ -7,381 +7,39 @@ # curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) +rootdir=$(readlink -f "$curdir/../../..") source "$rootdir/test/ocf/common.sh" -cache_modes=("wt" "wb" "wa" "wo" "wi" "pt") +# cache line sizes and cache modes: -promotion_policies=("always" "nhit") -promotion_nhit_insertion_threshold_range=(2 1000) -promotion_nhit_trigger_threshold_range=(0 100) - -cleaning_policies=("alru" "acp" "nop") -cleaning_acp_wake_up_time_range=(0 10000) -cleaning_acp_flush_max_buffers_range=(1 10000) -cleaning_alru_wake_up_time_range=(0 3600) -cleaning_alru_flush_max_buffers_range=(1 10000) -cleaning_alru_staleness_time_range=(1 3600) -cleaning_alru_activity_threshold_range=(0 1000000) -cleaning_alru_max_dirty_ratio_range=(0 100) - -seqcutoff_policies=("always" "full" "never") -seqcutoff_policy_default="full" -seqcutoff_threshold_range=(1 4194181) -seqcutoff_threshold_default=1 -seqcutoff_promotion_count_range=(1 65535) -seqcutoff_promotion_count_default=8 -seqcutoff_promote_on_threshold_range=(0 1) -seqcutoff_promote_on_threshold_default=false - -set_cache_mode() { - if [ $# -ne 1 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local cache_mode=$1 - - for i in {1..3}; do - $rpc_py bdev_ocf_set_cachemode Ocf_cache$i $cache_mode - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' -} - -set_promotion_always_params() { - if [ $# -ne 1 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - - for i in {1..3}; do - $rpc_py bdev_ocf_set_promotion Ocf_cache$i \ - --policy $policy - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' -} - -set_promotion_nhit_params() { - if [ $# -ne 3 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - local insertion_threshold=$2 - local trigger_threshold=$3 - - for i in {1..3}; do - $rpc_py bdev_ocf_set_promotion Ocf_cache$i \ - --policy $policy \ - --nhit-insertion-threshold $insertion_threshold \ - --nhit-trigger-threshold $trigger_threshold - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' -} - -set_cleaning_alru_params() { - if [ $# -ne 6 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - local wake_up_time=$2 - local flush_max_buffers=$3 - local staleness_time=$4 - local activity_threshold=$5 - local max_dirty_ratio=$6 - - for i in {1..3}; do - $rpc_py bdev_ocf_set_cleaning Ocf_cache$i \ - --policy $policy \ - --alru-wake-up-time $wake_up_time \ - --alru-flush-max-buffers $flush_max_buffers \ - --alru-staleness-time $staleness_time \ - --alru-activity-threshold $activity_threshold \ - --alru-max-dirty-ratio $max_dirty_ratio - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' -} - -set_cleaning_acp_params() { - if [ $# -ne 3 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - local wake_up_time=$2 - local flush_max_buffers=$3 - - for i in {1..3}; do - $rpc_py bdev_ocf_set_cleaning Ocf_cache$i \ - --policy $policy \ - --acp-wake-up-time $wake_up_time \ - --acp-flush-max-buffers $flush_max_buffers - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' -} - -set_cleaning_nop_params() { - if [ $# -ne 1 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - - for i in {1..3}; do - $rpc_py bdev_ocf_set_cleaning Ocf_cache$i \ - --policy $policy - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' -} - -set_seqcutoff_params() { - if [ $# -ne 4 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - local threshold=$2 - local promotion_count=$3 - local promote_on_threshold=$4 - - for i in {1..3}; do - $rpc_py bdev_ocf_set_seqcutoff Ocf_core$i-1 \ - --policy $policy \ - --threshold $threshold \ - --promotion-count $promotion_count \ - --promote-on-threshold $promote_on_threshold - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' -} - -set_seqcutoff_params_all() { - if [ $# -ne 4 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - local threshold=$2 - local promotion_count=$3 - local promote_on_threshold=$4 - - for i in {1..3}; do - $rpc_py bdev_ocf_set_seqcutoff Ocf_cache$i \ - --policy $policy \ - --threshold $threshold \ - --promotion-count $promotion_count \ - --promote-on-threshold $promote_on_threshold - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' -} - -__check_cache_mode() { - if [ $# -ne 1 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local cache_mode=$1 - - $rpc_py bdev_ocf_get_bdevs | jq -e --arg cache_mode $cache_mode \ - '.caches[].cache_mode == $cache_mode' -} - -__check_promotion_always_params() { - if [ $# -ne 1 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - - $rpc_py bdev_ocf_get_bdevs | jq -e \ - --arg policy $policy \ - '.caches[].promotion.policy == $policy' -} - -__check_promotion_nhit_params() { - if [ $# -ne 3 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - local insertion_threshold=$2 - local trigger_threshold=$3 - - $rpc_py bdev_ocf_get_bdevs | jq -e \ - --arg policy $policy \ - --argjson insertion_threshold $insertion_threshold \ - --argjson trigger_threshold $trigger_threshold \ - '.caches[].promotion | - (.policy == $policy) and - (.insertion_threshold == $insertion_threshold) and - (.trigger_threshold == $trigger_threshold)' -} - -__check_cleaning_alru_params() { - if [ $# -ne 6 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - local wake_up_time=$2 - local flush_max_buffers=$3 - local staleness_time=$4 - local activity_threshold=$5 - local max_dirty_ratio=$6 - - $rpc_py bdev_ocf_get_bdevs | jq -e \ - --arg policy $policy \ - --argjson wake_up_time $wake_up_time \ - --argjson flush_max_buffers $flush_max_buffers \ - --argjson staleness_time $staleness_time \ - --argjson activity_threshold $activity_threshold \ - --argjson max_dirty_ratio $max_dirty_ratio \ - '.caches[].cleaning | - (.policy == $policy) and - (.wake_up_time == $wake_up_time) and - (.flush_max_buffers == $flush_max_buffers) and - (.staleness_time == $staleness_time) and - (.activity_threshold == $activity_threshold) and - (.max_dirty_ratio == $max_dirty_ratio)' -} - -__check_cleaning_acp_params() { - if [ $# -ne 3 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - local wake_up_time=$2 - local flush_max_buffers=$3 - - $rpc_py bdev_ocf_get_bdevs | jq -e \ - --arg policy $policy \ - --argjson wake_up_time $wake_up_time \ - --argjson flush_max_buffers $flush_max_buffers \ - '.caches[].cleaning | - (.policy == $policy) and - (.wake_up_time == $wake_up_time) and - (.flush_max_buffers == $flush_max_buffers)' -} - -__check_cleaning_nop_params() { - if [ $# -ne 1 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - - $rpc_py bdev_ocf_get_bdevs | jq -e \ - --arg policy $policy \ - '.caches[].cleaning.policy == $policy' -} - -__check_seqcutoff_params() { - if [ $# -ne 4 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - local threshold=$2 - local promotion_count=$3 - if [ $4 -eq 0 ]; then - local promote_on_threshold=false - elif [ $4 -eq 1 ]; then - local promote_on_threshold=true - fi - - $rpc_py bdev_ocf_get_bdevs | jq -e \ - --arg policy $policy \ - --argjson threshold $threshold \ - --argjson promotion_count $promotion_count \ - --argjson promote_on_threshold $promote_on_threshold \ - '.caches[].cores[0].seq_cutoff | - (.policy == $policy) and - (.threshold == $threshold * 1024) and - (.promotion_count == $promotion_count) and - (.promote_on_threshold == $promote_on_threshold)' - - $rpc_py bdev_ocf_get_bdevs | jq -e \ - --arg policy $seqcutoff_policy_default \ - --argjson threshold $seqcutoff_threshold_default \ - --argjson promotion_count $seqcutoff_promotion_count_default \ - --argjson promote_on_threshold $seqcutoff_promote_on_threshold_default \ - '.caches[].cores[1].seq_cutoff | - (.policy == $policy) and - (.threshold == $threshold * 1024) and - (.promotion_count == $promotion_count) and - (.promote_on_threshold == $promote_on_threshold)' -} - -__check_seqcutoff_params_all() { - if [ $# -ne 4 ]; then - echo >&2 "invalid number of arguments" - exit 1 - fi - - local policy=$1 - local threshold=$2 - local promotion_count=$3 - if [ $4 -eq 0 ]; then - local promote_on_threshold=false - elif [ $4 -eq 1 ]; then - local promote_on_threshold=true - fi - - $rpc_py bdev_ocf_get_bdevs | jq -e \ - --arg policy $policy \ - --argjson threshold $threshold \ - --argjson promotion_count $promotion_count \ - --argjson promote_on_threshold $promote_on_threshold \ - '.caches[].cores[].seq_cutoff | - (.policy == $policy) and - (.threshold == $threshold * 1024) and - (.promotion_count == $promotion_count) and - (.promote_on_threshold == $promote_on_threshold)' -} - -# cache modes: - -for add_cores in false true; do - for create_caches in false true; do - for stop_caches in false true; do - start_spdk - start_caches - if [ $create_caches = true ]; then - create_caches - __check_caches_attached - if [ $add_cores = true ]; then - create_cores - add_cores - __check_cores_attached +for cache_line_size in "${cache_line_sizes[@]}"; do + for add_cores in false true; do + for create_caches in false true; do + for stop_caches in false true; do + start_spdk + start_caches_with_cache_line_size $cache_line_size + if [ $create_caches == true ]; then + create_caches + __check_caches_attached + if [ $add_cores == true ]; then + create_cores + add_cores + __check_cores_attached + fi + else + __check_caches_detached fi - else - __check_caches_detached - fi - for cache_mode in "${cache_modes[@]}"; do - set_cache_mode $cache_mode - __check_cache_mode $cache_mode + __check_cache_line_size $cache_line_size + for cache_mode in "${cache_modes[@]}"; do + set_cache_mode $cache_mode + __check_cache_mode $cache_mode + done + if [ $stop_caches == true ]; then + stop_caches + __check_caches_empty + fi + stop_spdk done - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk done done done @@ -395,21 +53,21 @@ for promotion_policy in "${promotion_policies[@]}"; do create_caches start_caches __check_caches_attached - if [ $add_cores = true ]; then + if [ $add_cores == true ]; then create_cores add_cores __check_cores_attached fi - if [ $promotion_policy = always ]; then - set_promotion_always_params $promotion_policy - __check_promotion_always_params $promotion_policy - elif [ $promotion_policy = nhit ]; then + if [ $promotion_policy == always ]; then + set_promotion_always_params + __check_promotion_always_params + elif [ $promotion_policy == nhit ]; then insertion_threshold=$(random_number ${promotion_nhit_insertion_threshold_range[0]} ${promotion_nhit_insertion_threshold_range[1]}) trigger_threshold=$(random_number ${promotion_nhit_trigger_threshold_range[0]} ${promotion_nhit_trigger_threshold_range[1]}) - set_promotion_nhit_params $promotion_policy $insertion_threshold $trigger_threshold - __check_promotion_nhit_params $promotion_policy $insertion_threshold $trigger_threshold + set_promotion_nhit_params $insertion_threshold $trigger_threshold + __check_promotion_nhit_params $insertion_threshold $trigger_threshold fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -427,29 +85,29 @@ for cleaning_policy in "${cleaning_policies[@]}"; do create_caches start_caches __check_caches_attached - if [ $add_cores = true ]; then + if [ $add_cores == true ]; then create_cores add_cores __check_cores_attached fi - if [ $cleaning_policy = alru ]; then + if [ $cleaning_policy == alru ]; then wake_up_time=$(random_number ${cleaning_alru_wake_up_time_range[0]} ${cleaning_alru_wake_up_time_range[1]}) flush_max_buffers=$(random_number ${cleaning_alru_flush_max_buffers_range[0]} ${cleaning_alru_flush_max_buffers_range[1]}) staleness_time=$(random_number ${cleaning_alru_staleness_time_range[0]} ${cleaning_alru_staleness_time_range[1]}) activity_threshold=$(random_number ${cleaning_alru_activity_threshold_range[0]} ${cleaning_alru_activity_threshold_range[1]}) max_dirty_ratio=$(random_number ${cleaning_alru_max_dirty_ratio_range[0]} ${cleaning_alru_max_dirty_ratio_range[1]}) - set_cleaning_alru_params $cleaning_policy $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio - __check_cleaning_alru_params $cleaning_policy $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio - elif [ $cleaning_policy = acp ]; then + set_cleaning_alru_params $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio + __check_cleaning_alru_params $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio + elif [ $cleaning_policy == acp ]; then wake_up_time=$(random_number ${cleaning_acp_wake_up_time_range[0]} ${cleaning_acp_wake_up_time_range[1]}) flush_max_buffers=$(random_number ${cleaning_acp_flush_max_buffers_range[0]} ${cleaning_acp_flush_max_buffers_range[1]}) - set_cleaning_acp_params $cleaning_policy $wake_up_time $flush_max_buffers - __check_cleaning_acp_params $cleaning_policy $wake_up_time $flush_max_buffers - elif [ $cleaning_policy = nop ]; then - set_cleaning_nop_params $cleaning_policy - __check_cleaning_nop_params $cleaning_policy + set_cleaning_acp_params $wake_up_time $flush_max_buffers + __check_cleaning_acp_params $wake_up_time $flush_max_buffers + elif [ $cleaning_policy == nop ]; then + set_cleaning_nop_params + __check_cleaning_nop_params fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -473,14 +131,14 @@ for device in core cache; do threshold=$(random_number ${seqcutoff_threshold_range[0]} ${seqcutoff_threshold_range[1]}) promotion_count=$(random_number ${seqcutoff_promotion_count_range[0]} ${seqcutoff_promotion_count_range[1]}) promote_on_threshold=$(random_number ${seqcutoff_promote_on_threshold_range[0]} ${seqcutoff_promote_on_threshold_range[1]}) - if [ $device = core ]; then + if [ $device == core ]; then set_seqcutoff_params $seqcutoff_policy $threshold $promotion_count $promote_on_threshold __check_seqcutoff_params $seqcutoff_policy $threshold $promotion_count $promote_on_threshold - elif [ $device = cache ]; then + elif [ $device == cache ]; then set_seqcutoff_params_all $seqcutoff_policy $threshold $promotion_count $promote_on_threshold __check_seqcutoff_params_all $seqcutoff_policy $threshold $promotion_count $promote_on_threshold fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi diff --git a/test/ocf/management/detach-attach.sh b/test/ocf/management/detach-attach.sh index 592fd974487..210953ee51a 100755 --- a/test/ocf/management/detach-attach.sh +++ b/test/ocf/management/detach-attach.sh @@ -7,7 +7,7 @@ # curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) +rootdir=$(readlink -f "$curdir/../../..") source "$rootdir/test/ocf/common.sh" # caches only: @@ -20,11 +20,11 @@ for attach_caches in false true; do __check_caches_attached detach_caches __check_caches_detached - if [ $attach_caches = true ]; then + if [ $attach_caches == true ]; then attach_caches __check_caches_attached fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -46,12 +46,12 @@ for attach_caches in false true; do detach_caches __check_caches_detached __check_cores_attached - if [ $attach_caches = true ]; then + if [ $attach_caches == true ]; then attach_caches __check_caches_attached __check_cores_attached fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -73,13 +73,13 @@ for attach_caches in false true; do add_cores __check_caches_detached __check_cores_waitlist_attached - if [ $attach_caches = true ]; then + if [ $attach_caches == true ]; then attach_caches __check_caches_attached __check_cores_attached __check_cores_waitlist_empty fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -104,12 +104,12 @@ for attach_caches in false true; do remove_cores __check_caches_detached __check_cores_empty - if [ $attach_caches = true ]; then + if [ $attach_caches == true ]; then attach_caches __check_caches_attached __check_cores_empty fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -136,13 +136,13 @@ for attach_caches in false true; do __check_caches_detached __check_cores_empty __check_cores_waitlist_empty - if [ $attach_caches = true ]; then + if [ $attach_caches == true ]; then attach_caches __check_caches_attached __check_cores_empty __check_cores_waitlist_empty fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -172,7 +172,7 @@ for stop_caches in false true; do __check_caches_attached __check_cores_empty __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi diff --git a/test/ocf/management/hotremove.sh b/test/ocf/management/hotremove.sh index 049248a7312..0fa0210526a 100755 --- a/test/ocf/management/hotremove.sh +++ b/test/ocf/management/hotremove.sh @@ -7,71 +7,9 @@ # curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) +rootdir=$(readlink -f "$curdir/../../..") source "$rootdir/test/ocf/common.sh" -__check_caches_detached_only_first() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' -} - -__check_caches_detached_all_but_first() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached | not' -} - -__check_cores_detached_only_first() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[0].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[0].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[1].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[1].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[2].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[2].loading | not' -} - -__check_cores_detached_all_but_first() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[0].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[0].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[1].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[1].loading | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[2].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[2].loading | not' -} - -__check_cores_waitlist_detached_only_first() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 9' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[0].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[1].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[2].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[3].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[4].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[5].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[6].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[7].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[8].base_attached' -} - -__check_cores_waitlist_detached_all_but_first() { - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 9' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[0].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[1].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[2].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[3].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[4].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[5].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[6].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[7].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[8].base_attached | not' -} - # cores in waitlist: for remove_cores in false true; do @@ -82,11 +20,11 @@ for remove_cores in false true; do __check_cores_waitlist_attached destroy_cores __check_cores_waitlist_detached - if [ $create_cores = true ]; then + if [ $create_cores == true ]; then create_cores __check_cores_waitlist_attached fi - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores __check_cores_waitlist_empty fi @@ -102,19 +40,13 @@ for remove_cores in false true; do create_cores add_cores __check_cores_waitlist_attached - for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + destroy_cores_only_first __check_cores_waitlist_detached_only_first - if [ $create_cores = true ]; then - for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + if [ $create_cores == true ]; then + create_cores_only_first __check_cores_waitlist_attached fi - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores __check_cores_waitlist_empty fi @@ -130,21 +62,13 @@ for remove_cores in false true; do create_cores add_cores __check_cores_waitlist_attached - for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + destroy_cores_all_but_first __check_cores_waitlist_detached_all_but_first - if [ $create_cores = true ]; then - for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 - $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + if [ $create_cores == true ]; then + create_cores_all_but_first __check_cores_waitlist_attached fi - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores __check_cores_waitlist_empty fi @@ -162,11 +86,11 @@ for create_caches in false true; do __check_caches_attached destroy_caches __check_caches_detached - if [ $create_caches = true ]; then + if [ $create_caches == true ]; then create_caches __check_caches_attached fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -182,15 +106,13 @@ for create_caches in false true; do create_caches start_caches __check_caches_attached - $rpc_py bdev_malloc_delete Cache_dev1 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + destroy_caches_only_first __check_caches_detached_only_first - if [ $create_caches = true ]; then - $rpc_py bdev_malloc_create -b Cache_dev1 100 512 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + if [ $create_caches == true ]; then + create_caches_only_first __check_caches_attached fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -206,17 +128,13 @@ for create_caches in false true; do create_caches start_caches __check_caches_attached - $rpc_py bdev_malloc_delete Cache_dev2 - $rpc_py bdev_malloc_delete Cache_dev3 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + destroy_caches_all_but_first __check_caches_detached_all_but_first - if [ $create_caches = true ]; then - $rpc_py bdev_malloc_create -b Cache_dev2 100 512 - $rpc_py bdev_malloc_create -b Cache_dev3 100 512 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + if [ $create_caches == true ]; then + create_caches_all_but_first __check_caches_attached fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -239,16 +157,16 @@ for remove_cores in false true; do destroy_caches __check_caches_detached __check_cores_attached - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores __check_caches_detached __check_cores_empty fi - if [ $create_caches = true ]; then + if [ $create_caches == true ]; then create_caches __check_caches_attached fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -269,21 +187,19 @@ for remove_cores in false true; do add_cores __check_caches_attached __check_cores_attached - $rpc_py bdev_malloc_delete Cache_dev1 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + destroy_caches_only_first __check_caches_detached_only_first __check_cores_attached - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores __check_caches_detached_only_first __check_cores_empty fi - if [ $create_caches = true ]; then - $rpc_py bdev_malloc_create -b Cache_dev1 100 512 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + if [ $create_caches == true ]; then + create_caches_only_first __check_caches_attached fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -304,23 +220,19 @@ for remove_cores in false true; do add_cores __check_caches_attached __check_cores_attached - $rpc_py bdev_malloc_delete Cache_dev2 - $rpc_py bdev_malloc_delete Cache_dev3 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + destroy_caches_all_but_first __check_caches_detached_all_but_first __check_cores_attached - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores __check_caches_detached_all_but_first __check_cores_empty fi - if [ $create_caches = true ]; then - $rpc_py bdev_malloc_create -b Cache_dev2 100 512 - $rpc_py bdev_malloc_create -b Cache_dev3 100 512 - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + if [ $create_caches == true ]; then + create_caches_all_but_first __check_caches_attached fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -344,17 +256,17 @@ for create_cores in false true; do destroy_cores __check_caches_attached __check_cores_detached - if [ $create_cores = true ]; then + if [ $create_cores == true ]; then create_cores __check_caches_attached __check_cores_attached fi - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores __check_caches_attached __check_cores_empty fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -375,26 +287,20 @@ for create_cores in false true; do add_cores __check_caches_attached __check_cores_attached - for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + destroy_cores_only_first __check_caches_attached __check_cores_detached_only_first - if [ $create_cores = true ]; then - for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-1 200 512 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + if [ $create_cores == true ]; then + create_cores_only_first __check_caches_attached __check_cores_attached fi - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores __check_caches_attached __check_cores_empty fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -415,28 +321,20 @@ for create_cores in false true; do add_cores __check_caches_attached __check_cores_attached - for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-2 - $rpc_py bdev_malloc_delete Core_dev$i-3 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + destroy_cores_all_but_first __check_caches_attached __check_cores_detached_all_but_first - if [ $create_cores = true ]; then - for i in {1..3}; do - $rpc_py bdev_malloc_create -b Core_dev$i-2 200 512 - $rpc_py bdev_malloc_create -b Core_dev$i-3 200 512 - done - $rpc_py bdev_ocf_get_bdevs | jq -e '.' + if [ $create_cores == true ]; then + create_cores_all_but_first __check_caches_attached __check_cores_attached fi - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores __check_caches_attached __check_cores_empty fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -460,13 +358,13 @@ for create_caches_and_cores in false true; do __check_caches_detached destroy_cores __check_cores_detached - if [ $create_caches_and_cores = true ]; then + if [ $create_caches_and_cores == true ]; then create_caches __check_caches_attached create_cores __check_cores_attached fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -492,12 +390,12 @@ for create_caches in false true; do remove_cores __check_caches_detached __check_cores_empty - if [ $create_caches = true ]; then + if [ $create_caches == true ]; then create_caches __check_caches_attached __check_cores_empty fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi @@ -523,7 +421,7 @@ for stop_caches in false true; do __check_cores_attached create_caches __check_caches_attached - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches __check_caches_empty fi diff --git a/test/ocf/management/info_dump.sh b/test/ocf/management/info_dump.sh index eedce4bcbf6..f9c56b06626 100755 --- a/test/ocf/management/info_dump.sh +++ b/test/ocf/management/info_dump.sh @@ -18,13 +18,11 @@ $rpc_py bdev_ocf_start_cache Ocf_cache3 Cache_dev3 --no-load --cache-mode pt --c create_cores add_cores -# First cache detached. -$rpc_py bdev_malloc_delete Cache_dev1 -# First core in each cache detached. -for i in {1..3}; do - $rpc_py bdev_malloc_delete Core_dev$i-1 -done -# First core in wait list detached. +# first cache detached +destroy_caches_only_first +# first core in each cache detached +destroy_cores_only_first +# first core in wait list detached for i in {2..3}; do $rpc_py bdev_malloc_create -b Core_dev_waitlist$i 200 512 done @@ -41,7 +39,7 @@ $rpc_py bdev_ocf_get_stats Ocf_cache1 | jq -e \ ([.requests[], .blocks[], .errors[] | .percentage] | any(. != "0.0"))' $rpc_py bdev_ocf_reset_stats Ocf_cache1 $rpc_py bdev_ocf_get_stats Ocf_cache1 | jq -e \ - '.requests[], .blocks[], .errors[] | .count == 0 and .percentage == "0.0"' + '[.requests[], .blocks[], .errors[]] | all(.count == 0 and .percentage == "0.0")' # Test OCF get bdevs: # remove UUID fields before comparison as they are changing each run @@ -53,7 +51,7 @@ diff <($rpc_py bdev_get_bdevs | jq -e '.[].driver_specific.ocf | select(. != nul <(jq -e . "$curdir/info_dump_driver_specific.json") # Test current config dump: -diff <($rpc_py save_subsystem_config -n bdev | jq -e '.config[] | select(.method|test("bdev_ocf_"))') \ +diff <($rpc_py save_subsystem_config -n bdev | jq -e '.config[] | select(.method | test("bdev_ocf_"))') \ <(jq -e . "$curdir/info_dump_save_config.json") stop_spdk diff --git a/test/ocf/management/run.sh b/test/ocf/management/run.sh index 337e1b38adc..701a6f2e2fa 100755 --- a/test/ocf/management/run.sh +++ b/test/ocf/management/run.sh @@ -7,9 +7,9 @@ # curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) +rootdir=$(readlink -f "$curdir/../../..") -source $rootdir/test/common/autotest_common.sh +source "$rootdir/test/common/autotest_common.sh" run_test "start-stop" "$curdir/start-stop/run.sh" run_test "info_dump" "$curdir/info_dump.sh" diff --git a/test/ocf/management/start-stop/basic.sh b/test/ocf/management/start-stop/basic.sh index 1d6072522e0..5f851a1c838 100755 --- a/test/ocf/management/start-stop/basic.sh +++ b/test/ocf/management/start-stop/basic.sh @@ -7,7 +7,7 @@ # curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../../..) +rootdir=$(readlink -f "$curdir/../../../..") source "$rootdir/test/ocf/common.sh" # start caches: @@ -15,11 +15,19 @@ source "$rootdir/test/ocf/common.sh" for stop_caches in false true; do start_spdk create_caches + __check_caches_base_not_claimed + __check_caches_empty + __check_cores_waitlist_empty start_caches + __check_caches_base_claimed __check_caches_attached - if [ $stop_caches = true ]; then + __check_cores_waitlist_empty + __check_cores_empty + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_waitlist_empty fi stop_spdk done @@ -29,10 +37,17 @@ done for remove_cores in false true; do start_spdk create_cores + __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty add_cores + __check_caches_empty + __check_cores_base_claimed __check_cores_waitlist_attached - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores + __check_caches_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty fi stop_spdk @@ -44,13 +59,27 @@ for stop_caches in false true; do start_spdk create_caches create_cores + __check_caches_base_not_claimed + __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty start_caches + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_not_claimed + __check_cores_waitlist_empty + __check_cores_empty add_cores + __check_caches_base_claimed __check_caches_attached + __check_cores_base_claimed + __check_cores_waitlist_empty __check_cores_attached - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty fi stop_spdk @@ -62,15 +91,26 @@ for stop_caches in false true; do start_spdk create_caches create_cores + __check_caches_base_not_claimed + __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty add_cores + __check_caches_base_not_claimed + __check_caches_empty + __check_cores_base_claimed __check_cores_waitlist_attached start_caches + __check_caches_base_claimed __check_caches_attached - __check_cores_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty fi stop_spdk @@ -84,17 +124,27 @@ for stop_caches in false true; do create_cores start_caches add_cores - __check_caches_attached - __check_cores_attached stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty start_caches + __check_caches_base_claimed __check_caches_attached + __check_cores_base_not_claimed + __check_cores_waitlist_empty add_cores + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_claimed + __check_cores_waitlist_empty __check_cores_attached - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty fi stop_spdk @@ -108,13 +158,18 @@ for stop_caches in false true; do create_cores start_caches add_cores - __check_caches_attached - __check_cores_attached remove_cores + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_not_claimed + __check_cores_waitlist_empty __check_cores_empty - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done @@ -127,17 +182,24 @@ for stop_caches in false true; do create_cores start_caches add_cores - __check_caches_attached - __check_cores_attached for i in {1..3}; do $rpc_py bdev_ocf_remove_core Ocf_core$i-1 done - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 2' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 2' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached' - if [ $stop_caches = true ]; then + __check_caches_base_claimed + __check_caches_attached + __check_cores_waitlist_empty + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Core_dev.-1"))] | any(.claimed) | not' + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Core_dev.-[23]"))] | all(.claimed)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores | length] | all(. == 2)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores_count] | all(. == 2)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[]] | all(.base_attached)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[]] | any(.loading) | not' + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done @@ -150,18 +212,25 @@ for stop_caches in false true; do create_cores start_caches add_cores - __check_caches_attached - __check_cores_attached for i in {1..3}; do $rpc_py bdev_ocf_remove_core Ocf_core$i-2 $rpc_py bdev_ocf_remove_core Ocf_core$i-3 done - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores | length == 1' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores_count == 1' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[].cores[].base_attached' - if [ $stop_caches = true ]; then + __check_caches_base_claimed + __check_caches_attached + __check_cores_waitlist_empty + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Core_dev.-1"))] | all(.claimed)' + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Core_dev.-[23]"))] | any(.claimed) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores | length] | all(. == 1)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores_count] | all(. == 1)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[]] | all(.base_attached)' + $rpc_py bdev_ocf_get_bdevs | jq -e '[.caches[].cores[]] | any(.loading) | not' + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done diff --git a/test/ocf/management/start-stop/examine.sh b/test/ocf/management/start-stop/examine.sh index c8ea6a3bf94..f56862317be 100755 --- a/test/ocf/management/start-stop/examine.sh +++ b/test/ocf/management/start-stop/examine.sh @@ -7,7 +7,7 @@ # curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../../..) +rootdir=$(readlink -f "$curdir/../../../..") source "$rootdir/test/ocf/common.sh" # found cache device (cache only): @@ -15,12 +15,16 @@ source "$rootdir/test/ocf/common.sh" for stop_caches in false true; do start_spdk start_caches - __check_caches_detached create_caches + __check_caches_base_claimed __check_caches_attached - if [ $stop_caches = true ]; then + __check_cores_waitlist_empty + __check_cores_empty + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_waitlist_empty fi stop_spdk done @@ -31,13 +35,14 @@ for stop_caches in false true; do start_spdk start_caches add_cores - __check_caches_detached - __check_cores_waitlist_detached create_caches + __check_caches_base_claimed __check_caches_attached __check_cores_waitlist_detached - if [ $stop_caches = true ]; then + __check_cores_empty + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty __check_cores_waitlist_detached fi @@ -49,11 +54,14 @@ done for remove_cores in false true; do start_spdk add_cores - __check_cores_waitlist_detached create_cores + __check_caches_empty + __check_cores_base_claimed __check_cores_waitlist_attached - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores + __check_caches_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty fi stop_spdk @@ -65,15 +73,15 @@ for stop_caches in false true; do start_spdk start_caches add_cores - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_detached create_cores - __check_cores_empty + __check_caches_detached + __check_cores_base_claimed __check_cores_waitlist_attached - if [ $stop_caches = true ]; then + __check_cores_empty + if [ $stop_caches == true ]; then stop_caches __check_caches_empty + __check_cores_base_claimed __check_cores_waitlist_attached fi stop_spdk @@ -85,63 +93,42 @@ for stop_caches in false true; do start_spdk add_cores start_caches - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_detached create_cores - __check_cores_empty + __check_caches_detached + __check_cores_base_claimed __check_cores_waitlist_attached - if [ $stop_caches = true ]; then + __check_cores_empty + if [ $stop_caches == true ]; then stop_caches __check_caches_empty + __check_cores_base_claimed __check_cores_waitlist_attached fi stop_spdk done -# found core device, then start cache: - -for remove_cores in false true; do - for stop_caches in false true; do - start_spdk - add_cores - __check_cores_waitlist_detached - create_cores - __check_cores_waitlist_attached - start_caches - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_attached - if [ $remove_cores = true ]; then - remove_cores - __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk - done -done - # found cache and then core device: for stop_caches in false true; do start_spdk start_caches add_cores - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_detached create_caches + __check_caches_base_claimed __check_caches_attached + __check_cores_waitlist_detached __check_cores_empty create_cores - __check_cores_attached + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty fi stop_spdk @@ -153,19 +140,22 @@ for stop_caches in false true; do start_spdk start_caches add_cores - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_detached create_cores - __check_cores_empty + __check_caches_detached + __check_cores_base_claimed __check_cores_waitlist_attached + __check_cores_empty create_caches + __check_caches_base_claimed __check_caches_attached - __check_cores_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty fi stop_spdk @@ -177,18 +167,22 @@ for stop_caches in false true; do start_spdk add_cores start_caches - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_detached create_caches + __check_caches_base_claimed __check_caches_attached + __check_cores_waitlist_detached __check_cores_empty create_cores - __check_cores_attached + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty fi stop_spdk @@ -200,24 +194,104 @@ for stop_caches in false true; do start_spdk add_cores start_caches - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_detached create_cores - __check_cores_empty + __check_caches_detached + __check_cores_base_claimed __check_cores_waitlist_attached + __check_cores_empty create_caches + __check_caches_base_claimed __check_caches_attached - __check_cores_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty fi stop_spdk done +# found core device, then start cache: + +for remove_cores in false true; do + for stop_caches in false true; do + start_spdk + add_cores + create_cores + __check_caches_empty + __check_cores_base_claimed + __check_cores_waitlist_attached + start_caches + __check_caches_detached + __check_cores_base_claimed + __check_cores_waitlist_attached + __check_cores_empty + if [ $remove_cores == true ]; then + remove_cores + __check_caches_detached + __check_cores_base_not_claimed + __check_cores_waitlist_empty + __check_cores_empty + fi + if [ $stop_caches == true ]; then + stop_caches + __check_caches_empty + if [ $remove_cores == true ]; then + __check_cores_base_not_claimed + __check_cores_waitlist_empty + else + __check_cores_base_claimed + __check_cores_waitlist_attached + fi + fi + stop_spdk + done +done + +# found core device, then start cache, then found cache device: + +for remove_cores in false true; do + for stop_caches in false true; do + start_spdk + add_cores + create_cores + __check_caches_empty + __check_cores_base_claimed + __check_cores_waitlist_attached + start_caches + __check_caches_detached + __check_cores_base_claimed + __check_cores_waitlist_attached + __check_cores_empty + create_caches + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_claimed + __check_cores_waitlist_empty + __check_cores_attached + if [ $remove_cores == true ]; then + remove_cores + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_not_claimed + __check_cores_waitlist_empty + __check_cores_empty + fi + if [ $stop_caches == true ]; then + stop_caches + __check_caches_base_not_claimed + __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty + fi + stop_spdk + done +done + # found core device (not yet added) on detached cache: for stop_caches in false true; do @@ -225,22 +299,33 @@ for stop_caches in false true; do create_caches start_caches add_cores + __check_caches_base_claimed __check_caches_attached - __check_cores_empty __check_cores_waitlist_detached + __check_cores_empty detach_caches + __check_caches_base_not_claimed __check_caches_detached + __check_cores_waitlist_detached + __check_cores_empty create_cores + __check_caches_base_not_claimed __check_caches_detached - __check_cores_empty + __check_cores_base_claimed __check_cores_waitlist_attached + __check_cores_empty attach_caches + __check_caches_base_claimed __check_caches_attached - __check_cores_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done @@ -253,21 +338,35 @@ for stop_caches in false true; do create_cores start_caches add_cores - __check_caches_attached - __check_cores_attached detach_caches + __check_caches_base_not_claimed __check_caches_detached + __check_cores_base_claimed + __check_cores_waitlist_empty + __check_cores_attached destroy_cores + __check_caches_base_not_claimed + __check_caches_detached + __check_cores_waitlist_empty __check_cores_detached create_cores + __check_caches_base_not_claimed __check_caches_detached + __check_cores_base_claimed + __check_cores_waitlist_empty __check_cores_attached attach_caches + __check_caches_base_claimed __check_caches_attached + __check_cores_base_claimed + __check_cores_waitlist_empty __check_cores_attached - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done diff --git a/test/ocf/management/start-stop/incomplete.sh b/test/ocf/management/start-stop/incomplete.sh index 02250f378f0..cae743409f6 100755 --- a/test/ocf/management/start-stop/incomplete.sh +++ b/test/ocf/management/start-stop/incomplete.sh @@ -7,7 +7,7 @@ # curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../../..) +rootdir=$(readlink -f "$curdir/../../../..") source "$rootdir/test/ocf/common.sh" # incomplete caches: @@ -16,9 +16,12 @@ for stop_caches in false true; do start_spdk start_caches __check_caches_detached - if [ $stop_caches = true ]; then + __check_cores_waitlist_empty + __check_cores_empty + if [ $stop_caches == true ]; then stop_caches __check_caches_empty + __check_cores_waitlist_empty fi stop_spdk done @@ -28,9 +31,11 @@ done for remove_cores in false true; do start_spdk add_cores + __check_caches_empty __check_cores_waitlist_detached - if [ $remove_cores = true ]; then + if [ $remove_cores == true ]; then remove_cores + __check_caches_empty __check_cores_waitlist_empty fi stop_spdk @@ -38,67 +43,117 @@ done # incomplete caches and incomplete cores: -for remove_cores in false true; do - for stop_caches in false true; do - start_spdk - start_caches - add_cores - __check_caches_detached - __check_cores_waitlist_detached - if [ $remove_cores = true ]; then - remove_cores - __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk +for inverse in false true; do + for remove_cores in false true; do + for stop_caches in false true; do + start_spdk + if [ $inverse == false ]; then + start_caches + add_cores + else + add_cores + start_caches + fi + __check_caches_detached + __check_cores_waitlist_detached + __check_cores_empty + if [ $remove_cores == true ]; then + remove_cores + __check_caches_detached + __check_cores_waitlist_empty + __check_cores_empty + fi + if [ $stop_caches == true ]; then + stop_caches + __check_caches_empty + if [ $remove_cores == true ]; then + __check_cores_waitlist_empty + else + __check_cores_waitlist_detached + fi + fi + stop_spdk + done done done # complete caches and incomplete cores: -for remove_cores in false true; do - for stop_caches in false true; do - start_spdk - create_caches - start_caches - add_cores - __check_caches_attached - __check_cores_empty - __check_cores_waitlist_detached - if [ $remove_cores = true ]; then - remove_cores - __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk +for inverse in false true; do + for remove_cores in false true; do + for stop_caches in false true; do + start_spdk + create_caches + if [ $inverse == false ]; then + start_caches + add_cores + else + add_cores + start_caches + fi + __check_caches_base_claimed + __check_caches_attached + __check_cores_waitlist_detached + __check_cores_empty + if [ $remove_cores == true ]; then + remove_cores + __check_caches_base_claimed + __check_caches_attached + __check_cores_waitlist_empty + __check_cores_empty + fi + if [ $stop_caches == true ]; then + stop_caches + __check_caches_base_not_claimed + __check_caches_empty + if [ $remove_cores == true ]; then + __check_cores_waitlist_empty + else + __check_cores_waitlist_detached + fi + fi + stop_spdk + done done done # incomplete caches and complete cores: -for remove_cores in false true; do - for stop_caches in false true; do - start_spdk - create_cores - start_caches - add_cores - __check_caches_detached - __check_cores_empty - __check_cores_waitlist_attached - if [ $remove_cores = true ]; then - remove_cores - __check_cores_waitlist_empty - fi - if [ $stop_caches = true ]; then - stop_caches - __check_caches_empty - fi - stop_spdk +for inverse in false true; do + for remove_cores in false true; do + for stop_caches in false true; do + start_spdk + create_cores + if [ $inverse == false ]; then + start_caches + add_cores + else + add_cores + start_caches + fi + __check_caches_detached + __check_cores_base_claimed + __check_cores_waitlist_attached + __check_cores_empty + if [ $remove_cores == true ]; then + remove_cores + __check_caches_detached + __check_cores_base_not_claimed + __check_cores_waitlist_empty + __check_cores_empty + fi + if [ $stop_caches == true ]; then + stop_caches + __check_caches_empty + if [ $remove_cores == true ]; then + __check_cores_base_not_claimed + __check_cores_waitlist_empty + else + __check_cores_base_claimed + __check_cores_waitlist_attached + fi + fi + stop_spdk + done done done diff --git a/test/ocf/management/start-stop/load.sh b/test/ocf/management/start-stop/load.sh index da7ca3a08dd..291b6994676 100755 --- a/test/ocf/management/start-stop/load.sh +++ b/test/ocf/management/start-stop/load.sh @@ -7,68 +7,38 @@ # curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../../..) +rootdir=$(readlink -f "$curdir/../../../..") source "$rootdir/test/ocf/common.sh" -create_caches_with_metadata() { - create_caches - start_caches - __check_caches_attached - stop_caches - __check_caches_empty -} - -create_caches_with_metadata_with_cores() { - create_caches - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - stop_caches - __check_caches_empty - destroy_cores -} - -create_caches_with_metadata_persistent() { - create_caches_persistent - start_caches - __check_caches_attached - stop_caches - __check_caches_empty -} - -create_caches_with_metadata_with_cores_persistent() { - create_caches_persistent - create_cores - start_caches - add_cores - __check_caches_attached - __check_cores_attached - stop_caches - __check_caches_empty - destroy_cores -} - # load caches without cores in metadata: for add_cores in false true; do for stop_caches in false true; do start_spdk create_caches_with_metadata + __check_caches_base_not_claimed create_cores + __check_cores_base_not_claimed start_caches_try_load + __check_caches_base_claimed __check_caches_attached - __check_cores_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty - if [ $add_cores = true ]; then + __check_cores_empty + if [ $add_cores == true ]; then add_cores - __check_cores_attached + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_claimed __check_cores_waitlist_empty + __check_cores_attached fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done @@ -80,20 +50,29 @@ for add_cores in false true; do for stop_caches in false true; do start_spdk create_caches_with_metadata_with_cores + __check_caches_base_not_claimed create_cores + __check_cores_base_not_claimed start_caches_try_load + __check_caches_base_claimed __check_caches_attached - __check_cores_loading + __check_cores_base_not_claimed __check_cores_waitlist_empty - if [ $add_cores = true ]; then + __check_cores_loading + if [ $add_cores == true ]; then add_cores + __check_caches_base_claimed __check_caches_attached - __check_cores_attached + __check_cores_base_claimed __check_cores_waitlist_empty + __check_cores_attached fi - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done @@ -101,35 +80,62 @@ done # load caches with cores in metadata (add cores first): -for stop_caches in false true; do - start_spdk - create_caches_with_metadata_with_cores - add_cores - __check_cores_waitlist_detached - start_caches_try_load - __check_caches_attached - __check_cores_loading - __check_cores_waitlist_detached - if [ $stop_caches = true ]; then - stop_caches +for create_cores in false true; do + for stop_caches in false true; do + start_spdk + create_caches_with_metadata_with_cores + add_cores + __check_caches_base_not_claimed __check_caches_empty - fi - stop_spdk + __check_cores_waitlist_detached + start_caches_try_load + __check_caches_base_claimed + __check_caches_attached + __check_cores_waitlist_detached + __check_cores_loading + if [ $create_cores == true ]; then + create_cores + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_claimed + __check_cores_waitlist_empty + __check_cores_attached + fi + if [ $stop_caches == true ]; then + stop_caches + __check_caches_base_not_claimed + __check_caches_empty + if [ $create_cores == true ]; then + __check_cores_base_not_claimed + __check_cores_waitlist_empty + else + __check_cores_waitlist_detached + fi + fi + stop_spdk + done done # examine found cache device with metadata (without cores in metadata): for stop_caches in false true; do start_spdk - create_caches_with_metadata_persistent + create_caches_persistent_with_metadata + __check_caches_base_not_claimed destroy_caches_persistent start_caches_try_load __check_caches_detached + __check_cores_waitlist_empty create_caches_persistent + __check_caches_base_claimed __check_caches_attached - if [ $stop_caches = true ]; then + __check_cores_waitlist_empty + __check_cores_empty + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_waitlist_empty fi stop_spdk done @@ -138,18 +144,29 @@ done for stop_caches in false true; do start_spdk - create_caches_with_metadata_with_cores_persistent + create_caches_persistent_with_metadata_with_cores + __check_caches_base_not_claimed create_cores + __check_cores_base_not_claimed destroy_caches_persistent + __check_cores_base_not_claimed start_caches_try_load __check_caches_detached + __check_cores_base_not_claimed + __check_cores_waitlist_empty __check_cores_empty create_caches_persistent + __check_caches_base_claimed __check_caches_attached + __check_cores_base_not_claimed + __check_cores_waitlist_empty __check_cores_loading - if [ $stop_caches = true ]; then + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done @@ -158,71 +175,102 @@ done for stop_caches in false true; do start_spdk - create_caches_with_metadata_with_cores_persistent + create_caches_persistent_with_metadata_with_cores + __check_caches_base_not_claimed create_cores + __check_cores_base_not_claimed destroy_caches_persistent + __check_cores_base_not_claimed start_caches_try_load __check_caches_detached - __check_cores_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty + __check_cores_empty create_caches_persistent + __check_caches_base_claimed __check_caches_attached - __check_cores_loading + __check_cores_base_not_claimed __check_cores_waitlist_empty + __check_cores_loading add_cores + __check_caches_base_claimed __check_caches_attached - __check_cores_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done for stop_caches in false true; do start_spdk - create_caches_with_metadata_with_cores_persistent + create_caches_persistent_with_metadata_with_cores + __check_caches_base_not_claimed create_cores + __check_cores_base_not_claimed destroy_caches_persistent + __check_cores_base_not_claimed start_caches_try_load __check_caches_detached - __check_cores_empty + __check_cores_base_not_claimed __check_cores_waitlist_empty + __check_cores_empty add_cores __check_caches_detached - __check_cores_empty + __check_cores_base_claimed __check_cores_waitlist_attached + __check_cores_empty create_caches_persistent + __check_caches_base_claimed __check_caches_attached - __check_cores_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done for stop_caches in false true; do start_spdk - create_caches_with_metadata_with_cores_persistent + create_caches_persistent_with_metadata_with_cores + __check_caches_base_not_claimed create_cores + __check_cores_base_not_claimed destroy_caches_persistent + __check_cores_base_not_claimed add_cores __check_caches_empty + __check_cores_base_claimed __check_cores_waitlist_attached start_caches_try_load __check_caches_detached - __check_cores_empty + __check_cores_base_claimed __check_cores_waitlist_attached + __check_cores_empty create_caches_persistent + __check_caches_base_claimed __check_caches_attached - __check_cores_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done @@ -232,18 +280,29 @@ done for stop_caches in false true; do start_spdk create_caches_with_metadata + __check_caches_base_not_claimed start_caches_try_load + __check_caches_base_claimed __check_caches_attached + __check_cores_waitlist_empty __check_cores_empty add_cores - __check_cores_empty + __check_caches_base_claimed + __check_caches_attached __check_cores_waitlist_detached + __check_cores_empty create_cores - __check_cores_attached + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done @@ -253,44 +312,99 @@ done for stop_caches in false true; do start_spdk create_caches_with_metadata_with_cores + __check_caches_base_not_claimed start_caches_try_load + __check_caches_base_claimed __check_caches_attached + __check_cores_waitlist_empty __check_cores_loading add_cores - __check_cores_loading + __check_caches_base_claimed + __check_caches_attached __check_cores_waitlist_detached + __check_cores_loading create_cores - __check_cores_attached + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done -# examine found cache with metadata and core devices after cache load: +# examine found cache with metadata first and then core devices after cache load: for stop_caches in false true; do start_spdk - create_caches_with_metadata_with_cores_persistent + create_caches_persistent_with_metadata_with_cores + __check_caches_base_not_claimed destroy_caches_persistent add_cores + __check_caches_empty __check_cores_waitlist_detached start_caches_try_load __check_caches_detached + __check_cores_waitlist_detached __check_cores_empty create_caches_persistent + __check_caches_base_claimed __check_caches_attached - __check_cores_loading __check_cores_waitlist_detached + __check_cores_loading create_cores + __check_caches_base_claimed __check_caches_attached + __check_cores_base_claimed + __check_cores_waitlist_empty __check_cores_attached + if [ $stop_caches == true ]; then + stop_caches + __check_caches_base_not_claimed + __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty + fi + stop_spdk +done + +# examine found core devices first and then cache with metadata after cache load: + +for stop_caches in false true; do + start_spdk + create_caches_persistent_with_metadata_with_cores + __check_caches_base_not_claimed + destroy_caches_persistent + add_cores + __check_caches_empty + __check_cores_waitlist_detached + start_caches_try_load + __check_caches_detached + __check_cores_waitlist_detached + __check_cores_empty + create_cores + __check_caches_detached + __check_cores_base_claimed + __check_cores_waitlist_attached + __check_cores_empty + create_caches_persistent + __check_caches_base_claimed + __check_caches_attached + __check_cores_base_claimed __check_cores_waitlist_empty - if [ $stop_caches = true ]; then + __check_cores_attached + if [ $stop_caches == true ]; then stop_caches + __check_caches_base_not_claimed __check_caches_empty + __check_cores_base_not_claimed + __check_cores_waitlist_empty fi stop_spdk done @@ -299,17 +413,22 @@ done for stop_caches in false true; do start_spdk - create_caches_with_metadata_with_cores_persistent + create_caches_persistent_with_metadata_with_cores + __check_caches_base_not_claimed destroy_caches_persistent add_cores + __check_caches_empty __check_cores_waitlist_detached start_caches_try_load __check_caches_detached + __check_cores_waitlist_detached __check_cores_empty - $rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[1]}" -d blk Cache_dev2 - $rpc_py bdev_virtio_attach_controller -t pci -a "${cache_dev_persistent_addr[2]}" -d blk Cache_dev3 - sleep 3 + + persistent_cache_create Cache_dev2 "${persistent_cache_addr[1]}" + persistent_cache_create Cache_dev3 "${persistent_cache_addr[2]}" + waitforcondition ocf_settled $rpc_py bdev_ocf_get_bdevs | jq -e '.' + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Cache_dev[23]"))] | all(.claimed)' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' @@ -317,19 +436,22 @@ for stop_caches in false true; do $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores_count == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | any(.base_attached) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | all(.loading)' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores_count == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | any(.base_attached) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | all(.loading)' __check_cores_waitlist_detached + for i in {1..3}; do $rpc_py bdev_malloc_create -b Core_dev2-$i 200 512 done - sleep 3 + waitforcondition ocf_settled $rpc_py bdev_ocf_get_bdevs | jq -e '.' + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Cache_dev[23]"))] | all(.claimed)' + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Core_dev2-."))] | all(.claimed)' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches | length == 3' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].base_attached | not' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[0].cores | length == 0' @@ -337,18 +459,23 @@ for stop_caches in false true; do $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].base_attached' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | length == 3' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores_count == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].base_attached' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores[].loading | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | all(.base_attached)' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[1].cores | any(.loading) | not' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].base_attached' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | length == 3' $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores_count == 3' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].base_attached | not' - $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores[].loading' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | any(.base_attached) | not' + $rpc_py bdev_ocf_get_bdevs | jq -e '.caches[2].cores | all(.loading)' $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 6' - $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist[].base_attached | not' - if [ $stop_caches = true ]; then + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | any(.base_attached) | not' + + if [ $stop_caches == true ]; then stop_caches + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Cache_dev[23]"))] | any(.claimed) | not' + $rpc_py bdev_get_bdevs | jq -e '[.[] | select(.name | test("Core_dev2-."))] | any(.claimed) | not' __check_caches_empty + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | length == 6' + $rpc_py bdev_ocf_get_bdevs | jq -e '.cores_waitlist | any(.base_attached) | not' fi stop_spdk done diff --git a/test/ocf/management/start-stop/run.sh b/test/ocf/management/start-stop/run.sh index e5019223a92..502cb08cc5f 100755 --- a/test/ocf/management/start-stop/run.sh +++ b/test/ocf/management/start-stop/run.sh @@ -7,9 +7,9 @@ # curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../../..) +rootdir=$(readlink -f "$curdir/../../../..") -source $rootdir/test/common/autotest_common.sh +source "$rootdir/test/common/autotest_common.sh" run_test "basic" "$curdir/basic.sh" run_test "incomplete" "$curdir/incomplete.sh" diff --git a/test/ocf/ocf.sh b/test/ocf/ocf.sh index d6bcc4b302a..1238e86703a 100755 --- a/test/ocf/ocf.sh +++ b/test/ocf/ocf.sh @@ -8,8 +8,8 @@ # curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../..) +rootdir=$(readlink -f "$curdir/../..") -source $rootdir/test/common/autotest_common.sh +source "$rootdir/test/common/autotest_common.sh" run_test "management" "$curdir/management/run.sh" From 6fcbbb92fd4daa805c4b9923afd221dddd241ea2 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Tue, 7 Oct 2025 17:32:01 +0200 Subject: [PATCH 25/34] Add integrity tests Change-Id: Ie01d40cebf469df6d84f5689b2767f6aa7f3721c Signed-off-by: Rafal Stefanowski --- test/ocf/common.sh | 13 +++ test/ocf/integrity/bdev_config.json | 62 ++++++++++++++ test/ocf/integrity/flush.sh | 112 +++++++++--------------- test/ocf/integrity/io_stats.sh | 80 +++++++++++++++++ test/ocf/integrity/mngt_during_io.sh | 123 +++++++++++++++++++++++++++ test/ocf/integrity/run.sh | 16 ++++ test/ocf/ocf.sh | 1 + 7 files changed, 334 insertions(+), 73 deletions(-) create mode 100644 test/ocf/integrity/bdev_config.json create mode 100755 test/ocf/integrity/io_stats.sh create mode 100755 test/ocf/integrity/mngt_during_io.sh create mode 100755 test/ocf/integrity/run.sh diff --git a/test/ocf/common.sh b/test/ocf/common.sh index 8c793c3a010..ef394559f2e 100644 --- a/test/ocf/common.sh +++ b/test/ocf/common.sh @@ -7,6 +7,7 @@ source "$rootdir/test/common/autotest_common.sh" rpc_py="$rootdir/scripts/rpc.py" +bdevperf_py="$rootdir/examples/bdev/bdevperf/bdevperf.py" cache_line_sizes=(4 8 16 32 64) cache_modes=("wt" "wb" "wa" "wo" "wi" "pt") @@ -117,6 +118,18 @@ stop_spdk() { killprocess $spdk_pid } +start_bdevperf() { + "$SPDK_EXAMPLE_DIR/bdevperf" -z -L vbdev_ocf "$@" & + bdevperf_pid=$! + trap 'killprocess $bdevperf_pid; exit 1' SIGINT SIGTERM EXIT + waitforlisten $bdevperf_pid +} + +stop_bdevperf() { + trap - SIGINT SIGTERM EXIT + killprocess $bdevperf_pid +} + ocf_settled() { # Check if there are no caches currently busy (under a lock). # Instead of the usual cache info, an empty JSON object is printed for such cache. diff --git a/test/ocf/integrity/bdev_config.json b/test/ocf/integrity/bdev_config.json new file mode 100644 index 00000000000..22bc8f2251d --- /dev/null +++ b/test/ocf/integrity/bdev_config.json @@ -0,0 +1,62 @@ +{ + "subsystems": [ + { + "subsystem": "bdev", + "config": [ + { + "method": "bdev_malloc_create", + "params": { + "name": "Mal_cache1", + "num_blocks": 409600, + "block_size": 512 + } + }, + { + "method": "bdev_malloc_create", + "params": { + "name": "Mal_core1-1", + "num_blocks": 819200, + "block_size": 512 + } + }, + { + "method": "bdev_malloc_create", + "params": { + "name": "Mal_core1-2", + "num_blocks": 819200, + "block_size": 512 + } + }, + { + "method": "bdev_ocf_start_cache", + "params": { + "cache_name": "Ocf_cache1", + "base_name": "Mal_cache1", + "cache_mode": "wt", + "cache_line_size": 4096, + "no_load": true + } + }, + { + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core1-1", + "base_name": "Mal_core1-1", + "cache_name": "Ocf_cache1" + } + }, + { + "method": "bdev_ocf_add_core", + "params": { + "core_name": "Ocf_core1-2", + "base_name": "Mal_core1-2", + "cache_name": "Ocf_cache1" + } + }, + { + "method": "bdev_wait_for_examine" + } + ] + } + ] +} diff --git a/test/ocf/integrity/flush.sh b/test/ocf/integrity/flush.sh index 29b1c7c82cf..30d044be93a 100755 --- a/test/ocf/integrity/flush.sh +++ b/test/ocf/integrity/flush.sh @@ -1,84 +1,50 @@ #!/usr/bin/env bash + +# # SPDX-License-Identifier: BSD-3-Clause -# Copyright (C) 2021 Intel Corporation. +# Copyright (C) 2025 Huawei Technologies # All rights reserved. +# curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -rootdir=$(readlink -f $curdir/../../..) -source $rootdir/test/common/autotest_common.sh +rootdir=$(readlink -f "$curdir/../../..") +source "$rootdir/test/ocf/common.sh" -bdevperf=$rootdir/build/examples/bdevperf -rpc_py="$rootdir/scripts/rpc.py -s /var/tmp/spdk.sock" +devices=("Ocf_cache1" "Ocf_core1-1" "Ocf_core1-2") +cache_modes=("wb" "wo") +io_patterns=("write" "rw" "randwrite" "randrw") +io_depth=128 +io_size=4096 +rw_mix=50 +runtime=10 check_flush_in_progress() { - $rpc_py bdev_ocf_flush_status MalCache0 \ - | jq -e '.in_progress' > /dev/null + $rpc_py bdev_ocf_get_bdevs $1 | jq -e '.flush.in_progress' > /dev/null } -bdevperf_config() { - local config - - config="$( - cat <<- JSON - { - "method": "bdev_malloc_create", - "params": { - "name": "Malloc0", - "num_blocks": 102400, - "block_size": 512 - } - }, - { - "method": "bdev_malloc_create", - "params": { - "name": "Malloc1", - "num_blocks": 1024000, - "block_size": 512 - } - }, - { - "method": "bdev_ocf_create", - "params": { - "name": "MalCache0", - "mode": "wb", - "cache_line_size": 4, - "cache_bdev_name": "Malloc0", - "core_bdev_name": "Malloc1" - } - } - JSON - )" - - jq . <<- JSON - { - "subsystems": [ - { - "subsystem": "bdev", - "config": [ - $( - IFS="," - printf '%s\n' "$config" - ), - { - "method": "bdev_wait_for_examine" - } - ] - } - ] - } - JSON -} - -$bdevperf --json <(bdevperf_config) -q 128 -o 4096 -w write -t 120 -r /var/tmp/spdk.sock & -bdevperf_pid=$! -trap 'killprocess $bdevperf_pid' SIGINT SIGTERM EXIT -waitforlisten $bdevperf_pid -sleep 5 - -$rpc_py bdev_ocf_flush_start MalCache0 -sleep 1 - -while check_flush_in_progress; do - sleep 1 +start_bdevperf -c "$curdir/bdev_config.json" +$rpc_py bdev_ocf_set_seqcutoff Ocf_cache1 --policy never +$rpc_py bdev_ocf_set_promotion Ocf_cache1 --policy always +$rpc_py bdev_ocf_set_cleaning Ocf_cache1 --policy nop + +for cache_mode in "${cache_modes[@]}"; do + $rpc_py bdev_ocf_set_cachemode Ocf_cache1 $cache_mode + for ip in "${io_patterns[@]}"; do + for dev in "${devices[@]}"; do + $bdevperf_py perform_tests -t $runtime -q $io_depth -o $io_size \ + -w $ip $([[ $ip =~ rw$ ]] && echo "-M $rw_mix") + for dv in "${devices[@]}"; do + $rpc_py bdev_ocf_get_stats $dv | jq -e '.usage | .dirty.count > .clean.count' + done + + $rpc_py bdev_ocf_flush_start $dev + while check_flush_in_progress $dev; do + sleep 1 + done + + $rpc_py bdev_ocf_get_bdevs $dev | jq -e '.flush.error == 0' + $rpc_py bdev_ocf_get_stats $dev | jq -e '.usage | .dirty.count == 0' + done + done done -$rpc_py bdev_ocf_flush_status MalCache0 | jq -e '.status == 0' +stop_bdevperf diff --git a/test/ocf/integrity/io_stats.sh b/test/ocf/integrity/io_stats.sh new file mode 100755 index 00000000000..779fc49deb3 --- /dev/null +++ b/test/ocf/integrity/io_stats.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f "$curdir/../../..") +source "$rootdir/test/ocf/common.sh" + +io_patterns=("read" "write" "rw" "randread" "randwrite" "randrw" "unmap" "flush") +io_depth=(1 16 128) +io_size=(512 4096) +rw_mix=50 +runtime=10 +cpu_mask=0xF + +start_bdevperf -c "$curdir/bdev_config.json" -m $cpu_mask -C +for cache_mode in "${cache_modes[@]}"; do + $rpc_py bdev_ocf_set_cachemode Ocf_cache1 $cache_mode + for id in "${io_depth[@]}"; do + for is in "${io_size[@]}"; do + for ip in "${io_patterns[@]}"; do + $rpc_py bdev_ocf_reset_stats Ocf_cache1 + test_results=$($bdevperf_py perform_tests -t $runtime -q $id \ + -o $is -w $ip $([[ $ip =~ rw$ ]] && echo "-M $rw_mix")) + stats_core1=$($rpc_py bdev_ocf_get_stats Ocf_core1-1) + stats_core2=$($rpc_py bdev_ocf_get_stats Ocf_core1-2) + + xtrace_disable + reqs_sent_core1=$(echo $test_results | jq -e \ + '[.results[] | select(.job == "Ocf_core1-1") | + .runtime * .iops] | add | round') + reqs_sent_core2=$(echo $test_results | jq -e \ + '[.results[] | select(.job == "Ocf_core1-2") | + .runtime * .iops] | add | round') + reqs_recv_rd_core1=$(echo $stats_core1 | jq -e \ + '.requests | .rd_total.count + .rd_pt.count') + reqs_recv_rd_core2=$(echo $stats_core2 | jq -e \ + '.requests | .rd_total.count + .rd_pt.count') + reqs_recv_wr_core1=$(echo $stats_core1 | jq -e \ + '.requests | .wr_total.count + .wr_pt.count') + reqs_recv_wr_core2=$(echo $stats_core2 | jq -e \ + '.requests | .wr_total.count + .wr_pt.count') + reqs_recv_total_core1=$(echo $stats_core1 | jq -e '.requests.total.count') + reqs_recv_total_core2=$(echo $stats_core2 | jq -e '.requests.total.count') + xtrace_restore + + if [[ $ip == "unmap" || $ip == "flush" ]]; then + continue + elif [[ $ip =~ read$ ]]; then + [ $reqs_recv_rd_core1 -eq $reqs_sent_core1 ] + [ $reqs_recv_rd_core2 -eq $reqs_sent_core2 ] + [ $reqs_recv_wr_core1 -eq 0 ] + [ $reqs_recv_wr_core2 -eq 0 ] + elif [[ $ip =~ write$ ]]; then + [ $reqs_recv_rd_core1 -eq 0 ] + [ $reqs_recv_rd_core2 -eq 0 ] + [ $reqs_recv_wr_core1 -eq $reqs_sent_core1 ] + [ $reqs_recv_wr_core2 -eq $reqs_sent_core2 ] + elif [[ $ip =~ rw$ ]]; then + [ $reqs_recv_rd_core1 -gt 0 ] + [ $reqs_recv_rd_core2 -gt 0 ] + [ $reqs_recv_wr_core1 -gt 0 ] + [ $reqs_recv_wr_core2 -gt 0 ] + reqs_rw_recv_core1=$((reqs_recv_rd_core1 + reqs_recv_wr_core1)) + reqs_rw_recv_core2=$((reqs_recv_rd_core2 + reqs_recv_wr_core2)) + [ $reqs_rw_recv_core1 -eq $reqs_sent_core1 ] + [ $reqs_rw_recv_core2 -eq $reqs_sent_core2 ] + fi + + [ $reqs_recv_total_core1 -eq $reqs_sent_core1 ] + [ $reqs_recv_total_core2 -eq $reqs_sent_core2 ] + done + done + done +done +stop_bdevperf diff --git a/test/ocf/integrity/mngt_during_io.sh b/test/ocf/integrity/mngt_during_io.sh new file mode 100755 index 00000000000..240ecf3d924 --- /dev/null +++ b/test/ocf/integrity/mngt_during_io.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f "$curdir/../../..") +source "$rootdir/test/ocf/common.sh" + +# change cache parameters and state during IO: + +for cache_mode in "${cache_modes[@]}"; do + start_bdevperf -c "$curdir/bdev_config.json" + $bdevperf_py perform_tests -t 600 -q 128 -o 4096 -w randrw -M 50 & + sleep 1 + + $rpc_py bdev_ocf_set_cachemode Ocf_cache1 $cache_mode + sleep 1 + + nhit_insertion_threshold=$(random_number ${promotion_nhit_insertion_threshold_range[0]} \ + ${promotion_nhit_insertion_threshold_range[1]}) + nhit_trigger_threshold=$(random_number ${promotion_nhit_trigger_threshold_range[0]} \ + ${promotion_nhit_trigger_threshold_range[1]}) + $rpc_py bdev_ocf_set_promotion Ocf_cache1 \ + --policy nhit \ + --nhit-insertion-threshold $nhit_insertion_threshold \ + --nhit-trigger-threshold $nhit_trigger_threshold + sleep 1 + $rpc_py bdev_ocf_set_promotion Ocf_cache1 --policy always + sleep 1 + + acp_wake_up_time=$(random_number ${cleaning_acp_wake_up_time_range[0]} \ + ${cleaning_acp_wake_up_time_range[1]}) + acp_flush_max_buffers=$(random_number ${cleaning_acp_flush_max_buffers_range[0]} \ + ${cleaning_acp_flush_max_buffers_range[1]}) + $rpc_py bdev_ocf_set_cleaning Ocf_cache1 \ + --policy acp \ + --acp-wake-up-time $acp_wake_up_time \ + --acp-flush-max-buffers $acp_flush_max_buffers + sleep 1 + $rpc_py bdev_ocf_set_cleaning Ocf_cache1 --policy alru + sleep 1 + + seqcutoff_threshold=$(random_number ${seqcutoff_threshold_range[0]} \ + ${seqcutoff_threshold_range[1]}) + seqcutoff_promotion_count=$(random_number ${seqcutoff_promotion_count_range[0]} \ + ${seqcutoff_promotion_count_range[1]}) + seqcutoff_promote_on_threshold=$(random_number ${seqcutoff_promote_on_threshold_range[0]} \ + ${seqcutoff_promote_on_threshold_range[1]}) + $rpc_py bdev_ocf_set_seqcutoff Ocf_cache1 \ + --policy always \ + --threshold $seqcutoff_threshold \ + --promotion-count $seqcutoff_promotion_count \ + --promote-on-threshold $seqcutoff_promote_on_threshold + sleep 1 + $rpc_py bdev_ocf_set_seqcutoff Ocf_cache1 --policy never + sleep 1 + + $rpc_py bdev_ocf_detach_cache Ocf_cache1 + sleep 1 + $rpc_py bdev_ocf_attach_cache Ocf_cache1 Mal_cache1 -f + sleep 1 + + $rpc_py bdev_ocf_remove_core Ocf_core1-1 + sleep 1 + $rpc_py bdev_ocf_add_core Ocf_core1-1 Mal_core1-1 Ocf_cache1 + sleep 1 + + $rpc_py bdev_ocf_get_stats Ocf_cache1 + sleep 1 + $rpc_py bdev_ocf_reset_stats Ocf_cache1 + sleep 1 + $rpc_py bdev_ocf_flush_start Ocf_cache1 + sleep 1 + $rpc_py bdev_ocf_get_bdevs + sleep 1 + $rpc_py bdev_ocf_stop_cache Ocf_cache1 + sleep 1 + + stop_bdevperf +done + +# hot remove and examine during IO: + +for cache_mode in "${cache_modes[@]}"; do + start_bdevperf -c "$curdir/bdev_config.json" + $bdevperf_py perform_tests -t 600 -q 128 -o 4096 -w randrw -M 50 & + bpy_pid=$! + sleep 1 + + $rpc_py bdev_ocf_set_cachemode Ocf_cache1 $cache_mode + sleep 1 + + $rpc_py bdev_malloc_delete Mal_cache1 + sleep 1 + $rpc_py bdev_malloc_create -b Mal_cache1 200 512 + sleep 1 + + $rpc_py bdev_malloc_delete Mal_core1-1 + sleep 1 + $rpc_py bdev_malloc_create -b Mal_core1-1 400 512 + sleep 1 + + $rpc_py bdev_malloc_delete Mal_cache1 + sleep 1 + $rpc_py bdev_malloc_create -b Mal_cache1 200 512 + sleep 1 + + $rpc_py bdev_malloc_delete Mal_core1-2 + sleep 1 + $rpc_py bdev_malloc_create -b Mal_core1-2 400 512 + sleep 1 + + killprocess $bpy_pid + sleep 1 + $bdevperf_py perform_tests -t 600 -q 128 -o 4096 -w randrw -M 50 & + sleep 5 + + stop_bdevperf +done diff --git a/test/ocf/integrity/run.sh b/test/ocf/integrity/run.sh new file mode 100755 index 00000000000..b923965dc0c --- /dev/null +++ b/test/ocf/integrity/run.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f "$curdir/../../..") + +source "$rootdir/test/common/autotest_common.sh" + +run_test "io_stats" "$curdir/io_stats.sh" +run_test "mngt_during_io" "$curdir/mngt_during_io.sh" +run_test "flush" "$curdir/flush.sh" diff --git a/test/ocf/ocf.sh b/test/ocf/ocf.sh index 1238e86703a..4d304993151 100755 --- a/test/ocf/ocf.sh +++ b/test/ocf/ocf.sh @@ -13,3 +13,4 @@ rootdir=$(readlink -f "$curdir/../..") source "$rootdir/test/common/autotest_common.sh" run_test "management" "$curdir/management/run.sh" +run_test "integrity" "$curdir/integrity/run.sh" From 0eaa11ad493f63189c4ab8e1f6f8b633742e6de5 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Tue, 14 Oct 2025 10:12:55 +0200 Subject: [PATCH 26/34] Read trylock cache in bdev_ocf_get_bdevs Change-Id: I915d5af1cce0ce38bca45462a7b66d14ce8be8e7 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 63 +++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index ef2f7aafa4c..f74b3de50cf 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -2977,6 +2977,17 @@ _get_bdevs_cache_visitor(ocf_cache_t cache, void *cb_arg) spdk_json_write_object_begin(w); + if ((rc = ocf_mngt_cache_read_trylock(cache))) { + if (rc == -OCF_ERR_NO_LOCK) { + SPDK_WARNLOG("OCF cache '%s': cache is busy - no info will be printed for it\n", + ocf_cache_get_name(cache)); + } else { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + } + goto end; + } + if ((rc = dump_cache_info(w, cache))) { SPDK_ERRLOG("OCF cache '%s': failed to get cache info: %s\n", ocf_cache_get_name(cache), spdk_strerror(-rc)); @@ -2986,9 +2997,11 @@ _get_bdevs_cache_visitor(ocf_cache_t cache, void *cb_arg) rc = ocf_core_visit(cache, _get_bdevs_core_visitor, w, false); spdk_json_write_array_end(w); - spdk_json_write_object_end(w); + ocf_mngt_cache_read_unlock(cache); - return rc; +end: + spdk_json_write_object_end(w); + return 0; } /* RPC entry point. */ @@ -2999,13 +3012,14 @@ vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, void struct spdk_json_write_ctx *w = rpc_cb_arg1; struct vbdev_ocf_core *core_ctx; struct vbdev_ocf_mngt_ctx *mngt_ctx; + ocf_cache_t cache; int rc; SPDK_DEBUGLOG(vbdev_ocf, "OCF: getting info about vbdevs\n"); if (!g_vbdev_ocf_module_is_running) { SPDK_ERRLOG("OCF: failed to handle the call - module stopping\n"); - goto end; + goto end_rpc; } if (!bdev_name) { @@ -3026,7 +3040,7 @@ vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, void } spdk_json_write_array_end(w); - goto end; + goto end_rpc; } if ((core_ctx = vbdev_ocf_core_waitlist_get_by_name(bdev_name))) { @@ -3035,14 +3049,14 @@ vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, void vbdev_ocf_core_get_name(core_ctx), spdk_strerror(-rc)); } - goto end; + goto end_rpc; } mngt_ctx = calloc(1, sizeof(struct vbdev_ocf_mngt_ctx)); if (!mngt_ctx) { SPDK_ERRLOG("OCF '%s': failed to allocate memory for getting bdevs info context\n", bdev_name); - goto end; + goto end_rpc; } mngt_ctx->bdev_name = bdev_name; /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ @@ -3052,8 +3066,31 @@ vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, void if ((rc = vbdev_ocf_bdev_resolve(mngt_ctx))) { SPDK_ERRLOG("OCF '%s': failed to find cache or core of that name: %s\n", bdev_name, spdk_strerror(-rc)); - free(mngt_ctx); - goto end; + goto end_free; + } + + cache = mngt_ctx->cache ? : ocf_core_get_cache(mngt_ctx->core); + if (!cache) { + /* This situation can happen only when ocf_core_get_cache() above + * returned NULL while core is being in the process of adding to its cache. + * However, this core was found previously by vbdev_ocf_bdev_resolve(), + * so it should not be the issue in this scenario. Nevertheless, check for this + * condition just in case, to avoid triggering BUG_ON() on trylock() below. */ + + SPDK_WARNLOG("OCF core '%s': adding to cache - no info will be printed for it\n", + ocf_core_get_name(mngt_ctx->core)); + goto end_free; + } + + if ((rc = ocf_mngt_cache_read_trylock(cache))) { + if (rc == -OCF_ERR_NO_LOCK) { + SPDK_WARNLOG("OCF cache '%s': cache is busy - no info will be printed for it\n", + ocf_cache_get_name(cache)); + } else { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + } + goto end_free; } if (mngt_ctx->core) { @@ -3066,11 +3103,17 @@ vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, void SPDK_ERRLOG("OCF cache '%s': failed to get cache info: %s\n", ocf_cache_get_name(mngt_ctx->cache), spdk_strerror(-rc)); } + + spdk_json_write_named_array_begin(w, "cores"); + rc = ocf_core_visit(mngt_ctx->cache, _get_bdevs_core_visitor, w, false); + spdk_json_write_array_end(w); } - free(mngt_ctx); + ocf_mngt_cache_read_unlock(cache); -end: +end_free: + free(mngt_ctx); +end_rpc: rpc_cb_fn(rpc_cb_arg1, rpc_cb_arg2); } From 3f2a8b849723857b3b84570a05046fdd123560ce Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Tue, 14 Oct 2025 10:17:44 +0200 Subject: [PATCH 27/34] Fix race condition when adding cores in examine Change-Id: I3607fd8a261755440fdd2948a027a37aaf66d67e Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 16 ++++++++++++---- module/bdev/ocf/vbdev_ocf_core.c | 29 +++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index f74b3de50cf..1d4921916d6 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -585,7 +585,7 @@ _core_add_examine_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int e SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n", ocf_core_get_name(core), ocf_cache_get_name(cache)); - /* If core was taken from waitlist, remove it from there. */ + /* If core was taken from wait list, remove it from there. */ if (vbdev_ocf_core_waitlist_get_by_name(vbdev_ocf_core_get_name(core_ctx))) { vbdev_ocf_core_waitlist_remove(core_ctx); } @@ -612,6 +612,13 @@ _core_add_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) return; } + /* Check if core is loaded from metadata only if its try_add flag was not + * set to true already during examine_config stage. That would mean that + * this core was hot removed before and now it is being attached back. */ + if (!core_ctx->core_cfg.try_add) { + core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx)); + } + ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _core_add_examine_add_cb, core_ctx); } @@ -717,7 +724,7 @@ _examine_disk_core_visitor(ocf_core_t core, void *cb_arg) return 0; } - /* Get cache once to be in sync with adding core from waitlist scenario. */ + /* Get cache once to be in sync with adding core from wait list scenario. */ ocf_mngt_cache_get(cache); ocf_mngt_cache_lock(cache, _core_add_examine_lock_cb, core_ctx); @@ -754,6 +761,9 @@ vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev) continue; } + SPDK_NOTICELOG("OCF core '%s': adding from wait list to cache '%s'\n", + vbdev_ocf_core_get_name(core_ctx), core_ctx->cache_name); + if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, core_ctx->cache_name, OCF_CACHE_NAME_SIZE, &cache)) { SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache '%s'\n", @@ -771,8 +781,6 @@ vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev) return; } - core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx)); - ocf_mngt_cache_lock(cache, _core_add_examine_lock_cb, core_ctx); return; } diff --git a/module/bdev/ocf/vbdev_ocf_core.c b/module/bdev/ocf/vbdev_ocf_core.c index 1edc85a32a5..ea2e7283b01 100644 --- a/module/bdev/ocf/vbdev_ocf_core.c +++ b/module/bdev/ocf/vbdev_ocf_core.c @@ -67,7 +67,7 @@ _core_is_loaded_cache_visitor(ocf_cache_t cache, void *cb_arg) if (!rc && !ocf_core_get_priv(core)) { /* Core context is assigned only after manual core add (either right * away if all devices are present, after corresponding cache start, - * or base bdev appearance). + * or base bdev showing up). * If there is no context, it means that this core was just added from * metadata during cache load and that's what we're looking for here. */ @@ -518,24 +518,35 @@ _core_add_from_waitlist_lock_cb(ocf_cache_t cache, void *cb_arg, int error) return; } + core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx)); + ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _core_add_from_waitlist_add_cb, core_ctx); } -void -vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache) +static void +_core_add_from_waitlist_read_lock_cb(ocf_cache_t cache, void *cb_arg, int error) { struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache); struct vbdev_ocf_core *core_ctx; uint32_t cache_block_size = spdk_bdev_get_block_size(cache_ctx->base.bdev); uint32_t core_block_size; + SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': looking for its cores in wait list\n", + ocf_cache_get_name(cache)); + + if (error) { + SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n", + ocf_cache_get_name(cache), error); + return; + } + vbdev_ocf_foreach_core_in_waitlist(core_ctx) { if (strcmp(ocf_cache_get_name(cache), core_ctx->cache_name) || !vbdev_ocf_core_is_base_attached(core_ctx)) { continue; } - SPDK_NOTICELOG("OCF core '%s': adding from waitlist to cache '%s'\n", + SPDK_NOTICELOG("OCF core '%s': adding from wait list to cache '%s'\n", vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache)); core_block_size = spdk_bdev_get_block_size(core_ctx->base.bdev); @@ -546,10 +557,16 @@ vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache) continue; } - core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx)); - ocf_mngt_cache_lock(cache, _core_add_from_waitlist_lock_cb, core_ctx); } + + ocf_mngt_cache_read_unlock(cache); +} + +void +vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache) +{ + ocf_mngt_cache_read_lock(cache, _core_add_from_waitlist_read_lock_cb, NULL); } static void From ca71913528f2b5680342e1ad5e1ad5699e3f2599 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Tue, 28 Oct 2025 15:29:34 +0100 Subject: [PATCH 28/34] Move cache IO channel update before attaching new device to OCF Change-Id: If00f029f084e5f500af7d7e6524cf33193fa1860 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 1d4921916d6..26ae0eb7821 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -473,9 +473,18 @@ _examine_config_cache_visitor(ocf_cache_t cache, void *cb_arg) return rc; } + /* Update cache IO channel in all cores before attaching new cache device to OCF. */ + if ((rc = vbdev_ocf_core_create_cache_channel(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to create IO channel for new cache device\n", + ocf_cache_get_name(cache)); + vbdev_ocf_cache_base_detach(cache); + return rc; + } + if ((rc = vbdev_ocf_cache_config_volume_create(cache))) { SPDK_ERRLOG("OCF cache '%s': failed to create config volume\n", ocf_cache_get_name(cache)); + vbdev_ocf_core_destroy_cache_channel(cache); vbdev_ocf_cache_base_detach(cache); return rc; } @@ -644,17 +653,12 @@ _cache_attach_cb(ocf_cache_t cache, void *cb_arg, int error) vbdev_ocf_mem_calculate(cache); } + vbdev_ocf_core_destroy_cache_channel(cache); vbdev_ocf_cache_base_detach(cache); } else { SPDK_NOTICELOG("OCF cache '%s': device attached\n", ocf_cache_get_name(cache)); - /* Update cache IO channel after new device attach. */ - if ((error = vbdev_ocf_core_create_cache_channel(cache))) { - SPDK_ERRLOG("OCF cache '%s': failed to create channel for newly attached cache: %s\n", - ocf_cache_get_name(cache), spdk_strerror(-error)); - } else { - vbdev_ocf_core_add_from_waitlist(cache); - } + vbdev_ocf_core_add_from_waitlist(cache); } if (mngt_ctx->rpc_cb_fn) { @@ -703,6 +707,7 @@ _cache_attach_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_mngt_cache_unlock(cache); err_lock: vbdev_ocf_cache_config_volume_destroy(cache); + vbdev_ocf_core_destroy_cache_channel(cache); vbdev_ocf_cache_base_detach(cache); spdk_bdev_module_examine_done(&ocf_if); } @@ -1459,6 +1464,7 @@ _cache_attach_rpc_lock_cb(ocf_cache_t cache, void *cb_arg, int error) ocf_cache_get_name(cache), error); mngt_ctx->rpc_cb_fn(ocf_cache_get_name(cache), mngt_ctx->rpc_cb_arg, error); vbdev_ocf_cache_config_volume_destroy(cache); + vbdev_ocf_core_destroy_cache_channel(cache); vbdev_ocf_cache_base_detach(cache); free(mngt_ctx); return; @@ -1516,6 +1522,13 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force goto err_base; } + /* Update cache IO channel in all cores before attaching new cache device to OCF. */ + if ((rc = vbdev_ocf_core_create_cache_channel(cache))) { + SPDK_ERRLOG("OCF cache '%s': failed to create IO channel for new cache device: %s\n", + cache_name, spdk_strerror(-rc)); + goto err_channel; + } + if ((rc = vbdev_ocf_cache_config_volume_create(cache))) { SPDK_ERRLOG("OCF cache '%s': failed to create config volume: %s\n", cache_name, spdk_strerror(-rc)); @@ -1540,6 +1553,8 @@ vbdev_ocf_cache_attach(const char *cache_name, const char *base_name, bool force err_alloc: vbdev_ocf_cache_config_volume_destroy(cache); err_volume: + vbdev_ocf_core_destroy_cache_channel(cache); +err_channel: vbdev_ocf_cache_base_detach(cache); err_base: err_state: From 6adce76738a12c8385edd5b2784697cdbc0e6117 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Tue, 28 Oct 2025 15:38:27 +0100 Subject: [PATCH 29/34] Make sure selected core still exists after taking a cache lock Change-Id: Ib038d093689c45fade8515861b50cb2b30f5601d Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 26ae0eb7821..56c7726f9b7 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -2308,6 +2308,15 @@ _seqcutoff_lock_cb(ocf_cache_t cache, void *cb_arg, int error) SPDK_DEBUGLOG(vbdev_ocf, "OCF '%s': setting sequential cut-off on core device\n", ocf_core_get_name(core)); + /* Once again check if selected core still exists in cache as it may have + * been hot removed between resolving its name and taking this lock. */ + if ((rc = ocf_core_get_by_name(cache, mngt_ctx->bdev_name, + OCF_CORE_NAME_SIZE, &core))) { + SPDK_ERRLOG("OCF core '%s': already removed from cache (OCF error: %d)\n", + mngt_ctx->bdev_name, rc); + goto err_param; + } + if (mngt_ctx->u.seqcutoff.policy >= ocf_seq_cutoff_policy_always && mngt_ctx->u.seqcutoff.policy < ocf_seq_cutoff_policy_max) { if ((rc = ocf_mngt_core_set_seq_cutoff_policy(core, mngt_ctx->u.seqcutoff.policy))) { @@ -2498,6 +2507,16 @@ _flush_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (core) { SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': flushing...\n", ocf_core_get_name(core)); + /* Once again check if selected core still exists in cache as it may have + * been hot removed between resolving its name and taking this lock. */ + if ((rc = ocf_core_get_by_name(cache, mngt_ctx->bdev_name, + OCF_CORE_NAME_SIZE, &core))) { + SPDK_ERRLOG("OCF core '%s': already removed from cache (OCF error: %d)\n", + mngt_ctx->bdev_name, rc); + ocf_mngt_cache_read_unlock(cache); + goto end; + } + core_ctx = ocf_core_get_priv(core); core_ctx->flush.in_progress = true; ocf_mngt_core_flush(core, _flush_core_cb, NULL); @@ -2575,6 +2594,16 @@ _get_stats_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (core) { SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': collecting statistics\n", ocf_core_get_name(core)); + /* Once again check if selected core still exists in cache as it may have + * been hot removed between resolving its name and taking this lock. */ + if ((rc = ocf_core_get_by_name(cache, mngt_ctx->bdev_name, + OCF_CORE_NAME_SIZE, &core))) { + SPDK_ERRLOG("OCF core '%s': already removed from cache (OCF error: %d)\n", + mngt_ctx->bdev_name, rc); + ocf_mngt_cache_read_unlock(cache); + goto end; + } + if ((rc = vbdev_ocf_stats_core_get(core, &stats))) { SPDK_ERRLOG("OCF core '%s': failed to collect statistics (OCF error: %d)\n", ocf_core_get_name(core), rc); @@ -2662,6 +2691,16 @@ _reset_stats_lock_cb(ocf_cache_t cache, void *cb_arg, int error) if (core) { SPDK_DEBUGLOG(vbdev_ocf, "OCF core '%s': resetting statistics\n", ocf_core_get_name(core)); + /* Once again check if selected core still exists in cache as it may have + * been hot removed between resolving its name and taking this lock. */ + if ((rc = ocf_core_get_by_name(cache, mngt_ctx->bdev_name, + OCF_CORE_NAME_SIZE, &core))) { + SPDK_ERRLOG("OCF core '%s': already removed from cache (OCF error: %d)\n", + mngt_ctx->bdev_name, rc); + ocf_mngt_cache_unlock(cache); + goto end; + } + if ((rc = vbdev_ocf_stats_core_reset(core))) { SPDK_ERRLOG("OCF core '%s': failed to reset statistics (OCF error: %d)\n", ocf_core_get_name(core), rc); From 06808370f9df8051cd611452108226da61fddaeb Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Tue, 28 Oct 2025 16:00:42 +0100 Subject: [PATCH 30/34] Add stress tests Change-Id: I323302f657e0d943acd3b4d4e4c8bd3efd4931d0 Signed-off-by: Rafal Stefanowski --- test/ocf/common.sh | 37 ++++- test/ocf/integrity/run.sh | 1 + test/ocf/integrity/stress.sh | 301 +++++++++++++++++++++++++++++++++++ 3 files changed, 333 insertions(+), 6 deletions(-) create mode 100755 test/ocf/integrity/stress.sh diff --git a/test/ocf/common.sh b/test/ocf/common.sh index ef394559f2e..795757420e2 100644 --- a/test/ocf/common.sh +++ b/test/ocf/common.sh @@ -196,10 +196,15 @@ create_caches_with_metadata_with_cores() { destroy_cores } -destroy_caches() { +destroy_caches_do() { for i in {1..3}; do malloc_cache_destroy Cache_dev$i done +} + +destroy_caches() { + destroy_caches_do + $rpc_py bdev_ocf_get_bdevs | jq -e '.' } @@ -282,12 +287,17 @@ create_cores() { $rpc_py bdev_ocf_get_bdevs | jq -e '.' } -destroy_cores() { +destroy_cores_do() { for i in {1..3}; do for j in {1..3}; do malloc_core_destroy Core_dev$i-$j done done +} + +destroy_cores() { + destroy_cores_do + $rpc_py bdev_ocf_get_bdevs | jq -e '.' } @@ -371,17 +381,27 @@ stop_caches() { $rpc_py bdev_ocf_get_bdevs | jq -e '.' } -detach_caches() { +detach_caches_do() { for i in {1..3}; do $rpc_py bdev_ocf_detach_cache Ocf_cache$i done +} + +detach_caches() { + detach_caches_do + $rpc_py bdev_ocf_get_bdevs | jq -e '.' } -attach_caches() { +attach_caches_do() { for i in {1..3}; do $rpc_py bdev_ocf_attach_cache Ocf_cache$i Cache_dev$i done +} + +attach_caches() { + attach_caches_do + $rpc_py bdev_ocf_get_bdevs | jq -e '.' } @@ -399,12 +419,17 @@ add_cores() { $rpc_py bdev_ocf_get_bdevs | jq -e '.' } -remove_cores() { +remove_cores_do() { for i in {1..3}; do for j in {1..3}; do $rpc_py bdev_ocf_remove_core Ocf_core$i-$j done done +} + +remove_cores() { + remove_cores_do + $rpc_py bdev_ocf_get_bdevs | jq -e '.' } @@ -675,7 +700,7 @@ __check_cores_waitlist_detached_all_but_first() { __check_setup_completed() { for i in {1..3}; do for j in {1..3}; do - waitforbdev Core_dev$i-$j 20000 + waitforbdev Core_dev$i-$j 20000 > /dev/null done done diff --git a/test/ocf/integrity/run.sh b/test/ocf/integrity/run.sh index b923965dc0c..86889add488 100755 --- a/test/ocf/integrity/run.sh +++ b/test/ocf/integrity/run.sh @@ -14,3 +14,4 @@ source "$rootdir/test/common/autotest_common.sh" run_test "io_stats" "$curdir/io_stats.sh" run_test "mngt_during_io" "$curdir/mngt_during_io.sh" run_test "flush" "$curdir/flush.sh" +run_test "stress" "$curdir/stress.sh" diff --git a/test/ocf/integrity/stress.sh b/test/ocf/integrity/stress.sh new file mode 100755 index 00000000000..c4c85d27877 --- /dev/null +++ b/test/ocf/integrity/stress.sh @@ -0,0 +1,301 @@ +#!/usr/bin/env bash + +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2025 Huawei Technologies +# All rights reserved. +# + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f "$curdir/../../..") +source "$rootdir/test/ocf/common.sh" + +app_pid= +bg_pids=() + +start_spdk_() { + "$SPDK_BIN_DIR/spdk_tgt" -L vbdev_ocf "$@" & + app_pid=$! + waitforlisten $app_pid +} + +stop_spdk_() { + killprocess $app_pid +} + +start_bdevperf_() { + "$SPDK_EXAMPLE_DIR/bdevperf" -z -L vbdev_ocf -c "$curdir/bdev_config.json" & + app_pid=$! + waitforlisten $app_pid + $bdevperf_py perform_tests -t 600 -q 128 -o 4096 -w randrw -M 50 & +} + +stop_bdevperf_() { + killprocess $app_pid +} + +run_in_bg() { + xtrace_disable + (while true; do + "$@" &> /dev/null || true + sleep 0.1 + done) & + bg_pids+=($!) + xtrace_restore +} + +cleanup() { + for pid in "${bg_pids[@]}"; do + killprocess $pid || true + done + killprocess $app_pid +} + +trap 'cleanup; exit 1' SIGINT SIGTERM EXIT + +run_in_bg $rpc_py bdev_ocf_get_bdevs +run_in_bg $rpc_py bdev_ocf_get_bdevs Ocf_cache1 +run_in_bg $rpc_py bdev_ocf_get_bdevs Ocf_core1-1 +run_in_bg $rpc_py bdev_ocf_get_bdevs Ocf_core1-2 +run_in_bg $rpc_py bdev_ocf_get_bdevs Ocf_core1-3 +run_in_bg $rpc_py bdev_ocf_get_bdevs Ocf_cache3 +run_in_bg $rpc_py bdev_ocf_get_bdevs Ocf_core3-1 +run_in_bg $rpc_py bdev_ocf_get_bdevs Ocf_core3-2 +run_in_bg $rpc_py bdev_ocf_get_bdevs Ocf_core3-3 +run_in_bg $rpc_py bdev_ocf_get_stats Ocf_cache1 +run_in_bg $rpc_py bdev_ocf_get_stats Ocf_core1-1 +run_in_bg $rpc_py bdev_ocf_get_stats Ocf_core1-2 +run_in_bg $rpc_py bdev_ocf_get_stats Ocf_core1-3 +run_in_bg $rpc_py bdev_ocf_get_stats Ocf_cache3 +run_in_bg $rpc_py bdev_ocf_get_stats Ocf_core3-1 +run_in_bg $rpc_py bdev_ocf_get_stats Ocf_core3-2 +run_in_bg $rpc_py bdev_ocf_get_stats Ocf_core3-3 + +# Test fast setup under the load of multiple info dumping RPC calls. + +# basic: + +start_spdk_ +create_caches_do +create_cores_do +start_caches_do +add_cores_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +create_caches_do +create_cores_do +add_cores_do +start_caches_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +create_cores_do +create_caches_do +start_caches_do +add_cores_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +create_cores_do +create_caches_do +add_cores_do +start_caches_do +__check_setup_completed +stop_spdk_ + +# examine: + +start_spdk_ +start_caches_do +add_cores_do +create_caches_do +create_cores_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +start_caches_do +add_cores_do +create_cores_do +create_caches_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +add_cores_do +start_caches_do +create_caches_do +create_cores_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +add_cores_do +start_caches_do +create_cores_do +create_caches_do +__check_setup_completed +stop_spdk_ + +# load: + +start_spdk_ +create_caches_persistent_with_metadata_with_cores +destroy_caches_persistent +start_caches_do +add_cores_do +create_caches_persistent_do +create_cores_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +create_caches_persistent_with_metadata_with_cores +destroy_caches_persistent +start_caches_do +add_cores_do +create_cores_do +create_caches_persistent_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +create_caches_persistent_with_metadata_with_cores +destroy_caches_persistent +add_cores_do +start_caches_do +create_caches_persistent_do +create_cores_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +create_caches_persistent_with_metadata_with_cores +destroy_caches_persistent +add_cores_do +start_caches_do +create_cores_do +create_caches_persistent_do +__check_setup_completed +stop_spdk_ + +# detach-attach: + +start_spdk_ +create_caches_do +create_cores_do +start_caches_do +add_cores_do +detach_caches_do +attach_caches_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +create_caches_do +create_cores_do +start_caches_do +detach_caches_do +add_cores_do +attach_caches_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +create_caches_do +create_cores_do +start_caches_do +add_cores_do +detach_caches_do +remove_cores_do +attach_caches_do +add_cores_do +__check_setup_completed +stop_spdk_ + +# hotremove: + +start_spdk_ +create_caches_do +start_caches_do +destroy_caches_do +stop_spdk_ + +start_spdk_ +create_cores_do +add_cores_do +destroy_cores_do +stop_spdk_ + +start_spdk_ +create_caches_do +create_cores_do +start_caches_do +add_cores_do +destroy_caches_do +create_caches_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +create_caches_do +create_cores_do +start_caches_do +add_cores_do +destroy_cores_do +create_cores_do +__check_setup_completed +stop_spdk_ + +start_spdk_ +create_caches_do +create_cores_do +start_caches_do +add_cores_do +destroy_caches_do +destroy_cores_do +create_caches_do +create_cores_do +__check_setup_completed +stop_spdk_ + +# management during IO: + +start_bdevperf_ +for cache_mode in "${cache_modes[@]}"; do + $rpc_py bdev_ocf_set_cachemode Ocf_cache1 $cache_mode + $rpc_py bdev_ocf_set_promotion Ocf_cache1 \ + --policy nhit \ + --nhit-insertion-threshold ${promotion_nhit_insertion_threshold_range[1]} \ + --nhit-trigger-threshold ${promotion_nhit_trigger_threshold_range[1]} + $rpc_py bdev_ocf_set_promotion Ocf_cache1 --policy always + $rpc_py bdev_ocf_set_cleaning Ocf_cache1 \ + --policy acp \ + --acp-wake-up-time ${cleaning_acp_wake_up_time_range[1]} \ + --acp-flush-max-buffers ${cleaning_acp_flush_max_buffers_range[1]} + $rpc_py bdev_ocf_set_cleaning Ocf_cache1 --policy alru + $rpc_py bdev_ocf_set_seqcutoff Ocf_cache1 \ + --policy always \ + --threshold ${seqcutoff_threshold_range[1]} \ + --promotion-count ${seqcutoff_promotion_count_range[1]} \ + --promote-on-threshold ${seqcutoff_promote_on_threshold_range[1]} + $rpc_py bdev_ocf_set_seqcutoff Ocf_cache1 --policy never + $rpc_py bdev_ocf_detach_cache Ocf_cache1 + $rpc_py bdev_ocf_attach_cache Ocf_cache1 Mal_cache1 -f + $rpc_py bdev_ocf_remove_core Ocf_core1-1 + $rpc_py bdev_ocf_add_core Ocf_core1-1 Mal_core1-1 Ocf_cache1 + $rpc_py bdev_ocf_reset_stats Ocf_cache1 + $rpc_py bdev_ocf_flush_start Ocf_cache1 + $rpc_py bdev_malloc_delete Mal_cache1 + $rpc_py bdev_malloc_create -b Mal_cache1 200 512 + $rpc_py bdev_malloc_delete Mal_core1-1 + $rpc_py bdev_malloc_create -b Mal_core1-1 400 512 +done +stop_bdevperf_ + +trap - SIGINT SIGTERM EXIT +cleanup From 2f21b8325087ca2e251841ead30634538bd8aa1b Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Wed, 29 Oct 2025 14:07:51 +0100 Subject: [PATCH 31/34] Make a copy of core context when dumping core info Change-Id: Iaeaf9a9af3c82d908efddf5947a55bf8b2f0fcc2 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 31 ++++++++++++++++--------------- test/ocf/common.sh | 10 ---------- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 56c7726f9b7..65b46925b74 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -2923,28 +2923,29 @@ dump_core_waitlist_info(struct spdk_json_write_ctx *w, struct vbdev_ocf_core *co static int dump_core_info(struct spdk_json_write_ctx *w, ocf_core_t core) { - struct vbdev_ocf_core *core_ctx = ocf_core_get_priv(core); + struct vbdev_ocf_core *core_ctx_priv = ocf_core_get_priv(core); + struct vbdev_ocf_core _core_ctx = {}; + struct vbdev_ocf_core *core_ctx = &_core_ctx; int rc; - /* If this core returns no cache it means that it is - * still in the process of adding itself to the cache. - * Do not print any core info until adding is finished. - * Return an empty JSON object instead. */ - if (!ocf_core_get_cache(core)) { - return 0; + /* If core context exists, make a local copy of it in + * case it is freed by other thread in the meantime. + * If it doesn't exist, a temporary empty context + * (initialized with zeros) will be used instead. */ + if (core_ctx_priv) { + _core_ctx = *core_ctx_priv; } spdk_json_write_named_string(w, "name", ocf_core_get_name(core)); - if (core_ctx && !spdk_uuid_is_null(spdk_bdev_get_uuid(&core_ctx->ocf_vbdev))) { + if (!spdk_uuid_is_null(spdk_bdev_get_uuid(&core_ctx->ocf_vbdev))) { spdk_json_write_named_uuid(w, "uuid", spdk_bdev_get_uuid(&core_ctx->ocf_vbdev)); } else { spdk_json_write_named_null(w, "uuid"); } spdk_json_write_named_string(w, "cache_name", ocf_cache_get_name(ocf_core_get_cache(core))); - spdk_json_write_named_string(w, "base_name", core_ctx ? core_ctx->base.name : ""); - spdk_json_write_named_bool(w, "base_attached", - core_ctx ? vbdev_ocf_core_is_base_attached(core_ctx) : false); - if (core_ctx && vbdev_ocf_core_is_base_attached(core_ctx)) { + spdk_json_write_named_string(w, "base_name", core_ctx->base.name); + spdk_json_write_named_bool(w, "base_attached", vbdev_ocf_core_is_base_attached(core_ctx)); + if (vbdev_ocf_core_is_base_attached(core_ctx)) { spdk_json_write_named_uint64(w, "size", spdk_bdev_get_block_size(core_ctx->base.bdev) * spdk_bdev_get_num_blocks(core_ctx->base.bdev)); spdk_json_write_named_uint32(w, "block_size", spdk_bdev_get_block_size(core_ctx->base.bdev)); @@ -2952,7 +2953,7 @@ dump_core_info(struct spdk_json_write_ctx *w, ocf_core_t core) spdk_json_write_named_null(w, "size"); spdk_json_write_named_null(w, "block_size"); } - spdk_json_write_named_bool(w, "loading", !core_ctx); + spdk_json_write_named_bool(w, "loading", !core_ctx_priv); spdk_json_write_named_object_begin(w, "seq_cutoff"); if ((rc = dump_seqcutoff_info(w, core))) { @@ -2962,8 +2963,8 @@ dump_core_info(struct spdk_json_write_ctx *w, ocf_core_t core) spdk_json_write_object_end(w); spdk_json_write_named_object_begin(w, "flush"); - spdk_json_write_named_bool(w, "in_progress", core_ctx ? core_ctx->flush.in_progress : false); - spdk_json_write_named_int32(w, "error", core_ctx ? core_ctx->flush.error : 0); + spdk_json_write_named_bool(w, "in_progress", core_ctx->flush.in_progress); + spdk_json_write_named_int32(w, "error", core_ctx->flush.error); spdk_json_write_object_end(w); return rc; diff --git a/test/ocf/common.sh b/test/ocf/common.sh index 795757420e2..f0e6a2d4afe 100644 --- a/test/ocf/common.sh +++ b/test/ocf/common.sh @@ -139,16 +139,6 @@ ocf_settled() { return 1 fi - # Check if there are no cores in caches still in the initialization process. - # Such condition can happen when base bdevs with OCF metadata were found for - # started caches and their configuration is loaded from this metadata in background. - if ! ./scripts/rpc.py bdev_ocf_get_bdevs | jq -e \ - 'if (.caches | length == 0) then true - else ([.caches[].cores[]] | any(. == {}) | not) end'; then - - return 1 - fi - # By using array arithmetic check if there is no attached # cores in wait list that belong to any attached cache. # Those cores should be automatically added to their caches, so if there From a2b2c133199c24eb53f0749701ce558e02e785ee Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Wed, 29 Oct 2025 14:12:39 +0100 Subject: [PATCH 32/34] Make a copy of bdev name given in RPC call Change-Id: Ibba93fc9803babcc9c3055aeb223825c580f7743 Signed-off-by: Rafal Stefanowski --- module/bdev/ocf/vbdev_ocf.c | 12 ++++++------ module/bdev/ocf/vbdev_ocf_cache.h | 2 +- module/bdev/ocf/volume.h | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 65b46925b74..4e93ea6bc6d 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -1867,7 +1867,7 @@ vbdev_ocf_core_remove(const char *core_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo rc = -ENOMEM; goto err_alloc; } - mngt_ctx->bdev_name = core_name; + strlcpy(mngt_ctx->bdev_name, core_name, OCF_CORE_NAME_SIZE); if ((rc = vbdev_ocf_bdev_resolve(mngt_ctx))) { SPDK_ERRLOG("OCF core '%s': failed to find core of that name: %s\n", @@ -2436,7 +2436,7 @@ vbdev_ocf_set_seqcutoff(const char *bdev_name, const char *policy, int32_t thres } mngt_ctx->rpc_cb_fn = rpc_cb_fn; mngt_ctx->rpc_cb_arg = rpc_cb_arg; - mngt_ctx->bdev_name = bdev_name; + strlcpy(mngt_ctx->bdev_name, bdev_name, VBDEV_OCF_BDEV_NAME_SIZE); /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ mngt_ctx->cache = NULL; mngt_ctx->core = NULL; @@ -2559,7 +2559,7 @@ vbdev_ocf_flush_start(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo } mngt_ctx->rpc_cb_fn = rpc_cb_fn; mngt_ctx->rpc_cb_arg = rpc_cb_arg; - mngt_ctx->bdev_name = bdev_name; + strlcpy(mngt_ctx->bdev_name, bdev_name, VBDEV_OCF_BDEV_NAME_SIZE); /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ mngt_ctx->cache = NULL; mngt_ctx->core = NULL; @@ -2652,7 +2652,7 @@ vbdev_ocf_get_stats(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, mngt_ctx->u.rpc_dump.rpc_cb_fn = rpc_cb_fn; mngt_ctx->u.rpc_dump.rpc_cb_arg = rpc_cb_arg1; mngt_ctx->rpc_cb_arg = rpc_cb_arg2; - mngt_ctx->bdev_name = bdev_name; + strlcpy(mngt_ctx->bdev_name, bdev_name, VBDEV_OCF_BDEV_NAME_SIZE); /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ mngt_ctx->cache = NULL; mngt_ctx->core = NULL; @@ -2745,7 +2745,7 @@ vbdev_ocf_reset_stats(const char *bdev_name, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, vo } mngt_ctx->rpc_cb_fn = rpc_cb_fn; mngt_ctx->rpc_cb_arg = rpc_cb_arg; - mngt_ctx->bdev_name = bdev_name; + strlcpy(mngt_ctx->bdev_name, bdev_name, VBDEV_OCF_BDEV_NAME_SIZE); /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ mngt_ctx->cache = NULL; mngt_ctx->core = NULL; @@ -3121,7 +3121,7 @@ vbdev_ocf_get_bdevs(const char *bdev_name, vbdev_ocf_rpc_dump_cb rpc_cb_fn, void bdev_name); goto end_rpc; } - mngt_ctx->bdev_name = bdev_name; + strlcpy(mngt_ctx->bdev_name, bdev_name, VBDEV_OCF_BDEV_NAME_SIZE); /* Cache or core will be set using vbdev_ocf_bdev_resolve(). */ mngt_ctx->cache = NULL; mngt_ctx->core = NULL; diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h index 90d3887b6bb..626a612f1a7 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.h +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -50,7 +50,7 @@ struct vbdev_ocf_mngt_ctx { void *rpc_cb_arg; /* Name of bdev. */ - const char *bdev_name; + char bdev_name[VBDEV_OCF_BDEV_NAME_SIZE]; /* OCF cache handle. */ ocf_cache_t cache; diff --git a/module/bdev/ocf/volume.h b/module/bdev/ocf/volume.h index 210649892bc..3545293e26a 100644 --- a/module/bdev/ocf/volume.h +++ b/module/bdev/ocf/volume.h @@ -7,8 +7,10 @@ #ifndef VBDEV_OCF_VOLUME_H #define VBDEV_OCF_VOLUME_H +#define VBDEV_OCF_BDEV_NAME_SIZE spdk_max(OCF_CACHE_NAME_SIZE, OCF_CORE_NAME_SIZE) + struct vbdev_ocf_base { - char name[spdk_max(OCF_CACHE_NAME_SIZE, OCF_CORE_NAME_SIZE)]; + char name[VBDEV_OCF_BDEV_NAME_SIZE]; bool is_cache; bool attached; struct spdk_bdev *bdev; From 91ba30ad3a8cbff28fcda393c7c4a9979f846bcb Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Thu, 30 Oct 2025 15:49:06 +0100 Subject: [PATCH 33/34] Add dirty_ratio_inertia cleaning param Change-Id: I80e5bc492f9025a13ba4cdc7ca12676bb0ef3f8f Signed-off-by: Rafal Stefanowski --- doc/jsonrpc.md.jinja2 | 6 ++- module/bdev/ocf/vbdev_ocf.c | 42 +++++++++++++++----- module/bdev/ocf/vbdev_ocf.h | 4 +- module/bdev/ocf/vbdev_ocf_cache.h | 3 +- module/bdev/ocf/vbdev_ocf_rpc.c | 10 +++-- python/spdk/cli/bdev.py | 14 +++++-- test/ocf/common.sh | 22 ++++++---- test/ocf/management/config_change.sh | 7 ++-- test/ocf/management/info_dump_get_bdevs.json | 9 +++-- 9 files changed, 81 insertions(+), 36 deletions(-) diff --git a/doc/jsonrpc.md.jinja2 b/doc/jsonrpc.md.jinja2 index 764b22c6615..f303c552184 100644 --- a/doc/jsonrpc.md.jinja2 +++ b/doc/jsonrpc.md.jinja2 @@ -3611,7 +3611,8 @@ Example response: "flush_max_buffers": 100, "staleness_time": 120, "activity_threshold": 10000, - "max_dirty_ratio": 100 + "dirty_ratio_threshold": 100, + "dirty_ratio_inertia": 134217728 }, "flush": { "in_progress": false, @@ -3657,7 +3658,8 @@ Example response: "flush_max_buffers": 100, "staleness_time": 120, "activity_threshold": 10000, - "max_dirty_ratio": 100 + "dirty_ratio_threshold": 100, + "dirty_ratio_inertia": 134217728 }, "flush": { "in_progress": false, diff --git a/module/bdev/ocf/vbdev_ocf.c b/module/bdev/ocf/vbdev_ocf.c index 4e93ea6bc6d..3df7bb0397e 100644 --- a/module/bdev/ocf/vbdev_ocf.c +++ b/module/bdev/ocf/vbdev_ocf.c @@ -2206,10 +2206,27 @@ _cleaning_lock_cb(ocf_cache_t cache, void *cb_arg, int error) } } - if (mngt_ctx->u.cleaning.alru_max_dirty_ratio >= 0) { - if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_max_dirty_ratio, - mngt_ctx->u.cleaning.alru_max_dirty_ratio))) { - SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_max_dirty_ratio param (OCF error: %d)\n", + if (mngt_ctx->u.cleaning.alru_dirty_ratio_threshold >= 0) { + if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, + ocf_alru_dirty_ratio_threshold, mngt_ctx->u.cleaning.alru_dirty_ratio_threshold))) { + SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_dirty_ratio_threshold param (OCF error: %d)\n", + ocf_cache_get_name(cache), rc); + goto err_param; + } + } + + if (mngt_ctx->u.cleaning.alru_dirty_ratio_inertia >= 0) { + /* Maximum value of dirty ratio inertia is the size of unsigned int, + * so check its value to prevent exceeding before even sending it to OCF. */ + if (mngt_ctx->u.cleaning.alru_dirty_ratio_inertia * MiB > OCF_ALRU_MAX_DIRTY_RATIO_INERTIA) { + SPDK_ERRLOG("OCF cache '%s': alru_dirty_ratio_inertia param out of range\n", + ocf_cache_get_name(cache)); + rc = -EINVAL; + goto err_param; + } + if ((rc = ocf_mngt_cache_cleaning_set_param(cache, ocf_cleaning_alru, ocf_alru_dirty_ratio_inertia, + mngt_ctx->u.cleaning.alru_dirty_ratio_inertia * MiB))) { + SPDK_ERRLOG("OCF cache '%s': failed to set cleaning alru_dirty_ratio_inertia param (OCF error: %d)\n", ocf_cache_get_name(cache), rc); goto err_param; } @@ -2240,8 +2257,8 @@ void vbdev_ocf_set_cleaning(const char *cache_name, const char *policy, int32_t acp_wake_up_time, int32_t acp_flush_max_buffers, int32_t alru_wake_up_time, int32_t alru_flush_max_buffers, int32_t alru_staleness_time, - int32_t alru_activity_threshold, int32_t alru_max_dirty_ratio, - vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) + int32_t alru_activity_threshold, int32_t alru_dirty_ratio_threshold, + int32_t alru_dirty_ratio_inertia, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg) { ocf_cache_t cache; struct vbdev_ocf_mngt_ctx *mngt_ctx; @@ -2278,7 +2295,8 @@ vbdev_ocf_set_cleaning(const char *cache_name, const char *policy, int32_t acp_w mngt_ctx->u.cleaning.alru_flush_max_buffers = alru_flush_max_buffers; mngt_ctx->u.cleaning.alru_staleness_time = alru_staleness_time; mngt_ctx->u.cleaning.alru_activity_threshold = alru_activity_threshold; - mngt_ctx->u.cleaning.alru_max_dirty_ratio = alru_max_dirty_ratio; + mngt_ctx->u.cleaning.alru_dirty_ratio_threshold = alru_dirty_ratio_threshold; + mngt_ctx->u.cleaning.alru_dirty_ratio_inertia = alru_dirty_ratio_inertia; ocf_mngt_cache_lock(cache, _cleaning_lock_cb, mngt_ctx); @@ -2857,10 +2875,16 @@ dump_cleaning_info(struct spdk_json_write_ctx *w, ocf_cache_t cache) spdk_json_write_named_uint32(w, "activity_threshold", param_val); if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, - ocf_alru_max_dirty_ratio, ¶m_val))) { + ocf_alru_dirty_ratio_threshold, ¶m_val))) { + return rc; + } + spdk_json_write_named_uint32(w, "dirty_ratio_threshold", param_val); + + if ((rc = ocf_mngt_cache_cleaning_get_param(cache, ocf_cleaning_alru, + ocf_alru_dirty_ratio_inertia, ¶m_val))) { return rc; } - spdk_json_write_named_uint32(w, "max_dirty_ratio", param_val); + spdk_json_write_named_uint32(w, "dirty_ratio_inertia", param_val); } return 0; diff --git a/module/bdev/ocf/vbdev_ocf.h b/module/bdev/ocf/vbdev_ocf.h index 8c429b2bf1d..ddf3c90e4ed 100644 --- a/module/bdev/ocf/vbdev_ocf.h +++ b/module/bdev/ocf/vbdev_ocf.h @@ -63,8 +63,8 @@ void vbdev_ocf_set_promotion(const char *cache_name, const char *policy, void vbdev_ocf_set_cleaning(const char *cache_name, const char *policy, int32_t acp_wake_up_time, int32_t acp_flush_max_buffers, int32_t alru_wake_up_time, int32_t alru_flush_max_buffers, int32_t alru_staleness_time, - int32_t alru_activity_threshold, int32_t alru_max_dirty_ratio, - vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg); + int32_t alru_activity_threshold, int32_t alru_dirty_ratio_threshold, + int32_t alru_dirty_ratio_inertia, vbdev_ocf_rpc_mngt_cb rpc_cb_fn, void *rpc_cb_arg); void vbdev_ocf_set_seqcutoff(const char *bdev_name, const char *policy, int32_t threshold, int32_t promotion_count, int32_t promote_on_threshold, diff --git a/module/bdev/ocf/vbdev_ocf_cache.h b/module/bdev/ocf/vbdev_ocf_cache.h index 626a612f1a7..bc13e88a500 100644 --- a/module/bdev/ocf/vbdev_ocf_cache.h +++ b/module/bdev/ocf/vbdev_ocf_cache.h @@ -91,7 +91,8 @@ struct vbdev_ocf_mngt_ctx { int32_t alru_flush_max_buffers; int32_t alru_staleness_time; int32_t alru_activity_threshold; - int32_t alru_max_dirty_ratio; + int32_t alru_dirty_ratio_threshold; + int32_t alru_dirty_ratio_inertia; } cleaning; /* Sequential cut-off parameters. */ diff --git a/module/bdev/ocf/vbdev_ocf_rpc.c b/module/bdev/ocf/vbdev_ocf_rpc.c index e90144307c1..1144ce6b523 100644 --- a/module/bdev/ocf/vbdev_ocf_rpc.c +++ b/module/bdev/ocf/vbdev_ocf_rpc.c @@ -456,7 +456,8 @@ struct rpc_bdev_ocf_set_cleaning { int32_t alru_flush_max_buffers; int32_t alru_staleness_time; int32_t alru_activity_threshold; - int32_t alru_max_dirty_ratio; + int32_t alru_dirty_ratio_threshold; + int32_t alru_dirty_ratio_inertia; }; static void @@ -475,7 +476,8 @@ static const struct spdk_json_object_decoder rpc_bdev_ocf_set_cleaning_decoders[ {"alru_flush_max_buffers", offsetof(struct rpc_bdev_ocf_set_cleaning, alru_flush_max_buffers), spdk_json_decode_int32, true}, {"alru_staleness_time", offsetof(struct rpc_bdev_ocf_set_cleaning, alru_staleness_time), spdk_json_decode_int32, true}, {"alru_activity_threshold", offsetof(struct rpc_bdev_ocf_set_cleaning, alru_activity_threshold), spdk_json_decode_int32, true}, - {"alru_max_dirty_ratio", offsetof(struct rpc_bdev_ocf_set_cleaning, alru_max_dirty_ratio), spdk_json_decode_int32, true}, + {"alru_dirty_ratio_threshold", offsetof(struct rpc_bdev_ocf_set_cleaning, alru_dirty_ratio_threshold), spdk_json_decode_int32, true}, + {"alru_dirty_ratio_inertia", offsetof(struct rpc_bdev_ocf_set_cleaning, alru_dirty_ratio_inertia), spdk_json_decode_int32, true}, }; static void @@ -510,8 +512,8 @@ rpc_bdev_ocf_set_cleaning(struct spdk_jsonrpc_request *request, const struct spd vbdev_ocf_set_cleaning(req.cache_name, req.policy, req.acp_wake_up_time, req.acp_flush_max_buffers, req.alru_wake_up_time, req.alru_flush_max_buffers, req.alru_staleness_time, - req.alru_activity_threshold, req.alru_max_dirty_ratio, - rpc_bdev_ocf_set_cleaning_cb, request); + req.alru_activity_threshold, req.alru_dirty_ratio_threshold, + req.alru_dirty_ratio_inertia, rpc_bdev_ocf_set_cleaning_cb, request); cleanup: free_rpc_bdev_ocf_set_cleaning(&req); diff --git a/python/spdk/cli/bdev.py b/python/spdk/cli/bdev.py index 0bcbf37259b..551c60f8f47 100644 --- a/python/spdk/cli/bdev.py +++ b/python/spdk/cli/bdev.py @@ -196,7 +196,7 @@ def bdev_ocf_set_promotion(args): type=int, default=-1) p.add_argument('-t', '--nhit-trigger-threshold', - help='cache occupancy value over which NHIT promotion is active (range <0-100> [%]; default 80)', + help='cache occupancy value over which NHIT promotion is active (range <0-100> [%%]; default 80)', type=int, default=-1) p.set_defaults(func=bdev_ocf_set_promotion) @@ -211,7 +211,8 @@ def bdev_ocf_set_cleaning(args): alru_flush_max_buffers=args.alru_flush_max_buffers, alru_staleness_time=args.alru_staleness_time, alru_activity_threshold=args.alru_activity_threshold, - alru_max_dirty_ratio=args.alru_max_dirty_ratio) + alru_dirty_ratio_threshold=args.alru_dirty_ratio_threshold, + alru_dirty_ratio_inertia=args.alru_dirty_ratio_inertia) p = subparsers.add_parser('bdev_ocf_set_cleaning', help='Set cleaning parameters for OCF cache') p.add_argument('cache_name', help='name of the cache vbdev') p.add_argument('-p', '--policy', @@ -244,8 +245,13 @@ def bdev_ocf_set_cleaning(args): help='cache idle time before flushing thread can start (range <0-1000000> [ms]; default 10000)', type=int, default=-1) - p.add_argument('-d', '--alru-max-dirty-ratio', - help='maximum percentage of cache occupancy at which cleaning is triggered immediately (range <0-100> [%]; default 100)', + p.add_argument('-d', '--alru-dirty-ratio-threshold', + help='dirty ratio of the cache device at which cleaning will be triggered (range <0-100> [%%]; default 100)', + type=int, + default=-1) + p.add_argument('-i', '--alru-dirty-ratio-inertia', + help='inertia for dirty ratio - cleaning triggered by exceeding dirty ratio threshold will stop \ + when dirty ratio drops below (threshold - inertia) (range <0-4095> [MiB]; default 128)', type=int, default=-1) p.set_defaults(func=bdev_ocf_set_cleaning) diff --git a/test/ocf/common.sh b/test/ocf/common.sh index f0e6a2d4afe..af6e1e5cd41 100644 --- a/test/ocf/common.sh +++ b/test/ocf/common.sh @@ -21,7 +21,8 @@ cleaning_alru_wake_up_time_range=(0 3600) cleaning_alru_flush_max_buffers_range=(1 10000) cleaning_alru_staleness_time_range=(1 3600) cleaning_alru_activity_threshold_range=(0 1000000) -cleaning_alru_max_dirty_ratio_range=(0 100) +cleaning_alru_dirty_ratio_threshold_range=(0 100) +cleaning_alru_dirty_ratio_inertia_range=(0 4095) seqcutoff_policies=("always" "full" "never") seqcutoff_policy_default="full" seqcutoff_threshold_range=(1 4194181) @@ -464,7 +465,7 @@ set_promotion_nhit_params() { } set_cleaning_alru_params() { - if [ $# -ne 5 ]; then + if [ $# -ne 6 ]; then echo >&2 "invalid number of arguments" exit 1 fi @@ -473,7 +474,8 @@ set_cleaning_alru_params() { local flush_max_buffers=$2 local staleness_time=$3 local activity_threshold=$4 - local max_dirty_ratio=$5 + local dirty_ratio_threshold=$5 + local dirty_ratio_inertia=$6 for i in {1..3}; do $rpc_py bdev_ocf_set_cleaning Ocf_cache$i \ @@ -482,7 +484,8 @@ set_cleaning_alru_params() { --alru-flush-max-buffers $flush_max_buffers \ --alru-staleness-time $staleness_time \ --alru-activity-threshold $activity_threshold \ - --alru-max-dirty-ratio $max_dirty_ratio + --alru-dirty-ratio-threshold $dirty_ratio_threshold \ + --alru-dirty-ratio-inertia $dirty_ratio_inertia done $rpc_py bdev_ocf_get_bdevs | jq -e '.' } @@ -750,7 +753,7 @@ __check_promotion_nhit_params() { } __check_cleaning_alru_params() { - if [ $# -ne 5 ]; then + if [ $# -ne 6 ]; then echo >&2 "invalid number of arguments" exit 1 fi @@ -759,21 +762,24 @@ __check_cleaning_alru_params() { local flush_max_buffers=$2 local staleness_time=$3 local activity_threshold=$4 - local max_dirty_ratio=$5 + local dirty_ratio_threshold=$5 + local dirty_ratio_inertia=$6 $rpc_py bdev_ocf_get_bdevs | jq -e \ --argjson wake_up_time $wake_up_time \ --argjson flush_max_buffers $flush_max_buffers \ --argjson staleness_time $staleness_time \ --argjson activity_threshold $activity_threshold \ - --argjson max_dirty_ratio $max_dirty_ratio \ + --argjson dirty_ratio_threshold $dirty_ratio_threshold \ + --argjson dirty_ratio_inertia $dirty_ratio_inertia \ '[.caches[].cleaning] | all( (.policy == "alru") and (.wake_up_time == $wake_up_time) and (.flush_max_buffers == $flush_max_buffers) and (.staleness_time == $staleness_time) and (.activity_threshold == $activity_threshold) and - (.max_dirty_ratio == $max_dirty_ratio))' + (.dirty_ratio_threshold == $dirty_ratio_threshold) and + (.dirty_ratio_inertia == $dirty_ratio_inertia * 1024 * 1024))' } __check_cleaning_acp_params() { diff --git a/test/ocf/management/config_change.sh b/test/ocf/management/config_change.sh index 5aeebbeecaa..4d9464ee069 100755 --- a/test/ocf/management/config_change.sh +++ b/test/ocf/management/config_change.sh @@ -95,9 +95,10 @@ for cleaning_policy in "${cleaning_policies[@]}"; do flush_max_buffers=$(random_number ${cleaning_alru_flush_max_buffers_range[0]} ${cleaning_alru_flush_max_buffers_range[1]}) staleness_time=$(random_number ${cleaning_alru_staleness_time_range[0]} ${cleaning_alru_staleness_time_range[1]}) activity_threshold=$(random_number ${cleaning_alru_activity_threshold_range[0]} ${cleaning_alru_activity_threshold_range[1]}) - max_dirty_ratio=$(random_number ${cleaning_alru_max_dirty_ratio_range[0]} ${cleaning_alru_max_dirty_ratio_range[1]}) - set_cleaning_alru_params $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio - __check_cleaning_alru_params $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $max_dirty_ratio + dirty_ratio_threshold=$(random_number ${cleaning_alru_dirty_ratio_threshold_range[0]} ${cleaning_alru_dirty_ratio_threshold_range[1]}) + dirty_ratio_inertia=$(random_number ${cleaning_alru_dirty_ratio_inertia_range[0]} ${cleaning_alru_dirty_ratio_inertia_range[1]}) + set_cleaning_alru_params $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $dirty_ratio_threshold $dirty_ratio_inertia + __check_cleaning_alru_params $wake_up_time $flush_max_buffers $staleness_time $activity_threshold $dirty_ratio_threshold $dirty_ratio_inertia elif [ $cleaning_policy == acp ]; then wake_up_time=$(random_number ${cleaning_acp_wake_up_time_range[0]} ${cleaning_acp_wake_up_time_range[1]}) flush_max_buffers=$(random_number ${cleaning_acp_flush_max_buffers_range[0]} ${cleaning_acp_flush_max_buffers_range[1]}) diff --git a/test/ocf/management/info_dump_get_bdevs.json b/test/ocf/management/info_dump_get_bdevs.json index d13a1190d23..d66dbc8a1da 100644 --- a/test/ocf/management/info_dump_get_bdevs.json +++ b/test/ocf/management/info_dump_get_bdevs.json @@ -43,7 +43,8 @@ "flush_max_buffers": 100, "staleness_time": 120, "activity_threshold": 10000, - "max_dirty_ratio": 100 + "dirty_ratio_threshold": 100, + "dirty_ratio_inertia": 134217728 }, "flush": { "in_progress": false, @@ -130,7 +131,8 @@ "flush_max_buffers": 100, "staleness_time": 120, "activity_threshold": 10000, - "max_dirty_ratio": 100 + "dirty_ratio_threshold": 100, + "dirty_ratio_inertia": 134217728 }, "flush": { "in_progress": false, @@ -217,7 +219,8 @@ "flush_max_buffers": 100, "staleness_time": 120, "activity_threshold": 10000, - "max_dirty_ratio": 100 + "dirty_ratio_threshold": 100, + "dirty_ratio_inertia": 134217728 }, "flush": { "in_progress": false, From 4a15191a01d3ad51c9edead641e3e991524daf7c Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Thu, 30 Oct 2025 15:53:10 +0100 Subject: [PATCH 34/34] Update OCF Change-Id: I03748efcc653d49fe561731046ff236fcd37314a Signed-off-by: Rafal Stefanowski --- ocf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocf b/ocf index 6a019be7f59..6b433080dc4 160000 --- a/ocf +++ b/ocf @@ -1 +1 @@ -Subproject commit 6a019be7f593c0978fa98c1b3814e6646113a276 +Subproject commit 6b433080dc4e0038bcc50e243105fb7430e7de54