From 0d6c408f48b4681da20972cec48f74e7f509dcbe Mon Sep 17 00:00:00 2001 From: David van Moolenbroek Date: Tue, 25 Aug 2015 08:36:39 +0200 Subject: [PATCH] libblockdriver: suspend threads for live update Same as for VFS. Change-Id: I0f09d43f24c32361af5e5658923140c79244d3d1 --- minix/drivers/storage/ahci/ahci.c | 3 + minix/drivers/storage/virtio_blk/virtio_blk.c | 3 + minix/include/minix/blockdriver_mt.h | 2 + minix/lib/libblockdriver/Makefile | 2 +- minix/lib/libblockdriver/driver_mt.c | 83 ++++++++++++++-- minix/lib/libblockdriver/driver_mt.h | 9 ++ minix/lib/libblockdriver/liveupdate.c | 94 +++++++++++++++++++ minix/lib/libblockdriver/mq.c | 17 +++- minix/lib/libblockdriver/mq.h | 1 + 9 files changed, 204 insertions(+), 10 deletions(-) create mode 100644 minix/lib/libblockdriver/driver_mt.h create mode 100644 minix/lib/libblockdriver/liveupdate.c diff --git a/minix/drivers/storage/ahci/ahci.c b/minix/drivers/storage/ahci/ahci.c index 26cc85841..2b148be2f 100644 --- a/minix/drivers/storage/ahci/ahci.c +++ b/minix/drivers/storage/ahci/ahci.c @@ -2378,6 +2378,9 @@ static void sef_local_startup(void) /* Register signal callbacks. */ sef_setcb_signal_handler(sef_cb_signal_handler); + /* Enable support for live update. */ + blockdriver_mt_support_lu(); + /* Let SEF perform startup. */ sef_startup(); } diff --git a/minix/drivers/storage/virtio_blk/virtio_blk.c b/minix/drivers/storage/virtio_blk/virtio_blk.c index bc4e1d7da..4e1efc49c 100644 --- a/minix/drivers/storage/virtio_blk/virtio_blk.c +++ b/minix/drivers/storage/virtio_blk/virtio_blk.c @@ -733,6 +733,9 @@ sef_local_startup(void) sef_setcb_init_fresh(sef_cb_init_fresh); sef_setcb_signal_handler(sef_cb_signal_handler); + /* Enable suppor for live update. */ + blockdriver_mt_support_lu(); + sef_startup(); } diff --git a/minix/include/minix/blockdriver_mt.h b/minix/include/minix/blockdriver_mt.h index 3c74515df..9396e657b 100644 --- a/minix/include/minix/blockdriver_mt.h +++ b/minix/include/minix/blockdriver_mt.h @@ -13,4 +13,6 @@ void blockdriver_mt_terminate(void); void blockdriver_mt_set_workers(device_id_t id, unsigned int workers); thread_id_t blockdriver_mt_get_tid(void); +void blockdriver_mt_support_lu(void); + #endif /* _MINIX_BLOCKDRIVER_MT_H */ diff --git a/minix/lib/libblockdriver/Makefile b/minix/lib/libblockdriver/Makefile index 42b999280..fbe4c0810 100644 --- a/minix/lib/libblockdriver/Makefile +++ b/minix/lib/libblockdriver/Makefile @@ -7,6 +7,6 @@ CPPFLAGS+= -D_MINIX_SYSTEM LIB= blockdriver -SRCS= driver.c drvlib.c driver_st.c driver_mt.c mq.c trace.c +SRCS= driver.c drvlib.c driver_st.c driver_mt.c liveupdate.c mq.c trace.c .include diff --git a/minix/lib/libblockdriver/driver_mt.c b/minix/lib/libblockdriver/driver_mt.c index 8d82876b1..0c439ed66 100644 --- a/minix/lib/libblockdriver/driver_mt.c +++ b/minix/lib/libblockdriver/driver_mt.c @@ -17,6 +17,7 @@ #include "const.h" #include "driver.h" +#include "driver_mt.h" #include "mq.h" /* A thread ID is composed of a device ID and a per-device worker thread ID. @@ -271,6 +272,20 @@ static void master_handle_exits(void) num_exited = 0; } +/*===========================================================================* + * master_yield * + *===========================================================================*/ +static void master_yield(void) +{ +/* Let worker threads run, and clean up any exited threads. + */ + + mthread_yield_all(); + + if (num_exited > 0) + master_handle_exits(); +} + /*===========================================================================* * master_handle_message * *===========================================================================*/ @@ -421,12 +436,8 @@ void blockdriver_mt_task(struct blockdriver *driver_tab) /* Dispatch the message. */ master_handle_message(&mess, ipc_status); - /* Let other threads run. */ - mthread_yield_all(); - - /* Clean up any exited threads. */ - if (num_exited > 0) - master_handle_exits(); + /* Let worker threads run. */ + master_yield(); } /* Free up resources. */ @@ -508,3 +519,63 @@ void blockdriver_mt_set_workers(device_id_t id, unsigned int workers) dp->workers = workers; } + +/*===========================================================================* + * blockdriver_mt_is_idle * + *===========================================================================*/ +int blockdriver_mt_is_idle(void) +{ +/* Return whether the block driver is idle. This means that it has no enqueued + * requests and no busy worker threads. Used for live update functionality. + */ + unsigned int did, wid; + + for (did = 0; did < MAX_DEVICES; did++) { + if (!mq_isempty(did)) + return FALSE; + + for (wid = 0; wid < device[did].workers; wid++) + if (device[did].worker[wid].state == STATE_BUSY) + return FALSE; + } + + return TRUE; +} + +/*===========================================================================* + * blockdriver_mt_suspend * + *===========================================================================*/ +void blockdriver_mt_suspend(void) +{ +/* Suspend the driver operation in order to facilitate a live update. + * Suspension involves shutting down all worker threads, because transferring + * thread stacks is currently not supported by the state transfer framework. + */ + unsigned int did; + + assert(running); + assert(blockdriver_mt_is_idle()); + + /* We terminate the worker threads by simulating a driver shutdown. */ + running = FALSE; + + for (did = 0; did < MAX_DEVICES; did++) + mthread_event_fire_all(&device[did].queue_event); + + master_yield(); +} + +/*===========================================================================* + * blockdriver_mt_resume * + *===========================================================================*/ +void blockdriver_mt_resume(void) +{ +/* Resume regular operation after a (successful or failed) live update. We do + * not recreate worker threads; instead, they are recreated lazily as new + * requests come in. + */ + + assert(!running); + + running = TRUE; +} diff --git a/minix/lib/libblockdriver/driver_mt.h b/minix/lib/libblockdriver/driver_mt.h new file mode 100644 index 000000000..a52e73807 --- /dev/null +++ b/minix/lib/libblockdriver/driver_mt.h @@ -0,0 +1,9 @@ +#ifndef _BLOCKDRIVER_DRIVER_MT_H +#define _BLOCKDRIVER_DRIVER_MT_H + +/* These functions are used for live update. */ +int blockdriver_mt_is_idle(void); +void blockdriver_mt_suspend(void); +void blockdriver_mt_resume(void); + +#endif /* !_BLOCKDRIVER_DRIVER_MT_H */ diff --git a/minix/lib/libblockdriver/liveupdate.c b/minix/lib/libblockdriver/liveupdate.c new file mode 100644 index 000000000..d01006469 --- /dev/null +++ b/minix/lib/libblockdriver/liveupdate.c @@ -0,0 +1,94 @@ +/* This file implements SEF hooks for live update of multithreaded block + * drivers. + */ + +#include +#include + +#include "driver_mt.h" + +/*===========================================================================* + * sef_cb_lu_prepare * + *===========================================================================*/ +static int sef_cb_lu_prepare(int state) +{ +/* This function is called to decide whether we can enter the given live + * update state, and to prepare for such an update. If we are requested to + * update to a request-free or protocol-free state, make sure there is no work + * pending or being processed, and shut down all worker threads. + */ + + switch (state) { + case SEF_LU_STATE_REQUEST_FREE: + case SEF_LU_STATE_PROTOCOL_FREE: + if (!blockdriver_mt_is_idle()) { + printf("libblockdriver(%d): not idle, blocking update\n", + sef_self()); + break; + } + + blockdriver_mt_suspend(); + + return OK; + } + + return ENOTREADY; +} + +/*===========================================================================* + * sef_cb_lu_state_changed * + *===========================================================================*/ +static void sef_cb_lu_state_changed(int old_state, int state) +{ +/* This function is called in the old driver instance when the state changes. + * We use it to resume normal operation after a failed live update. + */ + + if (state != SEF_LU_STATE_NULL) + return; + + switch (old_state) { + case SEF_LU_STATE_REQUEST_FREE: + case SEF_LU_STATE_PROTOCOL_FREE: + blockdriver_mt_resume(); + } +} + +/*===========================================================================* + * sef_cb_init_lu * + *===========================================================================*/ +static int sef_cb_init_lu(int type, sef_init_info_t *info) +{ +/* This function is called in the new driver instance during a live update. + */ + int r; + + /* Perform regular state transfer. */ + if ((r = SEF_CB_INIT_LU_DEFAULT(type, info)) != OK) + return r; + + /* Recreate worker threads, if necessary. */ + switch (info->prepare_state) { + case SEF_LU_STATE_REQUEST_FREE: + case SEF_LU_STATE_PROTOCOL_FREE: + blockdriver_mt_resume(); + } + + return OK; +} + +/*===========================================================================* + * blockdriver_mt_support_lu * + *===========================================================================*/ +void blockdriver_mt_support_lu(void) +{ +/* Enable suppor for live update of this driver. To be called before + * sef_startup(). + */ + + /* Register live update callbacks. */ + sef_setcb_init_lu(sef_cb_init_lu); + sef_setcb_lu_prepare(sef_cb_lu_prepare); + sef_setcb_lu_state_changed(sef_cb_lu_state_changed); + sef_setcb_lu_state_isvalid(sef_cb_lu_state_isvalid_standard); +} diff --git a/minix/lib/libblockdriver/mq.c b/minix/lib/libblockdriver/mq.c index af2fcc5ec..fe4e2d3e6 100644 --- a/minix/lib/libblockdriver/mq.c +++ b/minix/lib/libblockdriver/mq.c @@ -70,6 +70,19 @@ int mq_enqueue(device_id_t device_id, const message *mess, return TRUE; } +/*===========================================================================* + * mq_isempty * + *===========================================================================*/ +int mq_isempty(device_id_t device_id) +{ +/* Return whether the message queue for the given device is empty. + */ + + assert(device_id >= 0 && device_id < MAX_DEVICES); + + return STAILQ_EMPTY(&queue[device_id]); +} + /*===========================================================================* * mq_dequeue * *===========================================================================*/ @@ -80,9 +93,7 @@ int mq_dequeue(device_id_t device_id, message *mess, int *ipc_status) */ struct mq_cell *cell; - assert(device_id >= 0 && device_id < MAX_DEVICES); - - if (STAILQ_EMPTY(&queue[device_id])) + if (mq_isempty(device_id)) return FALSE; cell = STAILQ_FIRST(&queue[device_id]); diff --git a/minix/lib/libblockdriver/mq.h b/minix/lib/libblockdriver/mq.h index 124b0212e..a31cce60f 100644 --- a/minix/lib/libblockdriver/mq.h +++ b/minix/lib/libblockdriver/mq.h @@ -5,5 +5,6 @@ void mq_init(void); int mq_enqueue(device_id_t device_id, const message *mess, int ipc_status); int mq_dequeue(device_id_t device_id, message *mess, int *ipc_status); +int mq_isempty(device_id_t device_id); #endif /* _BLOCKDRIVER_MQ_H */