By decoupling synchronous drivers from VFS, we are a big step closer to supporting driver crashes under all circumstances. That is, VFS can't become stuck on IPC with a synchronous driver (e.g., INET) and can recover from crashing block drivers during open/close/ioctl or during communication with an FS. In order to maintain serialized communication with a synchronous driver, the communication is wrapped by a mutex on a per driver basis (not major numbers as there can be multiple majors with identical endpoints). Majors that share a driver endpoint point to a single mutex object. In order to support crashes from block drivers, the file reopen tactic had to be changed; first reopen files associated with the crashed driver, then send the new driver endpoint to FSes. This solves a deadlock between the FS and the block driver; - VFS would send REQ_NEW_DRIVER to an FS, but he FS only receives it after retrying the current request to the newly started driver. - The block driver would refuse the retried request until all files had been reopened. - VFS would reopen files only after getting a reply from the initial REQ_NEW_DRIVER. When a character special driver crashes, all associated files have to be marked invalid and closed (or reopened if flagged as such). However, they can only be closed if a thread holds exclusive access to it. To obtain exclusive access, the worker thread (which handles the new driver endpoint event from DS) schedules a new job to garbage collect invalid files. This way, we can signal the worker thread that was talking to the crashed driver and will release exclusive access to a file associated with the crashed driver and prevent the garbage collecting worker thread from dead locking on that file. Also, when a character special driver crashes, RS will unmap the driver and remap it upon restart. During unmapping, associated files are marked invalid instead of waiting for an endpoint up event from DS, as that event might come later than new read/write/select requests and thus cause confusion in the freshly started driver. When locking a filp, the usage counters are no longer checked. The usage counter can legally go down to zero during filp invalidation while there are locks pending. DS events are handled by a separate worker thread instead of the main thread as reopening files could lead to another crash and a stuck thread. An additional worker thread is then necessary to unlock it. Finally, with everything asynchronous a race condition in do_select surfaced. A select entry was only marked in use after succesfully sending initial select requests to drivers and having to wait. When multiple select() calls were handled there was opportunity that these entries were overwritten. This had as effect that some select results were ignored (and select() remained blocking instead if returning) or do_select tried to access filps that were not present (because thrown away by secondary select()). This bug manifested itself with sendrecs, but was very hard to reproduce. However, it became awfully easy to trigger with asynsends only.
178 lines
5 KiB
C
178 lines
5 KiB
C
#include "fs.h"
|
|
#include "glo.h"
|
|
#include "vmnt.h"
|
|
#include "fproc.h"
|
|
#include <minix/vfsif.h>
|
|
#include <assert.h>
|
|
|
|
static int sendmsg(struct vmnt *vmp, struct worker_thread *wp);
|
|
static int queuemsg(struct vmnt *vmp);
|
|
|
|
/*===========================================================================*
|
|
* sendmsg *
|
|
*===========================================================================*/
|
|
static int sendmsg(struct vmnt *vmp, struct worker_thread *wp)
|
|
{
|
|
/* This is the low level function that sends requests to FS processes.
|
|
*/
|
|
int r, transid;
|
|
|
|
vmp->m_comm.c_cur_reqs++; /* One more request awaiting a reply */
|
|
transid = wp->w_tid + VFS_TRANSID;
|
|
wp->w_fs_sendrec->m_type = TRNS_ADD_ID(wp->w_fs_sendrec->m_type, transid);
|
|
wp->w_task = vmp->m_fs_e;
|
|
if ((r = asynsend3(vmp->m_fs_e, wp->w_fs_sendrec, AMF_NOREPLY)) != OK) {
|
|
printf("VFS: sendmsg: error sending message. "
|
|
"FS_e: %d req_nr: %d err: %d\n", vmp->m_fs_e,
|
|
wp->w_fs_sendrec->m_type, r);
|
|
util_stacktrace();
|
|
return(r);
|
|
}
|
|
|
|
return(r);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* send_work *
|
|
*===========================================================================*/
|
|
void send_work(void)
|
|
{
|
|
/* Try to send out as many requests as possible */
|
|
struct vmnt *vmp;
|
|
|
|
if (sending == 0) return;
|
|
for (vmp = &vmnt[0]; vmp < &vmnt[NR_MNTS]; vmp++)
|
|
fs_sendmore(vmp);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* fs_cancel *
|
|
*===========================================================================*/
|
|
void fs_cancel(struct vmnt *vmp)
|
|
{
|
|
/* Cancel all pending requests for this vmp */
|
|
struct worker_thread *worker;
|
|
|
|
while ((worker = vmp->m_comm.c_req_queue) != NULL) {
|
|
vmp->m_comm.c_req_queue = worker->w_next;
|
|
worker->w_next = NULL;
|
|
sending--;
|
|
worker_stop(worker);
|
|
}
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* fs_sendmore *
|
|
*===========================================================================*/
|
|
void fs_sendmore(struct vmnt *vmp)
|
|
{
|
|
struct worker_thread *worker;
|
|
|
|
/* Can we send more requests? */
|
|
if (vmp->m_fs_e == NONE) return;
|
|
if ((worker = vmp->m_comm.c_req_queue) == NULL) /* No process is queued */
|
|
return;
|
|
if (vmp->m_comm.c_cur_reqs >= vmp->m_comm.c_max_reqs)/*No room to send more*/
|
|
return;
|
|
if (vmp->m_flags & VMNT_CALLBACK) /* Hold off for now */
|
|
return;
|
|
|
|
vmp->m_comm.c_req_queue = worker->w_next; /* Remove head */
|
|
worker->w_next = NULL;
|
|
sending--;
|
|
assert(sending >= 0);
|
|
(void) sendmsg(vmp, worker);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* drv_sendrec *
|
|
*===========================================================================*/
|
|
int drv_sendrec(endpoint_t drv_e, message *reqmp)
|
|
{
|
|
int r;
|
|
struct dmap *dp;
|
|
|
|
if ((dp = get_dmap(drv_e)) == NULL)
|
|
panic("driver endpoint %d invalid", drv_e);
|
|
|
|
lock_dmap(dp);
|
|
if (dp->dmap_servicing != NONE)
|
|
panic("driver locking inconsistency");
|
|
dp->dmap_servicing = self->w_tid;
|
|
self->w_task = drv_e;
|
|
self->w_drv_sendrec = reqmp;
|
|
|
|
if ((r = asynsend3(drv_e, self->w_drv_sendrec, AMF_NOREPLY)) == OK) {
|
|
/* Yield execution until we've received the reply */
|
|
worker_wait();
|
|
} else {
|
|
printf("VFS: drv_sendrec: error sending msg to driver %d: %d\n",
|
|
drv_e, r);
|
|
util_stacktrace();
|
|
}
|
|
|
|
dp->dmap_servicing = NONE;
|
|
self->w_task = NONE;
|
|
self->w_drv_sendrec = NULL;
|
|
unlock_dmap(dp);
|
|
return(OK);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* fs_sendrec *
|
|
*===========================================================================*/
|
|
int fs_sendrec(endpoint_t fs_e, message *reqmp)
|
|
{
|
|
struct vmnt *vmp;
|
|
int r;
|
|
|
|
if ((vmp = find_vmnt(fs_e)) == NULL) {
|
|
printf("Trying to talk to non-existent FS endpoint %d\n", fs_e);
|
|
return(EIO);
|
|
}
|
|
if (fs_e == fp->fp_endpoint) return(EDEADLK);
|
|
|
|
self->w_fs_sendrec = reqmp; /* Where to store request and reply */
|
|
|
|
/* Find out whether we can send right away or have to enqueue */
|
|
if ( !(vmp->m_flags & VMNT_CALLBACK) &&
|
|
vmp->m_comm.c_cur_reqs < vmp->m_comm.c_max_reqs) {
|
|
/* There's still room to send more and no proc is queued */
|
|
r = sendmsg(vmp, self);
|
|
} else {
|
|
r = queuemsg(vmp);
|
|
}
|
|
self->w_next = NULL; /* End of list */
|
|
|
|
if (r != OK) return(r);
|
|
|
|
worker_wait(); /* Yield execution until we've received the reply. */
|
|
|
|
return(reqmp->m_type);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* queuemsg *
|
|
*===========================================================================*/
|
|
static int queuemsg(struct vmnt *vmp)
|
|
{
|
|
/* Put request on queue for vmnt */
|
|
|
|
struct worker_thread *queue;
|
|
|
|
if (vmp->m_comm.c_req_queue == NULL) {
|
|
vmp->m_comm.c_req_queue = self;
|
|
} else {
|
|
/* Walk the list ... */
|
|
queue = vmp->m_comm.c_req_queue;
|
|
while (queue->w_next != NULL) queue = queue->w_next;
|
|
|
|
/* ... and append this worker */
|
|
queue->w_next = self;
|
|
}
|
|
|
|
self->w_next = NULL; /* End of list */
|
|
sending++;
|
|
|
|
return(OK);
|
|
}
|