CVS: winex/server iocomp.c, NONE, 1.1 Makefile.in, 1.20, 1.21 object.h, 1.15, 1.16 protocol.def, 1.41, 1.42 request.h, 1.27, 1.28 shm.c, 1.18, 1.19 thread.c, 1.54, 1.55 thread.h, 1.27, 1.28 trace.c, 1.42, 1.43
[email protected] 31 Jul 2007 18:00:24 -0000
| Newsgroups | gmane.comp.emulators.winex.cvs |
|---|---|
| Message-ID | <[email protected]> |
Subject: winex/server iocomp.c,NONE,1.1 Makefile.in,1.20,1.21 object.h,1.15,1.16 protocol.def,1.41,1.42 request.h,1.27,1.28 shm.c,1.18,1.19 thread.c,1.54,1.55 thread.h,1.27,1.28 trace.c,1.42,1.43Update of /var/lib/cvsd/cvsroot/winex/server
In directory agravaine:/tmp/cvs-serv24703/server
Modified Files:
Makefile.in object.h protocol.def request.h shm.c thread.c
thread.h trace.c
Added Files:
iocomp.c
Log Message:
Partially implement IO completion ports to the point that they can be used as task queues (ie, not associated with a file handle). Still to be done:
- finish thread accounting for blocking
- implement associating a file handle with an IO completion port
- modify relevant functions to send notifications to an associated IO completion port
- lots of testing of corner cases
--- NEW FILE: iocomp.c ---
/*
* IO Completion Port support
*
* Copyright (C) 2007 TransGaming Technologies
*
* Main reference:
* http://www.microsoft.com/technet/sysinternals/information/IoCompletionPorts.mspx
*
* Status: IO completion ports are only partially implemented. In particular,
* enough functionality is implemented to allow them to be used as task
* queues.
*
* Known TODOs:
* - keep num_active_threads current; we do proper accounting in here, but
* also need to do so anywhere a thread can block (Sleep, WaitFor*, etc)
* - implement associating a file handle (and its current thread) with a
* completion port
* - modify the following functions to be able to signal completion via
* completion ports (and check around MSDN for any others):
* ConnectNamedPipe
* DeviceIoControl
* LockFileEx
* ReadDirectoryChangesW
* ReadFile
* TransactNamedPipe
* WaitCommEvent
* WriteFile
*/
#include "config.h"
#include <assert.h>
#include <stdio.h>
#ifdef HAVE_SYS_TIME_H
#include <sys/time.h>
#endif
#include "winnt.h"
#include "handle.h"
#include "request.h"
#include "thread.h"
struct iocomp_task {
struct iocomp_task *next;
struct thread *assigned_thread;
void *completion_key;
void *overlapped;
unsigned int num_bytes;
};
struct iocomp
{
struct object obj; /* object header */
unsigned int max_active_threads; /* max number of concurrent threads */
unsigned int num_active_threads; /* num threads currently processing
a task */
int abandon_waits; /* used to prevent threads from
getting signalled prematurely */
struct iocomp_task *tasks; /* ptr to head of FIFO task list */
struct iocomp_task *last_task; /* ptr to last task in FIFO task list */
struct iocomp_task *assigned_tasks; /* ptr to head of assigned but not yet
retrieved task list */
};
static void iocomp_dump (struct object *obj, int verbose);
static int iocomp_signaled (struct object *obj,
struct wait_queue_entry *entry);
static int iocomp_satisfied (struct object *obj, struct thread *thread);
static void iocomp_destroy (struct object *obj);
static const struct object_ops iocomp_ops =
{
sizeof (struct iocomp), /* size */
iocomp_dump, /* dump */
add_queue, /* add_queue */
remove_queue, /* remove_queue */
iocomp_signaled, /* signaled */
iocomp_satisfied, /* satisfied */
NULL, /* get_poll_events */
NULL, /* poll_event */
no_get_fd, /* get_fd */
no_get_ctx_fd, /* get_ctx_fd */
no_flush, /* flush */
no_get_file_info, /* get_file_info */
NULL, /* queue_async */
iocomp_destroy /* destroy */
};
const struct object_ops* get_this_process_iocomp_ops (void) {
return &iocomp_ops;
}
static struct iocomp *create_iocomp (unsigned int MaxNumThreads)
{
struct iocomp *pIOComp;
if ((pIOComp = alloc_object (iocomp_ops_tag, -1)))
{
pIOComp->max_active_threads = MaxNumThreads;
pIOComp->num_active_threads = 0;
pIOComp->tasks = NULL;
pIOComp->last_task = NULL;
pIOComp->assigned_tasks = NULL;
pIOComp->abandon_waits = 1;
}
return pIOComp;
}
static void iocomp_dump (struct object *obj, int verbose)
{
struct iocomp *iocomp = (struct iocomp *)obj;
struct iocomp_task *task;
unsigned int tasks = 0, assigned_tasks = 0;
task = iocomp->tasks;
while (task)
{
tasks++;
task = task->next;
}
task = iocomp->assigned_tasks;
while (task)
{
assigned_tasks++;
task = task->next;
}
assert (obj->ops == iocomp_ops_tag);
fprintf (stderr, "IO Completion max=%u active=%u queued_tasks=%u assigned=%u ",
iocomp->max_active_threads, iocomp->num_active_threads,
tasks, assigned_tasks);
dump_object_name (&iocomp->obj);
fputc ('\n', stderr);
}
/* Return 1 if we have a task to be processed */
static int iocomp_signaled (struct object *obj,
struct wait_queue_entry *entry)
{
struct iocomp *iocomp = (struct iocomp *)obj;
struct iocomp_task *task;
assert (obj->ops == iocomp_ops_tag);
task = iocomp->assigned_tasks;
if (task == NULL)
return 0;
while (task)
{
if (task->assigned_thread == entry->thread)
return 1;
task = task->next;
}
return 0;
}
/* Return 1 if it is safe for a waiting thread to be woken up */
static int iocomp_satisfied (struct object *obj, struct thread *thread)
{
struct iocomp *iocomp = (struct iocomp *)obj;
assert (obj->ops == iocomp_ops_tag);
return iocomp->abandon_waits;
}
static void iocomp_destroy (struct object *obj)
{
struct iocomp *iocomp = (struct iocomp *)obj;
struct iocomp_task *task;
assert (obj->ops == iocomp_ops_tag);
while ((task = iocomp->tasks))
{
iocomp->tasks = task->next;
mem_free (task);
}
while ((task = iocomp->assigned_tasks))
{
iocomp->assigned_tasks = task->next;
mem_free (task);
}
}
/* Selects the task at the top of the FIFO queue and assigns it to
the passed thread (which then needs to retrieve it) */
static void assign_task_to_thread (struct iocomp *iocomp,
struct thread *thread)
{
struct iocomp_task *task;
assert (iocomp->tasks != NULL);
/* Remove task from tasks queue */
task = iocomp->tasks;
iocomp->tasks = task->next;
if (iocomp->last_task == task)
iocomp->last_task = NULL;
/* Add task to assigned list, where it'll wait for the assigned thread
to wake up and retrieve it */
task->next = iocomp->assigned_tasks;
iocomp->assigned_tasks = task;
/* Assign it */
task->assigned_thread = thread;
/* Active thread accounting */
iocomp->num_active_threads++;
thread->iocomp = (struct iocomp *)grab_object (iocomp);
}
/* Kick off a thread if needed and available, and if we don't have the
max number of threads running */
static void check_and_activate_thread (struct iocomp *iocomp)
{
if (iocomp->tasks && iocomp->obj.tail &&
(iocomp->num_active_threads < iocomp->max_active_threads))
{
struct wait_queue_entry * waiting = iocomp->obj.tail;
assign_task_to_thread (iocomp, waiting->thread);
wake_thread (waiting->thread);
}
}
/* Notification routine called on thread exiting or blocking */
void iocomp_notify_thread_inactive (struct iocomp *iocomp)
{
iocomp->num_active_threads--;
/* Kick off a thread if appropriate */
check_and_activate_thread (iocomp);
}
/* Notification routine called when a thread unblocks. Note that this can
result in (temporarily) > max_active_threads running, but that's
as expected */
void iocomp_notify_thread_active (struct iocomp *iocomp)
{
iocomp->num_active_threads++;
}
DECL_HANDLER (create_io_completion)
{
struct iocomp *iocomp;
reply->handle = 0;
if ((iocomp = create_iocomp (req->num_threads)))
{
reply->handle = alloc_handle (current->process, iocomp,
IO_COMPLETION_ALL_ACCESS, 0);
release_object (iocomp);
}
}
DECL_HANDLER (set_io_completion)
{
struct iocomp *iocomp;
if ((iocomp =
(struct iocomp *)get_handle_obj (current->process, req->handle,
IO_COMPLETION_MODIFY_STATE,
iocomp_ops_tag)))
{
struct iocomp_task *task;
/* Create a new task and add it to the queue */
task = mem_alloc (sizeof (struct iocomp_task));
if (!task)
{
release_object (iocomp);
return;
}
task->next = NULL;
task->assigned_thread = NULL;
task->completion_key = req->completion_key;
task->overlapped = req->overlapped;
task->num_bytes = req->num_bytes;
if (iocomp->last_task)
iocomp->last_task->next = task;
else
iocomp->tasks = task;
iocomp->last_task = task;
/* Kick off a thread if appropriate */
check_and_activate_thread (iocomp);
release_object (iocomp);
}
}
DECL_HANDLER (remove_io_completion)
{
struct iocomp *iocomp;
/* Active thread accounting */
if (current->iocomp)
{
current->iocomp->num_active_threads--;
release_object (current->iocomp);
}
if ((iocomp =
(struct iocomp *)get_handle_obj (current->process, req->handle,
IO_COMPLETION_MODIFY_STATE,
iocomp_ops_tag)))
{
/* Check for outstanding tasks; if so, and if there aren't too many
threads already running, grab the one from the top of
the FIFO queue and return */
if (iocomp->tasks &&
(iocomp->num_active_threads < iocomp->max_active_threads))
{
struct iocomp_task *task;
/* Remove top task from queue */
task = iocomp->tasks;
iocomp->tasks = task->next;
if (iocomp->last_task == task)
iocomp->last_task = NULL;
/* Return the task */
reply->completion_key = task->completion_key;
reply->overlapped = task->overlapped;
reply->num_bytes = task->num_bytes;
mem_free (task);
/* Active thread accounting */
iocomp->num_active_threads++;
current->iocomp = (struct iocomp *)grab_object (iocomp);
}
/* No task. Time to wait for one if the timeout isn't infinite and
is > 0 */
else if (!(req->select_flags & SELECT_TIMEOUT) ||
req->sec || req->usec)
{
struct timeval tv;
/* Convert time to absolute time */
gettimeofday (&tv, NULL);
tv.tv_usec += req->usec;
if (tv.tv_usec >= 1000000)
{
tv.tv_sec++;
tv.tv_usec -= 1000000;
}
tv.tv_sec += req->sec;
/* Need to ensure that existing waits on the port aren't
interrupted while we're adding a new one */
iocomp->abandon_waits = 0;
select_on (1, req->cookie, &req->handle, req->select_flags,
tv.tv_sec, tv.tv_usec, NULL);
iocomp->abandon_waits = 1;
reply->completion_key = NULL;
reply->overlapped = NULL;
reply->num_bytes = 0;
}
/* No task, and user specified timeout of 0 (meaning to return
immediately) */
else
{
set_error (STATUS_TIMEOUT);
reply->completion_key = NULL;
reply->overlapped = NULL;
reply->num_bytes = 0;
}
release_object (iocomp);
}
}
DECL_HANDLER (retrieve_assigned_io_completion)
{
struct iocomp *iocomp;
if ((iocomp =
(struct iocomp *)get_handle_obj (current->process, req->handle,
IO_COMPLETION_MODIFY_STATE,
iocomp_ops_tag)))
{
struct iocomp_task *task, *prev;
/* Walk assigned task list to find ours */
task = iocomp->assigned_tasks;
prev = NULL;
while (task)
{
if (task->assigned_thread != current)
{
prev = task;
task = task->next;
continue;
}
break;
}
/* WTF? */
if (!task)
{
fprintf (stderr,
"server: couldn't find assigned IO completion task??!!\n");
set_error (STATUS_INVALID_PARAMETER);
reply->completion_key = NULL;
reply->overlapped = NULL;
reply->num_bytes = 0;
}
else
{
/* Remove task from list */
if (prev)
prev->next = task->next;
else
iocomp->assigned_tasks = task->next;
reply->completion_key = task->completion_key;
reply->overlapped = task->overlapped;
reply->num_bytes = task->num_bytes;
mem_free (task);
}
release_object (iocomp);
}
}
Index: Makefile.in
===================================================================
RCS file: /var/lib/cvsd/cvsroot/winex/server/Makefile.in,v
retrieving revision 1.20
retrieving revision 1.21
diff -u -d -r1.20 -r1.21
--- Makefile.in 30 Mar 2007 15:55:03 -0000 1.20
+++ Makefile.in 31 Jul 2007 18:00:22 -0000 1.21
@@ -30,6 +30,7 @@
file.c \
gdbremote.c \
handle.c \
+ iocomp.c \
mapping.c \
mem_malloc.c \
mutex.c \
Index: object.h
===================================================================
RCS file: /var/lib/cvsd/cvsroot/winex/server/object.h,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- object.h 29 Mar 2007 14:03:56 -0000 1.15
+++ object.h 31 Jul 2007 18:00:22 -0000 1.16
@@ -291,7 +291,8 @@
#define fd_server_ops_tag (const struct object_ops*)0xdeadf01b
#define distributed_commands_ops_tag (const struct object_ops*)0xdeadf01c
#define dbus_table_ops_tag (const struct object_ops*)0xdeadf01d
-#define last_ops_tag_value 0xdeadf01e
+#define iocomp_ops_tag (const struct object_ops*)0xdeadf01e
+#define last_ops_tag_value 0xdeadf01f
#define token_ops_tag (const struct object_ops*)last_ops_tag_value
@@ -313,6 +314,7 @@
#endif
extern const struct object_ops* get_this_process_handle_table_ops(void);
+extern const struct object_ops* get_this_process_iocomp_ops(void);
extern const struct object_ops* get_this_process_mapping_ops(void);
extern const struct object_ops* get_this_process_mutex_ops(void);
extern const struct object_ops* get_this_process_named_pipe_ops(void);
Index: protocol.def
===================================================================
RCS file: /var/lib/cvsd/cvsroot/winex/server/protocol.def,v
retrieving revision 1.41
retrieving revision 1.42
diff -u -d -r1.41 -r1.42
--- protocol.def 30 Mar 2007 18:58:50 -0000 1.41
+++ protocol.def 31 Jul 2007 18:00:22 -0000 1.42
@@ -2004,3 +2004,40 @@
handle_t handle; /* handle to the token */
@END
+/* Create an IO completion */
+@REQ(create_io_completion)
+ unsigned int num_threads; /* max number of threads to run concurrently */
+@REPLY
+ handle_t handle; /* handle to created IO completion port */
+@END
+
+/* Add a task to an IO completion */
+@REQ(set_io_completion)
+ handle_t handle; /* IO completion port */
+ void* completion_key; /* completion key */
+ void* overlapped; /* overlapped data */
+ unsigned int num_bytes; /* number of bytes transferred */
+@END
+
+/* Notify wineserver that we're ready to receive a new task, and immediately
+ return if one is available and given to us */
+@REQ(remove_io_completion)
+ handle_t handle; /* IO completion port */
+ void* cookie; /* magic cookie for client if needs to wait */
+ int select_flags; /* flags for select_on() */
+ int sec; /* absolute timeout */
+ int usec; /* absolute timeout */
+@REPLY
+ void* completion_key; /* completion key */
+ void* overlapped; /* overlapped data */
+ unsigned int num_bytes; /* number of bytes transferred */
+@END
+
+/* Retrieve task given to us after having to wait */
+@REQ(retrieve_assigned_io_completion)
+ handle_t handle; /* IO completion port */
+@REPLY
+ void* completion_key; /* completion key */
+ void* overlapped; /* overlapped data */
+ unsigned int num_bytes; /* number of bytes transferred */
+@END
Index: request.h
===================================================================
RCS file: /var/lib/cvsd/cvsroot/winex/server/request.h,v
retrieving revision 1.27
retrieving revision 1.28
diff -u -d -r1.27 -r1.28
--- request.h 30 Mar 2007 15:55:03 -0000 1.27
+++ request.h 31 Jul 2007 18:00:22 -0000 1.28
@@ -304,6 +304,10 @@
DECL_HANDLER(get_cdrom_eject_fd_list);
DECL_HANDLER(add_cdrom_device_info);
DECL_HANDLER(open_token);
+DECL_HANDLER(create_io_completion);
+DECL_HANDLER(set_io_completion);
+DECL_HANDLER(remove_io_completion);
+DECL_HANDLER(retrieve_assigned_io_completion);
#ifdef WANT_REQUEST_HANDLERS
@@ -479,6 +483,10 @@
(req_handler)req_get_cdrom_eject_fd_list,
(req_handler)req_add_cdrom_device_info,
(req_handler)req_open_token,
+ (req_handler)req_create_io_completion,
+ (req_handler)req_set_io_completion,
+ (req_handler)req_remove_io_completion,
+ (req_handler)req_retrieve_assigned_io_completion,
};
#endif /* WANT_REQUEST_HANDLERS */
Index: shm.c
===================================================================
RCS file: /var/lib/cvsd/cvsroot/winex/server/shm.c,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- shm.c 30 Mar 2007 15:55:03 -0000 1.18
+++ shm.c 31 Jul 2007 18:00:22 -0000 1.19
@@ -783,6 +783,7 @@
OPS_TAG( fd_server_ops ),
OPS_TAG( distributed_commands_ops ),
OPS_TAG( dbus_table_ops ),
+ OPS_TAG( iocomp_ops ),
OPS_TAG( token_ops )
};
Index: thread.c
===================================================================
RCS file: /var/lib/cvsd/cvsroot/winex/server/thread.c,v
retrieving revision 1.54
retrieving revision 1.55
diff -u -d -r1.54 -r1.55
--- thread.c 30 Mar 2007 15:55:03 -0000 1.54
+++ thread.c 31 Jul 2007 18:00:22 -0000 1.55
@@ -127,6 +127,7 @@
thread->system_apc.tail = NULL;
thread->user_apc.head = NULL;
thread->user_apc.tail = NULL;
+ thread->iocomp = NULL;
thread->error = 0;
thread->req_data = NULL;
thread->req_toread = 0;
@@ -247,6 +248,13 @@
if (thread->request_fd != -1) close( thread->request_fd );
if (thread->reply_fd != -1) close( thread->reply_fd );
if( thread->wait_fd != -1 ) release_context_fd( thread->wait_fd, NULL, NULL );
+ if (thread->iocomp)
+ {
+ iocomp_notify_thread_inactive (thread->iocomp);
+ release_object (thread->iocomp);
+ thread->iocomp = NULL;
+ }
+
if (thread->queue)
{
if (thread->process->queue == thread->queue)
@@ -662,8 +670,8 @@
extern int set_handle_in_place( struct process* process, handle_t handle, context_fd ctx_fd );
/* select on a list of handles */
-static void select_on( int count, void *cookie, const handle_t *handles,
- int flags, int sec, int usec, struct object* objs[] )
+void select_on( int count, void *cookie, const handle_t *handles,
+ int flags, int sec, int usec, struct object* objs[] )
{
int ret, i;
struct object *objects[MAXIMUM_WAIT_OBJECTS];
Index: thread.h
===================================================================
RCS file: /var/lib/cvsd/cvsroot/winex/server/thread.h,v
retrieving revision 1.27
retrieving revision 1.28
diff -u -d -r1.27 -r1.28
--- thread.h 30 Mar 2007 15:55:03 -0000 1.27
+++ thread.h 31 Jul 2007 18:00:22 -0000 1.28
@@ -28,6 +28,7 @@
struct debug_event;
struct startup_info;
struct msg_queue;
+struct iocomp;
enum run_state
{
@@ -79,6 +80,7 @@
struct thread_wait *wait; /* current wait condition if sleeping */
struct apc_queue system_apc; /* queue of system async procedure calls */
struct apc_queue user_apc; /* queue of user async procedure calls */
+ struct iocomp *iocomp; /* pointer to IO Completion with which we're associated, if any */
struct inflight_fd inflight[MAX_INFLIGHT_FDS]; /* fds currently in flight */
unsigned int error; /* current error code */
const union generic_request *req; /* current request */
@@ -173,7 +175,10 @@
extern struct thread_snapshot *thread_snap( int *count );
extern int is_cookie_satisfied( const struct thread* thread, const void* cookie );
extern void send_unix_signal( struct thread *thread, int resume, int code, size_t size, void *data );
-
+extern void select_on( int count, void *cookie, const handle_t *handles,
+ int flags, int sec, int usec, struct object* objs[] );
+extern void iocomp_notify_thread_inactive (struct iocomp *iocomp);
+extern void iocomp_notify_thread_active (struct iocomp *iocomp);
/* ptrace functions */
Index: trace.c
===================================================================
RCS file: /var/lib/cvsd/cvsroot/winex/server/trace.c,v
retrieving revision 1.42
retrieving revision 1.43
diff -u -d -r1.42 -r1.43
--- trace.c 30 Mar 2007 18:58:50 -0000 1.42
+++ trace.c 31 Jul 2007 18:00:22 -0000 1.43
@@ -2264,6 +2264,52 @@
fprintf( stderr, " handle=%d", req->handle );
}
+static void dump_create_io_completion_request( const struct create_io_completion_request *req )
+{
+ fprintf( stderr, " num_threads=%08x", req->num_threads );
+}
+
+static void dump_create_io_completion_reply( const struct create_io_completion_reply *req )
+{
+ fprintf( stderr, " handle=%d", req->handle );
+}
+
+static void dump_set_io_completion_request( const struct set_io_completion_request *req )
+{
+ fprintf( stderr, " handle=%d,", req->handle );
+ fprintf( stderr, " completion_key=%p,", req->completion_key );
+ fprintf( stderr, " overlapped=%p,", req->overlapped );
+ fprintf( stderr, " num_bytes=%08x", req->num_bytes );
+}
+
+static void dump_remove_io_completion_request( const struct remove_io_completion_request *req )
+{
+ fprintf( stderr, " handle=%d,", req->handle );
+ fprintf( stderr, " cookie=%p,", req->cookie );
+ fprintf( stderr, " select_flags=%d,", req->select_flags );
+ fprintf( stderr, " sec=%d,", req->sec );
+ fprintf( stderr, " usec=%d", req->usec );
+}
+
+static void dump_remove_io_completion_reply( const struct remove_io_completion_reply *req )
+{
+ fprintf( stderr, " completion_key=%p,", req->completion_key );
+ fprintf( stderr, " overlapped=%p,", req->overlapped );
+ fprintf( stderr, " num_bytes=%08x", req->num_bytes );
+}
+
+static void dump_retrieve_assigned_io_completion_request( const struct retrieve_assigned_io_completion_request *req )
+{
+ fprintf( stderr, " handle=%d", req->handle );
+}
+
+static void dump_retrieve_assigned_io_completion_reply( const struct retrieve_assigned_io_completion_reply *req )
+{
+ fprintf( stderr, " completion_key=%p,", req->completion_key );
+ fprintf( stderr, " overlapped=%p,", req->overlapped );
+ fprintf( stderr, " num_bytes=%08x", req->num_bytes );
+}
+
static const dump_func req_dumpers[REQ_NB_REQUESTS] = {
(dump_func)dump_new_process_request,
(dump_func)dump_get_new_process_info_request,
@@ -2434,6 +2480,10 @@
(dump_func)dump_get_cdrom_eject_fd_list_request,
(dump_func)dump_add_cdrom_device_info_request,
(dump_func)dump_open_token_request,
+ (dump_func)dump_create_io_completion_request,
+ (dump_func)dump_set_io_completion_request,
+ (dump_func)dump_remove_io_completion_request,
+ (dump_func)dump_retrieve_assigned_io_completion_request,
};
static const dump_func reply_dumpers[REQ_NB_REQUESTS] = {
@@ -2606,6 +2656,10 @@
(dump_func)dump_get_cdrom_eject_fd_list_reply,
(dump_func)0,
(dump_func)dump_open_token_reply,
+ (dump_func)dump_create_io_completion_reply,
+ (dump_func)0,
+ (dump_func)dump_remove_io_completion_reply,
+ (dump_func)dump_retrieve_assigned_io_completion_reply,
};
static const char * const req_names[REQ_NB_REQUESTS] = {
@@ -2778,6 +2832,10 @@
"get_cdrom_eject_fd_list",
"add_cdrom_device_info",
"open_token",
+ "create_io_completion",
+ "set_io_completion",
+ "remove_io_completion",
+ "retrieve_assigned_io_completion",
};
/* ### make_requests end ### */