Re: final -- I think -- opencrypto mutex/condvar patch

Index: opencrypto/crypto.c
===================================================================
RCS file: /cvsroot/src/sys/opencrypto/crypto.c,v
retrieving revision 1.22
diff -u -p -r1.22 crypto.c
--- opencrypto/crypto.c	1 Feb 2008 04:52:35 -0000	1.22
+++ opencrypto/crypto.c	3 Feb 2008 20:25:58 -0000
@@ -26,9 +26,6 @@
 #include <sys/cdefs.h>
 __KERNEL_RCSID(0, "$NetBSD: crypto.c,v 1.22 2008/02/01 04:52:35 tls Exp $");
 
-/* XXX FIXME: should be defopt'ed */
-#define CRYPTO_TIMING			/* enable cryptop timing stuff */
-
 #include <sys/param.h>
 #include <sys/reboot.h>
 #include <sys/systm.h>
@@ -40,11 +37,14 @@ __KERNEL_RCSID(0, "$NetBSD: crypto.c,v 1
 #include <sys/sysctl.h>
 #include <sys/intr.h>
 
+#include "opt_ocf.h"
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>			/* XXX for M_XDATA */
 
-  #define splcrypto splnet
-  /* below is kludges to check whats still missing */
+kcondvar_t cryptoret_cv;
+kmutex_t crypto_mtx;
+
+/* below are kludges for residual code wrtitten to FreeBSD interfaces */
   #define SWI_CRYPTO 17
   #define register_swi(lvl, fn)  \
   softint_establish(SOFTINT_NET, (void (*)(void*))fn, NULL)
@@ -78,18 +78,57 @@ static	TAILQ_HEAD(,cryptkop) crp_kq =
  * but have two to avoid type futzing (cryptop vs. cryptkop).  See below
  * for how synchronization is handled.
  */
-static	TAILQ_HEAD(,cryptop) crp_ret_q =	/* callback queues */
+static	TAILQ_HEAD(crprethead, cryptop) crp_ret_q =	/* callback queues */
 		TAILQ_HEAD_INITIALIZER(crp_ret_q);
-static	TAILQ_HEAD(,cryptkop) crp_ret_kq =
+static	TAILQ_HEAD(krprethead, cryptkop) crp_ret_kq =
 		TAILQ_HEAD_INITIALIZER(crp_ret_kq);
 
 /*
+ * XXX these functions are ghastly hacks for when the submission
+ * XXX routines discover a request that was not CBIMM is already
+ * XXX done, and must be yanked from the retq (where _done) put it
+ * XXX as cryptoret won't get the chance.  The queue is walked backwards
+ * XXX as the request is generally the last one queued.
+ *
+ *	 call with the lock held, or else.
+ */
+int
+crypto_ret_q_remove(struct cryptop *crp)
+{
+	struct cryptop * acrp;
+
+	TAILQ_FOREACH_REVERSE(acrp, &crp_ret_q, crprethead, crp_next) {
+		if (acrp == crp) {
+			TAILQ_REMOVE(&crp_ret_q, crp, crp_next);
+			crp->crp_flags &= (~CRYPTO_F_ONRETQ);
+			return 1;
+		}
+	}
+	return 0;
+}
+
+int
+crypto_ret_kq_remove(struct cryptkop *krp)
+{
+	struct cryptkop * akrp;
+
+	TAILQ_FOREACH_REVERSE(akrp, &crp_ret_kq, krprethead, krp_next) {
+		if (akrp == krp) {
+			TAILQ_REMOVE(&crp_ret_kq, krp, krp_next);
+			krp->krp_flags &= (~CRYPTO_F_ONRETQ);
+			return 1;
+		}
+	}
+	return 0;
+}
+
+/*
  * Crypto op and desciptor data structures are allocated
  * from separate private zones(FreeBSD)/pools(netBSD/OpenBSD) .
  */
 struct pool cryptop_pool;
 struct pool cryptodesc_pool;
-int crypto_pool_initialized = 0;
+struct pool cryptkop_pool;
 
 int	crypto_usercrypto = 1;		/* userland may open /dev/crypto */
 int	crypto_userasymcrypto = 1;	/* userland may do asym crypto reqs */
@@ -170,14 +209,23 @@ static	int crypto_invoke(struct cryptop 
 static	int crypto_kinvoke(struct cryptkop *krp, int hint);
 
 static struct cryptostats cryptostats;
+#ifdef CRYPTO_TIMING
 static	int crypto_timing = 0;
-
+#endif
 
 static int
 crypto_init0(void)
 {
 	int error;
 
+	mutex_init(&crypto_mtx, MUTEX_DEFAULT, IPL_NET);
+	cv_init(&cryptoret_cv, "crypto_wait");
+	pool_init(&cryptop_pool, sizeof(struct cryptop), 0, 0,  
+		  0, "cryptop", NULL, IPL_NET); 
+	pool_init(&cryptodesc_pool, sizeof(struct cryptodesc), 0, 0,
+		  0, "cryptodesc", NULL, IPL_NET);
+	pool_init(&cryptkop_pool, sizeof(struct cryptkop), 0, 0,
+		  0, "cryptkop", NULL, IPL_NET);
 
 	crypto_drivers = malloc(CRYPTO_DRIVERS_INITIAL *
 	    sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
@@ -217,7 +265,7 @@ crypto_destroy(void)
 }
 
 /*
- * Create a new session.
+ * Create a new session.  Must be called with crypto_mtx held.
  */
 int
 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard)
@@ -225,9 +273,8 @@ crypto_newsession(u_int64_t *sid, struct
 	struct cryptoini *cr;
 	u_int32_t hid, lid;
 	int err = EINVAL;
-	int s;
 
-	s = splcrypto();
+	KASSERT(mutex_owned(&crypto_mtx));
 
 	if (crypto_drivers == NULL)
 		goto done;
@@ -288,22 +335,20 @@ crypto_newsession(u_int64_t *sid, struct
 		}
 	}
 done:
-	splx(s);
 	return err;
 }
 
 /*
  * Delete an existing session (or a reserved session on an unregistered
- * driver).
+ * driver).  Must be called with crypto_mtx mutex held.
  */
 int
 crypto_freesession(u_int64_t sid)
 {
 	u_int32_t hid;
 	int err = 0;
-	int s;
 
-	s = splcrypto();
+	KASSERT(mutex_owned(&crypto_mtx));
 
 	if (crypto_drivers == NULL) {
 		err = EINVAL;
@@ -322,9 +367,10 @@ crypto_freesession(u_int64_t sid)
 		crypto_drivers[hid].cc_sessions--;
 
 	/* Call the driver cleanup routine, if available. */
-	if (crypto_drivers[hid].cc_freesession)
+	if (crypto_drivers[hid].cc_freesession) {
 		err = crypto_drivers[hid].cc_freesession(
 				crypto_drivers[hid].cc_arg, sid);
+	}
 	else
 		err = 0;
 
@@ -337,7 +383,6 @@ crypto_freesession(u_int64_t sid)
 		bzero(&crypto_drivers[hid], sizeof(struct cryptocap));
 
 done:
-	splx(s);
 	return err;
 }
 
@@ -349,11 +394,11 @@ int32_t
 crypto_get_driverid(u_int32_t flags)
 {
 	struct cryptocap *newdrv;
-	int i, s;
+	int i;
 
-	crypto_init();
+	crypto_init();		/* XXX oh, this is foul! */
 
-	s = splcrypto();
+	mutex_spin_enter(&crypto_mtx);
 	for (i = 0; i < crypto_drivers_num; i++)
 		if (crypto_drivers[i].cc_process == NULL &&
 		    (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 &&
@@ -364,7 +409,7 @@ crypto_get_driverid(u_int32_t flags)
 	if (i == crypto_drivers_num) {
 		/* Be careful about wrap-around. */
 		if (2 * crypto_drivers_num <= crypto_drivers_num) {
-			splx(s);
+			mutex_spin_exit(&crypto_mtx);
 			printf("crypto: driver count wraparound!\n");
 			return -1;
 		}
@@ -372,7 +417,7 @@ crypto_get_driverid(u_int32_t flags)
 		newdrv = malloc(2 * crypto_drivers_num *
 		    sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
 		if (newdrv == NULL) {
-			splx(s);
+			mutex_spin_exit(&crypto_mtx);
 			printf("crypto: no space to expand driver table!\n");
 			return -1;
 		}
@@ -393,7 +438,7 @@ crypto_get_driverid(u_int32_t flags)
 	if (bootverbose)
 		printf("crypto: assign driver %u, flags %u\n", i, flags);
 
-	splx(s);
+	mutex_spin_exit(&crypto_mtx);
 
 	return i;
 }
@@ -415,11 +460,10 @@ crypto_kregister(u_int32_t driverid, int
     int (*kprocess)(void*, struct cryptkop *, int),
     void *karg)
 {
-	int s;
 	struct cryptocap *cap;
 	int err;
 
-	s = splcrypto();
+	mutex_spin_enter(&crypto_mtx);
 
 	cap = crypto_checkdriver(driverid);
 	if (cap != NULL &&
@@ -431,12 +475,14 @@ crypto_kregister(u_int32_t driverid, int
 		 */
 
 		cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
-		if (bootverbose)
-			printf("crypto: driver %u registers key alg %u flags %u\n"
-				, driverid
-				, kalg
-				, flags
+		if (bootverbose) {
+			printf("crypto: driver %u registers key alg %u "
+			       " flags %u\n",
+				driverid,
+				kalg,
+				flags
 			);
+		}
 
 		if (cap->cc_kprocess == NULL) {
 			cap->cc_karg = karg;
@@ -446,7 +492,7 @@ crypto_kregister(u_int32_t driverid, int
 	} else
 		err = EINVAL;
 
-	splx(s);
+	mutex_spin_exit(&crypto_mtx);
 	return err;
 }
 
@@ -463,9 +509,9 @@ crypto_register(u_int32_t driverid, int 
     void *arg)
 {
 	struct cryptocap *cap;
-	int s, err;
+	int err;
 
-	s = splcrypto();
+	mutex_spin_enter(&crypto_mtx);
 
 	cap = crypto_checkdriver(driverid);
 	/* NB: algorithms are in the range [1..max] */
@@ -479,13 +525,15 @@ crypto_register(u_int32_t driverid, int 
 
 		cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
 		cap->cc_max_op_len[alg] = maxoplen;
-		if (bootverbose)
-			printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n"
-				, driverid
-				, alg
-				, flags
-				, maxoplen
+		if (bootverbose) {
+			printf("crypto: driver %u registers alg %u "
+				"flags %u maxoplen %u\n",
+				driverid,
+				alg,
+				flags,
+				maxoplen
 			);
+		}
 
 		if (cap->cc_process == NULL) {
 			cap->cc_arg = arg;
@@ -498,7 +546,7 @@ crypto_register(u_int32_t driverid, int 
 	} else
 		err = EINVAL;
 
-	splx(s);
+	mutex_spin_exit(&crypto_mtx);
 	return err;
 }
 
@@ -511,11 +559,11 @@ crypto_register(u_int32_t driverid, int 
 int
 crypto_unregister(u_int32_t driverid, int alg)
 {
-	int i, err, s;
+	int i, err;
 	u_int32_t ses;
 	struct cryptocap *cap;
 
-	s = splcrypto();
+	mutex_spin_enter(&crypto_mtx);
 
 	cap = crypto_checkdriver(driverid);
 	if (cap != NULL &&
@@ -544,7 +592,7 @@ crypto_unregister(u_int32_t driverid, in
 	} else
 		err = EINVAL;
 
-	splx(s);
+	mutex_spin_exit(&crypto_mtx);
 	return err;
 }
 
@@ -554,14 +602,19 @@ crypto_unregister(u_int32_t driverid, in
  * around so that subsequent calls using those sessions will
  * correctly detect the driver has been unregistered and reroute
  * requests.
+ *
+ * XXX careful.  Don't change this to call crypto_unregister() for each
+ * XXX registered algorithm unless you drop the mutex across the calls;
+ * XXX you can't take it recursively.
  */
 int
 crypto_unregister_all(u_int32_t driverid)
 {
-	int i, err, s = splcrypto();
+	int i, err;
 	u_int32_t ses;
 	struct cryptocap *cap;
 
+	mutex_spin_enter(&crypto_mtx);
 	cap = crypto_checkdriver(driverid);
 	if (cap != NULL) {
 		for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) {
@@ -581,7 +634,7 @@ crypto_unregister_all(u_int32_t driverid
 	} else
 		err = EINVAL;
 
-	splx(s);
+	mutex_spin_exit(&crypto_mtx);
 	return err;
 }
 
@@ -593,9 +646,9 @@ int
 crypto_unblock(u_int32_t driverid, int what)
 {
 	struct cryptocap *cap;
-	int needwakeup, err, s;
+	int needwakeup, err;
 
-	s = splcrypto();
+	mutex_spin_enter(&crypto_mtx);
 	cap = crypto_checkdriver(driverid);
 	if (cap != NULL) {
 		needwakeup = 0;
@@ -608,12 +661,13 @@ crypto_unblock(u_int32_t driverid, int w
 			cap->cc_kqblocked = 0;
 		}
 		if (needwakeup) {
+			mutex_spin_exit(&crypto_mtx);
 			setsoftcrypto(softintr_cookie);
 		}
 		err = 0;
 	} else
 		err = EINVAL;
-	splx(s);
+	mutex_spin_exit(&crypto_mtx);
 
 	return err;
 }
@@ -626,9 +680,9 @@ int
 crypto_dispatch(struct cryptop *crp)
 {
 	u_int32_t hid = SESID2HID(crp->crp_sid);
-	int s, result;
+	int result;
 
-	s = splcrypto();
+	mutex_spin_enter(&crypto_mtx);
 
 	cryptostats.cs_ops++;
 
@@ -645,6 +699,7 @@ crypto_dispatch(struct cryptop *crp)
 		 */
 		cap = crypto_checkdriver(hid);
 		if (cap && !cap->cc_qblocked) {
+			mutex_spin_exit(&crypto_mtx);
 			result = crypto_invoke(crp, 0);
 			if (result == ERESTART) {
 				/*
@@ -652,10 +707,12 @@ crypto_dispatch(struct cryptop *crp)
 				 * driver ``blocked'' for cryptop's and put
 				 * the op on the queue.
 				 */
+				mutex_spin_enter(&crypto_mtx);
 				crypto_drivers[hid].cc_qblocked = 1;
 				TAILQ_INSERT_HEAD(&crp_q, crp, crp_next);
 				cryptostats.cs_blocks++;
 			}
+			goto out_released;
 		} else {
 			/*
 			 * The driver is blocked, just queue the op until
@@ -674,13 +731,17 @@ crypto_dispatch(struct cryptop *crp)
 		 */
 		TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
 		if (wasempty) {
+			mutex_spin_exit(&crypto_mtx);
 			setsoftcrypto(softintr_cookie);
+			result = 0;
+			goto out_released;
 		}
 
 		result = 0;
 	}
-	splx(s);
 
+	mutex_spin_exit(&crypto_mtx);
+out_released:
 	return result;
 }
 
@@ -692,13 +753,14 @@ int
 crypto_kdispatch(struct cryptkop *krp)
 {
 	struct cryptocap *cap;
-	int s, result;
+	int result;
 
-	s = splcrypto();
+	mutex_spin_enter(&crypto_mtx);
 	cryptostats.cs_kops++;
 
 	cap = crypto_checkdriver(krp->krp_hid);
 	if (cap && !cap->cc_kqblocked) {
+		mutex_spin_exit(&crypto_mtx);
 		result = crypto_kinvoke(krp, 0);
 		if (result == ERESTART) {
 			/*
@@ -706,6 +768,7 @@ crypto_kdispatch(struct cryptkop *krp)
 			 * driver ``blocked'' for cryptop's and put
 			 * the op on the queue.
 			 */
+			mutex_spin_enter(&crypto_mtx);
 			crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
 			TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
 			cryptostats.cs_kblocks++;
@@ -718,7 +781,7 @@ crypto_kdispatch(struct cryptkop *krp)
 		TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
 		result = 0;
 	}
-	splx(s);
+	mutex_spin_exit(&crypto_mtx);
 
 	return result;
 }
@@ -736,7 +799,7 @@ crypto_kinvoke(struct cryptkop *krp, int
 	if (krp == NULL)
 		return EINVAL;
 	if (krp->krp_callback == NULL) {
-		free(krp, M_XDATA);		/* XXX allocated in cryptodev */
+		pool_put(&cryptkop_pool, krp);
 		return EINVAL;
 	}
 
@@ -818,9 +881,11 @@ crypto_invoke(struct cryptop *crp, int h
 
 	hid = SESID2HID(crp->crp_sid);
 	if (hid < crypto_drivers_num) {
+		mutex_enter(&crypto_mtx);
 		if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP)
 			crypto_freesession(crp->crp_sid);
 		process = crypto_drivers[hid].cc_process;
+		mutex_exit(&crypto_mtx);
 	} else {
 		process = NULL;
 	}
@@ -833,6 +898,7 @@ crypto_invoke(struct cryptop *crp, int h
 		 * Driver has unregistered; migrate the session and return
 		 * an error to the caller so they'll resubmit the op.
 		 */
+		mutex_enter(&crypto_mtx);
 		for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
 			crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
 
@@ -840,12 +906,15 @@ crypto_invoke(struct cryptop *crp, int h
 			crp->crp_sid = nid;
 
 		crp->crp_etype = EAGAIN;
+		mutex_exit(&crypto_mtx);
+
 		crypto_done(crp);
 		return 0;
 	} else {
 		/*
 		 * Invoke the driver to process the request.
 		 */
+		DPRINTF(("calling process for %08x\n", (uint32_t)crp));
 		return (*process)(crypto_drivers[hid].cc_arg, crp, hint);
 	}
 }
@@ -857,20 +926,15 @@ void
 crypto_freereq(struct cryptop *crp)
 {
 	struct cryptodesc *crd;
-	int s;
 
 	if (crp == NULL)
 		return;
 
-	s = splcrypto();
-
 	while ((crd = crp->crp_desc) != NULL) {
 		crp->crp_desc = crd->crd_next;
 		pool_put(&cryptodesc_pool, crd);
 	}
-
 	pool_put(&cryptop_pool, crp);
-	splx(s);
 }
 
 /*
@@ -881,29 +945,17 @@ crypto_getreq(int num)
 {
 	struct cryptodesc *crd;
 	struct cryptop *crp;
-	int s;
-
-	s = splcrypto();
-
-	if (crypto_pool_initialized == 0) {
-		pool_init(&cryptop_pool, sizeof(struct cryptop), 0, 0,
-		    0, "cryptop", NULL, IPL_NET);
-		pool_init(&cryptodesc_pool, sizeof(struct cryptodesc), 0, 0,
-		    0, "cryptodesc", NULL, IPL_NET);
-		crypto_pool_initialized = 1;
-	}
 
 	crp = pool_get(&cryptop_pool, 0);
 	if (crp == NULL) {
-		splx(s);
 		return NULL;
 	}
 	bzero(crp, sizeof(struct cryptop));
+	cv_init(&crp->crp_cv, "crydev");
 
 	while (num--) {
 		crd = pool_get(&cryptodesc_pool, 0);
 		if (crd == NULL) {
-			splx(s);
 			crypto_freereq(crp);
 			return NULL;
 		}
@@ -913,7 +965,6 @@ crypto_getreq(int num)
 		crp->crp_desc = crd;
 	}
 
-	splx(s);
 	return crp;
 }
 
@@ -923,6 +974,8 @@ crypto_getreq(int num)
 void
 crypto_done(struct cryptop *crp)
 {
+	int wasempty;
+
 	if (crp->crp_etype != 0)
 		cryptostats.cs_errs++;
 #ifdef CRYPTO_TIMING
@@ -930,26 +983,24 @@ crypto_done(struct cryptop *crp)
 		crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
 #endif
 	/*
-	 * On netbsd 1.6O, CBIMM does its wake_one() before the requestor
-	 * has done its tsleep().
+	 * Normal case; queue the callback for the thread.
+	 *
+	 * The return queue is manipulated by the swi thread
+	 * and, potentially, by crypto device drivers calling
+	 * back to mark operations completed.  Thus we need
+	 * to mask both while manipulating the return queue.
 	 */
-	{
-		int s, wasempty;
-		/*
-		 * Normal case; queue the callback for the thread.
-		 *
-		 * The return queue is manipulated by the swi thread
-		 * and, potentially, by crypto device drivers calling
-		 * back to mark operations completed.  Thus we need
-		 * to mask both while manipulating the return queue.
-		 */
-		s = splcrypto();
-		wasempty = TAILQ_EMPTY(&crp_ret_q);
-		TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
-		if (wasempty)
-			wakeup_one(&crp_ret_q);
-		splx(s);
+	mutex_spin_enter(&crypto_mtx);
+	wasempty = TAILQ_EMPTY(&crp_ret_q);
+	DPRINTF(("crypto_done: queueing %08x\n", (uint32_t)crp));
+	crp->crp_flags |= CRYPTO_F_ONRETQ|CRYPTO_F_DONE;
+	TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
+	if (wasempty) {
+		DPRINTF(("crypto_done: waking cryptoret, %08x " \
+			"hit empty queue\n.", (uint32_t)crp));
+		cv_signal(&cryptoret_cv);
 	}
+	mutex_spin_exit(&crypto_mtx);
 }
 
 /*
@@ -958,7 +1009,7 @@ crypto_done(struct cryptop *crp)
 void
 crypto_kdone(struct cryptkop *krp)
 {
-	int s, wasempty;
+	int wasempty;
 
 	if (krp->krp_status != 0)
 		cryptostats.cs_kerrs++;
@@ -968,21 +1019,21 @@ crypto_kdone(struct cryptkop *krp)
 	 * back to mark operations completed.  Thus we need
 	 * to mask both while manipulating the return queue.
 	 */
-	s = splcrypto();
+	mutex_spin_enter(&crypto_mtx);
 	wasempty = TAILQ_EMPTY(&crp_ret_kq);
+	krp->krp_flags |= CRYPTO_F_ONRETQ|CRYPTO_F_DONE;
 	TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next);
 	if (wasempty)
-		wakeup_one(&crp_ret_q);
-	splx(s);
+		cv_signal(&cryptoret_cv);
+	mutex_spin_exit(&crypto_mtx);
 }
 
 int
 crypto_getfeat(int *featp)
 {
 	int hid, kalg, feat = 0;
-	int s;
 
-	s = splcrypto();
+	mutex_spin_enter(&crypto_mtx);
 
 	if (crypto_userasymcrypto == 0)
 		goto out;
@@ -1000,7 +1051,7 @@ crypto_getfeat(int *featp)
 				feat |=  1 << kalg;
 	}
 out:
-	splx(s);
+	mutex_spin_exit(&crypto_mtx);
 	*featp = feat;
 	return (0);
 }
@@ -1014,11 +1065,11 @@ cryptointr(void)
 	struct cryptop *crp, *submit;
 	struct cryptkop *krp;
 	struct cryptocap *cap;
-	int result, hint, s;
+	int result, hint;
 
 	printf("crypto softint\n");
 	cryptostats.cs_intrs++;
-	s = splcrypto();
+	mutex_spin_enter(&crypto_mtx);
 	do {
 		/*
 		 * Find the first element in the queue that can be
@@ -1059,7 +1110,11 @@ cryptointr(void)
 		}
 		if (submit != NULL) {
 			TAILQ_REMOVE(&crp_q, submit, crp_next);
+			mutex_spin_exit(&crypto_mtx);
 			result = crypto_invoke(submit, hint);
+			/* we must take here as the TAILQ op or kinvoke
+			   may need this mutex below.  sigh. */
+			mutex_spin_enter(&crypto_mtx);	
 			if (result == ERESTART) {
 				/*
 				 * The driver ran out of resources, mark the
@@ -1089,7 +1144,10 @@ cryptointr(void)
 		}
 		if (krp != NULL) {
 			TAILQ_REMOVE(&crp_kq, krp, krp_next);
+			mutex_spin_exit(&crypto_mtx);
 			result = crypto_kinvoke(krp, 0);
+			/* the next iteration will want the mutex. :-/ */
+			mutex_spin_enter(&crypto_mtx);
 			if (result == ERESTART) {
 				/*
 				 * The driver ran out of resources, mark the
@@ -1107,7 +1165,7 @@ cryptointr(void)
 			}
 		}
 	} while (submit != NULL || krp != NULL);
-	splx(s);
+	mutex_spin_exit(&crypto_mtx);
 }
 
 /*
@@ -1118,19 +1176,24 @@ cryptoret(void)
 {
 	struct cryptop *crp;
 	struct cryptkop *krp;
-	int s;
 
-	s = splcrypto();
 	for (;;) {
+		mutex_spin_enter(&crypto_mtx);
+
 		crp = TAILQ_FIRST(&crp_ret_q);
-		if (crp != NULL)
+		if (crp != NULL) {
 			TAILQ_REMOVE(&crp_ret_q, crp, crp_next);
+			crp->crp_flags &= ~CRYPTO_F_ONRETQ;
+		}
 		krp = TAILQ_FIRST(&crp_ret_kq);
-		if (krp != NULL)
+		if (krp != NULL) {
 			TAILQ_REMOVE(&crp_ret_kq, krp, krp_next);
+			krp->krp_flags &= ~CRYPTO_F_ONRETQ;
+		}
 
+		/* drop before calling any callbacks. */
+		mutex_spin_exit(&crypto_mtx);
 		if (crp != NULL || krp != NULL) {
-			splx(s);		/* lower ipl for callbacks */
 			if (crp != NULL) {
 #ifdef CRYPTO_TIMING
 				if (crypto_timing) {
@@ -1145,18 +1208,23 @@ cryptoret(void)
 					crypto_tstat(&cryptostats.cs_finis, &t);
 				} else
 #endif
+				{
+#if 0		/* too loud */
+					DCPRINTF(("cryptoret: calling back " \
+						"%08x for crp %08x\n", \
+						(uint32_t)crp->crp_callback, \
+						(uint32_t)crp));
+#endif
 					crp->crp_callback(crp);
+				}
 			}
 			if (krp != NULL)
 				krp->krp_callback(krp);
-			s  = splcrypto();
 		} else {
-			(void) tsleep(&crp_ret_q, PLOCK, "crypto_wait", 0);
+			mutex_spin_enter(&crypto_mtx);
+			cv_wait(&cryptoret_cv, &crypto_mtx);
+			mutex_spin_exit(&crypto_mtx);
 			cryptostats.cs_rets++;
 		}
 	}
 }
-
-
-
-
Index: opencrypto/cryptodev.c
===================================================================
RCS file: /cvsroot/src/sys/opencrypto/cryptodev.c,v
retrieving revision 1.32
diff -u -p -r1.32 cryptodev.c
--- opencrypto/cryptodev.c	2 Feb 2008 02:39:00 -0000	1.32
+++ opencrypto/cryptodev.c	3 Feb 2008 20:26:00 -0000
@@ -52,16 +52,10 @@ __KERNEL_RCSID(0, "$NetBSD: cryptodev.c,
 #include <sys/device.h>
 #include <sys/kauth.h>
 
+#include "opt_ocf.h"
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>
 
-  #define splcrypto splnet
-#ifdef CRYPTO_DEBUG
-#define DPRINTF(a) uprintf a
-#else
-#define DPRINTF(a)
-#endif
-
 struct csession {
 	TAILQ_ENTRY(csession) next;
 	u_int64_t	sid;
@@ -178,12 +172,14 @@ cryptof_ioctl(struct file *fp, u_long cm
                 if ((error = falloc(l, &criofp, &criofd)) != 0)
                         return error;
                 criofcr = pool_get(&fcrpl, PR_WAITOK);
+		mutex_spin_enter(&crypto_mtx);
                 TAILQ_INIT(&criofcr->csessions);
                 /*
                  * Don't ever return session 0, to allow detection of
                  * failed creation attempts with multi-create ioctl.
                  */
                 criofcr->sesn = 1;
+		mutex_spin_exit(&crypto_mtx);
                 (void)fdclone(l, criofp, criofd, (FREAD|FWRITE),
 			      &cryptofops, criofcr);
                 *(u_int32_t *)data = criofd;
@@ -310,7 +306,8 @@ cryptof_ioctl(struct file *fp, u_long cm
 					goto bail;
 			}
 		}
-
+		/* crypto_newsession requires that we hold the mutex. */
+		mutex_spin_enter(&crypto_mtx);
 		error = crypto_newsession(&sid, (txform ? &crie : &cria),
 			    crypto_devallowsoft);
 		if (error) {
@@ -332,6 +329,7 @@ cryptof_ioctl(struct file *fp, u_long cm
 		sop->ses = cse->ses;
 
 bail:
+		mutex_spin_exit(&crypto_mtx);
 		if (error) {
 			if (crie.cri_key)
 				FREE(crie.cri_key, M_XDATA);
@@ -340,16 +338,20 @@ bail:
 		}
 		break;
 	case CIOCFSESSION:
+		mutex_spin_enter(&crypto_mtx);
 		ses = *(u_int32_t *)data;
 		cse = csefind(fcr, ses);
 		if (cse == NULL)
 			return (EINVAL);
 		csedelete(fcr, cse);
 		error = csefree(cse);
+		mutex_spin_exit(&crypto_mtx);
 		break;
 	case CIOCCRYPT:
+		mutex_spin_enter(&crypto_mtx);
 		cop = (struct crypt_op *)data;
 		cse = csefind(fcr, cop->ses);
+		mutex_spin_exit(&crypto_mtx);
 		if (cse == NULL) {
 			DPRINTF(("csefind failed\n"));
 			return (EINVAL);
@@ -374,7 +376,7 @@ cryptodev_op(struct csession *cse, struc
 {
 	struct cryptop *crp = NULL;
 	struct cryptodesc *crde = NULL, *crda = NULL;
-	int error, s;
+	int error;
 
 	if (cop->len > 256*1024-4)
 		return (E2BIG);
@@ -480,14 +482,33 @@ cryptodev_op(struct csession *cse, struc
 		crp->crp_mac=cse->tmp_mac;
 	}
 
-	s = splcrypto();	/* NB: only needed with CRYPTO_F_CBIMM */
+	/*
+	 * XXX there was a comment here which said that we went to
+	 * XXX splcrypto() but needed to only if CRYPTO_F_CBIMM,
+	 * XXX disabled on NetBSD since 1.6O due to a race condition.
+	 * XXX But crypto_dispatch went to splcrypto() itself!  (And
+	 * XXX now takes the crypto_mtx mutex itself).  We do, however,
+	 * XXX need to hold the mutex across the call to cv_wait().
+	 * XXX     (should we arrange for crypto_dispatch to return to
+	 * XXX      us with it held?  it seems quite ugly to do so.)
+	 */
 	error = crypto_dispatch(crp);
-	if (error == 0 && (crp->crp_flags & CRYPTO_F_DONE) == 0)
-		error = tsleep(crp, PSOCK, "crydev", 0);
-	splx(s);
-	if (error) {
+	mutex_spin_enter(&crypto_mtx);
+	if (error != 0) {
+		DPRINTF(("cryptodev_op: not waiting, error.\n"));
+		mutex_spin_exit(&crypto_mtx);
 		goto bail;
 	}
+	while (!(crp->crp_flags & CRYPTO_F_DONE)) {
+		DPRINTF(("cryptodev_op: sleeping on cv %08x for crp %08x\n", \
+			(uint32_t)&crp->crp_cv, (uint32_t)crp));
+		cv_wait(&crp->crp_cv, &crypto_mtx);	/* XXX cv_wait_sig? */
+	}
+	if (crp->crp_flags & CRYPTO_F_ONRETQ) {
+		DPRINTF(("cryptodev_op: DONE, not woken by cryptoret.\n"));
+		(void)crypto_ret_q_remove(crp);
+	}
+	mutex_spin_exit(&crypto_mtx);
 
 	if (crp->crp_etype != 0) {
 		error = crp->crp_etype;
@@ -521,11 +542,20 @@ cryptodev_cb(void *op)
 {
 	struct cryptop *crp = (struct cryptop *) op;
 	struct csession *cse = (struct csession *)crp->crp_opaque;
+	int error = 0;
 
+	mutex_spin_enter(&crypto_mtx);
 	cse->error = crp->crp_etype;
-	if (crp->crp_etype == EAGAIN)
-		return crypto_dispatch(crp);
-	wakeup_one(crp);
+	if (crp->crp_etype == EAGAIN) {
+		/* always drop mutex to call dispatch routine */
+		mutex_spin_exit(&crypto_mtx);
+		error = crypto_dispatch(crp);
+		mutex_spin_enter(&crypto_mtx);
+	}
+	if (error != 0 || (crp->crp_flags & CRYPTO_F_DONE)) {
+		cv_signal(&crp->crp_cv);
+	}
+	mutex_spin_exit(&crypto_mtx);
 	return (0);
 }
 
@@ -534,7 +564,9 @@ cryptodevkey_cb(void *op)
 {
 	struct cryptkop *krp = (struct cryptkop *) op;
 
-	wakeup_one(krp);
+	mutex_spin_enter(&crypto_mtx);
+	cv_signal(&krp->krp_cv);
+	mutex_spin_exit(&crypto_mtx);
 	return (0);
 }
 
@@ -600,10 +632,11 @@ cryptodev_key(struct crypt_kop *kop)
 		return (EINVAL);
 	}
 
-	krp = (struct cryptkop *)malloc(sizeof *krp, M_XDATA, M_WAITOK);
+	krp = pool_get(&cryptkop_pool, PR_WAITOK);
 	if (!krp)
 		return (ENOMEM);
 	bzero(krp, sizeof *krp);
+	cv_init(&krp->krp_cv, "crykdev");
 	krp->krp_op = kop->crk_op;
 	krp->krp_status = kop->crk_status;
 	krp->krp_iparams = kop->crk_iparams;
@@ -626,10 +659,19 @@ cryptodev_key(struct crypt_kop *kop)
 	}
 
 	error = crypto_kdispatch(krp);
-	if (error == 0)
-		error = tsleep(krp, PSOCK, "crydev", 0);
-	if (error)
+	if (error != 0) {
 		goto fail;
+	}
+
+	mutex_spin_enter(&crypto_mtx);
+	while (!(krp->krp_flags & CRYPTO_F_DONE)) {
+		cv_wait(&krp->krp_cv, &crypto_mtx);	/* XXX cv_wait_sig? */
+	}
+	if (krp->krp_flags & CRYPTO_F_ONRETQ) {
+		DPRINTF(("cryptodev_key: DONE early, not via cryptoret.\n"));
+		(void)crypto_ret_kq_remove(krp);
+	}
+	mutex_spin_exit(&crypto_mtx);
 
 	if (krp->krp_status != 0) {
 		error = krp->krp_status;
@@ -652,7 +694,7 @@ fail:
 			if (krp->krp_param[i].crp_p)
 				FREE(krp->krp_param[i].crp_p, M_XDATA);
 		}
-		free(krp, M_XDATA);
+		pool_put(&cryptkop_pool, krp);
 	}
 	return (error);
 }
@@ -664,6 +706,7 @@ cryptof_close(struct file *fp, struct lw
 	struct fcrypt *fcr = (struct fcrypt *)fp->f_data;
 	struct csession *cse;
 
+	mutex_spin_enter(&crypto_mtx);
 	while ((cse = TAILQ_FIRST(&fcr->csessions))) {
 		TAILQ_REMOVE(&fcr->csessions, cse, next);
 		(void)csefree(cse);
@@ -671,43 +714,54 @@ cryptof_close(struct file *fp, struct lw
 	pool_put(&fcrpl, fcr);
 
 	fp->f_data = NULL;
+	mutex_spin_exit(&crypto_mtx);
 
 	return 0;
 }
 
+/* csefind: call with crypto_mtx held. */
 static struct csession *
 csefind(struct fcrypt *fcr, u_int ses)
 {
-	struct csession *cse;
+	struct csession *cse, *ret = NULL;
 
+	KASSERT(mutex_owned(&crypto_mtx));
 	TAILQ_FOREACH(cse, &fcr->csessions, next)
 		if (cse->ses == ses)
-			return (cse);
-	return (NULL);
+			ret = cse;
+	return (ret);
 }
 
+/* csedelete: call with crypto_mtx held. */
 static int
 csedelete(struct fcrypt *fcr, struct csession *cse_del)
 {
 	struct csession *cse;
+	int ret = 0;
 
+	KASSERT(mutex_owned(&crypto_mtx));
 	TAILQ_FOREACH(cse, &fcr->csessions, next) {
 		if (cse == cse_del) {
 			TAILQ_REMOVE(&fcr->csessions, cse, next);
-			return (1);
+			ret = 1;
 		}
 	}
-	return (0);
+	return (ret);
 }
 
+/* cseadd: call with crypto_mtx held. */
 static struct csession *
 cseadd(struct fcrypt *fcr, struct csession *cse)
 {
+	KASSERT(mutex_owned(&crypto_mtx));
+	/* don't let session ID wrap! */
+	if (fcr->sesn + 1 == 0) return NULL;
 	TAILQ_INSERT_TAIL(&fcr->csessions, cse, next);
 	cse->ses = fcr->sesn++;
 	return (cse);
 }
 
+/* csecreate: call with crypto_mtx held. */
 static struct csession *
 csecreate(struct fcrypt *fcr, u_int64_t sid, void *key, u_int64_t keylen,
     void *mackey, u_int64_t mackeylen, u_int32_t cipher, u_int32_t mac,
@@ -715,10 +769,7 @@ csecreate(struct fcrypt *fcr, u_int64_t 
 {
 	struct csession *cse;
 
-	/* Don't let the session ID wrap! */
-	if (fcr->sesn + 1 == 0)
-		return NULL;
-
+	KASSERT(mutex_owned(&crypto_mtx));
 	cse = pool_get(&csepl, PR_NOWAIT);
 	if (cse == NULL)
 		return NULL;
@@ -734,16 +785,18 @@ csecreate(struct fcrypt *fcr, u_int64_t 
 	if (cseadd(fcr, cse))
 		return (cse);
 	else {
-		FREE(cse, M_XDATA);
+		pool_put(&csepl, cse);
 		return NULL;
 	}
 }
 
+/* csefree: call with crypto_mtx held. */
 static int
 csefree(struct csession *cse)
 {
 	int error;
 
+	KASSERT(mutex_owned(&crypto_mtx));
 	error = crypto_freesession(cse->sid);
 	if (cse->key)
 		FREE(cse->key, M_XDATA);
@@ -768,12 +821,14 @@ cryptoopen(dev_t dev, int flag, int mode
 		return error;
 
 	fcr = pool_get(&fcrpl, PR_WAITOK);
+	mutex_spin_enter(&crypto_mtx);
 	TAILQ_INIT(&fcr->csessions);
 	/*
 	 * Don't ever return session 0, to allow detection of
 	 * failed creation attempts with multi-create ioctl.
 	 */
 	fcr->sesn = 1;
+	mutex_spin_exit(&crypto_mtx);
 	return fdclone(l, fp, fd, flag, &cryptofops, fcr);
 }
 
Index: opencrypto/cryptodev.h
===================================================================
RCS file: /cvsroot/src/sys/opencrypto/cryptodev.h,v
retrieving revision 1.11
diff -u -p -r1.11 cryptodev.h
--- opencrypto/cryptodev.h	2 Feb 2008 02:39:00 -0000	1.11
+++ opencrypto/cryptodev.h	3 Feb 2008 20:26:00 -0000
@@ -282,6 +282,7 @@ struct cryptop {
 #define	CRYPTO_F_CBIMM		0x0010	/* Do callback immediately */
 #define	CRYPTO_F_DONE		0x0020	/* Operation completed */
 #define	CRYPTO_F_CBIFSYNC	0x0040	/* Do CBIMM if op is synchronous */
+#define	CRYPTO_F_ONRETQ		0x0080	/* Request is on return queue */
 
 	void *		crp_buf;	/* Data to be processed */
 	void *		crp_opaque;	/* Opaque pointer, passed along */
@@ -291,6 +292,7 @@ struct cryptop {
 
 	void *		crp_mac;
 	struct timespec	crp_tstamp;	/* performance time stamp */
+	kcondvar_t	crp_cv;
 };
 
 #define CRYPTO_BUF_CONTIG	0x0
@@ -315,6 +317,8 @@ struct cryptkop {
 	u_int32_t	krp_hid;
 	struct crparam	krp_param[CRK_MAXPARAM];	/* kvm */
 	int		(*krp_callback)(struct cryptkop *);
+	int		krp_flags;	/* same values as crp_flags */
+	kcondvar_t	krp_cv;
 };
 
 /* Crypto capabilities structure */
@@ -387,6 +391,8 @@ void	cuio_copyback(struct uio *, int, in
 int	cuio_apply(struct uio *, int, int,
 	    int (*f)(void *, void *, unsigned int), void *);
 
+extern	int crypto_ret_q_remove(struct cryptop *);
+extern	int crypto_ret_kq_remove(struct cryptkop *);
 extern	void crypto_freereq(struct cryptop *crp);
 extern	struct cryptop *crypto_getreq(int num);
 
@@ -394,6 +400,17 @@ extern	int crypto_usercrypto;		/* userla
 extern	int crypto_userasymcrypto;	/* userland may do asym crypto reqs */
 extern	int crypto_devallowsoft;	/* only use hardware crypto */
 
+/*
+ * Asymmetric operations are allocated in cryptodev.c but can be
+ * freed in crypto.c.
+ */
+extern	struct pool	cryptkop_pool;
+
+/*
+ * Mutual exclusion and its unwelcome friends.
+ */
+
+extern	kmutex_t	crypto_mtx;
 
 /*
  * initialize the crypto framework subsystem (not the pseudo-device).
@@ -418,6 +435,21 @@ extern	void cuio_copydata(struct uio* ui
 extern	void cuio_copyback(struct uio* uio, int off, int len, void *cp);
 extern int	cuio_getptr(struct uio *, int loc, int *off);
 
+#ifdef CRYPTO_DEBUG	/* yuck, netipsec defines these differently */
+#ifndef DPRINTF
+#define DPRINTF(a) uprintf a
+#endif
+#ifndef DCPRINTF
+#define DCPRINTF(a) printf a
+#endif
+#else
+#ifndef DPRINTF
+#define DPRINTF(a)
+#endif
+#ifndef DCPRINTF
+#define DCPRINTF(a)
+#endif
+#endif
 
 #endif /* _KERNEL */
 #endif /* _CRYPTO_CRYPTO_H_ */
Index: opencrypto/cryptosoft.c
===================================================================
RCS file: /cvsroot/src/sys/opencrypto/cryptosoft.c,v
retrieving revision 1.19
diff -u -p -r1.19 cryptosoft.c
--- opencrypto/cryptosoft.c	2 Feb 2008 04:46:29 -0000	1.19
+++ opencrypto/cryptosoft.c	3 Feb 2008 20:26:00 -0000
@@ -33,6 +33,7 @@ __KERNEL_RCSID(0, "$NetBSD: cryptosoft.c
 #include <sys/sysctl.h>
 #include <sys/errno.h>
 
+#include "opt_ocf.h"
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/cryptosoft.h>
 #include <opencrypto/xform.h>
@@ -1008,6 +1009,7 @@ swcr_process(void *arg, struct cryptop *
 	}
 
 done:
+	DPRINTF(("request %08x done\n", (uint32_t)crp));
 	crypto_done(crp);
 	return 0;
 }
Index: opencrypto/cryptosoft_xform.c
===================================================================
RCS file: /cvsroot/src/sys/opencrypto/cryptosoft_xform.c,v
retrieving revision 1.7
diff -u -p -r1.7 cryptosoft_xform.c
--- opencrypto/cryptosoft_xform.c	2 Feb 2008 02:39:00 -0000	1.7
+++ opencrypto/cryptosoft_xform.c	3 Feb 2008 20:26:00 -0000
@@ -326,9 +326,8 @@ des1_setkey(u_int8_t **sched, const u_in
 	int err;
 
 	MALLOC(p, des_key_schedule *, sizeof (des_key_schedule),
-		M_CRYPTO_DATA, M_NOWAIT);
+		M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
 	if (p != NULL) {
-		bzero(p, sizeof(des_key_schedule));
 		des_set_key((des_cblock *)__UNCONST(key), p[0]);
 		err = 0;
 	} else
@@ -370,9 +369,8 @@ des3_setkey(u_int8_t **sched, const u_in
 	int err;
 
 	MALLOC(p, des_key_schedule *, 3*sizeof (des_key_schedule),
-		M_CRYPTO_DATA, M_NOWAIT);
+		M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
 	if (p != NULL) {
-		bzero(p, 3*sizeof(des_key_schedule));
 		des_set_key((des_cblock *)__UNCONST(key +  0), p[0]);
 		des_set_key((des_cblock *)__UNCONST(key +  8), p[1]);
 		des_set_key((des_cblock *)__UNCONST(key + 16), p[2]);
@@ -395,22 +393,14 @@ static void
 blf_encrypt(void *key, u_int8_t *blk)
 {
 
-#if defined(__NetBSD__)
 	BF_ecb_encrypt(blk, blk, (BF_KEY *)key, 1);
-#else
-	blf_ecb_encrypt((blf_ctx *) key, blk, 8);
-#endif
 }
 
 static void
 blf_decrypt(void *key, u_int8_t *blk)
 {
 
-#if defined(__NetBSD__)
 	BF_ecb_encrypt(blk, blk, (BF_KEY *)key, 0);
-#else
-	blf_ecb_decrypt((blf_ctx *) key, blk, 8);
-#endif
 }
 
 static int
@@ -418,21 +408,10 @@ blf_setkey(u_int8_t **sched, const u_int
 {
 	int err;
 
-#if defined(__FreeBSD__) || defined(__NetBSD__)
-#define	BLF_SIZ	sizeof(BF_KEY)
-#else
-#define	BLF_SIZ	sizeof(blf_ctx)
-#endif
-
-	MALLOC(*sched, u_int8_t *, BLF_SIZ,
-		M_CRYPTO_DATA, M_NOWAIT);
+	MALLOC(*sched, u_int8_t *, sizeof(BF_KEY),
+		M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
 	if (*sched != NULL) {
-		bzero(*sched, BLF_SIZ);
-#if defined(__FreeBSD__) || defined(__NetBSD__)
 		BF_set_key((BF_KEY *) *sched, len, key);
-#else
-		blf_key((blf_ctx *)*sched, key, len);
-#endif
 		err = 0;
 	} else
 		err = ENOMEM;
@@ -442,7 +421,7 @@ blf_setkey(u_int8_t **sched, const u_int
 static void
 blf_zerokey(u_int8_t **sched)
 {
-	bzero(*sched, BLF_SIZ);
+	bzero(*sched, sizeof(BF_KEY));
 	FREE(*sched, M_CRYPTO_DATA);
 	*sched = NULL;
 }
@@ -465,9 +444,8 @@ cast5_setkey(u_int8_t **sched, const u_i
 	int err;
 
 	MALLOC(*sched, u_int8_t *, sizeof(cast128_key), M_CRYPTO_DATA,
-	       M_NOWAIT);
+	       M_NOWAIT|M_ZERO);
 	if (*sched != NULL) {
-		bzero(*sched, sizeof(cast128_key));
 		cast128_setkey((cast128_key *)*sched, key, len);
 		err = 0;
 	} else
@@ -505,7 +483,7 @@ skipjack_setkey(u_int8_t **sched, const 
 	 * Will this break a pdp-10, Cray-1, or GE-645 port?
 	 */
 	MALLOC(*sched, u_int8_t *, 10 * (sizeof(u_int8_t *) + 0x100),
-		M_CRYPTO_DATA, M_NOWAIT);
+		M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
 
 	if (*sched != NULL) {
 
@@ -513,8 +491,6 @@ skipjack_setkey(u_int8_t **sched, const 
 		u_int8_t* table = (u_int8_t*) &key_tables[10];
 		int k;
 
-		bzero(*sched, 10 * sizeof(u_int8_t *)+0x100);
-
 		for (k = 0; k < 10; k++) {
 			key_tables[k] = table;
 			table += 0x100;
@@ -553,9 +529,8 @@ rijndael128_setkey(u_int8_t **sched, con
 	int err;
 
 	MALLOC(*sched, u_int8_t *, sizeof(rijndael_ctx), M_CRYPTO_DATA,
-	    M_NOWAIT);
+	    M_NOWAIT|M_ZERO);
 	if (*sched != NULL) {
-		bzero(*sched, sizeof(rijndael_ctx));
 		rijndael_set_key((rijndael_ctx *) *sched, key, len * 8);
 		err = 0;
 	} else
Index: opencrypto/files.opencrypto
===================================================================
RCS file: /cvsroot/src/sys/opencrypto/files.opencrypto,v
retrieving revision 1.18
diff -u -p -r1.18 files.opencrypto
--- opencrypto/files.opencrypto	27 Oct 2006 21:20:48 -0000	1.18
+++ opencrypto/files.opencrypto	3 Feb 2008 20:26:00 -0000
@@ -23,3 +23,5 @@ file	opencrypto/deflate.c		swcrypto	# wr
 # (and thus crypto hardware accelerators).
 defpseudo crypto: opencrypto
 file	opencrypto/cryptodev.c		crypto
+
+defflag opt_ocf.h	CRYPTO_DEBUG CRYPTO_TIMING
Index: netipsec/xform_ah.c
===================================================================
RCS file: /cvsroot/src/sys/netipsec/xform_ah.c,v
retrieving revision 1.19
diff -u -p -r1.19 xform_ah.c
--- netipsec/xform_ah.c	28 Oct 2007 15:48:23 -0000	1.19
+++ netipsec/xform_ah.c	3 Feb 2008 20:26:01 -0000
@@ -230,8 +230,13 @@ ah_init(struct secasvar *sav, struct xfo
 	int error;
 
 	error = ah_init0(sav, xsp, &cria);
-	return error ? error :
-		 crypto_newsession(&sav->tdb_cryptoid, &cria, crypto_support);
+	if (!error) {
+		mutex_spin_enter(&crypto_mtx);
+		error = crypto_newsession(&sav->tdb_cryptoid,
+					   &cria, crypto_support);
+		mutex_spin_exit(&crypto_mtx);
+	}
+	return error;
 }
 
 /*
@@ -247,7 +252,9 @@ ah_zeroize(struct secasvar *sav)
 	if (sav->key_auth)
 		bzero(_KEYBUF(sav->key_auth), _KEYLEN(sav->key_auth));
 
+	mutex_spin_enter(&crypto_mtx);
 	err = crypto_freesession(sav->tdb_cryptoid);
+	mutex_spin_exit(&crypto_mtx);
 	sav->tdb_cryptoid = 0;
 	sav->tdb_authalgxform = NULL;
 	sav->tdb_xform = NULL;
Index: netipsec/xform_esp.c
===================================================================
RCS file: /cvsroot/src/sys/netipsec/xform_esp.c,v
retrieving revision 1.16
diff -u -p -r1.16 xform_esp.c
--- netipsec/xform_esp.c	27 Jun 2007 20:38:33 -0000	1.16
+++ netipsec/xform_esp.c	3 Feb 2008 20:26:03 -0000
@@ -231,6 +231,7 @@ esp_init(struct secasvar *sav, struct xf
 	crie.cri_key = _KEYBUF(sav->key_enc);
 	/* XXX Rounds ? */
 
+	mutex_spin_enter(&crypto_mtx);
 	if (sav->tdb_authalgxform && sav->tdb_encalgxform) {
 		/* init both auth & enc */
 		crie.cri_next = &cria;
@@ -247,6 +248,7 @@ esp_init(struct secasvar *sav, struct xf
 		DPRINTF(("esp_init: no encoding OR authentication xform!\n"));
 		error = EINVAL;
 	}
+	mutex_spin_exit(&crypto_mtx);
 	return error;
 }
 
Index: netipsec/xform_ipcomp.c
===================================================================
RCS file: /cvsroot/src/sys/netipsec/xform_ipcomp.c,v
retrieving revision 1.16
diff -u -p -r1.16 xform_ipcomp.c
--- netipsec/xform_ipcomp.c	29 Dec 2007 14:56:35 -0000	1.16
+++ netipsec/xform_ipcomp.c	3 Feb 2008 20:26:03 -0000
@@ -106,6 +106,7 @@ ipcomp_init(struct secasvar *sav, struct
 {
 	struct comp_algo *tcomp;
 	struct cryptoini cric;
+	int ses;
 
 	/* NB: algorithm really comes in alg_enc and not alg_comp! */
 	tcomp = ipcomp_algorithm_lookup(sav->alg_enc);
@@ -122,7 +123,10 @@ ipcomp_init(struct secasvar *sav, struct
 	bzero(&cric, sizeof (cric));
 	cric.cri_alg = sav->tdb_compalgxform->type;
 
-	return crypto_newsession(&sav->tdb_cryptoid, &cric, crypto_support);
+	mutex_spin_enter(&crypto_mtx);
+	ses = crypto_newsession(&sav->tdb_cryptoid, &cric, crypto_support);
+	mutex_spin_exit(&crypto_mtx);
+	return ses;
 }
 
 /*
@@ -133,7 +137,9 @@ ipcomp_zeroize(struct secasvar *sav)
 {
 	int err;
 
+	mutex_spin_enter(&crypto_mtx);
 	err = crypto_freesession(sav->tdb_cryptoid);
+	mutex_spin_exit(&crypto_mtx);
 	sav->tdb_cryptoid = 0;
 	return err;
 }