OpenWrt XBurst

OpenWrt XBurst Git Source Tree

Root/package/ubsec_ssb/src/ubsec_ssb.c

1
2	/*
3	* Copyright (c) 2008 Daniel Mueller (daniel@danm.de)
4	* Copyright (c) 2007 David McCullough (david_mccullough@securecomputing.com)
5	* Copyright (c) 2000 Jason L. Wright (jason@thought.net)
6	* Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
7	* Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
8	*
9	* Redistribution and use in source and binary forms, with or without
10	* modification, are permitted provided that the following conditions
11	* are met:
12	* 1. Redistributions of source code must retain the above copyright
13	* notice, this list of conditions and the following disclaimer.
14	* 2. Redistributions in binary form must reproduce the above copyright
15	* notice, this list of conditions and the following disclaimer in the
16	* documentation and/or other materials provided with the distribution.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
22	* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24	* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26	* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
27	* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28	* POSSIBILITY OF SUCH DAMAGE.
29	*
30	* Effort sponsored in part by the Defense Advanced Research Projects
31	* Agency (DARPA) and Air Force Research Laboratory, Air Force
32	* Materiel Command, USAF, under agreement number F30602-01-2-0537.
33	*
34	*/
35	#undef UBSEC_DEBUG
36	#undef UBSEC_VERBOSE_DEBUG
37
38	#ifdef UBSEC_VERBOSE_DEBUG
39	#define UBSEC_DEBUG
40	#endif
41
42	/*
43	* uBsec BCM5365 hardware crypto accelerator
44	*/
45
46	#include <linux/kernel.h>
47	#include <linux/module.h>
48	#include <linux/moduleparam.h>
49	#include <linux/proc_fs.h>
50	#include <linux/types.h>
51	#include <linux/init.h>
52	#include <linux/delay.h>
53	#include <linux/interrupt.h>
54	#include <linux/fs.h>
55	#include <linux/random.h>
56	#include <linux/skbuff.h>
57	#include <linux/stat.h>
58	#include <asm/io.h>
59
60	#include <linux/ssb/ssb.h>
61
62	/*
63	* BSD queue
64	*/
65	#include "bsdqueue.h"
66
67	/*
68	* OCF
69	*/
70	#include "cryptodev.h"
71	#include "uio.h"
72
73	#define HMAC_HACK 1
74
75	#ifdef HMAC_HACK
76	#include "hmachack.h"
77	#include "md5.h"
78	#include "md5.c"
79	#include "sha1.h"
80	#include "sha1.c"
81	#endif
82
83	#include "ubsecreg.h"
84	#include "ubsecvar.h"
85
86	#define DRV_MODULE_NAME "ubsec_ssb"
87	#define PFX DRV_MODULE_NAME ": "
88	#define DRV_MODULE_VERSION "0.02"
89	#define DRV_MODULE_RELDATE "Feb 21, 2009"
90
91	#if 1
92	#define DPRINTF(a...) \
93	if (debug) \
94	{ \
95	printk(DRV_MODULE_NAME ": " a); \
96	}
97	#else
98	#define DPRINTF(a...)
99	#endif
100
101	/*
102	* Prototypes
103	*/
104	static irqreturn_t ubsec_ssb_isr(int, void , struct pt_regs );
105	static int __devinit ubsec_ssb_probe(struct ssb_device *sdev,
106	const struct ssb_device_id *ent);
107	static void __devexit ubsec_ssb_remove(struct ssb_device *sdev);
108	int ubsec_attach(struct ssb_device sdev, const struct ssb_device_id ent,
109	struct device *self);
110	static void ubsec_setup_mackey(struct ubsec_session *ses, int algo,
111	caddr_t key, int klen);
112	static int dma_map_skb(struct ubsec_softc *sc,
113	struct ubsec_dma_alloc* q_map, struct sk_buff skb, int mlen);
114	static int dma_map_uio(struct ubsec_softc *sc,
115	struct ubsec_dma_alloc q_map, struct uio uio, int *mlen);
116	static void dma_unmap(struct ubsec_softc *sc,
117	struct ubsec_dma_alloc *q_map, int mlen);
118	static int ubsec_dmamap_aligned(struct ubsec_softc *sc,
119	const struct ubsec_dma_alloc *q_map, int mlen);
120
121	#ifdef UBSEC_DEBUG
122	static int proc_read(char buf, char *start, off_t offset,
123	int size, int peof, void data);
124	#endif
125
126	void ubsec_reset_board(struct ubsec_softc *);
127	void ubsec_init_board(struct ubsec_softc *);
128	void ubsec_cleanchip(struct ubsec_softc *);
129	void ubsec_totalreset(struct ubsec_softc *);
130	int ubsec_free_q(struct ubsec_softc, struct ubsec_q );
131
132	static int ubsec_newsession(device_t, u_int32_t , struct cryptoini );
133	static int ubsec_freesession(device_t, u_int64_t);
134	static int ubsec_process(device_t, struct cryptop *, int);
135
136	void ubsec_callback(struct ubsec_softc , struct ubsec_q );
137	void ubsec_feed(struct ubsec_softc *);
138	void ubsec_mcopy(struct sk_buff , struct sk_buff , int, int);
139	void ubsec_dma_free(struct ubsec_softc , struct ubsec_dma_alloc );
140	int ubsec_dma_malloc(struct ubsec_softc , struct ubsec_dma_alloc ,
141	size_t, int);
142
143	/* DEBUG crap... */
144	void ubsec_dump_pb(struct ubsec_pktbuf *);
145	void ubsec_dump_mcr(struct ubsec_mcr *);
146
147	#define READ_REG(sc,r) \
148	ssb_read32((sc)->sdev, (r));
149	#define WRITE_REG(sc,r,val) \
150	ssb_write32((sc)->sdev, (r), (val));
151	#define READ_REG_SDEV(sdev,r) \
152	ssb_read32((sdev), (r));
153	#define WRITE_REG_SDEV(sdev,r,val) \
154	ssb_write32((sdev), (r), (val));
155
156	#define SWAP32(x) (x) = htole32(ntohl((x)))
157	#define HTOLE32(x) (x) = htole32(x)
158
159	#ifdef __LITTLE_ENDIAN
160	#define letoh16(x) (x)
161	#define letoh32(x) (x)
162	#endif
163
164	static int debug;
165	module_param(debug, int, 0644);
166	MODULE_PARM_DESC(debug, "Enable debug output");
167
168	#define UBSEC_SSB_MAX_CHIPS 1
169	static struct ubsec_softc *ubsec_chip_idx[UBSEC_SSB_MAX_CHIPS];
170	static struct ubsec_stats ubsecstats;
171
172	#ifdef UBSEC_DEBUG
173	static struct proc_dir_entry *procdebug;
174	#endif
175
176	static struct ssb_device_id ubsec_ssb_tbl[] = {
177	/* Broadcom BCM5365P IPSec Core */
178	SSB_DEVICE(SSB_VENDOR_BROADCOM, SSB_DEV_IPSEC, SSB_ANY_REV),
179	SSB_DEVTABLE_END
180	};
181
182	static struct ssb_driver ubsec_ssb_driver = {
183	.name = DRV_MODULE_NAME,
184	.id_table = ubsec_ssb_tbl,
185	.probe = ubsec_ssb_probe,
186	.remove = __devexit_p(ubsec_ssb_remove),
187	/*
188	.suspend = ubsec_ssb_suspend,
189	.resume = ubsec_ssb_resume
190	*/
191	};
192
193	static device_method_t ubsec_ssb_methods = {
194	/* crypto device methods */
195	DEVMETHOD(cryptodev_newsession, ubsec_newsession),
196	DEVMETHOD(cryptodev_freesession,ubsec_freesession),
197	DEVMETHOD(cryptodev_process, ubsec_process),
198	};
199
200	#ifdef UBSEC_DEBUG
201	static int
202	proc_read(char buf, char *start, off_t offset,
203	int size, int peof, void data)
204	{
205	int i = 0, byteswritten = 0, ret;
206	unsigned int stat, ctrl;
207	#ifdef UBSEC_VERBOSE_DEBUG
208	struct ubsec_q *q;
209	struct ubsec_dma *dmap;
210	#endif
211
212	while ((i < UBSEC_SSB_MAX_CHIPS) && (ubsec_chip_idx[i] != NULL))
213	{
214	struct ubsec_softc *sc = ubsec_chip_idx[i];
215
216	stat = READ_REG(sc, BS_STAT);
217	ctrl = READ_REG(sc, BS_CTRL);
218	ret = snprintf((buf + byteswritten),
219	(size - byteswritten) ,
220	"DEV %d, DMASTAT %08x, DMACTRL %08x\n", i, stat, ctrl);
221
222	byteswritten += ret;
223
224	#ifdef UBSEC_VERBOSE_DEBUG
225	printf("DEV %d, DMASTAT %08x, DMACTRL %08x\n", i, stat, ctrl);
226
227	/* Dump all queues MCRs */
228	if (!BSD_SIMPLEQ_EMPTY(&sc->sc_qchip)) {
229	BSD_SIMPLEQ_FOREACH(q, &sc->sc_qchip, q_next)
230	{
231	dmap = q->q_dma;
232	ubsec_dump_mcr(&dmap->d_dma->d_mcr);
233	}
234	}
235	#endif
236
237	i++;
238	}
239
240	*peof = 1;
241
242	return byteswritten;
243	}
244	#endif
245
246	/*
247	* map in a given sk_buff
248	*/
249	static int
250	dma_map_skb(struct ubsec_softc sc, struct ubsec_dma_alloc q_map, struct sk_buff skb, int mlen)
251	{
252	int i = 0;
253	dma_addr_t tmp;
254
255	#ifdef UBSEC_DEBUG
256	DPRINTF("%s()\n", __FUNCTION__);
257	#endif
258
259	/*
260	* We support only a limited number of fragments.
261	*/
262	if (unlikely((skb_shinfo(skb)->nr_frags + 1) >= UBS_MAX_SCATTER))
263	{
264	printk(KERN_ERR "Only %d scatter fragments are supported.\n", UBS_MAX_SCATTER);
265	return (-ENOMEM);
266	}
267
268	#ifdef UBSEC_VERBOSE_DEBUG
269	DPRINTF("%s - map %d 0x%x %d\n", __FUNCTION__, 0, (unsigned int)skb->data, skb_headlen(skb));
270	#endif
271
272	/* first data package */
273	tmp = dma_map_single(sc->sc_dv,
274	skb->data,
275	skb_headlen(skb),
276	DMA_BIDIRECTIONAL);
277
278	q_map[i].dma_paddr = tmp;
279	q_map[i].dma_vaddr = skb->data;
280	q_map[i].dma_size = skb_headlen(skb);
281
282	if (unlikely(tmp == 0))
283	{
284	printk(KERN_ERR "Could not map memory region for dma.\n");
285	return (-EINVAL);
286	}
287
288	#ifdef UBSEC_VERBOSE_DEBUG
289	DPRINTF("%s - map %d done physical addr 0x%x\n", __FUNCTION__, 0, (unsigned int)tmp);
290	#endif
291
292
293	/* all other data packages */
294	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
295
296	#ifdef UBSEC_VERBOSE_DEBUG
297	DPRINTF("%s - map %d 0x%x %d\n", __FUNCTION__, i + 1,
298	(unsigned int)page_address(skb_shinfo(skb)->frags[i].page) +
299	skb_shinfo(skb)->frags[i].page_offset, skb_shinfo(skb)->frags[i].size);
300	#endif
301
302	tmp = dma_map_single(sc->sc_dv,
303	page_address(skb_shinfo(skb)->frags[i].page) +
304	skb_shinfo(skb)->frags[i].page_offset,
305	skb_shinfo(skb)->frags[i].size,
306	DMA_BIDIRECTIONAL);
307
308	q_map[i + 1].dma_paddr = tmp;
309	q_map[i + 1].dma_vaddr = (void*)(page_address(skb_shinfo(skb)->frags[i].page) +
310	skb_shinfo(skb)->frags[i].page_offset);
311	q_map[i + 1].dma_size = skb_shinfo(skb)->frags[i].size;
312
313	if (unlikely(tmp == 0))
314	{
315	printk(KERN_ERR "Could not map memory region for dma.\n");
316	return (-EINVAL);
317	}
318
319	#ifdef UBSEC_VERBOSE_DEBUG
320	DPRINTF("%s - map %d done physical addr 0x%x\n", __FUNCTION__, i + 1, (unsigned int)tmp);
321	#endif
322
323	}
324	*mlen = i + 1;
325
326	return(0);
327	}
328
329	/*
330	* map in a given uio buffer
331	*/
332
333	static int
334	dma_map_uio(struct ubsec_softc sc, struct ubsec_dma_alloc q_map, struct uio uio, int mlen)
335	{
336	struct iovec *iov = uio->uio_iov;
337	int n;
338	dma_addr_t tmp;
339
340	#ifdef UBSEC_DEBUG
341	DPRINTF("%s()\n", __FUNCTION__);
342	#endif
343
344	/*
345	* We support only a limited number of fragments.
346	*/
347	if (unlikely(uio->uio_iovcnt >= UBS_MAX_SCATTER))
348	{
349	printk(KERN_ERR "Only %d scatter fragments are supported.\n", UBS_MAX_SCATTER);
350	return (-ENOMEM);
351	}
352
353	for (n = 0; n < uio->uio_iovcnt; n++) {
354	#ifdef UBSEC_VERBOSE_DEBUG
355	DPRINTF("%s - map %d 0x%x %d\n", __FUNCTION__, n, (unsigned int)iov->iov_base, iov->iov_len);
356	#endif
357	tmp = dma_map_single(sc->sc_dv,
358	iov->iov_base,
359	iov->iov_len,
360	DMA_BIDIRECTIONAL);
361
362	q_map[n].dma_paddr = tmp;
363	q_map[n].dma_vaddr = iov->iov_base;
364	q_map[n].dma_size = iov->iov_len;
365
366	if (unlikely(tmp == 0))
367	{
368	printk(KERN_ERR "Could not map memory region for dma.\n");
369	return (-EINVAL);
370	}
371
372	#ifdef UBSEC_VERBOSE_DEBUG
373	DPRINTF("%s - map %d done physical addr 0x%x\n", __FUNCTION__, n, (unsigned int)tmp);
374	#endif
375
376	iov++;
377	}
378	*mlen = n;
379
380	return(0);
381	}
382
383	static void
384	dma_unmap(struct ubsec_softc sc, struct ubsec_dma_alloc q_map, int mlen)
385	{
386	int i;
387
388	#ifdef UBSEC_DEBUG
389	DPRINTF("%s()\n", __FUNCTION__);
390	#endif
391
392	for(i = 0; i < mlen; i++)
393	{
394	#ifdef UBSEC_VERBOSE_DEBUG
395	DPRINTF("%s - unmap %d 0x%x %d\n", __FUNCTION__, i, (unsigned int)q_map[i].dma_paddr, q_map[i].dma_size);
396	#endif
397	dma_unmap_single(sc->sc_dv,
398	q_map[i].dma_paddr,
399	q_map[i].dma_size,
400	DMA_BIDIRECTIONAL);
401	}
402	return;
403	}
404
405	/*
406	* Is the operand suitable aligned for direct DMA. Each
407	* segment must be aligned on a 32-bit boundary and all
408	* but the last segment must be a multiple of 4 bytes.
409	*/
410	static int
411	ubsec_dmamap_aligned(struct ubsec_softc sc, const struct ubsec_dma_alloc q_map, int mlen)
412	{
413	int i;
414
415	#ifdef UBSEC_DEBUG
416	DPRINTF("%s()\n", __FUNCTION__);
417	#endif
418
419	for (i = 0; i < mlen; i++) {
420	if (q_map[i].dma_paddr & 3)
421	return (0);
422	if (i != (mlen - 1) && (q_map[i].dma_size & 3))
423	return (0);
424	}
425	return (1);
426	}
427
428
429	#define N(a) (sizeof(a) / sizeof (a[0]))
430	static void
431	ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen)
432	{
433	#ifdef HMAC_HACK
434	MD5_CTX md5ctx;
435	SHA1_CTX sha1ctx;
436	int i;
437
438	#ifdef UBSEC_DEBUG
439	DPRINTF("%s()\n", __FUNCTION__);
440	#endif
441
442	for (i = 0; i < klen; i++)
443	key[i] ^= HMAC_IPAD_VAL;
444
445	if (algo == CRYPTO_MD5_HMAC) {
446	MD5Init(&md5ctx);
447	MD5Update(&md5ctx, key, klen);
448	MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
449	bcopy(md5ctx.md5_st8, ses->ses_hminner, sizeof(md5ctx.md5_st8));
450	} else {
451	SHA1Init(&sha1ctx);
452	SHA1Update(&sha1ctx, key, klen);
453	SHA1Update(&sha1ctx, hmac_ipad_buffer,
454	SHA1_HMAC_BLOCK_LEN - klen);
455	bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
456	}
457
458	for (i = 0; i < klen; i++)
459	key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
460
461	if (algo == CRYPTO_MD5_HMAC) {
462	MD5Init(&md5ctx);
463	MD5Update(&md5ctx, key, klen);
464	MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
465	bcopy(md5ctx.md5_st8, ses->ses_hmouter, sizeof(md5ctx.md5_st8));
466	} else {
467	SHA1Init(&sha1ctx);
468	SHA1Update(&sha1ctx, key, klen);
469	SHA1Update(&sha1ctx, hmac_opad_buffer,
470	SHA1_HMAC_BLOCK_LEN - klen);
471	bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
472	}
473
474	for (i = 0; i < klen; i++)
475	key[i] ^= HMAC_OPAD_VAL;
476
477	#else /* HMAC_HACK */
478	DPRINTF("md5/sha not implemented\n");
479	#endif /* HMAC_HACK */
480	}
481	#undef N
482
483	static int
484	__devinit ubsec_ssb_probe(struct ssb_device *sdev,
485	const struct ssb_device_id *ent)
486	{
487	int err;
488
489	#ifdef UBSEC_DEBUG
490	DPRINTF("%s()\n", __FUNCTION__);
491	#endif
492
493	err = ssb_bus_powerup(sdev->bus, 0);
494	if (err) {
495	dev_err(sdev->dev, "Failed to powerup the bus\n");
496	goto err_out;
497	}
498
499	err = request_irq(sdev->irq, (irq_handler_t)ubsec_ssb_isr,
500	IRQF_DISABLED \| IRQF_SHARED, DRV_MODULE_NAME, sdev);
501	if (err) {
502	dev_err(sdev->dev, "Could not request irq\n");
503	goto err_out_powerdown;
504	}
505
506	err = ssb_dma_set_mask(sdev, DMA_32BIT_MASK);
507	if (err) {
508	dev_err(sdev->dev,
509	"Required 32BIT DMA mask unsupported by the system.\n");
510	goto err_out_free_irq;
511	}
512
513	printk(KERN_INFO "Sentry5(tm) ROBOGateway(tm) IPSec Core at IRQ %u\n",
514	sdev->irq);
515
516	DPRINTF("Vendor: %x, core id: %x, revision: %x\n",
517	sdev->id.vendor, sdev->id.coreid, sdev->id.revision);
518
519	ssb_device_enable(sdev, 0);
520
521	if (ubsec_attach(sdev, ent, sdev->dev) != 0)
522	goto err_out_disable;
523
524	#ifdef UBSEC_DEBUG
525	procdebug = create_proc_entry(DRV_MODULE_NAME, S_IRUSR, NULL);
526	if (procdebug)
527	{
528	procdebug->read_proc = proc_read;
529	procdebug->data = NULL;
530	} else
531	DPRINTF("Unable to create proc file.\n");
532	#endif
533
534	return 0;
535
536	err_out_disable:
537	ssb_device_disable(sdev, 0);
538
539	err_out_free_irq:
540	free_irq(sdev->irq, sdev);
541
542	err_out_powerdown:
543	ssb_bus_may_powerdown(sdev->bus);
544
545	err_out:
546	return err;
547	}
548
549	static void __devexit ubsec_ssb_remove(struct ssb_device *sdev) {
550
551	struct ubsec_softc *sc;
552	unsigned int ctrlflgs;
553	struct ubsec_dma *dmap;
554	u_int32_t i;
555
556	#ifdef UBSEC_DEBUG
557	DPRINTF("%s()\n", __FUNCTION__);
558	#endif
559
560	ctrlflgs = READ_REG_SDEV(sdev, BS_CTRL);
561	/* disable all IPSec Core interrupts globally */
562	ctrlflgs ^= (BS_CTRL_MCR1INT \| BS_CTRL_MCR2INT \|
563	BS_CTRL_DMAERR);
564	WRITE_REG_SDEV(sdev, BS_CTRL, ctrlflgs);
565
566	free_irq(sdev->irq, sdev);
567
568	sc = (struct ubsec_softc *)ssb_get_drvdata(sdev);
569
570	/* unregister all crypto algorithms */
571	crypto_unregister_all(sc->sc_cid);
572
573	/* Free queue / dma memory */
574	for (i = 0; i < UBS_MAX_NQUEUE; i++) {
575	struct ubsec_q *q;
576
577	q = sc->sc_queuea[i];
578	if (q != NULL)
579	{
580	dmap = q->q_dma;
581	if (dmap != NULL)
582	{
583	ubsec_dma_free(sc, &dmap->d_alloc);
584	q->q_dma = NULL;
585	}
586	kfree(q);
587	}
588	sc->sc_queuea[i] = NULL;
589	}
590
591	ssb_device_disable(sdev, 0);
592	ssb_bus_may_powerdown(sdev->bus);
593	ssb_set_drvdata(sdev, NULL);
594
595	#ifdef UBSEC_DEBUG
596	if (procdebug)
597	remove_proc_entry(DRV_MODULE_NAME, NULL);
598	#endif
599
600	}
601
602
603	int
604	ubsec_attach(struct ssb_device sdev, const struct ssb_device_id ent,
605	struct device *self)
606	{
607	struct ubsec_softc *sc = NULL;
608	struct ubsec_dma *dmap;
609	u_int32_t i;
610	static int num_chips = 0;
611
612	#ifdef UBSEC_DEBUG
613	DPRINTF("%s()\n", __FUNCTION__);
614	#endif
615
616	sc = (struct ubsec_softc ) kmalloc(sizeof(sc), GFP_KERNEL);
617	if (!sc)
618	return(-ENOMEM);
619	memset(sc, 0, sizeof(*sc));
620
621	sc->sc_dv = sdev->dev;
622	sc->sdev = sdev;
623
624	spin_lock_init(&sc->sc_ringmtx);
625
626	softc_device_init(sc, "ubsec_ssb", num_chips, ubsec_ssb_methods);
627
628	/* Maybe someday there are boards with more than one chip available */
629	if (num_chips < UBSEC_SSB_MAX_CHIPS) {
630	ubsec_chip_idx[device_get_unit(sc->sc_dev)] = sc;
631	num_chips++;
632	}
633
634	ssb_set_drvdata(sdev, sc);
635
636	BSD_SIMPLEQ_INIT(&sc->sc_queue);
637	BSD_SIMPLEQ_INIT(&sc->sc_qchip);
638	BSD_SIMPLEQ_INIT(&sc->sc_queue2);
639	BSD_SIMPLEQ_INIT(&sc->sc_qchip2);
640	BSD_SIMPLEQ_INIT(&sc->sc_q2free);
641
642	sc->sc_statmask = BS_STAT_MCR1_DONE \| BS_STAT_DMAERR;
643
644	sc->sc_cid = crypto_get_driverid(softc_get_device(sc), CRYPTOCAP_F_HARDWARE);
645	if (sc->sc_cid < 0) {
646	device_printf(sc->sc_dev, "could not get crypto driver id\n");
647	return -1;
648	}
649
650	BSD_SIMPLEQ_INIT(&sc->sc_freequeue);
651	dmap = sc->sc_dmaa;
652	for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
653	struct ubsec_q *q;
654
655	q = (struct ubsec_q *)kmalloc(sizeof(struct ubsec_q), GFP_KERNEL);
656	if (q == NULL) {
657	printf(": can't allocate queue buffers\n");
658	break;
659	}
660
661	if (ubsec_dma_malloc(sc, &dmap->d_alloc, sizeof(struct ubsec_dmachunk),0)) {
662	printf(": can't allocate dma buffers\n");
663	kfree(q);
664	break;
665	}
666	dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
667
668	q->q_dma = dmap;
669	sc->sc_queuea[i] = q;
670
671	BSD_SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
672	}
673
674	/*
675	* Reset Broadcom chip
676	*/
677	ubsec_reset_board(sc);
678
679	/*
680	* Init Broadcom chip
681	*/
682	ubsec_init_board(sc);
683
684	/* supported crypto algorithms */
685	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
686	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
687
688	if (sc->sc_flags & UBS_FLAGS_AES) {
689	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
690	printf(KERN_INFO DRV_MODULE_NAME ": DES 3DES AES128 AES192 AES256 MD5_HMAC SHA1_HMAC\n");
691	}
692	else
693	printf(KERN_INFO DRV_MODULE_NAME ": DES 3DES MD5_HMAC SHA1_HMAC\n");
694
695	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
696	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
697
698	return 0;
699	}
700
701	/*
702	* UBSEC Interrupt routine
703	*/
704	static irqreturn_t
705	ubsec_ssb_isr(int irq, void arg, struct pt_regs regs)
706	{
707	struct ubsec_softc *sc = NULL;
708	volatile u_int32_t stat;
709	struct ubsec_q *q;
710	struct ubsec_dma *dmap;
711	int npkts = 0, i;
712
713	#ifdef UBSEC_VERBOSE_DEBUG
714	DPRINTF("%s()\n", __FUNCTION__);
715	#endif
716
717	sc = (struct ubsec_softc *)ssb_get_drvdata(arg);
718
719	stat = READ_REG(sc, BS_STAT);
720
721	stat &= sc->sc_statmask;
722	if (stat == 0)
723	return IRQ_NONE;
724
725	WRITE_REG(sc, BS_STAT, stat); /* IACK */
726
727	/*
728	* Check to see if we have any packets waiting for us
729	*/
730	if ((stat & BS_STAT_MCR1_DONE)) {
731	while (!BSD_SIMPLEQ_EMPTY(&sc->sc_qchip)) {
732	q = BSD_SIMPLEQ_FIRST(&sc->sc_qchip);
733	dmap = q->q_dma;
734
735	if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
736	{
737	DPRINTF("error while processing MCR. Flags = %x\n", dmap->d_dma->d_mcr.mcr_flags);
738	break;
739	}
740
741	BSD_SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
742
743	npkts = q->q_nstacked_mcrs;
744	/*
745	* search for further sc_qchip ubsec_q's that share
746	* the same MCR, and complete them too, they must be
747	* at the top.
748	*/
749	for (i = 0; i < npkts; i++) {
750	if(q->q_stacked_mcr[i])
751	ubsec_callback(sc, q->q_stacked_mcr[i]);
752	else
753	break;
754	}
755	ubsec_callback(sc, q);
756	}
757
758	/*
759	* Don't send any more packet to chip if there has been
760	* a DMAERR.
761	*/
762	if (likely(!(stat & BS_STAT_DMAERR)))
763	ubsec_feed(sc);
764	else
765	DPRINTF("DMA error occurred. Stop feeding crypto chip.\n");
766	}
767
768	/*
769	* Check to see if we got any DMA Error
770	*/
771	if (stat & BS_STAT_DMAERR) {
772	volatile u_int32_t a = READ_REG(sc, BS_ERR);
773
774	printf(KERN_ERR "%s: dmaerr %s@%08x\n", DRV_MODULE_NAME,
775	(a & BS_ERR_READ) ? "read" : "write", a & BS_ERR_ADDR);
776
777	ubsecstats.hst_dmaerr++;
778	ubsec_totalreset(sc);
779	ubsec_feed(sc);
780	}
781
782	return IRQ_HANDLED;
783	}
784
785	/*
786	* ubsec_feed() - aggregate and post requests to chip
787	* It is assumed that the caller set splnet()
788	*/
789	void
790	ubsec_feed(struct ubsec_softc *sc)
791	{
792	#ifdef UBSEC_VERBOSE_DEBUG
793	static int max;
794	#endif
795	struct ubsec_q q, q2;
796	int npkts, i;
797	void *v;
798	u_int32_t stat;
799
800	npkts = sc->sc_nqueue;
801	if (npkts > UBS_MAX_AGGR)
802	npkts = UBS_MAX_AGGR;
803	if (npkts < 2)
804	goto feed1;
805
806	stat = READ_REG(sc, BS_STAT);
807
808	if (stat & (BS_STAT_MCR1_FULL \| BS_STAT_DMAERR)) {
809	if(stat & BS_STAT_DMAERR) {
810	ubsec_totalreset(sc);
811	ubsecstats.hst_dmaerr++;
812	}
813	return;
814	}
815
816	#ifdef UBSEC_VERBOSE_DEBUG
817	DPRINTF("merging %d records\n", npkts);
818
819	/* XXX temporary aggregation statistics reporting code */
820	if (max < npkts) {
821	max = npkts;
822	DPRINTF("%s: new max aggregate %d\n", DRV_MODULE_NAME, max);
823	}
824	#endif /* UBSEC_VERBOSE_DEBUG */
825
826	q = BSD_SIMPLEQ_FIRST(&sc->sc_queue);
827	BSD_SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
828	--sc->sc_nqueue;
829
830	#if 0
831	/*
832	* XXX
833	* We use dma_map_single() - no sync required!
834	*/
835
836	bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
837	0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
838	if (q->q_dst_map != NULL)
839	bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
840	0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
841	#endif
842
843	q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */
844
845	for (i = 0; i < q->q_nstacked_mcrs; i++) {
846	q2 = BSD_SIMPLEQ_FIRST(&sc->sc_queue);
847
848	#if 0
849	bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
850	0, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
851	if (q2->q_dst_map != NULL)
852	bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
853	0, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
854	#endif
855	BSD_SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
856	--sc->sc_nqueue;
857
858	v = ((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
859	sizeof(struct ubsec_mcr_add);
860	bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
861	q->q_stacked_mcr[i] = q2;
862	}
863	q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
864	BSD_SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
865	#if 0
866	bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
867	0, q->q_dma->d_alloc.dma_map->dm_mapsize,
868	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
869	#endif
870	WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
871	offsetof(struct ubsec_dmachunk, d_mcr));
872	#ifdef UBSEC_VERBOSE_DEBUG
873	DPRINTF("feed (1): q->chip %p %08x %08x\n", q,
874	(u_int32_t)q->q_dma->d_alloc.dma_paddr,
875	(u_int32_t)(q->q_dma->d_alloc.dma_paddr +
876	offsetof(struct ubsec_dmachunk, d_mcr)));
877	#endif /* UBSEC_DEBUG */
878	return;
879
880	feed1:
881	while (!BSD_SIMPLEQ_EMPTY(&sc->sc_queue)) {
882	stat = READ_REG(sc, BS_STAT);
883
884	if (stat & (BS_STAT_MCR1_FULL \| BS_STAT_DMAERR)) {
885	if(stat & BS_STAT_DMAERR) {
886	ubsec_totalreset(sc);
887	ubsecstats.hst_dmaerr++;
888	}
889	break;
890	}
891
892	q = BSD_SIMPLEQ_FIRST(&sc->sc_queue);
893
894	#if 0
895	bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
896	0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
897	if (q->q_dst_map != NULL)
898	bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
899	0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
900	bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
901	0, q->q_dma->d_alloc.dma_map->dm_mapsize,
902	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
903	#endif
904
905	WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
906	offsetof(struct ubsec_dmachunk, d_mcr));
907	#ifdef UBSEC_VERBOSE_DEBUG
908	DPRINTF("feed (2): q->chip %p %08x %08x\n", q,
909	(u_int32_t)q->q_dma->d_alloc.dma_paddr,
910	(u_int32_t)(q->q_dma->d_alloc.dma_paddr +
911	offsetof(struct ubsec_dmachunk, d_mcr)));
912	#endif /* UBSEC_DEBUG */
913	BSD_SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
914	--sc->sc_nqueue;
915	BSD_SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
916	}
917	}
918
919	/*
920	* Allocate a new 'session' and return an encoded session id. 'sidp'
921	* contains our registration id, and should contain an encoded session
922	* id on successful allocation.
923	*/
924	static int
925	ubsec_newsession(device_t dev, u_int32_t sidp, struct cryptoini cri)
926	{
927	struct cryptoini c, encini = NULL, *macini = NULL;
928	struct ubsec_softc *sc = NULL;
929	struct ubsec_session *ses = NULL;
930	int sesn, i;
931
932	#ifdef UBSEC_DEBUG
933	DPRINTF("%s()\n", __FUNCTION__);
934	#endif
935
936	if (sidp == NULL \|\| cri == NULL)
937	return (EINVAL);
938
939	sc = device_get_softc(dev);
940
941	if (sc == NULL)
942	return (EINVAL);
943
944	for (c = cri; c != NULL; c = c->cri_next) {
945	if (c->cri_alg == CRYPTO_MD5_HMAC \|\|
946	c->cri_alg == CRYPTO_SHA1_HMAC) {
947	if (macini)
948	return (EINVAL);
949	macini = c;
950	} else if (c->cri_alg == CRYPTO_DES_CBC \|\|
951	c->cri_alg == CRYPTO_3DES_CBC \|\|
952	c->cri_alg == CRYPTO_AES_CBC) {
953	if (encini)
954	return (EINVAL);
955	encini = c;
956	} else
957	return (EINVAL);
958	}
959	if (encini == NULL && macini == NULL)
960	return (EINVAL);
961
962	if (sc->sc_sessions == NULL) {
963	ses = sc->sc_sessions = (struct ubsec_session *)kmalloc(
964	sizeof(struct ubsec_session), SLAB_ATOMIC);
965	if (ses == NULL)
966	return (ENOMEM);
967	memset(ses, 0, sizeof(struct ubsec_session));
968	sesn = 0;
969	sc->sc_nsessions = 1;
970	} else {
971	for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
972	if (sc->sc_sessions[sesn].ses_used == 0) {
973	ses = &sc->sc_sessions[sesn];
974	break;
975	}
976	}
977
978	if (ses == NULL) {
979	sesn = sc->sc_nsessions;
980	ses = (struct ubsec_session )kmalloc((sesn + 1)
981	sizeof(struct ubsec_session), SLAB_ATOMIC);
982	if (ses == NULL)
983	return (ENOMEM);
984	memset(ses, 0, (sesn + 1) * sizeof(struct ubsec_session));
985	bcopy(sc->sc_sessions, ses, sesn *
986	sizeof(struct ubsec_session));
987	bzero(sc->sc_sessions, sesn *
988	sizeof(struct ubsec_session));
989	kfree(sc->sc_sessions);
990	sc->sc_sessions = ses;
991	ses = &sc->sc_sessions[sesn];
992	sc->sc_nsessions++;
993	}
994	}
995
996	bzero(ses, sizeof(struct ubsec_session));
997	ses->ses_used = 1;
998	if (encini) {
999	/* get an IV */
1000	/* XXX may read fewer than requested */
1001	read_random(ses->ses_iv, sizeof(ses->ses_iv));
1002
1003	/* Go ahead and compute key in ubsec's byte order */
1004	if (encini->cri_alg == CRYPTO_DES_CBC) {
1005	/* DES uses the same key three times:
1006	* 1st encrypt -> 2nd decrypt -> 3nd encrypt */
1007	bcopy(encini->cri_key, &ses->ses_key[0], 8);
1008	bcopy(encini->cri_key, &ses->ses_key[2], 8);
1009	bcopy(encini->cri_key, &ses->ses_key[4], 8);
1010	ses->ses_keysize = 192; /* Fake! Actually its only 64bits ..
1011	oh no it is even less: 54bits. */
1012	} else if(encini->cri_alg == CRYPTO_3DES_CBC) {
1013	bcopy(encini->cri_key, ses->ses_key, 24);
1014	ses->ses_keysize = 192;
1015	} else if(encini->cri_alg == CRYPTO_AES_CBC) {
1016	ses->ses_keysize = encini->cri_klen;
1017
1018	if (ses->ses_keysize != 128 &&
1019	ses->ses_keysize != 192 &&
1020	ses->ses_keysize != 256)
1021	{
1022	DPRINTF("unsupported AES key size: %d\n", ses->ses_keysize);
1023	return (EINVAL);
1024	}
1025	bcopy(encini->cri_key, ses->ses_key, (ses->ses_keysize / 8));
1026	}
1027
1028	/* Hardware requires the keys in little endian byte order */
1029	for (i=0; i < (ses->ses_keysize / 32); i++)
1030	SWAP32(ses->ses_key[i]);
1031	}
1032
1033	if (macini) {
1034	ses->ses_mlen = macini->cri_mlen;
1035
1036	if (ses->ses_mlen == 0 \|\|
1037	ses->ses_mlen > SHA1_HASH_LEN) {
1038
1039	if (macini->cri_alg == CRYPTO_MD5_HMAC \|\|
1040	macini->cri_alg == CRYPTO_SHA1_HMAC)
1041	{
1042	ses->ses_mlen = DEFAULT_HMAC_LEN;
1043	} else
1044	{
1045	/*
1046	* Reserved for future usage. MD5/SHA1 calculations have
1047	* different hash sizes.
1048	*/
1049	printk(KERN_ERR DRV_MODULE_NAME ": unsupported hash operation with mac/hash len: %d\n", ses->ses_mlen);
1050	return (EINVAL);
1051	}
1052
1053	}
1054
1055	if (macini->cri_key != NULL) {
1056	ubsec_setup_mackey(ses, macini->cri_alg, macini->cri_key,
1057	macini->cri_klen / 8);
1058	}
1059	}
1060
1061	*sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn);
1062	return (0);
1063	}
1064
1065	/*
1066	* Deallocate a session.
1067	*/
1068	static int
1069	ubsec_freesession(device_t dev, u_int64_t tid)
1070	{
1071	struct ubsec_softc *sc = device_get_softc(dev);
1072	int session;
1073	u_int32_t sid = ((u_int32_t)tid) & 0xffffffff;
1074
1075	#ifdef UBSEC_DEBUG
1076	DPRINTF("%s()\n", __FUNCTION__);
1077	#endif
1078
1079	if (sc == NULL)
1080	return (EINVAL);
1081
1082	session = UBSEC_SESSION(sid);
1083	if (session < sc->sc_nsessions) {
1084	bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
1085	return (0);
1086	} else
1087	return (EINVAL);
1088	}
1089
1090	static int
1091	ubsec_process(device_t dev, struct cryptop *crp, int hint)
1092	{
1093	struct ubsec_q *q = NULL;
1094	int err = 0, i, j, nicealign;
1095	struct ubsec_softc *sc = device_get_softc(dev);
1096	struct cryptodesc crd1, crd2, maccrd, enccrd;
1097	int encoffset = 0, macoffset = 0, cpskip, cpoffset;
1098	int sskip, dskip, stheend, dtheend, ivsize = 8;
1099	int16_t coffset;
1100	struct ubsec_session *ses;
1101	struct ubsec_generic_ctx ctx;
1102	struct ubsec_dma *dmap = NULL;
1103	unsigned long flags;
1104
1105	#ifdef UBSEC_DEBUG
1106	DPRINTF("%s()\n", __FUNCTION__);
1107	#endif
1108
1109	if (unlikely(crp == NULL \|\| crp->crp_callback == NULL)) {
1110	ubsecstats.hst_invalid++;
1111	return (EINVAL);
1112	}
1113
1114	if (unlikely(sc == NULL))
1115	return (EINVAL);
1116
1117	#ifdef UBSEC_VERBOSE_DEBUG
1118	DPRINTF("spin_lock_irqsave\n");
1119	#endif
1120	spin_lock_irqsave(&sc->sc_ringmtx, flags);
1121	//spin_lock_irq(&sc->sc_ringmtx);
1122
1123	if (BSD_SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1124	ubsecstats.hst_queuefull++;
1125	#ifdef UBSEC_VERBOSE_DEBUG
1126	DPRINTF("spin_unlock_irqrestore\n");
1127	#endif
1128	spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1129	//spin_unlock_irq(&sc->sc_ringmtx);
1130	err = ENOMEM;
1131	goto errout2;
1132	}
1133
1134	q = BSD_SIMPLEQ_FIRST(&sc->sc_freequeue);
1135	BSD_SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
1136	#ifdef UBSEC_VERBOSE_DEBUG
1137	DPRINTF("spin_unlock_irqrestore\n");
1138	#endif
1139	spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1140	//spin_unlock_irq(&sc->sc_ringmtx);
1141
1142	dmap = q->q_dma; /* Save dma pointer */
1143	bzero(q, sizeof(struct ubsec_q));
1144	bzero(&ctx, sizeof(ctx));
1145
1146	q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1147	q->q_dma = dmap;
1148	ses = &sc->sc_sessions[q->q_sesn];
1149
1150	if (crp->crp_flags & CRYPTO_F_SKBUF) {
1151	q->q_src_m = (struct sk_buff *)crp->crp_buf;
1152	q->q_dst_m = (struct sk_buff *)crp->crp_buf;
1153	} else if (crp->crp_flags & CRYPTO_F_IOV) {
1154	q->q_src_io = (struct uio *)crp->crp_buf;
1155	q->q_dst_io = (struct uio *)crp->crp_buf;
1156	} else {
1157	err = EINVAL;
1158	goto errout; /* XXX we don't handle contiguous blocks! */
1159	}
1160
1161	bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
1162
1163	dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1164	dmap->d_dma->d_mcr.mcr_flags = 0;
1165	q->q_crp = crp;
1166
1167	crd1 = crp->crp_desc;
1168	if (crd1 == NULL) {
1169	err = EINVAL;
1170	goto errout;
1171	}
1172	crd2 = crd1->crd_next;
1173
1174	if (crd2 == NULL) {
1175	if (crd1->crd_alg == CRYPTO_MD5_HMAC \|\|
1176	crd1->crd_alg == CRYPTO_SHA1_HMAC) {
1177	maccrd = crd1;
1178	enccrd = NULL;
1179	} else if (crd1->crd_alg == CRYPTO_DES_CBC \|\|
1180	crd1->crd_alg == CRYPTO_3DES_CBC \|\|
1181	crd1->crd_alg == CRYPTO_AES_CBC) {
1182	maccrd = NULL;
1183	enccrd = crd1;
1184	} else {
1185	err = EINVAL;
1186	goto errout;
1187	}
1188	} else {
1189	if ((crd1->crd_alg == CRYPTO_MD5_HMAC \|\|
1190	crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
1191	(crd2->crd_alg == CRYPTO_DES_CBC \|\|
1192	crd2->crd_alg == CRYPTO_3DES_CBC \|\|
1193	crd2->crd_alg == CRYPTO_AES_CBC) &&
1194	((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
1195	maccrd = crd1;
1196	enccrd = crd2;
1197	} else if ((crd1->crd_alg == CRYPTO_DES_CBC \|\|
1198	crd1->crd_alg == CRYPTO_3DES_CBC \|\|
1199	crd1->crd_alg == CRYPTO_AES_CBC) &&
1200	(crd2->crd_alg == CRYPTO_MD5_HMAC \|\|
1201	crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
1202	(crd1->crd_flags & CRD_F_ENCRYPT)) {
1203	enccrd = crd1;
1204	maccrd = crd2;
1205	} else {
1206	/*
1207	* We cannot order the ubsec as requested
1208	*/
1209	printk(KERN_ERR DRV_MODULE_NAME ": got wrong algorithm/signature order.\n");
1210	err = EINVAL;
1211	goto errout;
1212	}
1213	}
1214
1215	/* Encryption/Decryption requested */
1216	if (enccrd) {
1217	encoffset = enccrd->crd_skip;
1218
1219	if (enccrd->crd_alg == CRYPTO_DES_CBC \|\|
1220	enccrd->crd_alg == CRYPTO_3DES_CBC)
1221	{
1222	ctx.pc_flags \|= htole16(UBS_PKTCTX_ENC_3DES);
1223	ctx.pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC_DES);
1224	ivsize = 8; /* [3]DES uses 64bit IVs */
1225	} else {
1226	ctx.pc_flags \|= htole16(UBS_PKTCTX_ENC_AES);
1227	ctx.pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC_AES);
1228	ivsize = 16; /* AES uses 128bit IVs / [3]DES 64bit IVs */
1229
1230	switch(ses->ses_keysize)
1231	{
1232	case 128:
1233	ctx.pc_flags \|= htole16(UBS_PKTCTX_AES128);
1234	break;
1235	case 192:
1236	ctx.pc_flags \|= htole16(UBS_PKTCTX_AES192);
1237	break;
1238	case 256:
1239	ctx.pc_flags \|= htole16(UBS_PKTCTX_AES256);
1240	break;
1241	default:
1242	DPRINTF("invalid AES key size: %d\n", ses->ses_keysize);
1243	err = EINVAL;
1244	goto errout;
1245	}
1246	}
1247
1248	if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1249	/* Direction: Outbound */
1250
1251	q->q_flags \|= UBSEC_QFLAGS_COPYOUTIV;
1252
1253	if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
1254	bcopy(enccrd->crd_iv, ctx.pc_iv, ivsize);
1255	} else {
1256	for(i=0; i < (ivsize / 4); i++)
1257	ctx.pc_iv[i] = ses->ses_iv[i];
1258	}
1259
1260	/* If there is no IV in the buffer -> copy it here */
1261	if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
1262	if (crp->crp_flags & CRYPTO_F_SKBUF)
1263	/*
1264	m_copyback(q->q_src_m,
1265	enccrd->crd_inject,
1266	8, ctx.pc_iv);
1267	*/
1268	crypto_copyback(crp->crp_flags, (caddr_t)q->q_src_m,
1269	enccrd->crd_inject, ivsize, (caddr_t)ctx.pc_iv);
1270	else if (crp->crp_flags & CRYPTO_F_IOV)
1271	/*
1272	cuio_copyback(q->q_src_io,
1273	enccrd->crd_inject,
1274	8, ctx.pc_iv);
1275	*/
1276	crypto_copyback(crp->crp_flags, (caddr_t)q->q_src_io,
1277	enccrd->crd_inject, ivsize, (caddr_t)ctx.pc_iv);
1278	}
1279	} else {
1280	/* Direction: Inbound */
1281
1282	ctx.pc_flags \|= htole16(UBS_PKTCTX_INBOUND);
1283
1284	if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1285	bcopy(enccrd->crd_iv, ctx.pc_iv, ivsize);
1286	else if (crp->crp_flags & CRYPTO_F_SKBUF)
1287	/*
1288	m_copydata(q->q_src_m, enccrd->crd_inject,
1289	8, (caddr_t)ctx.pc_iv);
1290	*/
1291	crypto_copydata(crp->crp_flags, (caddr_t)q->q_src_m,
1292	enccrd->crd_inject, ivsize,
1293	(caddr_t)ctx.pc_iv);
1294	else if (crp->crp_flags & CRYPTO_F_IOV)
1295	/*
1296	cuio_copydata(q->q_src_io,
1297	enccrd->crd_inject, 8,
1298	(caddr_t)ctx.pc_iv);
1299	*/
1300	crypto_copydata(crp->crp_flags, (caddr_t)q->q_src_io,
1301	enccrd->crd_inject, ivsize,
1302	(caddr_t)ctx.pc_iv);
1303
1304	}
1305
1306	/* Even though key & IV sizes differ from cipher to cipher
1307	* copy / swap the full array lengths. Let the compiler unroll
1308	* the loop to increase the cpu pipeline performance... */
1309	for(i=0; i < 8; i++)
1310	ctx.pc_key[i] = ses->ses_key[i];
1311	for(i=0; i < 4; i++)
1312	SWAP32(ctx.pc_iv[i]);
1313	}
1314
1315	/* Authentication requested */
1316	if (maccrd) {
1317	macoffset = maccrd->crd_skip;
1318
1319	if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
1320	ctx.pc_flags \|= htole16(UBS_PKTCTX_AUTH_MD5);
1321	else
1322	ctx.pc_flags \|= htole16(UBS_PKTCTX_AUTH_SHA1);
1323
1324	for (i = 0; i < 5; i++) {
1325	ctx.pc_hminner[i] = ses->ses_hminner[i];
1326	ctx.pc_hmouter[i] = ses->ses_hmouter[i];
1327
1328	HTOLE32(ctx.pc_hminner[i]);
1329	HTOLE32(ctx.pc_hmouter[i]);
1330	}
1331	}
1332
1333	if (enccrd && maccrd) {
1334	/*
1335	* ubsec cannot handle packets where the end of encryption
1336	* and authentication are not the same, or where the
1337	* encrypted part begins before the authenticated part.
1338	*/
1339	if (((encoffset + enccrd->crd_len) !=
1340	(macoffset + maccrd->crd_len)) \|\|
1341	(enccrd->crd_skip < maccrd->crd_skip)) {
1342	err = EINVAL;
1343	goto errout;
1344	}
1345	sskip = maccrd->crd_skip;
1346	cpskip = dskip = enccrd->crd_skip;
1347	stheend = maccrd->crd_len;
1348	dtheend = enccrd->crd_len;
1349	coffset = enccrd->crd_skip - maccrd->crd_skip;
1350	cpoffset = cpskip + dtheend;
1351	#ifdef UBSEC_DEBUG
1352	DPRINTF("mac: skip %d, len %d, inject %d\n",
1353	maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
1354	DPRINTF("enc: skip %d, len %d, inject %d\n",
1355	enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
1356	DPRINTF("src: skip %d, len %d\n", sskip, stheend);
1357	DPRINTF("dst: skip %d, len %d\n", dskip, dtheend);
1358	DPRINTF("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1359	coffset, stheend, cpskip, cpoffset);
1360	#endif
1361	} else {
1362	cpskip = dskip = sskip = macoffset + encoffset;
1363	dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1364	cpoffset = cpskip + dtheend;
1365	coffset = 0;
1366	}
1367	ctx.pc_offset = htole16(coffset >> 2);
1368
1369	#if 0
1370	if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER,
1371	0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) {
1372	err = ENOMEM;
1373	goto errout;
1374	}
1375	#endif
1376
1377	if (crp->crp_flags & CRYPTO_F_SKBUF) {
1378	#if 0
1379	if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1380	q->q_src_m, BUS_DMA_NOWAIT) != 0) {
1381	bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1382	q->q_src_map = NULL;
1383	err = ENOMEM;
1384	goto errout;
1385	}
1386	#endif
1387	err = dma_map_skb(sc, q->q_src_map, q->q_src_m, &q->q_src_len);
1388	if (unlikely(err != 0))
1389	goto errout;
1390
1391	} else if (crp->crp_flags & CRYPTO_F_IOV) {
1392	#if 0
1393	if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1394	q->q_src_io, BUS_DMA_NOWAIT) != 0) {
1395	bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1396	q->q_src_map = NULL;
1397	err = ENOMEM;
1398	goto errout;
1399	}
1400	#endif
1401	err = dma_map_uio(sc, q->q_src_map, q->q_src_io, &q->q_src_len);
1402	if (unlikely(err != 0))
1403	goto errout;
1404	}
1405
1406	/*
1407	* Check alignment
1408	*/
1409	nicealign = ubsec_dmamap_aligned(sc, q->q_src_map, q->q_src_len);
1410
1411	dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1412
1413	#ifdef UBSEC_DEBUG
1414	DPRINTF("src skip: %d\n", sskip);
1415	#endif
1416	for (i = j = 0; i < q->q_src_len; i++) {
1417	struct ubsec_pktbuf *pb;
1418	size_t packl = q->q_src_map[i].dma_size;
1419	dma_addr_t packp = q->q_src_map[i].dma_paddr;
1420
1421	if (sskip >= packl) {
1422	sskip -= packl;
1423	continue;
1424	}
1425
1426	packl -= sskip;
1427	packp += sskip;
1428	sskip = 0;
1429
1430	/* maximum fragment size is 0xfffc */
1431	if (packl > 0xfffc) {
1432	DPRINTF("Error: fragment size is bigger than 0xfffc.\n");
1433	err = EIO;
1434	goto errout;
1435	}
1436
1437	if (j == 0)
1438	pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1439	else
1440	pb = &dmap->d_dma->d_sbuf[j - 1];
1441
1442	pb->pb_addr = htole32(packp);
1443
1444	if (stheend) {
1445	if (packl > stheend) {
1446	pb->pb_len = htole32(stheend);
1447	stheend = 0;
1448	} else {
1449	pb->pb_len = htole32(packl);
1450	stheend -= packl;
1451	}
1452	} else
1453	pb->pb_len = htole32(packl);
1454
1455	if ((i + 1) == q->q_src_len)
1456	pb->pb_next = 0;
1457	else
1458	pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1459	offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1460	j++;
1461	}
1462
1463	if (enccrd == NULL && maccrd != NULL) {
1464	/* Authentication only */
1465	dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
1466	dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
1467	dmap->d_dma->d_mcr.mcr_opktbuf.pb_next =
1468	htole32(dmap->d_alloc.dma_paddr +
1469	offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1470	#ifdef UBSEC_DEBUG
1471	DPRINTF("opkt: %x %x %x\n",
1472	dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1473	dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1474	dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1475	#endif
1476	} else {
1477	if (crp->crp_flags & CRYPTO_F_IOV) {
1478	if (!nicealign) {
1479	err = EINVAL;
1480	goto errout;
1481	}
1482	#if 0
1483	if (bus_dmamap_create(sc->sc_dmat, 0xfff0,
1484	UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT,
1485	&q->q_dst_map) != 0) {
1486	err = ENOMEM;
1487	goto errout;
1488	}
1489	if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1490	q->q_dst_io, BUS_DMA_NOWAIT) != 0) {
1491	bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1492	q->q_dst_map = NULL;
1493	goto errout;
1494	}
1495	#endif
1496
1497	/* HW shall copy the result into the source memory */
1498	for(i = 0; i < q->q_src_len; i++)
1499	q->q_dst_map[i] = q->q_src_map[i];
1500
1501	q->q_dst_len = q->q_src_len;
1502	q->q_has_dst = 0;
1503
1504	} else if (crp->crp_flags & CRYPTO_F_SKBUF) {
1505	if (nicealign) {
1506
1507	/* HW shall copy the result into the source memory */
1508	q->q_dst_m = q->q_src_m;
1509	for(i = 0; i < q->q_src_len; i++)
1510	q->q_dst_map[i] = q->q_src_map[i];
1511
1512	q->q_dst_len = q->q_src_len;
1513	q->q_has_dst = 0;
1514
1515	} else {
1516	#ifdef NOTYET
1517	int totlen, len;
1518	struct sk_buff m, top, **mp;
1519
1520	totlen = q->q_src_map->dm_mapsize;
1521	if (q->q_src_m->m_flags & M_PKTHDR) {
1522	len = MHLEN;
1523	MGETHDR(m, M_DONTWAIT, MT_DATA);
1524	} else {
1525	len = MLEN;
1526	MGET(m, M_DONTWAIT, MT_DATA);
1527	}
1528	if (m == NULL) {
1529	err = ENOMEM;
1530	goto errout;
1531	}
1532	if (len == MHLEN)
1533	M_DUP_PKTHDR(m, q->q_src_m);
1534	if (totlen >= MINCLSIZE) {
1535	MCLGET(m, M_DONTWAIT);
1536	if (m->m_flags & M_EXT)
1537	len = MCLBYTES;
1538	}
1539	m->m_len = len;
1540	top = NULL;
1541	mp = &top;
1542
1543	while (totlen > 0) {
1544	if (top) {
1545	MGET(m, M_DONTWAIT, MT_DATA);
1546	if (m == NULL) {
1547	m_freem(top);
1548	err = ENOMEM;
1549	goto errout;
1550	}
1551	len = MLEN;
1552	}
1553	if (top && totlen >= MINCLSIZE) {
1554	MCLGET(m, M_DONTWAIT);
1555	if (m->m_flags & M_EXT)
1556	len = MCLBYTES;
1557	}
1558	m->m_len = len = min(totlen, len);
1559	totlen -= len;
1560	*mp = m;
1561	mp = &m->m_next;
1562	}
1563	q->q_dst_m = top;
1564	ubsec_mcopy(q->q_src_m, q->q_dst_m,
1565	cpskip, cpoffset);
1566	if (bus_dmamap_create(sc->sc_dmat, 0xfff0,
1567	UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT,
1568	&q->q_dst_map) != 0) {
1569	err = ENOMEM;
1570	goto errout;
1571	}
1572	if (bus_dmamap_load_mbuf(sc->sc_dmat,
1573	q->q_dst_map, q->q_dst_m,
1574	BUS_DMA_NOWAIT) != 0) {
1575	bus_dmamap_destroy(sc->sc_dmat,
1576	q->q_dst_map);
1577	q->q_dst_map = NULL;
1578	err = ENOMEM;
1579	goto errout;
1580	}
1581	#else
1582	device_printf(sc->sc_dev,
1583	"%s,%d: CRYPTO_F_SKBUF unaligned not implemented\n",
1584	__FILE__, __LINE__);
1585	err = EINVAL;
1586	goto errout;
1587	#endif
1588	}
1589	} else {
1590	err = EINVAL;
1591	goto errout;
1592	}
1593
1594	#ifdef UBSEC_DEBUG
1595	DPRINTF("dst skip: %d\n", dskip);
1596	#endif
1597	for (i = j = 0; i < q->q_dst_len; i++) {
1598	struct ubsec_pktbuf *pb;
1599	size_t packl = q->q_dst_map[i].dma_size;
1600	dma_addr_t packp = q->q_dst_map[i].dma_paddr;
1601
1602	if (dskip >= packl) {
1603	dskip -= packl;
1604	continue;
1605	}
1606
1607	packl -= dskip;
1608	packp += dskip;
1609	dskip = 0;
1610
1611	if (packl > 0xfffc) {
1612	DPRINTF("Error: fragment size is bigger than 0xfffc.\n");
1613	err = EIO;
1614	goto errout;
1615	}
1616
1617	if (j == 0)
1618	pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1619	else
1620	pb = &dmap->d_dma->d_dbuf[j - 1];
1621
1622	pb->pb_addr = htole32(packp);
1623
1624	if (dtheend) {
1625	if (packl > dtheend) {
1626	pb->pb_len = htole32(dtheend);
1627	dtheend = 0;
1628	} else {
1629	pb->pb_len = htole32(packl);
1630	dtheend -= packl;
1631	}
1632	} else
1633	pb->pb_len = htole32(packl);
1634
1635	if ((i + 1) == q->q_dst_len) {
1636	if (maccrd)
1637	/* Authentication:
1638	* The last fragment of the output buffer
1639	* contains the HMAC. */
1640	pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1641	offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1642	else
1643	pb->pb_next = 0;
1644	} else
1645	pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1646	offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1647	j++;
1648	}
1649	}
1650
1651	dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1652	offsetof(struct ubsec_dmachunk, d_ctx));
1653
1654	if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1655	/* new Broadcom cards with dynamic long command context structure */
1656
1657	if (enccrd != NULL &&
1658	enccrd->crd_alg == CRYPTO_AES_CBC)
1659	{
1660	struct ubsec_pktctx_aes128 *ctxaes128;
1661	struct ubsec_pktctx_aes192 *ctxaes192;
1662	struct ubsec_pktctx_aes256 *ctxaes256;
1663
1664	switch(ses->ses_keysize)
1665	{
1666	/* AES 128bit */
1667	case 128:
1668	ctxaes128 = (struct ubsec_pktctx_aes128 *)
1669	(dmap->d_alloc.dma_vaddr +
1670	offsetof(struct ubsec_dmachunk, d_ctx));
1671
1672	ctxaes128->pc_len = htole16(sizeof(struct ubsec_pktctx_aes128));
1673	ctxaes128->pc_type = ctx.pc_type;
1674	ctxaes128->pc_flags = ctx.pc_flags;
1675	ctxaes128->pc_offset = ctx.pc_offset;
1676	for (i = 0; i < 4; i++)
1677	ctxaes128->pc_aeskey[i] = ctx.pc_key[i];
1678	for (i = 0; i < 5; i++)
1679	ctxaes128->pc_hminner[i] = ctx.pc_hminner[i];
1680	for (i = 0; i < 5; i++)
1681	ctxaes128->pc_hmouter[i] = ctx.pc_hmouter[i];
1682	for (i = 0; i < 4; i++)
1683	ctxaes128->pc_iv[i] = ctx.pc_iv[i];
1684	break;
1685
1686	/* AES 192bit */
1687	case 192:
1688	ctxaes192 = (struct ubsec_pktctx_aes192 *)
1689	(dmap->d_alloc.dma_vaddr +
1690	offsetof(struct ubsec_dmachunk, d_ctx));
1691
1692	ctxaes192->pc_len = htole16(sizeof(struct ubsec_pktctx_aes192));
1693	ctxaes192->pc_type = ctx.pc_type;
1694	ctxaes192->pc_flags = ctx.pc_flags;
1695	ctxaes192->pc_offset = ctx.pc_offset;
1696	for (i = 0; i < 6; i++)
1697	ctxaes192->pc_aeskey[i] = ctx.pc_key[i];
1698	for (i = 0; i < 5; i++)
1699	ctxaes192->pc_hminner[i] = ctx.pc_hminner[i];
1700	for (i = 0; i < 5; i++)
1701	ctxaes192->pc_hmouter[i] = ctx.pc_hmouter[i];
1702	for (i = 0; i < 4; i++)
1703	ctxaes192->pc_iv[i] = ctx.pc_iv[i];
1704	break;
1705
1706	/* AES 256bit */
1707	case 256:
1708	ctxaes256 = (struct ubsec_pktctx_aes256 *)
1709	(dmap->d_alloc.dma_vaddr +
1710	offsetof(struct ubsec_dmachunk, d_ctx));
1711
1712	ctxaes256->pc_len = htole16(sizeof(struct ubsec_pktctx_aes256));
1713	ctxaes256->pc_type = ctx.pc_type;
1714	ctxaes256->pc_flags = ctx.pc_flags;
1715	ctxaes256->pc_offset = ctx.pc_offset;
1716	for (i = 0; i < 8; i++)
1717	ctxaes256->pc_aeskey[i] = ctx.pc_key[i];
1718	for (i = 0; i < 5; i++)
1719	ctxaes256->pc_hminner[i] = ctx.pc_hminner[i];
1720	for (i = 0; i < 5; i++)
1721	ctxaes256->pc_hmouter[i] = ctx.pc_hmouter[i];
1722	for (i = 0; i < 4; i++)
1723	ctxaes256->pc_iv[i] = ctx.pc_iv[i];
1724	break;
1725
1726	}
1727	} else {
1728	/*
1729	* [3]DES / MD5_HMAC / SHA1_HMAC
1730	*
1731	* MD5_HMAC / SHA1_HMAC can use the IPSEC 3DES operation without
1732	* encryption.
1733	*/
1734	struct ubsec_pktctx_des *ctxdes;
1735
1736	ctxdes = (struct ubsec_pktctx_des *)(dmap->d_alloc.dma_vaddr +
1737	offsetof(struct ubsec_dmachunk, d_ctx));
1738
1739	ctxdes->pc_len = htole16(sizeof(struct ubsec_pktctx_des));
1740	ctxdes->pc_type = ctx.pc_type;
1741	ctxdes->pc_flags = ctx.pc_flags;
1742	ctxdes->pc_offset = ctx.pc_offset;
1743	for (i = 0; i < 6; i++)
1744	ctxdes->pc_deskey[i] = ctx.pc_key[i];
1745	for (i = 0; i < 5; i++)
1746	ctxdes->pc_hminner[i] = ctx.pc_hminner[i];
1747	for (i = 0; i < 5; i++)
1748	ctxdes->pc_hmouter[i] = ctx.pc_hmouter[i];
1749	ctxdes->pc_iv[0] = ctx.pc_iv[0];
1750	ctxdes->pc_iv[1] = ctx.pc_iv[1];
1751	}
1752	} else
1753	{
1754	/* old Broadcom card with fixed small command context structure */
1755
1756	/*
1757	* [3]DES / MD5_HMAC / SHA1_HMAC
1758	*/
1759	struct ubsec_pktctx *ctxs;
1760
1761	ctxs = (struct ubsec_pktctx *)(dmap->d_alloc.dma_vaddr +
1762	offsetof(struct ubsec_dmachunk, d_ctx));
1763
1764	/* transform generic context into small context */
1765	for (i = 0; i < 6; i++)
1766	ctxs->pc_deskey[i] = ctx.pc_key[i];
1767	for (i = 0; i < 5; i++)
1768	ctxs->pc_hminner[i] = ctx.pc_hminner[i];
1769	for (i = 0; i < 5; i++)
1770	ctxs->pc_hmouter[i] = ctx.pc_hmouter[i];
1771	ctxs->pc_iv[0] = ctx.pc_iv[0];
1772	ctxs->pc_iv[1] = ctx.pc_iv[1];
1773	ctxs->pc_flags = ctx.pc_flags;
1774	ctxs->pc_offset = ctx.pc_offset;
1775	}
1776
1777	#ifdef UBSEC_VERBOSE_DEBUG
1778	DPRINTF("spin_lock_irqsave\n");
1779	#endif
1780	spin_lock_irqsave(&sc->sc_ringmtx, flags);
1781	//spin_lock_irq(&sc->sc_ringmtx);
1782
1783	BSD_SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1784	sc->sc_nqueue++;
1785	ubsecstats.hst_ipackets++;
1786	ubsecstats.hst_ibytes += stheend;
1787	ubsec_feed(sc);
1788
1789	#ifdef UBSEC_VERBOSE_DEBUG
1790	DPRINTF("spin_unlock_irqrestore\n");
1791	#endif
1792	spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1793	//spin_unlock_irq(&sc->sc_ringmtx);
1794
1795	return (0);
1796
1797	errout:
1798	if (q != NULL) {
1799	#ifdef NOTYET
1800	if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1801	m_freem(q->q_dst_m);
1802	#endif
1803
1804	if ((q->q_has_dst == 1) && q->q_dst_len > 0) {
1805	#if 0
1806	bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1807	bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1808	#endif
1809	dma_unmap(sc, q->q_dst_map, q->q_dst_len);
1810	}
1811	if (q->q_src_len > 0) {
1812	#if 0
1813	bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1814	bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1815	#endif
1816	dma_unmap(sc, q->q_src_map, q->q_src_len);
1817	}
1818
1819	#ifdef UBSEC_VERBOSE_DEBUG
1820	DPRINTF("spin_lock_irqsave\n");
1821	#endif
1822	spin_lock_irqsave(&sc->sc_ringmtx, flags);
1823	//spin_lock_irq(&sc->sc_ringmtx);
1824
1825	BSD_SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1826
1827	#ifdef UBSEC_VERBOSE_DEBUG
1828	DPRINTF("spin_unlock_irqrestore\n");
1829	#endif
1830	spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1831	//spin_unlock_irq(&sc->sc_ringmtx);
1832
1833	}
1834	if (err == EINVAL)
1835	ubsecstats.hst_invalid++;
1836	else
1837	ubsecstats.hst_nomem++;
1838	errout2:
1839	crp->crp_etype = err;
1840	crypto_done(crp);
1841
1842	#ifdef UBSEC_DEBUG
1843	DPRINTF("%s() err = %x\n", __FUNCTION__, err);
1844	#endif
1845
1846	return (0);
1847	}
1848
1849	void
1850	ubsec_callback(struct ubsec_softc sc, struct ubsec_q q)
1851	{
1852	struct cryptop crp = (struct cryptop )q->q_crp;
1853	struct cryptodesc *crd;
1854	struct ubsec_dma *dmap = q->q_dma;
1855	int ivsize = 8;
1856
1857	#ifdef UBSEC_DEBUG
1858	DPRINTF("%s()\n", __FUNCTION__);
1859	#endif
1860
1861	ubsecstats.hst_opackets++;
1862	ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1863
1864	#if 0
1865	bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 0,
1866	dmap->d_alloc.dma_map->dm_mapsize,
1867	BUS_DMASYNC_POSTREAD\|BUS_DMASYNC_POSTWRITE);
1868	if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1869	bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1870	0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1871	bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1872	bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1873	}
1874	bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
1875	0, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1876	bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1877	bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1878	#endif
1879
1880	if ((q->q_has_dst == 1) && q->q_dst_len > 0)
1881	dma_unmap(sc, q->q_dst_map, q->q_dst_len);
1882
1883	dma_unmap(sc, q->q_src_map, q->q_src_len);
1884
1885	#ifdef NOTYET
1886	if ((crp->crp_flags & CRYPTO_F_SKBUF) && (q->q_src_m != q->q_dst_m)) {
1887	m_freem(q->q_src_m);
1888	crp->crp_buf = (caddr_t)q->q_dst_m;
1889	}
1890	#endif
1891
1892	/* copy out IV for future use */
1893	if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1894	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1895	if (crd->crd_alg != CRYPTO_DES_CBC &&
1896	crd->crd_alg != CRYPTO_3DES_CBC &&
1897	crd->crd_alg != CRYPTO_AES_CBC)
1898	continue;
1899
1900	if (crd->crd_alg == CRYPTO_AES_CBC)
1901	ivsize = 16;
1902	else
1903	ivsize = 8;
1904
1905	if (crp->crp_flags & CRYPTO_F_SKBUF)
1906	#if 0
1907	m_copydata((struct sk_buff *)crp->crp_buf,
1908	crd->crd_skip + crd->crd_len - 8, 8,
1909	(caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1910	#endif
1911	crypto_copydata(crp->crp_flags, (caddr_t)crp->crp_buf,
1912	crd->crd_skip + crd->crd_len - ivsize, ivsize,
1913	(caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1914
1915	else if (crp->crp_flags & CRYPTO_F_IOV) {
1916	#if 0
1917	cuio_copydata((struct uio *)crp->crp_buf,
1918	crd->crd_skip + crd->crd_len - 8, 8,
1919	(caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1920	#endif
1921	crypto_copydata(crp->crp_flags, (caddr_t)crp->crp_buf,
1922	crd->crd_skip + crd->crd_len - ivsize, ivsize,
1923	(caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1924
1925	}
1926	break;
1927	}
1928	}
1929
1930	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1931	if (crd->crd_alg != CRYPTO_MD5_HMAC &&
1932	crd->crd_alg != CRYPTO_SHA1_HMAC)
1933	continue;
1934	#if 0
1935	if (crp->crp_flags & CRYPTO_F_SKBUF)
1936	m_copyback((struct sk_buff *)crp->crp_buf,
1937	crd->crd_inject, 12,
1938	dmap->d_dma->d_macbuf);
1939	#endif
1940	#if 0
1941	/* BUG? it does not honor the mac len.. */
1942	crypto_copyback(crp->crp_flags, crp->crp_buf,
1943	crd->crd_inject, 12,
1944	(caddr_t)dmap->d_dma->d_macbuf);
1945	#endif
1946	crypto_copyback(crp->crp_flags, crp->crp_buf,
1947	crd->crd_inject,
1948	sc->sc_sessions[q->q_sesn].ses_mlen,
1949	(caddr_t)dmap->d_dma->d_macbuf);
1950	#if 0
1951	else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac)
1952	bcopy((caddr_t)dmap->d_dma->d_macbuf,
1953	crp->crp_mac, 12);
1954	#endif
1955	break;
1956	}
1957	BSD_SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1958	crypto_done(crp);
1959	}
1960
1961	void
1962	ubsec_mcopy(struct sk_buff srcm, struct sk_buff dstm, int hoffset, int toffset)
1963	{
1964	int i, j, dlen, slen;
1965	caddr_t dptr, sptr;
1966
1967	j = 0;
1968	sptr = srcm->data;
1969	slen = srcm->len;
1970	dptr = dstm->data;
1971	dlen = dstm->len;
1972
1973	while (1) {
1974	for (i = 0; i < min(slen, dlen); i++) {
1975	if (j < hoffset \|\| j >= toffset)
1976	dptr++ = sptr++;
1977	slen--;
1978	dlen--;
1979	j++;
1980	}
1981	if (slen == 0) {
1982	srcm = srcm->next;
1983	if (srcm == NULL)
1984	return;
1985	sptr = srcm->data;
1986	slen = srcm->len;
1987	}
1988	if (dlen == 0) {
1989	dstm = dstm->next;
1990	if (dstm == NULL)
1991	return;
1992	dptr = dstm->data;
1993	dlen = dstm->len;
1994	}
1995	}
1996	}
1997
1998	int
1999	ubsec_dma_malloc(struct ubsec_softc sc, struct ubsec_dma_alloc dma,
2000	size_t size, int mapflags)
2001	{
2002	dma->dma_vaddr = dma_alloc_coherent(sc->sc_dv,
2003	size, &dma->dma_paddr, GFP_KERNEL);
2004
2005	if (likely(dma->dma_vaddr))
2006	{
2007	dma->dma_size = size;
2008	return (0);
2009	}
2010
2011	DPRINTF("could not allocate %d bytes of coherent memory.\n", size);
2012
2013	return (1);
2014	}
2015
2016	void
2017	ubsec_dma_free(struct ubsec_softc sc, struct ubsec_dma_alloc dma)
2018	{
2019	dma_free_coherent(sc->sc_dv, dma->dma_size, dma->dma_vaddr,
2020	dma->dma_paddr);
2021	}
2022
2023	/*
2024	* Resets the board. Values in the regesters are left as is
2025	* from the reset (i.e. initial values are assigned elsewhere).
2026	*/
2027	void
2028	ubsec_reset_board(struct ubsec_softc *sc)
2029	{
2030	volatile u_int32_t ctrl;
2031
2032	#ifdef UBSEC_DEBUG
2033	DPRINTF("%s()\n", __FUNCTION__);
2034	#endif
2035	DPRINTF("Send reset signal to chip.\n");
2036
2037	ctrl = READ_REG(sc, BS_CTRL);
2038	ctrl \|= BS_CTRL_RESET;
2039	WRITE_REG(sc, BS_CTRL, ctrl);
2040
2041	/*
2042	* Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
2043	*/
2044	DELAY(10);
2045	}
2046
2047	/*
2048	* Init Broadcom registers
2049	*/
2050	void
2051	ubsec_init_board(struct ubsec_softc *sc)
2052	{
2053	u_int32_t ctrl;
2054
2055	#ifdef UBSEC_DEBUG
2056	DPRINTF("%s()\n", __FUNCTION__);
2057	#endif
2058	DPRINTF("Initialize chip.\n");
2059
2060	ctrl = READ_REG(sc, BS_CTRL);
2061	ctrl &= ~(BS_CTRL_BE32 \| BS_CTRL_BE64);
2062	ctrl \|= BS_CTRL_LITTLE_ENDIAN \| BS_CTRL_MCR1INT \| BS_CTRL_DMAERR;
2063
2064	WRITE_REG(sc, BS_CTRL, ctrl);
2065
2066	/* Set chip capabilities (BCM5365P) */
2067	sc->sc_flags \|= UBS_FLAGS_LONGCTX \| UBS_FLAGS_AES;
2068	}
2069
2070	/*
2071	* Clean up after a chip crash.
2072	* It is assumed that the caller has spin_lock_irq(sc_ringmtx).
2073	*/
2074	void
2075	ubsec_cleanchip(struct ubsec_softc *sc)
2076	{
2077	struct ubsec_q *q;
2078
2079	#ifdef UBSEC_DEBUG
2080	DPRINTF("%s()\n", __FUNCTION__);
2081	#endif
2082	DPRINTF("Clean up queues after chip crash.\n");
2083
2084	while (!BSD_SIMPLEQ_EMPTY(&sc->sc_qchip)) {
2085	q = BSD_SIMPLEQ_FIRST(&sc->sc_qchip);
2086	BSD_SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
2087	ubsec_free_q(sc, q);
2088	}
2089	}
2090
2091	/*
2092	* free a ubsec_q
2093	* It is assumed that the caller has spin_lock_irq(sc_ringmtx).
2094	*/
2095	int
2096	ubsec_free_q(struct ubsec_softc sc, struct ubsec_q q)
2097	{
2098	struct ubsec_q *q2;
2099	struct cryptop *crp;
2100	int npkts;
2101	int i;
2102
2103	#ifdef UBSEC_DEBUG
2104	DPRINTF("%s()\n", __FUNCTION__);
2105	#endif
2106
2107	npkts = q->q_nstacked_mcrs;
2108
2109	for (i = 0; i < npkts; i++) {
2110	if(q->q_stacked_mcr[i]) {
2111	q2 = q->q_stacked_mcr[i];
2112
2113	if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
2114	#ifdef NOTYET
2115	m_freem(q2->q_dst_m);
2116	#else
2117	printk(KERN_ERR "%s,%d: SKB not supported\n", __FILE__, __LINE__);
2118	#endif
2119
2120	crp = (struct cryptop *)q2->q_crp;
2121
2122	BSD_SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2123
2124	crp->crp_etype = EFAULT;
2125	crypto_done(crp);
2126	} else {
2127	break;
2128	}
2129	}
2130
2131	/*
2132	* Free header MCR
2133	*/
2134	if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2135	#ifdef NOTYET
2136	m_freem(q->q_dst_m);
2137	#else
2138	printk(KERN_ERR "%s,%d: SKB not supported\n", __FILE__, __LINE__);
2139	#endif
2140
2141	crp = (struct cryptop *)q->q_crp;
2142
2143	BSD_SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2144
2145	crp->crp_etype = EFAULT;
2146	crypto_done(crp);
2147	return(0);
2148	}
2149
2150	/*
2151	* Routine to reset the chip and clean up.
2152	* It is assumed that the caller has spin_lock_irq(sc_ringmtx).
2153	*/
2154	void
2155	ubsec_totalreset(struct ubsec_softc *sc)
2156	{
2157
2158	#ifdef UBSEC_DEBUG
2159	DPRINTF("%s()\n", __FUNCTION__);
2160	#endif
2161	DPRINTF("initiate total chip reset.. \n");
2162	ubsec_reset_board(sc);
2163	ubsec_init_board(sc);
2164	ubsec_cleanchip(sc);
2165	}
2166
2167	void
2168	ubsec_dump_pb(struct ubsec_pktbuf *pb)
2169	{
2170	printf("addr 0x%x (0x%x) next 0x%x\n",
2171	pb->pb_addr, pb->pb_len, pb->pb_next);
2172	}
2173
2174	void
2175	ubsec_dump_mcr(struct ubsec_mcr *mcr)
2176	{
2177	struct ubsec_mcr_add *ma;
2178	int i;
2179
2180	printf("MCR:\n");
2181	printf(" pkts: %u, flags 0x%x\n",
2182	letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
2183	ma = (struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
2184	for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
2185	printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
2186	letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
2187	letoh16(ma->mcr_reserved));
2188	printf(" %d: ipkt ", i);
2189	ubsec_dump_pb(&ma->mcr_ipktbuf);
2190	printf(" %d: opkt ", i);
2191	ubsec_dump_pb(&ma->mcr_opktbuf);
2192	ma++;
2193	}
2194	printf("END MCR\n");
2195	}
2196
2197	static int __init mod_init(void) {
2198	return ssb_driver_register(&ubsec_ssb_driver);
2199	}
2200
2201	static void __exit mod_exit(void) {
2202	ssb_driver_unregister(&ubsec_ssb_driver);
2203	}
2204
2205	module_init(mod_init);
2206	module_exit(mod_exit);
2207
2208	// Meta information
2209	MODULE_AUTHOR("Daniel Mueller <daniel@danm.de>");
2210	MODULE_LICENSE("BSD");
2211	MODULE_DESCRIPTION("OCF driver for BCM5365P IPSec Core");
2212	MODULE_VERSION(DRV_MODULE_VERSION);
2213
2214

Download this file

Branches:
backfire
history
master
release_2010-11-17
release_2010-12-14
release_2011-02-23
release_2011-05-28
release_2011-08-27
release_2011-11-13
release_2012-03-18
release_2012-04-09
release_2012-10-11
uclibc_nptl

OpenWrt XBurst

OpenWrt XBurst Git Source Tree

Root/package/ubsec_ssb/src/ubsec_ssb.c

interactive