Root/target/linux/generic/files/crypto/ocf/safe/safe.c

1/*-
2 * Linux port done by David McCullough <david_mccullough@mcafee.com>
3 * Copyright (C) 2004-2010 David McCullough
4 * The license and original author are listed below.
5 *
6 * Copyright (c) 2003 Sam Leffler, Errno Consulting
7 * Copyright (c) 2003 Global Technology Associates, Inc.
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31__FBSDID("$FreeBSD: src/sys/dev/safe/safe.c,v 1.18 2007/03/21 03:42:50 sam Exp $");
32 */
33
34#include <linux/version.h>
35#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,33))
36#include <generated/autoconf.h>
37#else
38#include <linux/autoconf.h>
39#endif
40#include <linux/module.h>
41#include <linux/kernel.h>
42#include <linux/init.h>
43#include <linux/list.h>
44#include <linux/slab.h>
45#include <linux/wait.h>
46#include <linux/sched.h>
47#include <linux/pci.h>
48#include <linux/delay.h>
49#include <linux/interrupt.h>
50#include <linux/spinlock.h>
51#include <linux/random.h>
52#include <linux/version.h>
53#include <linux/skbuff.h>
54#include <asm/io.h>
55
56/*
57 * SafeNet SafeXcel-1141 hardware crypto accelerator
58 */
59
60#include <cryptodev.h>
61#include <uio.h>
62#include <safe/safereg.h>
63#include <safe/safevar.h>
64
65#if 1
66#define DPRINTF(a) do { \
67                        if (debug) { \
68                            printk("%s: ", sc ? \
69                                device_get_nameunit(sc->sc_dev) : "safe"); \
70                            printk a; \
71                        } \
72                    } while (0)
73#else
74#define DPRINTF(a)
75#endif
76
77/*
78 * until we find a cleaner way, include the BSD md5/sha1 code
79 * here
80 */
81#define HMAC_HACK 1
82#ifdef HMAC_HACK
83#define LITTLE_ENDIAN 1234
84#define BIG_ENDIAN 4321
85#ifdef __LITTLE_ENDIAN
86#define BYTE_ORDER LITTLE_ENDIAN
87#endif
88#ifdef __BIG_ENDIAN
89#define BYTE_ORDER BIG_ENDIAN
90#endif
91#include <safe/md5.h>
92#include <safe/md5.c>
93#include <safe/sha1.h>
94#include <safe/sha1.c>
95
96u_int8_t hmac_ipad_buffer[64] = {
97    0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
98    0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
99    0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
100    0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
101    0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
102    0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
103    0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
104    0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36
105};
106
107u_int8_t hmac_opad_buffer[64] = {
108    0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
109    0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
110    0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
111    0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
112    0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
113    0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
114    0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
115    0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C
116};
117#endif /* HMAC_HACK */
118
119/* add proc entry for this */
120struct safe_stats safestats;
121
122#define debug safe_debug
123int safe_debug = 0;
124module_param(safe_debug, int, 0644);
125MODULE_PARM_DESC(safe_debug, "Enable debug");
126
127static void safe_callback(struct safe_softc *, struct safe_ringentry *);
128static void safe_feed(struct safe_softc *, struct safe_ringentry *);
129#if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
130static void safe_rng_init(struct safe_softc *);
131int safe_rngbufsize = 8; /* 32 bytes each read */
132module_param(safe_rngbufsize, int, 0644);
133MODULE_PARM_DESC(safe_rngbufsize, "RNG polling buffer size (32-bit words)");
134int safe_rngmaxalarm = 8; /* max alarms before reset */
135module_param(safe_rngmaxalarm, int, 0644);
136MODULE_PARM_DESC(safe_rngmaxalarm, "RNG max alarms before reset");
137#endif /* SAFE_NO_RNG */
138
139static void safe_totalreset(struct safe_softc *sc);
140static int safe_dmamap_aligned(struct safe_softc *sc, const struct safe_operand *op);
141static int safe_dmamap_uniform(struct safe_softc *sc, const struct safe_operand *op);
142static int safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re);
143static int safe_kprocess(device_t dev, struct cryptkop *krp, int hint);
144static int safe_kstart(struct safe_softc *sc);
145static int safe_ksigbits(struct safe_softc *sc, struct crparam *cr);
146static void safe_kfeed(struct safe_softc *sc);
147static void safe_kpoll(unsigned long arg);
148static void safe_kload_reg(struct safe_softc *sc, u_int32_t off,
149                                u_int32_t len, struct crparam *n);
150
151static int safe_newsession(device_t, u_int32_t *, struct cryptoini *);
152static int safe_freesession(device_t, u_int64_t);
153static int safe_process(device_t, struct cryptop *, int);
154
155static device_method_t safe_methods = {
156    /* crypto device methods */
157    DEVMETHOD(cryptodev_newsession, safe_newsession),
158    DEVMETHOD(cryptodev_freesession,safe_freesession),
159    DEVMETHOD(cryptodev_process, safe_process),
160    DEVMETHOD(cryptodev_kprocess, safe_kprocess),
161};
162
163#define READ_REG(sc,r) readl((sc)->sc_base_addr + (r))
164#define WRITE_REG(sc,r,val) writel((val), (sc)->sc_base_addr + (r))
165
166#define SAFE_MAX_CHIPS 8
167static struct safe_softc *safe_chip_idx[SAFE_MAX_CHIPS];
168
169/*
170 * split our buffers up into safe DMAable byte fragments to avoid lockup
171 * bug in 1141 HW on rev 1.0.
172 */
173
174static int
175pci_map_linear(
176    struct safe_softc *sc,
177    struct safe_operand *buf,
178    void *addr,
179    int len)
180{
181    dma_addr_t tmp;
182    int chunk, tlen = len;
183
184    tmp = pci_map_single(sc->sc_pcidev, addr, len, PCI_DMA_BIDIRECTIONAL);
185
186    buf->mapsize += len;
187    while (len > 0) {
188        chunk = (len > sc->sc_max_dsize) ? sc->sc_max_dsize : len;
189        buf->segs[buf->nsegs].ds_addr = tmp;
190        buf->segs[buf->nsegs].ds_len = chunk;
191        buf->segs[buf->nsegs].ds_tlen = tlen;
192        buf->nsegs++;
193        tmp += chunk;
194        len -= chunk;
195        tlen = 0;
196    }
197    return 0;
198}
199
200/*
201 * map in a given uio buffer (great on some arches :-)
202 */
203
204static int
205pci_map_uio(struct safe_softc *sc, struct safe_operand *buf, struct uio *uio)
206{
207    struct iovec *iov = uio->uio_iov;
208    int n;
209
210    DPRINTF(("%s()\n", __FUNCTION__));
211
212    buf->mapsize = 0;
213    buf->nsegs = 0;
214
215    for (n = 0; n < uio->uio_iovcnt; n++) {
216        pci_map_linear(sc, buf, iov->iov_base, iov->iov_len);
217        iov++;
218    }
219
220    /* identify this buffer by the first segment */
221    buf->map = (void *) buf->segs[0].ds_addr;
222    return(0);
223}
224
225/*
226 * map in a given sk_buff
227 */
228
229static int
230pci_map_skb(struct safe_softc *sc,struct safe_operand *buf,struct sk_buff *skb)
231{
232    int i;
233
234    DPRINTF(("%s()\n", __FUNCTION__));
235
236    buf->mapsize = 0;
237    buf->nsegs = 0;
238
239    pci_map_linear(sc, buf, skb->data, skb_headlen(skb));
240
241    for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
242        pci_map_linear(sc, buf,
243                page_address(skb_shinfo(skb)->frags[i].page) +
244                                        skb_shinfo(skb)->frags[i].page_offset,
245                skb_shinfo(skb)->frags[i].size);
246    }
247
248    /* identify this buffer by the first segment */
249    buf->map = (void *) buf->segs[0].ds_addr;
250    return(0);
251}
252
253
254#if 0 /* not needed at this time */
255static void
256pci_sync_operand(struct safe_softc *sc, struct safe_operand *buf)
257{
258    int i;
259
260    DPRINTF(("%s()\n", __FUNCTION__));
261    for (i = 0; i < buf->nsegs; i++)
262        pci_dma_sync_single_for_cpu(sc->sc_pcidev, buf->segs[i].ds_addr,
263                buf->segs[i].ds_len, PCI_DMA_BIDIRECTIONAL);
264}
265#endif
266
267static void
268pci_unmap_operand(struct safe_softc *sc, struct safe_operand *buf)
269{
270    int i;
271    DPRINTF(("%s()\n", __FUNCTION__));
272    for (i = 0; i < buf->nsegs; i++) {
273        if (buf->segs[i].ds_tlen) {
274            DPRINTF(("%s - unmap %d 0x%x %d\n", __FUNCTION__, i, buf->segs[i].ds_addr, buf->segs[i].ds_tlen));
275            pci_unmap_single(sc->sc_pcidev, buf->segs[i].ds_addr,
276                    buf->segs[i].ds_tlen, PCI_DMA_BIDIRECTIONAL);
277            DPRINTF(("%s - unmap %d 0x%x %d done\n", __FUNCTION__, i, buf->segs[i].ds_addr, buf->segs[i].ds_tlen));
278        }
279        buf->segs[i].ds_addr = 0;
280        buf->segs[i].ds_len = 0;
281        buf->segs[i].ds_tlen = 0;
282    }
283    buf->nsegs = 0;
284    buf->mapsize = 0;
285    buf->map = 0;
286}
287
288
289/*
290 * SafeXcel Interrupt routine
291 */
292static irqreturn_t
293#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)
294safe_intr(int irq, void *arg)
295#else
296safe_intr(int irq, void *arg, struct pt_regs *regs)
297#endif
298{
299    struct safe_softc *sc = arg;
300    int stat;
301    unsigned long flags;
302
303    stat = READ_REG(sc, SAFE_HM_STAT);
304
305    DPRINTF(("%s(stat=0x%x)\n", __FUNCTION__, stat));
306
307    if (stat == 0) /* shared irq, not for us */
308        return IRQ_NONE;
309
310    WRITE_REG(sc, SAFE_HI_CLR, stat); /* IACK */
311
312    if ((stat & SAFE_INT_PE_DDONE)) {
313        /*
314         * Descriptor(s) done; scan the ring and
315         * process completed operations.
316         */
317        spin_lock_irqsave(&sc->sc_ringmtx, flags);
318        while (sc->sc_back != sc->sc_front) {
319            struct safe_ringentry *re = sc->sc_back;
320
321#ifdef SAFE_DEBUG
322            if (debug) {
323                safe_dump_ringstate(sc, __func__);
324                safe_dump_request(sc, __func__, re);
325            }
326#endif
327            /*
328             * safe_process marks ring entries that were allocated
329             * but not used with a csr of zero. This insures the
330             * ring front pointer never needs to be set backwards
331             * in the event that an entry is allocated but not used
332             * because of a setup error.
333             */
334            DPRINTF(("%s re->re_desc.d_csr=0x%x\n", __FUNCTION__, re->re_desc.d_csr));
335            if (re->re_desc.d_csr != 0) {
336                if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr)) {
337                    DPRINTF(("%s !CSR_IS_DONE\n", __FUNCTION__));
338                    break;
339                }
340                if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len)) {
341                    DPRINTF(("%s !LEN_IS_DONE\n", __FUNCTION__));
342                    break;
343                }
344                sc->sc_nqchip--;
345                safe_callback(sc, re);
346            }
347            if (++(sc->sc_back) == sc->sc_ringtop)
348                sc->sc_back = sc->sc_ring;
349        }
350        spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
351    }
352
353    /*
354     * Check to see if we got any DMA Error
355     */
356    if (stat & SAFE_INT_PE_ERROR) {
357        printk("%s: dmaerr dmastat %08x\n", device_get_nameunit(sc->sc_dev),
358                (int)READ_REG(sc, SAFE_PE_DMASTAT));
359        safestats.st_dmaerr++;
360        safe_totalreset(sc);
361#if 0
362        safe_feed(sc);
363#endif
364    }
365
366    if (sc->sc_needwakeup) { /* XXX check high watermark */
367        int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
368        DPRINTF(("%s: wakeup crypto %x\n", __func__,
369            sc->sc_needwakeup));
370        sc->sc_needwakeup &= ~wakeup;
371        crypto_unblock(sc->sc_cid, wakeup);
372    }
373    
374    return IRQ_HANDLED;
375}
376
377/*
378 * safe_feed() - post a request to chip
379 */
380static void
381safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
382{
383    DPRINTF(("%s()\n", __FUNCTION__));
384#ifdef SAFE_DEBUG
385    if (debug) {
386        safe_dump_ringstate(sc, __func__);
387        safe_dump_request(sc, __func__, re);
388    }
389#endif
390    sc->sc_nqchip++;
391    if (sc->sc_nqchip > safestats.st_maxqchip)
392        safestats.st_maxqchip = sc->sc_nqchip;
393    /* poke h/w to check descriptor ring, any value can be written */
394    WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
395}
396
397#define N(a) (sizeof(a) / sizeof (a[0]))
398static void
399safe_setup_enckey(struct safe_session *ses, caddr_t key)
400{
401    int i;
402
403    bcopy(key, ses->ses_key, ses->ses_klen / 8);
404
405    /* PE is little-endian, insure proper byte order */
406    for (i = 0; i < N(ses->ses_key); i++)
407        ses->ses_key[i] = htole32(ses->ses_key[i]);
408}
409
410static void
411safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen)
412{
413#ifdef HMAC_HACK
414    MD5_CTX md5ctx;
415    SHA1_CTX sha1ctx;
416    int i;
417
418
419    for (i = 0; i < klen; i++)
420        key[i] ^= HMAC_IPAD_VAL;
421
422    if (algo == CRYPTO_MD5_HMAC) {
423        MD5Init(&md5ctx);
424        MD5Update(&md5ctx, key, klen);
425        MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
426        bcopy(md5ctx.md5_st8, ses->ses_hminner, sizeof(md5ctx.md5_st8));
427    } else {
428        SHA1Init(&sha1ctx);
429        SHA1Update(&sha1ctx, key, klen);
430        SHA1Update(&sha1ctx, hmac_ipad_buffer,
431            SHA1_HMAC_BLOCK_LEN - klen);
432        bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
433    }
434
435    for (i = 0; i < klen; i++)
436        key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
437
438    if (algo == CRYPTO_MD5_HMAC) {
439        MD5Init(&md5ctx);
440        MD5Update(&md5ctx, key, klen);
441        MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
442        bcopy(md5ctx.md5_st8, ses->ses_hmouter, sizeof(md5ctx.md5_st8));
443    } else {
444        SHA1Init(&sha1ctx);
445        SHA1Update(&sha1ctx, key, klen);
446        SHA1Update(&sha1ctx, hmac_opad_buffer,
447            SHA1_HMAC_BLOCK_LEN - klen);
448        bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
449    }
450
451    for (i = 0; i < klen; i++)
452        key[i] ^= HMAC_OPAD_VAL;
453
454#if 0
455    /*
456     * this code prevents SHA working on a BE host,
457     * so it is obviously wrong. I think the byte
458     * swap setup we do with the chip fixes this for us
459     */
460
461    /* PE is little-endian, insure proper byte order */
462    for (i = 0; i < N(ses->ses_hminner); i++) {
463        ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
464        ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
465    }
466#endif
467#else /* HMAC_HACK */
468    printk("safe: md5/sha not implemented\n");
469#endif /* HMAC_HACK */
470}
471#undef N
472
473/*
474 * Allocate a new 'session' and return an encoded session id. 'sidp'
475 * contains our registration id, and should contain an encoded session
476 * id on successful allocation.
477 */
478static int
479safe_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
480{
481    struct safe_softc *sc = device_get_softc(dev);
482    struct cryptoini *c, *encini = NULL, *macini = NULL;
483    struct safe_session *ses = NULL;
484    int sesn;
485
486    DPRINTF(("%s()\n", __FUNCTION__));
487
488    if (sidp == NULL || cri == NULL || sc == NULL)
489        return (EINVAL);
490
491    for (c = cri; c != NULL; c = c->cri_next) {
492        if (c->cri_alg == CRYPTO_MD5_HMAC ||
493            c->cri_alg == CRYPTO_SHA1_HMAC ||
494            c->cri_alg == CRYPTO_NULL_HMAC) {
495            if (macini)
496                return (EINVAL);
497            macini = c;
498        } else if (c->cri_alg == CRYPTO_DES_CBC ||
499            c->cri_alg == CRYPTO_3DES_CBC ||
500            c->cri_alg == CRYPTO_AES_CBC ||
501            c->cri_alg == CRYPTO_NULL_CBC) {
502            if (encini)
503                return (EINVAL);
504            encini = c;
505        } else
506            return (EINVAL);
507    }
508    if (encini == NULL && macini == NULL)
509        return (EINVAL);
510    if (encini) { /* validate key length */
511        switch (encini->cri_alg) {
512        case CRYPTO_DES_CBC:
513            if (encini->cri_klen != 64)
514                return (EINVAL);
515            break;
516        case CRYPTO_3DES_CBC:
517            if (encini->cri_klen != 192)
518                return (EINVAL);
519            break;
520        case CRYPTO_AES_CBC:
521            if (encini->cri_klen != 128 &&
522                encini->cri_klen != 192 &&
523                encini->cri_klen != 256)
524                return (EINVAL);
525            break;
526        }
527    }
528
529    if (sc->sc_sessions == NULL) {
530        ses = sc->sc_sessions = (struct safe_session *)
531            kmalloc(sizeof(struct safe_session), SLAB_ATOMIC);
532        if (ses == NULL)
533            return (ENOMEM);
534        memset(ses, 0, sizeof(struct safe_session));
535        sesn = 0;
536        sc->sc_nsessions = 1;
537    } else {
538        for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
539            if (sc->sc_sessions[sesn].ses_used == 0) {
540                ses = &sc->sc_sessions[sesn];
541                break;
542            }
543        }
544
545        if (ses == NULL) {
546            sesn = sc->sc_nsessions;
547            ses = (struct safe_session *)
548                kmalloc((sesn + 1) * sizeof(struct safe_session), SLAB_ATOMIC);
549            if (ses == NULL)
550                return (ENOMEM);
551            memset(ses, 0, (sesn + 1) * sizeof(struct safe_session));
552            bcopy(sc->sc_sessions, ses, sesn *
553                sizeof(struct safe_session));
554            bzero(sc->sc_sessions, sesn *
555                sizeof(struct safe_session));
556            kfree(sc->sc_sessions);
557            sc->sc_sessions = ses;
558            ses = &sc->sc_sessions[sesn];
559            sc->sc_nsessions++;
560        }
561    }
562
563    bzero(ses, sizeof(struct safe_session));
564    ses->ses_used = 1;
565
566    if (encini) {
567        /* get an IV */
568        /* XXX may read fewer than requested */
569        read_random(ses->ses_iv, sizeof(ses->ses_iv));
570
571        ses->ses_klen = encini->cri_klen;
572        if (encini->cri_key != NULL)
573            safe_setup_enckey(ses, encini->cri_key);
574    }
575
576    if (macini) {
577        ses->ses_mlen = macini->cri_mlen;
578        if (ses->ses_mlen == 0) {
579            if (macini->cri_alg == CRYPTO_MD5_HMAC)
580                ses->ses_mlen = MD5_HASH_LEN;
581            else
582                ses->ses_mlen = SHA1_HASH_LEN;
583        }
584
585        if (macini->cri_key != NULL) {
586            safe_setup_mackey(ses, macini->cri_alg, macini->cri_key,
587                macini->cri_klen / 8);
588        }
589    }
590
591    *sidp = SAFE_SID(device_get_unit(sc->sc_dev), sesn);
592    return (0);
593}
594
595/*
596 * Deallocate a session.
597 */
598static int
599safe_freesession(device_t dev, u_int64_t tid)
600{
601    struct safe_softc *sc = device_get_softc(dev);
602    int session, ret;
603    u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
604
605    DPRINTF(("%s()\n", __FUNCTION__));
606
607    if (sc == NULL)
608        return (EINVAL);
609
610    session = SAFE_SESSION(sid);
611    if (session < sc->sc_nsessions) {
612        bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
613        ret = 0;
614    } else
615        ret = EINVAL;
616    return (ret);
617}
618
619
620static int
621safe_process(device_t dev, struct cryptop *crp, int hint)
622{
623    struct safe_softc *sc = device_get_softc(dev);
624    int err = 0, i, nicealign, uniform;
625    struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
626    int bypass, oplen, ivsize;
627    caddr_t iv;
628    int16_t coffset;
629    struct safe_session *ses;
630    struct safe_ringentry *re;
631    struct safe_sarec *sa;
632    struct safe_pdesc *pd;
633    u_int32_t cmd0, cmd1, staterec;
634    unsigned long flags;
635
636    DPRINTF(("%s()\n", __FUNCTION__));
637
638    if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
639        safestats.st_invalid++;
640        return (EINVAL);
641    }
642    if (SAFE_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
643        safestats.st_badsession++;
644        return (EINVAL);
645    }
646
647    spin_lock_irqsave(&sc->sc_ringmtx, flags);
648    if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
649        safestats.st_ringfull++;
650        sc->sc_needwakeup |= CRYPTO_SYMQ;
651        spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
652        return (ERESTART);
653    }
654    re = sc->sc_front;
655
656    staterec = re->re_sa.sa_staterec; /* save */
657    /* NB: zero everything but the PE descriptor */
658    bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
659    re->re_sa.sa_staterec = staterec; /* restore */
660
661    re->re_crp = crp;
662    re->re_sesn = SAFE_SESSION(crp->crp_sid);
663
664    re->re_src.nsegs = 0;
665    re->re_dst.nsegs = 0;
666
667    if (crp->crp_flags & CRYPTO_F_SKBUF) {
668        re->re_src_skb = (struct sk_buff *)crp->crp_buf;
669        re->re_dst_skb = (struct sk_buff *)crp->crp_buf;
670    } else if (crp->crp_flags & CRYPTO_F_IOV) {
671        re->re_src_io = (struct uio *)crp->crp_buf;
672        re->re_dst_io = (struct uio *)crp->crp_buf;
673    } else {
674        safestats.st_badflags++;
675        err = EINVAL;
676        goto errout; /* XXX we don't handle contiguous blocks! */
677    }
678
679    sa = &re->re_sa;
680    ses = &sc->sc_sessions[re->re_sesn];
681
682    crd1 = crp->crp_desc;
683    if (crd1 == NULL) {
684        safestats.st_nodesc++;
685        err = EINVAL;
686        goto errout;
687    }
688    crd2 = crd1->crd_next;
689
690    cmd0 = SAFE_SA_CMD0_BASIC; /* basic group operation */
691    cmd1 = 0;
692    if (crd2 == NULL) {
693        if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
694            crd1->crd_alg == CRYPTO_SHA1_HMAC ||
695            crd1->crd_alg == CRYPTO_NULL_HMAC) {
696            maccrd = crd1;
697            enccrd = NULL;
698            cmd0 |= SAFE_SA_CMD0_OP_HASH;
699        } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
700            crd1->crd_alg == CRYPTO_3DES_CBC ||
701            crd1->crd_alg == CRYPTO_AES_CBC ||
702            crd1->crd_alg == CRYPTO_NULL_CBC) {
703            maccrd = NULL;
704            enccrd = crd1;
705            cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
706        } else {
707            safestats.st_badalg++;
708            err = EINVAL;
709            goto errout;
710        }
711    } else {
712        if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
713            crd1->crd_alg == CRYPTO_SHA1_HMAC ||
714            crd1->crd_alg == CRYPTO_NULL_HMAC) &&
715            (crd2->crd_alg == CRYPTO_DES_CBC ||
716            crd2->crd_alg == CRYPTO_3DES_CBC ||
717                crd2->crd_alg == CRYPTO_AES_CBC ||
718                crd2->crd_alg == CRYPTO_NULL_CBC) &&
719            ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
720            maccrd = crd1;
721            enccrd = crd2;
722        } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
723            crd1->crd_alg == CRYPTO_3DES_CBC ||
724            crd1->crd_alg == CRYPTO_AES_CBC ||
725            crd1->crd_alg == CRYPTO_NULL_CBC) &&
726            (crd2->crd_alg == CRYPTO_MD5_HMAC ||
727            crd2->crd_alg == CRYPTO_SHA1_HMAC ||
728            crd2->crd_alg == CRYPTO_NULL_HMAC) &&
729            (crd1->crd_flags & CRD_F_ENCRYPT)) {
730            enccrd = crd1;
731            maccrd = crd2;
732        } else {
733            safestats.st_badalg++;
734            err = EINVAL;
735            goto errout;
736        }
737        cmd0 |= SAFE_SA_CMD0_OP_BOTH;
738    }
739
740    if (enccrd) {
741        if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
742            safe_setup_enckey(ses, enccrd->crd_key);
743
744        if (enccrd->crd_alg == CRYPTO_DES_CBC) {
745            cmd0 |= SAFE_SA_CMD0_DES;
746            cmd1 |= SAFE_SA_CMD1_CBC;
747            ivsize = 2*sizeof(u_int32_t);
748        } else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
749            cmd0 |= SAFE_SA_CMD0_3DES;
750            cmd1 |= SAFE_SA_CMD1_CBC;
751            ivsize = 2*sizeof(u_int32_t);
752        } else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
753            cmd0 |= SAFE_SA_CMD0_AES;
754            cmd1 |= SAFE_SA_CMD1_CBC;
755            if (ses->ses_klen == 128)
756                 cmd1 |= SAFE_SA_CMD1_AES128;
757            else if (ses->ses_klen == 192)
758                 cmd1 |= SAFE_SA_CMD1_AES192;
759            else
760                 cmd1 |= SAFE_SA_CMD1_AES256;
761            ivsize = 4*sizeof(u_int32_t);
762        } else {
763            cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
764            ivsize = 0;
765        }
766
767        /*
768         * Setup encrypt/decrypt state. When using basic ops
769         * we can't use an inline IV because hash/crypt offset
770         * must be from the end of the IV to the start of the
771         * crypt data and this leaves out the preceding header
772         * from the hash calculation. Instead we place the IV
773         * in the state record and set the hash/crypt offset to
774         * copy both the header+IV.
775         */
776        if (enccrd->crd_flags & CRD_F_ENCRYPT) {
777            cmd0 |= SAFE_SA_CMD0_OUTBOUND;
778
779            if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
780                iv = enccrd->crd_iv;
781            else
782                iv = (caddr_t) ses->ses_iv;
783            if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
784                crypto_copyback(crp->crp_flags, crp->crp_buf,
785                    enccrd->crd_inject, ivsize, iv);
786            }
787            bcopy(iv, re->re_sastate.sa_saved_iv, ivsize);
788            /* make iv LE */
789            for (i = 0; i < ivsize/sizeof(re->re_sastate.sa_saved_iv[0]); i++)
790                re->re_sastate.sa_saved_iv[i] =
791                    cpu_to_le32(re->re_sastate.sa_saved_iv[i]);
792            cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
793            re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
794        } else {
795            cmd0 |= SAFE_SA_CMD0_INBOUND;
796
797            if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
798                bcopy(enccrd->crd_iv,
799                    re->re_sastate.sa_saved_iv, ivsize);
800            } else {
801                crypto_copydata(crp->crp_flags, crp->crp_buf,
802                    enccrd->crd_inject, ivsize,
803                    (caddr_t)re->re_sastate.sa_saved_iv);
804            }
805            /* make iv LE */
806            for (i = 0; i < ivsize/sizeof(re->re_sastate.sa_saved_iv[0]); i++)
807                re->re_sastate.sa_saved_iv[i] =
808                    cpu_to_le32(re->re_sastate.sa_saved_iv[i]);
809            cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
810        }
811        /*
812         * For basic encryption use the zero pad algorithm.
813         * This pads results to an 8-byte boundary and
814         * suppresses padding verification for inbound (i.e.
815         * decrypt) operations.
816         *
817         * NB: Not sure if the 8-byte pad boundary is a problem.
818         */
819        cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
820
821        /* XXX assert key bufs have the same size */
822        bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
823    }
824
825    if (maccrd) {
826        if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
827            safe_setup_mackey(ses, maccrd->crd_alg,
828                maccrd->crd_key, maccrd->crd_klen / 8);
829        }
830
831        if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
832            cmd0 |= SAFE_SA_CMD0_MD5;
833            cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
834        } else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
835            cmd0 |= SAFE_SA_CMD0_SHA1;
836            cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
837        } else {
838            cmd0 |= SAFE_SA_CMD0_HASH_NULL;
839        }
840        /*
841         * Digest data is loaded from the SA and the hash
842         * result is saved to the state block where we
843         * retrieve it for return to the caller.
844         */
845        /* XXX assert digest bufs have the same size */
846        bcopy(ses->ses_hminner, sa->sa_indigest,
847            sizeof(sa->sa_indigest));
848        bcopy(ses->ses_hmouter, sa->sa_outdigest,
849            sizeof(sa->sa_outdigest));
850
851        cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
852        re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
853    }
854
855    if (enccrd && maccrd) {
856        /*
857         * The offset from hash data to the start of
858         * crypt data is the difference in the skips.
859         */
860        bypass = maccrd->crd_skip;
861        coffset = enccrd->crd_skip - maccrd->crd_skip;
862        if (coffset < 0) {
863            DPRINTF(("%s: hash does not precede crypt; "
864                "mac skip %u enc skip %u\n",
865                __func__, maccrd->crd_skip, enccrd->crd_skip));
866            safestats.st_skipmismatch++;
867            err = EINVAL;
868            goto errout;
869        }
870        oplen = enccrd->crd_skip + enccrd->crd_len;
871        if (maccrd->crd_skip + maccrd->crd_len != oplen) {
872            DPRINTF(("%s: hash amount %u != crypt amount %u\n",
873                __func__, maccrd->crd_skip + maccrd->crd_len,
874                oplen));
875            safestats.st_lenmismatch++;
876            err = EINVAL;
877            goto errout;
878        }
879#ifdef SAFE_DEBUG
880        if (debug) {
881            printf("mac: skip %d, len %d, inject %d\n",
882                maccrd->crd_skip, maccrd->crd_len,
883                maccrd->crd_inject);
884            printf("enc: skip %d, len %d, inject %d\n",
885                enccrd->crd_skip, enccrd->crd_len,
886                enccrd->crd_inject);
887            printf("bypass %d coffset %d oplen %d\n",
888                bypass, coffset, oplen);
889        }
890#endif
891        if (coffset & 3) { /* offset must be 32-bit aligned */
892            DPRINTF(("%s: coffset %u misaligned\n",
893                __func__, coffset));
894            safestats.st_coffmisaligned++;
895            err = EINVAL;
896            goto errout;
897        }
898        coffset >>= 2;
899        if (coffset > 255) { /* offset must be <256 dwords */
900            DPRINTF(("%s: coffset %u too big\n",
901                __func__, coffset));
902            safestats.st_cofftoobig++;
903            err = EINVAL;
904            goto errout;
905        }
906        /*
907         * Tell the hardware to copy the header to the output.
908         * The header is defined as the data from the end of
909         * the bypass to the start of data to be encrypted.
910         * Typically this is the inline IV. Note that you need
911         * to do this even if src+dst are the same; it appears
912         * that w/o this bit the crypted data is written
913         * immediately after the bypass data.
914         */
915        cmd1 |= SAFE_SA_CMD1_HDRCOPY;
916        /*
917         * Disable IP header mutable bit handling. This is
918         * needed to get correct HMAC calculations.
919         */
920        cmd1 |= SAFE_SA_CMD1_MUTABLE;
921    } else {
922        if (enccrd) {
923            bypass = enccrd->crd_skip;
924            oplen = bypass + enccrd->crd_len;
925        } else {
926            bypass = maccrd->crd_skip;
927            oplen = bypass + maccrd->crd_len;
928        }
929        coffset = 0;
930    }
931    /* XXX verify multiple of 4 when using s/g */
932    if (bypass > 96) { /* bypass offset must be <= 96 bytes */
933        DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
934        safestats.st_bypasstoobig++;
935        err = EINVAL;
936        goto errout;
937    }
938
939    if (crp->crp_flags & CRYPTO_F_SKBUF) {
940        if (pci_map_skb(sc, &re->re_src, re->re_src_skb)) {
941            safestats.st_noload++;
942            err = ENOMEM;
943            goto errout;
944        }
945    } else if (crp->crp_flags & CRYPTO_F_IOV) {
946        if (pci_map_uio(sc, &re->re_src, re->re_src_io)) {
947            safestats.st_noload++;
948            err = ENOMEM;
949            goto errout;
950        }
951    }
952    nicealign = safe_dmamap_aligned(sc, &re->re_src);
953    uniform = safe_dmamap_uniform(sc, &re->re_src);
954
955    DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
956        nicealign, uniform, re->re_src.nsegs));
957    if (re->re_src.nsegs > 1) {
958        re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
959            ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
960        for (i = 0; i < re->re_src_nsegs; i++) {
961            /* NB: no need to check if there's space */
962            pd = sc->sc_spfree;
963            if (++(sc->sc_spfree) == sc->sc_springtop)
964                sc->sc_spfree = sc->sc_spring;
965
966            KASSERT((pd->pd_flags&3) == 0 ||
967                (pd->pd_flags&3) == SAFE_PD_DONE,
968                ("bogus source particle descriptor; flags %x",
969                pd->pd_flags));
970            pd->pd_addr = re->re_src_segs[i].ds_addr;
971            pd->pd_size = re->re_src_segs[i].ds_len;
972            pd->pd_flags = SAFE_PD_READY;
973        }
974        cmd0 |= SAFE_SA_CMD0_IGATHER;
975    } else {
976        /*
977         * No need for gather, reference the operand directly.
978         */
979        re->re_desc.d_src = re->re_src_segs[0].ds_addr;
980    }
981
982    if (enccrd == NULL && maccrd != NULL) {
983        /*
984         * Hash op; no destination needed.
985         */
986    } else {
987        if (crp->crp_flags & (CRYPTO_F_IOV|CRYPTO_F_SKBUF)) {
988            if (!nicealign) {
989                safestats.st_iovmisaligned++;
990                err = EINVAL;
991                goto errout;
992            }
993            if (uniform != 1) {
994                device_printf(sc->sc_dev, "!uniform source\n");
995                if (!uniform) {
996                    /*
997                     * There's no way to handle the DMA
998                     * requirements with this uio. We
999                     * could create a separate DMA area for
1000                     * the result and then copy it back,
1001                     * but for now we just bail and return
1002                     * an error. Note that uio requests
1003                     * > SAFE_MAX_DSIZE are handled because
1004                     * the DMA map and segment list for the
1005                     * destination wil result in a
1006                     * destination particle list that does
1007                     * the necessary scatter DMA.
1008                     */
1009                    safestats.st_iovnotuniform++;
1010                    err = EINVAL;
1011                    goto errout;
1012                }
1013            } else
1014                re->re_dst = re->re_src;
1015        } else {
1016            safestats.st_badflags++;
1017            err = EINVAL;
1018            goto errout;
1019        }
1020
1021        if (re->re_dst.nsegs > 1) {
1022            re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
1023                ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
1024            for (i = 0; i < re->re_dst_nsegs; i++) {
1025                pd = sc->sc_dpfree;
1026                KASSERT((pd->pd_flags&3) == 0 ||
1027                    (pd->pd_flags&3) == SAFE_PD_DONE,
1028                    ("bogus dest particle descriptor; flags %x",
1029                        pd->pd_flags));
1030                if (++(sc->sc_dpfree) == sc->sc_dpringtop)
1031                    sc->sc_dpfree = sc->sc_dpring;
1032                pd->pd_addr = re->re_dst_segs[i].ds_addr;
1033                pd->pd_flags = SAFE_PD_READY;
1034            }
1035            cmd0 |= SAFE_SA_CMD0_OSCATTER;
1036        } else {
1037            /*
1038             * No need for scatter, reference the operand directly.
1039             */
1040            re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
1041        }
1042    }
1043
1044    /*
1045     * All done with setup; fillin the SA command words
1046     * and the packet engine descriptor. The operation
1047     * is now ready for submission to the hardware.
1048     */
1049    sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
1050    sa->sa_cmd1 = cmd1
1051            | (coffset << SAFE_SA_CMD1_OFFSET_S)
1052            | SAFE_SA_CMD1_SAREV1 /* Rev 1 SA data structure */
1053            | SAFE_SA_CMD1_SRPCI
1054            ;
1055    /*
1056     * NB: the order of writes is important here. In case the
1057     * chip is scanning the ring because of an outstanding request
1058     * it might nab this one too. In that case we need to make
1059     * sure the setup is complete before we write the length
1060     * field of the descriptor as it signals the descriptor is
1061     * ready for processing.
1062     */
1063    re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
1064    if (maccrd)
1065        re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
1066    wmb();
1067    re->re_desc.d_len = oplen
1068              | SAFE_PE_LEN_READY
1069              | (bypass << SAFE_PE_LEN_BYPASS_S)
1070              ;
1071
1072    safestats.st_ipackets++;
1073    safestats.st_ibytes += oplen;
1074
1075    if (++(sc->sc_front) == sc->sc_ringtop)
1076        sc->sc_front = sc->sc_ring;
1077
1078    /* XXX honor batching */
1079    safe_feed(sc, re);
1080    spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1081    return (0);
1082
1083errout:
1084    if (re->re_src.map != re->re_dst.map)
1085        pci_unmap_operand(sc, &re->re_dst);
1086    if (re->re_src.map)
1087        pci_unmap_operand(sc, &re->re_src);
1088    spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1089    if (err != ERESTART) {
1090        crp->crp_etype = err;
1091        crypto_done(crp);
1092    } else {
1093        sc->sc_needwakeup |= CRYPTO_SYMQ;
1094    }
1095    return (err);
1096}
1097
1098static void
1099safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
1100{
1101    struct cryptop *crp = (struct cryptop *)re->re_crp;
1102    struct cryptodesc *crd;
1103
1104    DPRINTF(("%s()\n", __FUNCTION__));
1105
1106    safestats.st_opackets++;
1107    safestats.st_obytes += re->re_dst.mapsize;
1108
1109    if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
1110        device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n",
1111            re->re_desc.d_csr,
1112            re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
1113        safestats.st_peoperr++;
1114        crp->crp_etype = EIO; /* something more meaningful? */
1115    }
1116
1117    if (re->re_dst.map != NULL && re->re_dst.map != re->re_src.map)
1118        pci_unmap_operand(sc, &re->re_dst);
1119    pci_unmap_operand(sc, &re->re_src);
1120
1121    /*
1122     * If result was written to a differet mbuf chain, swap
1123     * it in as the return value and reclaim the original.
1124     */
1125    if ((crp->crp_flags & CRYPTO_F_SKBUF) && re->re_src_skb != re->re_dst_skb) {
1126        device_printf(sc->sc_dev, "no CRYPTO_F_SKBUF swapping support\n");
1127        /* kfree_skb(skb) */
1128        /* crp->crp_buf = (caddr_t)re->re_dst_skb */
1129        return;
1130    }
1131
1132    if (re->re_flags & SAFE_QFLAGS_COPYOUTIV) {
1133        /* copy out IV for future use */
1134        for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1135            int i;
1136            int ivsize;
1137
1138            if (crd->crd_alg == CRYPTO_DES_CBC ||
1139                crd->crd_alg == CRYPTO_3DES_CBC) {
1140                ivsize = 2*sizeof(u_int32_t);
1141            } else if (crd->crd_alg == CRYPTO_AES_CBC) {
1142                ivsize = 4*sizeof(u_int32_t);
1143            } else
1144                continue;
1145            crypto_copydata(crp->crp_flags, crp->crp_buf,
1146                crd->crd_skip + crd->crd_len - ivsize, ivsize,
1147                (caddr_t)sc->sc_sessions[re->re_sesn].ses_iv);
1148            for (i = 0;
1149                    i < ivsize/sizeof(sc->sc_sessions[re->re_sesn].ses_iv[0]);
1150                    i++)
1151                sc->sc_sessions[re->re_sesn].ses_iv[i] =
1152                    cpu_to_le32(sc->sc_sessions[re->re_sesn].ses_iv[i]);
1153            break;
1154        }
1155    }
1156
1157    if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
1158        /* copy out ICV result */
1159        for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1160            if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
1161                crd->crd_alg == CRYPTO_SHA1_HMAC ||
1162                crd->crd_alg == CRYPTO_NULL_HMAC))
1163                continue;
1164            if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
1165                /*
1166                 * SHA-1 ICV's are byte-swapped; fix 'em up
1167                 * before copy them to their destination.
1168                 */
1169                re->re_sastate.sa_saved_indigest[0] =
1170                    cpu_to_be32(re->re_sastate.sa_saved_indigest[0]);
1171                re->re_sastate.sa_saved_indigest[1] =
1172                    cpu_to_be32(re->re_sastate.sa_saved_indigest[1]);
1173                re->re_sastate.sa_saved_indigest[2] =
1174                    cpu_to_be32(re->re_sastate.sa_saved_indigest[2]);
1175            } else {
1176                re->re_sastate.sa_saved_indigest[0] =
1177                    cpu_to_le32(re->re_sastate.sa_saved_indigest[0]);
1178                re->re_sastate.sa_saved_indigest[1] =
1179                    cpu_to_le32(re->re_sastate.sa_saved_indigest[1]);
1180                re->re_sastate.sa_saved_indigest[2] =
1181                    cpu_to_le32(re->re_sastate.sa_saved_indigest[2]);
1182            }
1183            crypto_copyback(crp->crp_flags, crp->crp_buf,
1184                crd->crd_inject,
1185                sc->sc_sessions[re->re_sesn].ses_mlen,
1186                (caddr_t)re->re_sastate.sa_saved_indigest);
1187            break;
1188        }
1189    }
1190    crypto_done(crp);
1191}
1192
1193
1194#if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
1195#define SAFE_RNG_MAXWAIT 1000
1196
1197static void
1198safe_rng_init(struct safe_softc *sc)
1199{
1200    u_int32_t w, v;
1201    int i;
1202
1203    DPRINTF(("%s()\n", __FUNCTION__));
1204
1205    WRITE_REG(sc, SAFE_RNG_CTRL, 0);
1206    /* use default value according to the manual */
1207    WRITE_REG(sc, SAFE_RNG_CNFG, 0x834); /* magic from SafeNet */
1208    WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1209
1210    /*
1211     * There is a bug in rev 1.0 of the 1140 that when the RNG
1212     * is brought out of reset the ready status flag does not
1213     * work until the RNG has finished its internal initialization.
1214     *
1215     * So in order to determine the device is through its
1216     * initialization we must read the data register, using the
1217     * status reg in the read in case it is initialized. Then read
1218     * the data register until it changes from the first read.
1219     * Once it changes read the data register until it changes
1220     * again. At this time the RNG is considered initialized.
1221     * This could take between 750ms - 1000ms in time.
1222     */
1223    i = 0;
1224    w = READ_REG(sc, SAFE_RNG_OUT);
1225    do {
1226        v = READ_REG(sc, SAFE_RNG_OUT);
1227        if (v != w) {
1228            w = v;
1229            break;
1230        }
1231        DELAY(10);
1232    } while (++i < SAFE_RNG_MAXWAIT);
1233
1234    /* Wait Until data changes again */
1235    i = 0;
1236    do {
1237        v = READ_REG(sc, SAFE_RNG_OUT);
1238        if (v != w)
1239            break;
1240        DELAY(10);
1241    } while (++i < SAFE_RNG_MAXWAIT);
1242}
1243
1244static __inline void
1245safe_rng_disable_short_cycle(struct safe_softc *sc)
1246{
1247    DPRINTF(("%s()\n", __FUNCTION__));
1248
1249    WRITE_REG(sc, SAFE_RNG_CTRL,
1250        READ_REG(sc, SAFE_RNG_CTRL) &~ SAFE_RNG_CTRL_SHORTEN);
1251}
1252
1253static __inline void
1254safe_rng_enable_short_cycle(struct safe_softc *sc)
1255{
1256    DPRINTF(("%s()\n", __FUNCTION__));
1257
1258    WRITE_REG(sc, SAFE_RNG_CTRL,
1259        READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
1260}
1261
1262static __inline u_int32_t
1263safe_rng_read(struct safe_softc *sc)
1264{
1265    int i;
1266
1267    i = 0;
1268    while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
1269        ;
1270    return READ_REG(sc, SAFE_RNG_OUT);
1271}
1272
1273static int
1274safe_read_random(void *arg, u_int32_t *buf, int maxwords)
1275{
1276    struct safe_softc *sc = (struct safe_softc *) arg;
1277    int i, rc;
1278
1279    DPRINTF(("%s()\n", __FUNCTION__));
1280    
1281    safestats.st_rng++;
1282    /*
1283     * Fetch the next block of data.
1284     */
1285    if (maxwords > safe_rngbufsize)
1286        maxwords = safe_rngbufsize;
1287    if (maxwords > SAFE_RNG_MAXBUFSIZ)
1288        maxwords = SAFE_RNG_MAXBUFSIZ;
1289retry:
1290    /* read as much as we can */
1291    for (rc = 0; rc < maxwords; rc++) {
1292        if (READ_REG(sc, SAFE_RNG_STAT) != 0)
1293            break;
1294        buf[rc] = READ_REG(sc, SAFE_RNG_OUT);
1295    }
1296    if (rc == 0)
1297        return 0;
1298    /*
1299     * Check the comparator alarm count and reset the h/w if
1300     * it exceeds our threshold. This guards against the
1301     * hardware oscillators resonating with external signals.
1302     */
1303    if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
1304        u_int32_t freq_inc, w;
1305
1306        DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
1307            (unsigned)READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
1308        safestats.st_rngalarm++;
1309        safe_rng_enable_short_cycle(sc);
1310        freq_inc = 18;
1311        for (i = 0; i < 64; i++) {
1312            w = READ_REG(sc, SAFE_RNG_CNFG);
1313            freq_inc = ((w + freq_inc) & 0x3fL);
1314            w = ((w & ~0x3fL) | freq_inc);
1315            WRITE_REG(sc, SAFE_RNG_CNFG, w);
1316
1317            WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1318
1319            (void) safe_rng_read(sc);
1320            DELAY(25);
1321
1322            if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
1323                safe_rng_disable_short_cycle(sc);
1324                goto retry;
1325            }
1326            freq_inc = 1;
1327        }
1328        safe_rng_disable_short_cycle(sc);
1329    } else
1330        WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1331
1332    return(rc);
1333}
1334#endif /* defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG) */
1335
1336
1337/*
1338 * Resets the board. Values in the regesters are left as is
1339 * from the reset (i.e. initial values are assigned elsewhere).
1340 */
1341static void
1342safe_reset_board(struct safe_softc *sc)
1343{
1344    u_int32_t v;
1345    /*
1346     * Reset the device. The manual says no delay
1347     * is needed between marking and clearing reset.
1348     */
1349    DPRINTF(("%s()\n", __FUNCTION__));
1350
1351    v = READ_REG(sc, SAFE_PE_DMACFG) &~
1352        (SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
1353         SAFE_PE_DMACFG_SGRESET);
1354    WRITE_REG(sc, SAFE_PE_DMACFG, v
1355                    | SAFE_PE_DMACFG_PERESET
1356                    | SAFE_PE_DMACFG_PDRRESET
1357                    | SAFE_PE_DMACFG_SGRESET);
1358    WRITE_REG(sc, SAFE_PE_DMACFG, v);
1359}
1360
1361/*
1362 * Initialize registers we need to touch only once.
1363 */
1364static void
1365safe_init_board(struct safe_softc *sc)
1366{
1367    u_int32_t v, dwords;
1368
1369    DPRINTF(("%s()\n", __FUNCTION__));
1370
1371    v = READ_REG(sc, SAFE_PE_DMACFG);
1372    v &=~ ( SAFE_PE_DMACFG_PEMODE
1373            | SAFE_PE_DMACFG_FSENA /* failsafe enable */
1374            | SAFE_PE_DMACFG_GPRPCI /* gather ring on PCI */
1375            | SAFE_PE_DMACFG_SPRPCI /* scatter ring on PCI */
1376            | SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */
1377            | SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */
1378            | SAFE_PE_DMACFG_ESSA /* endian-swap SA's */
1379            | SAFE_PE_DMACFG_ESPACKET /* swap the packet data */
1380          );
1381    v |= SAFE_PE_DMACFG_FSENA /* failsafe enable */
1382      | SAFE_PE_DMACFG_GPRPCI /* gather ring on PCI */
1383      | SAFE_PE_DMACFG_SPRPCI /* scatter ring on PCI */
1384      | SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */
1385      | SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */
1386      | SAFE_PE_DMACFG_ESSA /* endian-swap SA's */
1387#if 0
1388      | SAFE_PE_DMACFG_ESPACKET /* swap the packet data */
1389#endif
1390      ;
1391    WRITE_REG(sc, SAFE_PE_DMACFG, v);
1392
1393#ifdef __BIG_ENDIAN
1394    /* tell the safenet that we are 4321 and not 1234 */
1395    WRITE_REG(sc, SAFE_ENDIAN, 0xe4e41b1b);
1396#endif
1397
1398    if (sc->sc_chiprev == SAFE_REV(1,0)) {
1399        /*
1400         * Avoid large PCI DMA transfers. Rev 1.0 has a bug where
1401         * "target mode transfers" done while the chip is DMA'ing
1402         * >1020 bytes cause the hardware to lockup. To avoid this
1403         * we reduce the max PCI transfer size and use small source
1404         * particle descriptors (<= 256 bytes).
1405         */
1406        WRITE_REG(sc, SAFE_DMA_CFG, 256);
1407        device_printf(sc->sc_dev,
1408            "Reduce max DMA size to %u words for rev %u.%u WAR\n",
1409            (unsigned) ((READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff),
1410            (unsigned) SAFE_REV_MAJ(sc->sc_chiprev),
1411            (unsigned) SAFE_REV_MIN(sc->sc_chiprev));
1412        sc->sc_max_dsize = 256;
1413    } else {
1414        sc->sc_max_dsize = SAFE_MAX_DSIZE;
1415    }
1416
1417    /* NB: operands+results are overlaid */
1418    WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
1419    WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
1420    /*
1421     * Configure ring entry size and number of items in the ring.
1422     */
1423    KASSERT((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
1424        ("PE ring entry not 32-bit aligned!"));
1425    dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
1426    WRITE_REG(sc, SAFE_PE_RINGCFG,
1427        (dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
1428    WRITE_REG(sc, SAFE_PE_RINGPOLL, 0); /* disable polling */
1429
1430    WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
1431    WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
1432    WRITE_REG(sc, SAFE_PE_PARTSIZE,
1433        (SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
1434    /*
1435     * NB: destination particles are fixed size. We use
1436     * an mbuf cluster and require all results go to
1437     * clusters or smaller.
1438     */
1439    WRITE_REG(sc, SAFE_PE_PARTCFG, sc->sc_max_dsize);
1440
1441    /* it's now safe to enable PE mode, do it */
1442    WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
1443
1444    /*
1445     * Configure hardware to use level-triggered interrupts and
1446     * to interrupt after each descriptor is processed.
1447     */
1448    WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
1449    WRITE_REG(sc, SAFE_HI_CLR, 0xffffffff);
1450    WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
1451    WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
1452}
1453
1454
1455/*
1456 * Clean up after a chip crash.
1457 * It is assumed that the caller in splimp()
1458 */
1459static void
1460safe_cleanchip(struct safe_softc *sc)
1461{
1462    DPRINTF(("%s()\n", __FUNCTION__));
1463
1464    if (sc->sc_nqchip != 0) {
1465        struct safe_ringentry *re = sc->sc_back;
1466
1467        while (re != sc->sc_front) {
1468            if (re->re_desc.d_csr != 0)
1469                safe_free_entry(sc, re);
1470            if (++re == sc->sc_ringtop)
1471                re = sc->sc_ring;
1472        }
1473        sc->sc_back = re;
1474        sc->sc_nqchip = 0;
1475    }
1476}
1477
1478/*
1479 * free a safe_q
1480 * It is assumed that the caller is within splimp().
1481 */
1482static int
1483safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
1484{
1485    struct cryptop *crp;
1486
1487    DPRINTF(("%s()\n", __FUNCTION__));
1488
1489    /*
1490     * Free header MCR
1491     */
1492    if ((re->re_dst_skb != NULL) && (re->re_src_skb != re->re_dst_skb))
1493#ifdef NOTYET
1494        m_freem(re->re_dst_m);
1495#else
1496        printk("%s,%d: SKB not supported\n", __FILE__, __LINE__);
1497#endif
1498
1499    crp = (struct cryptop *)re->re_crp;
1500    
1501    re->re_desc.d_csr = 0;
1502    
1503    crp->crp_etype = EFAULT;
1504    crypto_done(crp);
1505    return(0);
1506}
1507
1508/*
1509 * Routine to reset the chip and clean up.
1510 * It is assumed that the caller is in splimp()
1511 */
1512static void
1513safe_totalreset(struct safe_softc *sc)
1514{
1515    DPRINTF(("%s()\n", __FUNCTION__));
1516
1517    safe_reset_board(sc);
1518    safe_init_board(sc);
1519    safe_cleanchip(sc);
1520}
1521
1522/*
1523 * Is the operand suitable aligned for direct DMA. Each
1524 * segment must be aligned on a 32-bit boundary and all
1525 * but the last segment must be a multiple of 4 bytes.
1526 */
1527static int
1528safe_dmamap_aligned(struct safe_softc *sc, const struct safe_operand *op)
1529{
1530    int i;
1531
1532    DPRINTF(("%s()\n", __FUNCTION__));
1533
1534    for (i = 0; i < op->nsegs; i++) {
1535        if (op->segs[i].ds_addr & 3)
1536            return (0);
1537        if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3))
1538            return (0);
1539    }
1540    return (1);
1541}
1542
1543/*
1544 * Is the operand suitable for direct DMA as the destination
1545 * of an operation. The hardware requires that each ``particle''
1546 * but the last in an operation result have the same size. We
1547 * fix that size at SAFE_MAX_DSIZE bytes. This routine returns
1548 * 0 if some segment is not a multiple of of this size, 1 if all
1549 * segments are exactly this size, or 2 if segments are at worst
1550 * a multple of this size.
1551 */
1552static int
1553safe_dmamap_uniform(struct safe_softc *sc, const struct safe_operand *op)
1554{
1555    int result = 1;
1556
1557    DPRINTF(("%s()\n", __FUNCTION__));
1558
1559    if (op->nsegs > 0) {
1560        int i;
1561
1562        for (i = 0; i < op->nsegs-1; i++) {
1563            if (op->segs[i].ds_len % sc->sc_max_dsize)
1564                return (0);
1565            if (op->segs[i].ds_len != sc->sc_max_dsize)
1566                result = 2;
1567        }
1568    }
1569    return (result);
1570}
1571
1572static int
1573safe_kprocess(device_t dev, struct cryptkop *krp, int hint)
1574{
1575    struct safe_softc *sc = device_get_softc(dev);
1576    struct safe_pkq *q;
1577    unsigned long flags;
1578
1579    DPRINTF(("%s()\n", __FUNCTION__));
1580
1581    if (sc == NULL) {
1582        krp->krp_status = EINVAL;
1583        goto err;
1584    }
1585
1586    if (krp->krp_op != CRK_MOD_EXP) {
1587        krp->krp_status = EOPNOTSUPP;
1588        goto err;
1589    }
1590
1591    q = (struct safe_pkq *) kmalloc(sizeof(*q), GFP_KERNEL);
1592    if (q == NULL) {
1593        krp->krp_status = ENOMEM;
1594        goto err;
1595    }
1596    memset(q, 0, sizeof(*q));
1597    q->pkq_krp = krp;
1598    INIT_LIST_HEAD(&q->pkq_list);
1599
1600    spin_lock_irqsave(&sc->sc_pkmtx, flags);
1601    list_add_tail(&q->pkq_list, &sc->sc_pkq);
1602    safe_kfeed(sc);
1603    spin_unlock_irqrestore(&sc->sc_pkmtx, flags);
1604    return (0);
1605
1606err:
1607    crypto_kdone(krp);
1608    return (0);
1609}
1610
1611#define SAFE_CRK_PARAM_BASE 0
1612#define SAFE_CRK_PARAM_EXP 1
1613#define SAFE_CRK_PARAM_MOD 2
1614
1615static int
1616safe_kstart(struct safe_softc *sc)
1617{
1618    struct cryptkop *krp = sc->sc_pkq_cur->pkq_krp;
1619    int exp_bits, mod_bits, base_bits;
1620    u_int32_t op, a_off, b_off, c_off, d_off;
1621
1622    DPRINTF(("%s()\n", __FUNCTION__));
1623
1624    if (krp->krp_iparams < 3 || krp->krp_oparams != 1) {
1625        krp->krp_status = EINVAL;
1626        return (1);
1627    }
1628
1629    base_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_BASE]);
1630    if (base_bits > 2048)
1631        goto too_big;
1632    if (base_bits <= 0) /* 5. base not zero */
1633        goto too_small;
1634
1635    exp_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_EXP]);
1636    if (exp_bits > 2048)
1637        goto too_big;
1638    if (exp_bits <= 0) /* 1. exponent word length > 0 */
1639        goto too_small; /* 4. exponent not zero */
1640
1641    mod_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_MOD]);
1642    if (mod_bits > 2048)
1643        goto too_big;
1644    if (mod_bits <= 32) /* 2. modulus word length > 1 */
1645        goto too_small; /* 8. MSW of modulus != zero */
1646    if (mod_bits < exp_bits) /* 3 modulus len >= exponent len */
1647        goto too_small;
1648    if ((krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p[0] & 1) == 0)
1649        goto bad_domain; /* 6. modulus is odd */
1650    if (mod_bits > krp->krp_param[krp->krp_iparams].crp_nbits)
1651        goto too_small; /* make sure result will fit */
1652
1653    /* 7. modulus > base */
1654    if (mod_bits < base_bits)
1655        goto too_small;
1656    if (mod_bits == base_bits) {
1657        u_int8_t *basep, *modp;
1658        int i;
1659
1660        basep = krp->krp_param[SAFE_CRK_PARAM_BASE].crp_p +
1661            ((base_bits + 7) / 8) - 1;
1662        modp = krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p +
1663            ((mod_bits + 7) / 8) - 1;
1664        
1665        for (i = 0; i < (mod_bits + 7) / 8; i++, basep--, modp--) {
1666            if (*modp < *basep)
1667                goto too_small;
1668            if (*modp > *basep)
1669                break;
1670        }
1671    }
1672
1673    /* And on the 9th step, he rested. */
1674
1675    WRITE_REG(sc, SAFE_PK_A_LEN, (exp_bits + 31) / 32);
1676    WRITE_REG(sc, SAFE_PK_B_LEN, (mod_bits + 31) / 32);
1677    if (mod_bits > 1024) {
1678        op = SAFE_PK_FUNC_EXP4;
1679        a_off = 0x000;
1680        b_off = 0x100;
1681        c_off = 0x200;
1682        d_off = 0x300;
1683    } else {
1684        op = SAFE_PK_FUNC_EXP16;
1685        a_off = 0x000;
1686        b_off = 0x080;
1687        c_off = 0x100;
1688        d_off = 0x180;
1689    }
1690    sc->sc_pk_reslen = b_off - a_off;
1691    sc->sc_pk_resoff = d_off;
1692
1693    /* A is exponent, B is modulus, C is base, D is result */
1694    safe_kload_reg(sc, a_off, b_off - a_off,
1695        &krp->krp_param[SAFE_CRK_PARAM_EXP]);
1696    WRITE_REG(sc, SAFE_PK_A_ADDR, a_off >> 2);
1697    safe_kload_reg(sc, b_off, b_off - a_off,
1698        &krp->krp_param[SAFE_CRK_PARAM_MOD]);
1699    WRITE_REG(sc, SAFE_PK_B_ADDR, b_off >> 2);
1700    safe_kload_reg(sc, c_off, b_off - a_off,
1701        &krp->krp_param[SAFE_CRK_PARAM_BASE]);
1702    WRITE_REG(sc, SAFE_PK_C_ADDR, c_off >> 2);
1703    WRITE_REG(sc, SAFE_PK_D_ADDR, d_off >> 2);
1704
1705    WRITE_REG(sc, SAFE_PK_FUNC, op | SAFE_PK_FUNC_RUN);
1706
1707    return (0);
1708
1709too_big:
1710    krp->krp_status = E2BIG;
1711    return (1);
1712too_small:
1713    krp->krp_status = ERANGE;
1714    return (1);
1715bad_domain:
1716    krp->krp_status = EDOM;
1717    return (1);
1718}
1719
1720static int
1721safe_ksigbits(struct safe_softc *sc, struct crparam *cr)
1722{
1723    u_int plen = (cr->crp_nbits + 7) / 8;
1724    int i, sig = plen * 8;
1725    u_int8_t c, *p = cr->crp_p;
1726
1727    DPRINTF(("%s()\n", __FUNCTION__));
1728
1729    for (i = plen - 1; i >= 0; i--) {
1730        c = p[i];
1731        if (c != 0) {
1732            while ((c & 0x80) == 0) {
1733                sig--;
1734                c <<= 1;
1735            }
1736            break;
1737        }
1738        sig -= 8;
1739    }
1740    return (sig);
1741}
1742
1743static void
1744safe_kfeed(struct safe_softc *sc)
1745{
1746    struct safe_pkq *q, *tmp;
1747
1748    DPRINTF(("%s()\n", __FUNCTION__));
1749
1750    if (list_empty(&sc->sc_pkq) && sc->sc_pkq_cur == NULL)
1751        return;
1752    if (sc->sc_pkq_cur != NULL)
1753        return;
1754    list_for_each_entry_safe(q, tmp, &sc->sc_pkq, pkq_list) {
1755        sc->sc_pkq_cur = q;
1756        list_del(&q->pkq_list);
1757        if (safe_kstart(sc) != 0) {
1758            crypto_kdone(q->pkq_krp);
1759            kfree(q);
1760            sc->sc_pkq_cur = NULL;
1761        } else {
1762            /* op started, start polling */
1763            mod_timer(&sc->sc_pkto, jiffies + 1);
1764            break;
1765        }
1766    }
1767}
1768
1769static void
1770safe_kpoll(unsigned long arg)
1771{
1772    struct safe_softc *sc = NULL;
1773    struct safe_pkq *q;
1774    struct crparam *res;
1775    int i;
1776    u_int32_t buf[64];
1777    unsigned long flags;
1778
1779    DPRINTF(("%s()\n", __FUNCTION__));
1780
1781    if (arg >= SAFE_MAX_CHIPS)
1782        return;
1783    sc = safe_chip_idx[arg];
1784    if (!sc) {
1785        DPRINTF(("%s() - bad callback\n", __FUNCTION__));
1786        return;
1787    }
1788
1789    spin_lock_irqsave(&sc->sc_pkmtx, flags);
1790    if (sc->sc_pkq_cur == NULL)
1791        goto out;
1792    if (READ_REG(sc, SAFE_PK_FUNC) & SAFE_PK_FUNC_RUN) {
1793        /* still running, check back later */
1794        mod_timer(&sc->sc_pkto, jiffies + 1);
1795        goto out;
1796    }
1797
1798    q = sc->sc_pkq_cur;
1799    res = &q->pkq_krp->krp_param[q->pkq_krp->krp_iparams];
1800    bzero(buf, sizeof(buf));
1801    bzero(res->crp_p, (res->crp_nbits + 7) / 8);
1802    for (i = 0; i < sc->sc_pk_reslen >> 2; i++)
1803        buf[i] = le32_to_cpu(READ_REG(sc, SAFE_PK_RAM_START +
1804            sc->sc_pk_resoff + (i << 2)));
1805    bcopy(buf, res->crp_p, (res->crp_nbits + 7) / 8);
1806    /*
1807     * reduce the bits that need copying if possible
1808     */
1809    res->crp_nbits = min(res->crp_nbits,sc->sc_pk_reslen * 8);
1810    res->crp_nbits = safe_ksigbits(sc, res);
1811
1812    for (i = SAFE_PK_RAM_START; i < SAFE_PK_RAM_END; i += 4)
1813        WRITE_REG(sc, i, 0);
1814
1815    crypto_kdone(q->pkq_krp);
1816    kfree(q);
1817    sc->sc_pkq_cur = NULL;
1818
1819    safe_kfeed(sc);
1820out:
1821    spin_unlock_irqrestore(&sc->sc_pkmtx, flags);
1822}
1823
1824static void
1825safe_kload_reg(struct safe_softc *sc, u_int32_t off, u_int32_t len,
1826    struct crparam *n)
1827{
1828    u_int32_t buf[64], i;
1829
1830    DPRINTF(("%s()\n", __FUNCTION__));
1831
1832    bzero(buf, sizeof(buf));
1833    bcopy(n->crp_p, buf, (n->crp_nbits + 7) / 8);
1834
1835    for (i = 0; i < len >> 2; i++)
1836        WRITE_REG(sc, SAFE_PK_RAM_START + off + (i << 2),
1837            cpu_to_le32(buf[i]));
1838}
1839
1840#ifdef SAFE_DEBUG
1841static void
1842safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
1843{
1844    printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n"
1845        , tag
1846        , READ_REG(sc, SAFE_DMA_ENDIAN)
1847        , READ_REG(sc, SAFE_DMA_SRCADDR)
1848        , READ_REG(sc, SAFE_DMA_DSTADDR)
1849        , READ_REG(sc, SAFE_DMA_STAT)
1850    );
1851}
1852
1853static void
1854safe_dump_intrstate(struct safe_softc *sc, const char *tag)
1855{
1856    printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n"
1857        , tag
1858        , READ_REG(sc, SAFE_HI_CFG)
1859        , READ_REG(sc, SAFE_HI_MASK)
1860        , READ_REG(sc, SAFE_HI_DESC_CNT)
1861        , READ_REG(sc, SAFE_HU_STAT)
1862        , READ_REG(sc, SAFE_HM_STAT)
1863    );
1864}
1865
1866static void
1867safe_dump_ringstate(struct safe_softc *sc, const char *tag)
1868{
1869    u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
1870
1871    /* NB: assume caller has lock on ring */
1872    printf("%s: ERNGSTAT %x (next %u) back %lu front %lu\n",
1873        tag,
1874        estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
1875        (unsigned long)(sc->sc_back - sc->sc_ring),
1876        (unsigned long)(sc->sc_front - sc->sc_ring));
1877}
1878
1879static void
1880safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
1881{
1882    int ix, nsegs;
1883
1884    ix = re - sc->sc_ring;
1885    printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n"
1886        , tag
1887        , re, ix
1888        , re->re_desc.d_csr
1889        , re->re_desc.d_src
1890        , re->re_desc.d_dst
1891        , re->re_desc.d_sa
1892        , re->re_desc.d_len
1893    );
1894    if (re->re_src.nsegs > 1) {
1895        ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
1896            sizeof(struct safe_pdesc);
1897        for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {
1898            printf(" spd[%u] %p: %p size %u flags %x"
1899                , ix, &sc->sc_spring[ix]
1900                , (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr
1901                , sc->sc_spring[ix].pd_size
1902                , sc->sc_spring[ix].pd_flags
1903            );
1904            if (sc->sc_spring[ix].pd_size == 0)
1905                printf(" (zero!)");
1906            printf("\n");
1907            if (++ix == SAFE_TOTAL_SPART)
1908                ix = 0;
1909        }
1910    }
1911    if (re->re_dst.nsegs > 1) {
1912        ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
1913            sizeof(struct safe_pdesc);
1914        for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {
1915            printf(" dpd[%u] %p: %p flags %x\n"
1916                , ix, &sc->sc_dpring[ix]
1917                , (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr
1918                , sc->sc_dpring[ix].pd_flags
1919            );
1920            if (++ix == SAFE_TOTAL_DPART)
1921                ix = 0;
1922        }
1923    }
1924    printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
1925        re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
1926    printf("sa: key %x %x %x %x %x %x %x %x\n"
1927        , re->re_sa.sa_key[0]
1928        , re->re_sa.sa_key[1]
1929        , re->re_sa.sa_key[2]
1930        , re->re_sa.sa_key[3]
1931        , re->re_sa.sa_key[4]
1932        , re->re_sa.sa_key[5]
1933        , re->re_sa.sa_key[6]
1934        , re->re_sa.sa_key[7]
1935    );
1936    printf("sa: indigest %x %x %x %x %x\n"
1937        , re->re_sa.sa_indigest[0]
1938        , re->re_sa.sa_indigest[1]
1939        , re->re_sa.sa_indigest[2]
1940        , re->re_sa.sa_indigest[3]
1941        , re->re_sa.sa_indigest[4]
1942    );
1943    printf("sa: outdigest %x %x %x %x %x\n"
1944        , re->re_sa.sa_outdigest[0]
1945        , re->re_sa.sa_outdigest[1]
1946        , re->re_sa.sa_outdigest[2]
1947        , re->re_sa.sa_outdigest[3]
1948        , re->re_sa.sa_outdigest[4]
1949    );
1950    printf("sr: iv %x %x %x %x\n"
1951        , re->re_sastate.sa_saved_iv[0]
1952        , re->re_sastate.sa_saved_iv[1]
1953        , re->re_sastate.sa_saved_iv[2]
1954        , re->re_sastate.sa_saved_iv[3]
1955    );
1956    printf("sr: hashbc %u indigest %x %x %x %x %x\n"
1957        , re->re_sastate.sa_saved_hashbc
1958        , re->re_sastate.sa_saved_indigest[0]
1959        , re->re_sastate.sa_saved_indigest[1]
1960        , re->re_sastate.sa_saved_indigest[2]
1961        , re->re_sastate.sa_saved_indigest[3]
1962        , re->re_sastate.sa_saved_indigest[4]
1963    );
1964}
1965
1966static void
1967safe_dump_ring(struct safe_softc *sc, const char *tag)
1968{
1969    unsigned long flags;
1970
1971    spin_lock_irqsave(&sc->sc_ringmtx, flags);
1972    printf("\nSafeNet Ring State:\n");
1973    safe_dump_intrstate(sc, tag);
1974    safe_dump_dmastatus(sc, tag);
1975    safe_dump_ringstate(sc, tag);
1976    if (sc->sc_nqchip) {
1977        struct safe_ringentry *re = sc->sc_back;
1978        do {
1979            safe_dump_request(sc, tag, re);
1980            if (++re == sc->sc_ringtop)
1981                re = sc->sc_ring;
1982        } while (re != sc->sc_front);
1983    }
1984    spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1985}
1986#endif /* SAFE_DEBUG */
1987
1988
1989static int safe_probe(struct pci_dev *dev, const struct pci_device_id *ent)
1990{
1991    struct safe_softc *sc = NULL;
1992    u32 mem_start, mem_len, cmd;
1993    int i, rc, devinfo;
1994    dma_addr_t raddr;
1995    static int num_chips = 0;
1996
1997    DPRINTF(("%s()\n", __FUNCTION__));
1998
1999    if (pci_enable_device(dev) < 0)
2000        return(-ENODEV);
2001
2002    if (!dev->irq) {
2003        printk("safe: found device with no IRQ assigned. check BIOS settings!");
2004        pci_disable_device(dev);
2005        return(-ENODEV);
2006    }
2007
2008    if (pci_set_mwi(dev)) {
2009        printk("safe: pci_set_mwi failed!");
2010        return(-ENODEV);
2011    }
2012
2013    sc = (struct safe_softc *) kmalloc(sizeof(*sc), GFP_KERNEL);
2014    if (!sc)
2015        return(-ENOMEM);
2016    memset(sc, 0, sizeof(*sc));
2017
2018    softc_device_init(sc, "safe", num_chips, safe_methods);
2019
2020    sc->sc_irq = -1;
2021    sc->sc_cid = -1;
2022    sc->sc_pcidev = dev;
2023    if (num_chips < SAFE_MAX_CHIPS) {
2024        safe_chip_idx[device_get_unit(sc->sc_dev)] = sc;
2025        num_chips++;
2026    }
2027
2028    INIT_LIST_HEAD(&sc->sc_pkq);
2029    spin_lock_init(&sc->sc_pkmtx);
2030
2031    pci_set_drvdata(sc->sc_pcidev, sc);
2032
2033    /* we read its hardware registers as memory */
2034    mem_start = pci_resource_start(sc->sc_pcidev, 0);
2035    mem_len = pci_resource_len(sc->sc_pcidev, 0);
2036
2037    sc->sc_base_addr = (ocf_iomem_t) ioremap(mem_start, mem_len);
2038    if (!sc->sc_base_addr) {
2039        device_printf(sc->sc_dev, "failed to ioremap 0x%x-0x%x\n",
2040                mem_start, mem_start + mem_len - 1);
2041        goto out;
2042    }
2043
2044    /* fix up the bus size */
2045    if (pci_set_dma_mask(sc->sc_pcidev, DMA_32BIT_MASK)) {
2046        device_printf(sc->sc_dev, "No usable DMA configuration, aborting.\n");
2047        goto out;
2048    }
2049    if (pci_set_consistent_dma_mask(sc->sc_pcidev, DMA_32BIT_MASK)) {
2050        device_printf(sc->sc_dev, "No usable consistent DMA configuration, aborting.\n");
2051        goto out;
2052    }
2053
2054    pci_set_master(sc->sc_pcidev);
2055
2056    pci_read_config_dword(sc->sc_pcidev, PCI_COMMAND, &cmd);
2057
2058    if (!(cmd & PCI_COMMAND_MEMORY)) {
2059        device_printf(sc->sc_dev, "failed to enable memory mapping\n");
2060        goto out;
2061    }
2062
2063    if (!(cmd & PCI_COMMAND_MASTER)) {
2064        device_printf(sc->sc_dev, "failed to enable bus mastering\n");
2065        goto out;
2066    }
2067
2068    rc = request_irq(dev->irq, safe_intr, IRQF_SHARED, "safe", sc);
2069    if (rc) {
2070        device_printf(sc->sc_dev, "failed to hook irq %d\n", sc->sc_irq);
2071        goto out;
2072    }
2073    sc->sc_irq = dev->irq;
2074
2075    sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
2076            (SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
2077
2078    /*
2079     * Allocate packet engine descriptors.
2080     */
2081    sc->sc_ringalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2082            SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2083            &sc->sc_ringalloc.dma_paddr);
2084    if (!sc->sc_ringalloc.dma_vaddr) {
2085        device_printf(sc->sc_dev, "cannot allocate PE descriptor ring\n");
2086        goto out;
2087    }
2088
2089    /*
2090     * Hookup the static portion of all our data structures.
2091     */
2092    sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
2093    sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
2094    sc->sc_front = sc->sc_ring;
2095    sc->sc_back = sc->sc_ring;
2096    raddr = sc->sc_ringalloc.dma_paddr;
2097    bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
2098    for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
2099        struct safe_ringentry *re = &sc->sc_ring[i];
2100
2101        re->re_desc.d_sa = raddr +
2102            offsetof(struct safe_ringentry, re_sa);
2103        re->re_sa.sa_staterec = raddr +
2104            offsetof(struct safe_ringentry, re_sastate);
2105
2106        raddr += sizeof (struct safe_ringentry);
2107    }
2108    spin_lock_init(&sc->sc_ringmtx);
2109
2110    /*
2111     * Allocate scatter and gather particle descriptors.
2112     */
2113    sc->sc_spalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2114            SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
2115            &sc->sc_spalloc.dma_paddr);
2116    if (!sc->sc_spalloc.dma_vaddr) {
2117        device_printf(sc->sc_dev, "cannot allocate source particle descriptor ring\n");
2118        goto out;
2119    }
2120    sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
2121    sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
2122    sc->sc_spfree = sc->sc_spring;
2123    bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
2124
2125    sc->sc_dpalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2126            SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2127            &sc->sc_dpalloc.dma_paddr);
2128    if (!sc->sc_dpalloc.dma_vaddr) {
2129        device_printf(sc->sc_dev, "cannot allocate destination particle descriptor ring\n");
2130        goto out;
2131    }
2132    sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
2133    sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
2134    sc->sc_dpfree = sc->sc_dpring;
2135    bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
2136
2137    sc->sc_cid = crypto_get_driverid(softc_get_device(sc), CRYPTOCAP_F_HARDWARE);
2138    if (sc->sc_cid < 0) {
2139        device_printf(sc->sc_dev, "could not get crypto driver id\n");
2140        goto out;
2141    }
2142
2143    printf("%s:", device_get_nameunit(sc->sc_dev));
2144
2145    devinfo = READ_REG(sc, SAFE_DEVINFO);
2146    if (devinfo & SAFE_DEVINFO_RNG) {
2147        sc->sc_flags |= SAFE_FLAGS_RNG;
2148        printf(" rng");
2149    }
2150    if (devinfo & SAFE_DEVINFO_PKEY) {
2151        printf(" key");
2152        sc->sc_flags |= SAFE_FLAGS_KEY;
2153        crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
2154#if 0
2155        crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
2156#endif
2157        init_timer(&sc->sc_pkto);
2158        sc->sc_pkto.function = safe_kpoll;
2159        sc->sc_pkto.data = (unsigned long) device_get_unit(sc->sc_dev);
2160    }
2161    if (devinfo & SAFE_DEVINFO_DES) {
2162        printf(" des/3des");
2163        crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
2164        crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
2165    }
2166    if (devinfo & SAFE_DEVINFO_AES) {
2167        printf(" aes");
2168        crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
2169    }
2170    if (devinfo & SAFE_DEVINFO_MD5) {
2171        printf(" md5");
2172        crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
2173    }
2174    if (devinfo & SAFE_DEVINFO_SHA1) {
2175        printf(" sha1");
2176        crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
2177    }
2178    printf(" null");
2179    crypto_register(sc->sc_cid, CRYPTO_NULL_CBC, 0, 0);
2180    crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0);
2181    /* XXX other supported algorithms */
2182    printf("\n");
2183
2184    safe_reset_board(sc); /* reset h/w */
2185    safe_init_board(sc); /* init h/w */
2186
2187#if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
2188    if (sc->sc_flags & SAFE_FLAGS_RNG) {
2189        safe_rng_init(sc);
2190        crypto_rregister(sc->sc_cid, safe_read_random, sc);
2191    }
2192#endif /* SAFE_NO_RNG */
2193
2194    return (0);
2195
2196out:
2197    if (sc->sc_cid >= 0)
2198        crypto_unregister_all(sc->sc_cid);
2199    if (sc->sc_irq != -1)
2200        free_irq(sc->sc_irq, sc);
2201    if (sc->sc_ringalloc.dma_vaddr)
2202        pci_free_consistent(sc->sc_pcidev,
2203                SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2204                sc->sc_ringalloc.dma_vaddr, sc->sc_ringalloc.dma_paddr);
2205    if (sc->sc_spalloc.dma_vaddr)
2206        pci_free_consistent(sc->sc_pcidev,
2207                SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2208                sc->sc_spalloc.dma_vaddr, sc->sc_spalloc.dma_paddr);
2209    if (sc->sc_dpalloc.dma_vaddr)
2210        pci_free_consistent(sc->sc_pcidev,
2211                SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2212                sc->sc_dpalloc.dma_vaddr, sc->sc_dpalloc.dma_paddr);
2213    kfree(sc);
2214    return(-ENODEV);
2215}
2216
2217static void safe_remove(struct pci_dev *dev)
2218{
2219    struct safe_softc *sc = pci_get_drvdata(dev);
2220
2221    DPRINTF(("%s()\n", __FUNCTION__));
2222
2223    /* XXX wait/abort active ops */
2224
2225    WRITE_REG(sc, SAFE_HI_MASK, 0); /* disable interrupts */
2226
2227    del_timer_sync(&sc->sc_pkto);
2228
2229    crypto_unregister_all(sc->sc_cid);
2230
2231    safe_cleanchip(sc);
2232
2233    if (sc->sc_irq != -1)
2234        free_irq(sc->sc_irq, sc);
2235    if (sc->sc_ringalloc.dma_vaddr)
2236        pci_free_consistent(sc->sc_pcidev,
2237                SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2238                sc->sc_ringalloc.dma_vaddr, sc->sc_ringalloc.dma_paddr);
2239    if (sc->sc_spalloc.dma_vaddr)
2240        pci_free_consistent(sc->sc_pcidev,
2241                SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2242                sc->sc_spalloc.dma_vaddr, sc->sc_spalloc.dma_paddr);
2243    if (sc->sc_dpalloc.dma_vaddr)
2244        pci_free_consistent(sc->sc_pcidev,
2245                SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2246                sc->sc_dpalloc.dma_vaddr, sc->sc_dpalloc.dma_paddr);
2247    sc->sc_irq = -1;
2248    sc->sc_ringalloc.dma_vaddr = NULL;
2249    sc->sc_spalloc.dma_vaddr = NULL;
2250    sc->sc_dpalloc.dma_vaddr = NULL;
2251}
2252
2253static struct pci_device_id safe_pci_tbl[] = {
2254    { PCI_VENDOR_SAFENET, PCI_PRODUCT_SAFEXCEL,
2255      PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
2256    { },
2257};
2258MODULE_DEVICE_TABLE(pci, safe_pci_tbl);
2259
2260static struct pci_driver safe_driver = {
2261    .name = "safe",
2262    .id_table = safe_pci_tbl,
2263    .probe = safe_probe,
2264    .remove = safe_remove,
2265    /* add PM stuff here one day */
2266};
2267
2268static int __init safe_init (void)
2269{
2270    struct safe_softc *sc = NULL;
2271    int rc;
2272
2273    DPRINTF(("%s(%p)\n", __FUNCTION__, safe_init));
2274
2275    rc = pci_register_driver(&safe_driver);
2276    pci_register_driver_compat(&safe_driver, rc);
2277
2278    return rc;
2279}
2280
2281static void __exit safe_exit (void)
2282{
2283    pci_unregister_driver(&safe_driver);
2284}
2285
2286module_init(safe_init);
2287module_exit(safe_exit);
2288
2289MODULE_LICENSE("BSD");
2290MODULE_AUTHOR("David McCullough <david_mccullough@mcafee.com>");
2291MODULE_DESCRIPTION("OCF driver for safenet PCI crypto devices");
2292

Archive Download this file

Branches:
backfire
history
master
release_2010-11-17
release_2010-12-14
release_2011-02-23
release_2011-05-28
release_2011-08-27
release_2011-11-13
release_2012-03-18
release_2012-04-09
release_2012-10-11
uclibc_nptl



interactive