Root/target/linux/generic/files/crypto/ocf/safe/safe.c

1/*-
2 * Linux port done by David McCullough <david_mccullough@mcafee.com>
3 * Copyright (C) 2004-2010 David McCullough
4 * The license and original author are listed below.
5 *
6 * Copyright (c) 2003 Sam Leffler, Errno Consulting
7 * Copyright (c) 2003 Global Technology Associates, Inc.
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31__FBSDID("$FreeBSD: src/sys/dev/safe/safe.c,v 1.18 2007/03/21 03:42:50 sam Exp $");
32 */
33
34#include <linux/version.h>
35#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38) && !defined(AUTOCONF_INCLUDED)
36#include <linux/config.h>
37#endif
38#include <linux/module.h>
39#include <linux/kernel.h>
40#include <linux/init.h>
41#include <linux/list.h>
42#include <linux/slab.h>
43#include <linux/wait.h>
44#include <linux/sched.h>
45#include <linux/pci.h>
46#include <linux/delay.h>
47#include <linux/interrupt.h>
48#include <linux/spinlock.h>
49#include <linux/random.h>
50#include <linux/skbuff.h>
51#include <asm/io.h>
52
53/*
54 * SafeNet SafeXcel-1141 hardware crypto accelerator
55 */
56
57#include <cryptodev.h>
58#include <uio.h>
59#include <safe/safereg.h>
60#include <safe/safevar.h>
61
62#if 1
63#define DPRINTF(a) do { \
64                        if (debug) { \
65                            printk("%s: ", sc ? \
66                                device_get_nameunit(sc->sc_dev) : "safe"); \
67                            printk a; \
68                        } \
69                    } while (0)
70#else
71#define DPRINTF(a)
72#endif
73
74/*
75 * until we find a cleaner way, include the BSD md5/sha1 code
76 * here
77 */
78#define HMAC_HACK 1
79#ifdef HMAC_HACK
80#include <safe/hmachack.h>
81#include <safe/md5.h>
82#include <safe/md5.c>
83#include <safe/sha1.h>
84#include <safe/sha1.c>
85#endif /* HMAC_HACK */
86
87/* add proc entry for this */
88struct safe_stats safestats;
89
90#define debug safe_debug
91int safe_debug = 0;
92module_param(safe_debug, int, 0644);
93MODULE_PARM_DESC(safe_debug, "Enable debug");
94
95static void safe_callback(struct safe_softc *, struct safe_ringentry *);
96static void safe_feed(struct safe_softc *, struct safe_ringentry *);
97#if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
98static void safe_rng_init(struct safe_softc *);
99int safe_rngbufsize = 8; /* 32 bytes each read */
100module_param(safe_rngbufsize, int, 0644);
101MODULE_PARM_DESC(safe_rngbufsize, "RNG polling buffer size (32-bit words)");
102int safe_rngmaxalarm = 8; /* max alarms before reset */
103module_param(safe_rngmaxalarm, int, 0644);
104MODULE_PARM_DESC(safe_rngmaxalarm, "RNG max alarms before reset");
105#endif /* SAFE_NO_RNG */
106
107static void safe_totalreset(struct safe_softc *sc);
108static int safe_dmamap_aligned(struct safe_softc *sc, const struct safe_operand *op);
109static int safe_dmamap_uniform(struct safe_softc *sc, const struct safe_operand *op);
110static int safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re);
111static int safe_kprocess(device_t dev, struct cryptkop *krp, int hint);
112static int safe_kstart(struct safe_softc *sc);
113static int safe_ksigbits(struct safe_softc *sc, struct crparam *cr);
114static void safe_kfeed(struct safe_softc *sc);
115static void safe_kpoll(unsigned long arg);
116static void safe_kload_reg(struct safe_softc *sc, u_int32_t off,
117                                u_int32_t len, struct crparam *n);
118
119static int safe_newsession(device_t, u_int32_t *, struct cryptoini *);
120static int safe_freesession(device_t, u_int64_t);
121static int safe_process(device_t, struct cryptop *, int);
122
123static device_method_t safe_methods = {
124    /* crypto device methods */
125    DEVMETHOD(cryptodev_newsession, safe_newsession),
126    DEVMETHOD(cryptodev_freesession,safe_freesession),
127    DEVMETHOD(cryptodev_process, safe_process),
128    DEVMETHOD(cryptodev_kprocess, safe_kprocess),
129};
130
131#define READ_REG(sc,r) readl((sc)->sc_base_addr + (r))
132#define WRITE_REG(sc,r,val) writel((val), (sc)->sc_base_addr + (r))
133
134#define SAFE_MAX_CHIPS 8
135static struct safe_softc *safe_chip_idx[SAFE_MAX_CHIPS];
136
137/*
138 * split our buffers up into safe DMAable byte fragments to avoid lockup
139 * bug in 1141 HW on rev 1.0.
140 */
141
142static int
143pci_map_linear(
144    struct safe_softc *sc,
145    struct safe_operand *buf,
146    void *addr,
147    int len)
148{
149    dma_addr_t tmp;
150    int chunk, tlen = len;
151
152    tmp = pci_map_single(sc->sc_pcidev, addr, len, PCI_DMA_BIDIRECTIONAL);
153
154    buf->mapsize += len;
155    while (len > 0) {
156        chunk = (len > sc->sc_max_dsize) ? sc->sc_max_dsize : len;
157        buf->segs[buf->nsegs].ds_addr = tmp;
158        buf->segs[buf->nsegs].ds_len = chunk;
159        buf->segs[buf->nsegs].ds_tlen = tlen;
160        buf->nsegs++;
161        tmp += chunk;
162        len -= chunk;
163        tlen = 0;
164    }
165    return 0;
166}
167
168/*
169 * map in a given uio buffer (great on some arches :-)
170 */
171
172static int
173pci_map_uio(struct safe_softc *sc, struct safe_operand *buf, struct uio *uio)
174{
175    struct iovec *iov = uio->uio_iov;
176    int n;
177
178    DPRINTF(("%s()\n", __FUNCTION__));
179
180    buf->mapsize = 0;
181    buf->nsegs = 0;
182
183    for (n = 0; n < uio->uio_iovcnt; n++) {
184        pci_map_linear(sc, buf, iov->iov_base, iov->iov_len);
185        iov++;
186    }
187
188    /* identify this buffer by the first segment */
189    buf->map = (void *) buf->segs[0].ds_addr;
190    return(0);
191}
192
193/*
194 * map in a given sk_buff
195 */
196
197static int
198pci_map_skb(struct safe_softc *sc,struct safe_operand *buf,struct sk_buff *skb)
199{
200    int i;
201
202    DPRINTF(("%s()\n", __FUNCTION__));
203
204    buf->mapsize = 0;
205    buf->nsegs = 0;
206
207    pci_map_linear(sc, buf, skb->data, skb_headlen(skb));
208
209    for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
210        pci_map_linear(sc, buf,
211                page_address(skb_frag_page(&skb_shinfo(skb)->frags[i])) +
212                                        skb_shinfo(skb)->frags[i].page_offset,
213                skb_shinfo(skb)->frags[i].size);
214    }
215
216    /* identify this buffer by the first segment */
217    buf->map = (void *) buf->segs[0].ds_addr;
218    return(0);
219}
220
221
222#if 0 /* not needed at this time */
223static void
224pci_sync_operand(struct safe_softc *sc, struct safe_operand *buf)
225{
226    int i;
227
228    DPRINTF(("%s()\n", __FUNCTION__));
229    for (i = 0; i < buf->nsegs; i++)
230        pci_dma_sync_single_for_cpu(sc->sc_pcidev, buf->segs[i].ds_addr,
231                buf->segs[i].ds_len, PCI_DMA_BIDIRECTIONAL);
232}
233#endif
234
235static void
236pci_unmap_operand(struct safe_softc *sc, struct safe_operand *buf)
237{
238    int i;
239    DPRINTF(("%s()\n", __FUNCTION__));
240    for (i = 0; i < buf->nsegs; i++) {
241        if (buf->segs[i].ds_tlen) {
242            DPRINTF(("%s - unmap %d 0x%x %d\n", __FUNCTION__, i, buf->segs[i].ds_addr, buf->segs[i].ds_tlen));
243            pci_unmap_single(sc->sc_pcidev, buf->segs[i].ds_addr,
244                    buf->segs[i].ds_tlen, PCI_DMA_BIDIRECTIONAL);
245            DPRINTF(("%s - unmap %d 0x%x %d done\n", __FUNCTION__, i, buf->segs[i].ds_addr, buf->segs[i].ds_tlen));
246        }
247        buf->segs[i].ds_addr = 0;
248        buf->segs[i].ds_len = 0;
249        buf->segs[i].ds_tlen = 0;
250    }
251    buf->nsegs = 0;
252    buf->mapsize = 0;
253    buf->map = 0;
254}
255
256
257/*
258 * SafeXcel Interrupt routine
259 */
260static irqreturn_t
261#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)
262safe_intr(int irq, void *arg)
263#else
264safe_intr(int irq, void *arg, struct pt_regs *regs)
265#endif
266{
267    struct safe_softc *sc = arg;
268    int stat;
269    unsigned long flags;
270
271    stat = READ_REG(sc, SAFE_HM_STAT);
272
273    DPRINTF(("%s(stat=0x%x)\n", __FUNCTION__, stat));
274
275    if (stat == 0) /* shared irq, not for us */
276        return IRQ_NONE;
277
278    WRITE_REG(sc, SAFE_HI_CLR, stat); /* IACK */
279
280    if ((stat & SAFE_INT_PE_DDONE)) {
281        /*
282         * Descriptor(s) done; scan the ring and
283         * process completed operations.
284         */
285        spin_lock_irqsave(&sc->sc_ringmtx, flags);
286        while (sc->sc_back != sc->sc_front) {
287            struct safe_ringentry *re = sc->sc_back;
288
289#ifdef SAFE_DEBUG
290            if (debug) {
291                safe_dump_ringstate(sc, __func__);
292                safe_dump_request(sc, __func__, re);
293            }
294#endif
295            /*
296             * safe_process marks ring entries that were allocated
297             * but not used with a csr of zero. This insures the
298             * ring front pointer never needs to be set backwards
299             * in the event that an entry is allocated but not used
300             * because of a setup error.
301             */
302            DPRINTF(("%s re->re_desc.d_csr=0x%x\n", __FUNCTION__, re->re_desc.d_csr));
303            if (re->re_desc.d_csr != 0) {
304                if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr)) {
305                    DPRINTF(("%s !CSR_IS_DONE\n", __FUNCTION__));
306                    break;
307                }
308                if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len)) {
309                    DPRINTF(("%s !LEN_IS_DONE\n", __FUNCTION__));
310                    break;
311                }
312                sc->sc_nqchip--;
313                safe_callback(sc, re);
314            }
315            if (++(sc->sc_back) == sc->sc_ringtop)
316                sc->sc_back = sc->sc_ring;
317        }
318        spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
319    }
320
321    /*
322     * Check to see if we got any DMA Error
323     */
324    if (stat & SAFE_INT_PE_ERROR) {
325        printk("%s: dmaerr dmastat %08x\n", device_get_nameunit(sc->sc_dev),
326                (int)READ_REG(sc, SAFE_PE_DMASTAT));
327        safestats.st_dmaerr++;
328        safe_totalreset(sc);
329#if 0
330        safe_feed(sc);
331#endif
332    }
333
334    if (sc->sc_needwakeup) { /* XXX check high watermark */
335        int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
336        DPRINTF(("%s: wakeup crypto %x\n", __func__,
337            sc->sc_needwakeup));
338        sc->sc_needwakeup &= ~wakeup;
339        crypto_unblock(sc->sc_cid, wakeup);
340    }
341    
342    return IRQ_HANDLED;
343}
344
345/*
346 * safe_feed() - post a request to chip
347 */
348static void
349safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
350{
351    DPRINTF(("%s()\n", __FUNCTION__));
352#ifdef SAFE_DEBUG
353    if (debug) {
354        safe_dump_ringstate(sc, __func__);
355        safe_dump_request(sc, __func__, re);
356    }
357#endif
358    sc->sc_nqchip++;
359    if (sc->sc_nqchip > safestats.st_maxqchip)
360        safestats.st_maxqchip = sc->sc_nqchip;
361    /* poke h/w to check descriptor ring, any value can be written */
362    WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
363}
364
365#define N(a) (sizeof(a) / sizeof (a[0]))
366static void
367safe_setup_enckey(struct safe_session *ses, caddr_t key)
368{
369    int i;
370
371    bcopy(key, ses->ses_key, ses->ses_klen / 8);
372
373    /* PE is little-endian, insure proper byte order */
374    for (i = 0; i < N(ses->ses_key); i++)
375        ses->ses_key[i] = htole32(ses->ses_key[i]);
376}
377
378static void
379safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen)
380{
381#ifdef HMAC_HACK
382    MD5_CTX md5ctx;
383    SHA1_CTX sha1ctx;
384    int i;
385
386
387    for (i = 0; i < klen; i++)
388        key[i] ^= HMAC_IPAD_VAL;
389
390    if (algo == CRYPTO_MD5_HMAC) {
391        MD5Init(&md5ctx);
392        MD5Update(&md5ctx, key, klen);
393        MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
394        bcopy(md5ctx.md5_st8, ses->ses_hminner, sizeof(md5ctx.md5_st8));
395    } else {
396        SHA1Init(&sha1ctx);
397        SHA1Update(&sha1ctx, key, klen);
398        SHA1Update(&sha1ctx, hmac_ipad_buffer,
399            SHA1_HMAC_BLOCK_LEN - klen);
400        bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
401    }
402
403    for (i = 0; i < klen; i++)
404        key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
405
406    if (algo == CRYPTO_MD5_HMAC) {
407        MD5Init(&md5ctx);
408        MD5Update(&md5ctx, key, klen);
409        MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
410        bcopy(md5ctx.md5_st8, ses->ses_hmouter, sizeof(md5ctx.md5_st8));
411    } else {
412        SHA1Init(&sha1ctx);
413        SHA1Update(&sha1ctx, key, klen);
414        SHA1Update(&sha1ctx, hmac_opad_buffer,
415            SHA1_HMAC_BLOCK_LEN - klen);
416        bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
417    }
418
419    for (i = 0; i < klen; i++)
420        key[i] ^= HMAC_OPAD_VAL;
421
422#if 0
423    /*
424     * this code prevents SHA working on a BE host,
425     * so it is obviously wrong. I think the byte
426     * swap setup we do with the chip fixes this for us
427     */
428
429    /* PE is little-endian, insure proper byte order */
430    for (i = 0; i < N(ses->ses_hminner); i++) {
431        ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
432        ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
433    }
434#endif
435#else /* HMAC_HACK */
436    printk("safe: md5/sha not implemented\n");
437#endif /* HMAC_HACK */
438}
439#undef N
440
441/*
442 * Allocate a new 'session' and return an encoded session id. 'sidp'
443 * contains our registration id, and should contain an encoded session
444 * id on successful allocation.
445 */
446static int
447safe_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
448{
449    struct safe_softc *sc = device_get_softc(dev);
450    struct cryptoini *c, *encini = NULL, *macini = NULL;
451    struct safe_session *ses = NULL;
452    int sesn;
453
454    DPRINTF(("%s()\n", __FUNCTION__));
455
456    if (sidp == NULL || cri == NULL || sc == NULL)
457        return (EINVAL);
458
459    for (c = cri; c != NULL; c = c->cri_next) {
460        if (c->cri_alg == CRYPTO_MD5_HMAC ||
461            c->cri_alg == CRYPTO_SHA1_HMAC ||
462            c->cri_alg == CRYPTO_NULL_HMAC) {
463            if (macini)
464                return (EINVAL);
465            macini = c;
466        } else if (c->cri_alg == CRYPTO_DES_CBC ||
467            c->cri_alg == CRYPTO_3DES_CBC ||
468            c->cri_alg == CRYPTO_AES_CBC ||
469            c->cri_alg == CRYPTO_NULL_CBC) {
470            if (encini)
471                return (EINVAL);
472            encini = c;
473        } else
474            return (EINVAL);
475    }
476    if (encini == NULL && macini == NULL)
477        return (EINVAL);
478    if (encini) { /* validate key length */
479        switch (encini->cri_alg) {
480        case CRYPTO_DES_CBC:
481            if (encini->cri_klen != 64)
482                return (EINVAL);
483            break;
484        case CRYPTO_3DES_CBC:
485            if (encini->cri_klen != 192)
486                return (EINVAL);
487            break;
488        case CRYPTO_AES_CBC:
489            if (encini->cri_klen != 128 &&
490                encini->cri_klen != 192 &&
491                encini->cri_klen != 256)
492                return (EINVAL);
493            break;
494        }
495    }
496
497    if (sc->sc_sessions == NULL) {
498        ses = sc->sc_sessions = (struct safe_session *)
499            kmalloc(sizeof(struct safe_session), SLAB_ATOMIC);
500        if (ses == NULL)
501            return (ENOMEM);
502        memset(ses, 0, sizeof(struct safe_session));
503        sesn = 0;
504        sc->sc_nsessions = 1;
505    } else {
506        for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
507            if (sc->sc_sessions[sesn].ses_used == 0) {
508                ses = &sc->sc_sessions[sesn];
509                break;
510            }
511        }
512
513        if (ses == NULL) {
514            sesn = sc->sc_nsessions;
515            ses = (struct safe_session *)
516                kmalloc((sesn + 1) * sizeof(struct safe_session), SLAB_ATOMIC);
517            if (ses == NULL)
518                return (ENOMEM);
519            memset(ses, 0, (sesn + 1) * sizeof(struct safe_session));
520            bcopy(sc->sc_sessions, ses, sesn *
521                sizeof(struct safe_session));
522            bzero(sc->sc_sessions, sesn *
523                sizeof(struct safe_session));
524            kfree(sc->sc_sessions);
525            sc->sc_sessions = ses;
526            ses = &sc->sc_sessions[sesn];
527            sc->sc_nsessions++;
528        }
529    }
530
531    bzero(ses, sizeof(struct safe_session));
532    ses->ses_used = 1;
533
534    if (encini) {
535        ses->ses_klen = encini->cri_klen;
536        if (encini->cri_key != NULL)
537            safe_setup_enckey(ses, encini->cri_key);
538    }
539
540    if (macini) {
541        ses->ses_mlen = macini->cri_mlen;
542        if (ses->ses_mlen == 0) {
543            if (macini->cri_alg == CRYPTO_MD5_HMAC)
544                ses->ses_mlen = MD5_HASH_LEN;
545            else
546                ses->ses_mlen = SHA1_HASH_LEN;
547        }
548
549        if (macini->cri_key != NULL) {
550            safe_setup_mackey(ses, macini->cri_alg, macini->cri_key,
551                macini->cri_klen / 8);
552        }
553    }
554
555    *sidp = SAFE_SID(device_get_unit(sc->sc_dev), sesn);
556    return (0);
557}
558
559/*
560 * Deallocate a session.
561 */
562static int
563safe_freesession(device_t dev, u_int64_t tid)
564{
565    struct safe_softc *sc = device_get_softc(dev);
566    int session, ret;
567    u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
568
569    DPRINTF(("%s()\n", __FUNCTION__));
570
571    if (sc == NULL)
572        return (EINVAL);
573
574    session = SAFE_SESSION(sid);
575    if (session < sc->sc_nsessions) {
576        bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
577        ret = 0;
578    } else
579        ret = EINVAL;
580    return (ret);
581}
582
583
584static int
585safe_process(device_t dev, struct cryptop *crp, int hint)
586{
587    struct safe_softc *sc = device_get_softc(dev);
588    int err = 0, i, nicealign, uniform;
589    struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
590    int bypass, oplen, ivsize;
591    caddr_t iv;
592    int16_t coffset;
593    struct safe_session *ses;
594    struct safe_ringentry *re;
595    struct safe_sarec *sa;
596    struct safe_pdesc *pd;
597    u_int32_t cmd0, cmd1, staterec, rand_iv[4];
598    unsigned long flags;
599
600    DPRINTF(("%s()\n", __FUNCTION__));
601
602    if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
603        safestats.st_invalid++;
604        return (EINVAL);
605    }
606    if (SAFE_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
607        safestats.st_badsession++;
608        return (EINVAL);
609    }
610
611    spin_lock_irqsave(&sc->sc_ringmtx, flags);
612    if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
613        safestats.st_ringfull++;
614        sc->sc_needwakeup |= CRYPTO_SYMQ;
615        spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
616        return (ERESTART);
617    }
618    re = sc->sc_front;
619
620    staterec = re->re_sa.sa_staterec; /* save */
621    /* NB: zero everything but the PE descriptor */
622    bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
623    re->re_sa.sa_staterec = staterec; /* restore */
624
625    re->re_crp = crp;
626    re->re_sesn = SAFE_SESSION(crp->crp_sid);
627
628    re->re_src.nsegs = 0;
629    re->re_dst.nsegs = 0;
630
631    if (crp->crp_flags & CRYPTO_F_SKBUF) {
632        re->re_src_skb = (struct sk_buff *)crp->crp_buf;
633        re->re_dst_skb = (struct sk_buff *)crp->crp_buf;
634    } else if (crp->crp_flags & CRYPTO_F_IOV) {
635        re->re_src_io = (struct uio *)crp->crp_buf;
636        re->re_dst_io = (struct uio *)crp->crp_buf;
637    } else {
638        safestats.st_badflags++;
639        err = EINVAL;
640        goto errout; /* XXX we don't handle contiguous blocks! */
641    }
642
643    sa = &re->re_sa;
644    ses = &sc->sc_sessions[re->re_sesn];
645
646    crd1 = crp->crp_desc;
647    if (crd1 == NULL) {
648        safestats.st_nodesc++;
649        err = EINVAL;
650        goto errout;
651    }
652    crd2 = crd1->crd_next;
653
654    cmd0 = SAFE_SA_CMD0_BASIC; /* basic group operation */
655    cmd1 = 0;
656    if (crd2 == NULL) {
657        if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
658            crd1->crd_alg == CRYPTO_SHA1_HMAC ||
659            crd1->crd_alg == CRYPTO_NULL_HMAC) {
660            maccrd = crd1;
661            enccrd = NULL;
662            cmd0 |= SAFE_SA_CMD0_OP_HASH;
663        } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
664            crd1->crd_alg == CRYPTO_3DES_CBC ||
665            crd1->crd_alg == CRYPTO_AES_CBC ||
666            crd1->crd_alg == CRYPTO_NULL_CBC) {
667            maccrd = NULL;
668            enccrd = crd1;
669            cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
670        } else {
671            safestats.st_badalg++;
672            err = EINVAL;
673            goto errout;
674        }
675    } else {
676        if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
677            crd1->crd_alg == CRYPTO_SHA1_HMAC ||
678            crd1->crd_alg == CRYPTO_NULL_HMAC) &&
679            (crd2->crd_alg == CRYPTO_DES_CBC ||
680            crd2->crd_alg == CRYPTO_3DES_CBC ||
681                crd2->crd_alg == CRYPTO_AES_CBC ||
682                crd2->crd_alg == CRYPTO_NULL_CBC) &&
683            ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
684            maccrd = crd1;
685            enccrd = crd2;
686        } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
687            crd1->crd_alg == CRYPTO_3DES_CBC ||
688            crd1->crd_alg == CRYPTO_AES_CBC ||
689            crd1->crd_alg == CRYPTO_NULL_CBC) &&
690            (crd2->crd_alg == CRYPTO_MD5_HMAC ||
691            crd2->crd_alg == CRYPTO_SHA1_HMAC ||
692            crd2->crd_alg == CRYPTO_NULL_HMAC) &&
693            (crd1->crd_flags & CRD_F_ENCRYPT)) {
694            enccrd = crd1;
695            maccrd = crd2;
696        } else {
697            safestats.st_badalg++;
698            err = EINVAL;
699            goto errout;
700        }
701        cmd0 |= SAFE_SA_CMD0_OP_BOTH;
702    }
703
704    if (enccrd) {
705        if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
706            safe_setup_enckey(ses, enccrd->crd_key);
707
708        if (enccrd->crd_alg == CRYPTO_DES_CBC) {
709            cmd0 |= SAFE_SA_CMD0_DES;
710            cmd1 |= SAFE_SA_CMD1_CBC;
711            ivsize = 2*sizeof(u_int32_t);
712        } else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
713            cmd0 |= SAFE_SA_CMD0_3DES;
714            cmd1 |= SAFE_SA_CMD1_CBC;
715            ivsize = 2*sizeof(u_int32_t);
716        } else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
717            cmd0 |= SAFE_SA_CMD0_AES;
718            cmd1 |= SAFE_SA_CMD1_CBC;
719            if (ses->ses_klen == 128)
720                 cmd1 |= SAFE_SA_CMD1_AES128;
721            else if (ses->ses_klen == 192)
722                 cmd1 |= SAFE_SA_CMD1_AES192;
723            else
724                 cmd1 |= SAFE_SA_CMD1_AES256;
725            ivsize = 4*sizeof(u_int32_t);
726        } else {
727            cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
728            ivsize = 0;
729        }
730
731        /*
732         * Setup encrypt/decrypt state. When using basic ops
733         * we can't use an inline IV because hash/crypt offset
734         * must be from the end of the IV to the start of the
735         * crypt data and this leaves out the preceding header
736         * from the hash calculation. Instead we place the IV
737         * in the state record and set the hash/crypt offset to
738         * copy both the header+IV.
739         */
740        if (enccrd->crd_flags & CRD_F_ENCRYPT) {
741            cmd0 |= SAFE_SA_CMD0_OUTBOUND;
742
743            if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
744                iv = enccrd->crd_iv;
745            else
746                read_random((iv = (caddr_t) &rand_iv[0]), sizeof(rand_iv));
747            if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
748                crypto_copyback(crp->crp_flags, crp->crp_buf,
749                    enccrd->crd_inject, ivsize, iv);
750            }
751            bcopy(iv, re->re_sastate.sa_saved_iv, ivsize);
752            /* make iv LE */
753            for (i = 0; i < ivsize/sizeof(re->re_sastate.sa_saved_iv[0]); i++)
754                re->re_sastate.sa_saved_iv[i] =
755                    cpu_to_le32(re->re_sastate.sa_saved_iv[i]);
756            cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
757            re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
758        } else {
759            cmd0 |= SAFE_SA_CMD0_INBOUND;
760
761            if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
762                bcopy(enccrd->crd_iv,
763                    re->re_sastate.sa_saved_iv, ivsize);
764            } else {
765                crypto_copydata(crp->crp_flags, crp->crp_buf,
766                    enccrd->crd_inject, ivsize,
767                    (caddr_t)re->re_sastate.sa_saved_iv);
768            }
769            /* make iv LE */
770            for (i = 0; i < ivsize/sizeof(re->re_sastate.sa_saved_iv[0]); i++)
771                re->re_sastate.sa_saved_iv[i] =
772                    cpu_to_le32(re->re_sastate.sa_saved_iv[i]);
773            cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
774        }
775        /*
776         * For basic encryption use the zero pad algorithm.
777         * This pads results to an 8-byte boundary and
778         * suppresses padding verification for inbound (i.e.
779         * decrypt) operations.
780         *
781         * NB: Not sure if the 8-byte pad boundary is a problem.
782         */
783        cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
784
785        /* XXX assert key bufs have the same size */
786        bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
787    }
788
789    if (maccrd) {
790        if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
791            safe_setup_mackey(ses, maccrd->crd_alg,
792                maccrd->crd_key, maccrd->crd_klen / 8);
793        }
794
795        if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
796            cmd0 |= SAFE_SA_CMD0_MD5;
797            cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
798        } else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
799            cmd0 |= SAFE_SA_CMD0_SHA1;
800            cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
801        } else {
802            cmd0 |= SAFE_SA_CMD0_HASH_NULL;
803        }
804        /*
805         * Digest data is loaded from the SA and the hash
806         * result is saved to the state block where we
807         * retrieve it for return to the caller.
808         */
809        /* XXX assert digest bufs have the same size */
810        bcopy(ses->ses_hminner, sa->sa_indigest,
811            sizeof(sa->sa_indigest));
812        bcopy(ses->ses_hmouter, sa->sa_outdigest,
813            sizeof(sa->sa_outdigest));
814
815        cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
816        re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
817    }
818
819    if (enccrd && maccrd) {
820        /*
821         * The offset from hash data to the start of
822         * crypt data is the difference in the skips.
823         */
824        bypass = maccrd->crd_skip;
825        coffset = enccrd->crd_skip - maccrd->crd_skip;
826        if (coffset < 0) {
827            DPRINTF(("%s: hash does not precede crypt; "
828                "mac skip %u enc skip %u\n",
829                __func__, maccrd->crd_skip, enccrd->crd_skip));
830            safestats.st_skipmismatch++;
831            err = EINVAL;
832            goto errout;
833        }
834        oplen = enccrd->crd_skip + enccrd->crd_len;
835        if (maccrd->crd_skip + maccrd->crd_len != oplen) {
836            DPRINTF(("%s: hash amount %u != crypt amount %u\n",
837                __func__, maccrd->crd_skip + maccrd->crd_len,
838                oplen));
839            safestats.st_lenmismatch++;
840            err = EINVAL;
841            goto errout;
842        }
843#ifdef SAFE_DEBUG
844        if (debug) {
845            printf("mac: skip %d, len %d, inject %d\n",
846                maccrd->crd_skip, maccrd->crd_len,
847                maccrd->crd_inject);
848            printf("enc: skip %d, len %d, inject %d\n",
849                enccrd->crd_skip, enccrd->crd_len,
850                enccrd->crd_inject);
851            printf("bypass %d coffset %d oplen %d\n",
852                bypass, coffset, oplen);
853        }
854#endif
855        if (coffset & 3) { /* offset must be 32-bit aligned */
856            DPRINTF(("%s: coffset %u misaligned\n",
857                __func__, coffset));
858            safestats.st_coffmisaligned++;
859            err = EINVAL;
860            goto errout;
861        }
862        coffset >>= 2;
863        if (coffset > 255) { /* offset must be <256 dwords */
864            DPRINTF(("%s: coffset %u too big\n",
865                __func__, coffset));
866            safestats.st_cofftoobig++;
867            err = EINVAL;
868            goto errout;
869        }
870        /*
871         * Tell the hardware to copy the header to the output.
872         * The header is defined as the data from the end of
873         * the bypass to the start of data to be encrypted.
874         * Typically this is the inline IV. Note that you need
875         * to do this even if src+dst are the same; it appears
876         * that w/o this bit the crypted data is written
877         * immediately after the bypass data.
878         */
879        cmd1 |= SAFE_SA_CMD1_HDRCOPY;
880        /*
881         * Disable IP header mutable bit handling. This is
882         * needed to get correct HMAC calculations.
883         */
884        cmd1 |= SAFE_SA_CMD1_MUTABLE;
885    } else {
886        if (enccrd) {
887            bypass = enccrd->crd_skip;
888            oplen = bypass + enccrd->crd_len;
889        } else {
890            bypass = maccrd->crd_skip;
891            oplen = bypass + maccrd->crd_len;
892        }
893        coffset = 0;
894    }
895    /* XXX verify multiple of 4 when using s/g */
896    if (bypass > 96) { /* bypass offset must be <= 96 bytes */
897        DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
898        safestats.st_bypasstoobig++;
899        err = EINVAL;
900        goto errout;
901    }
902
903    if (crp->crp_flags & CRYPTO_F_SKBUF) {
904        if (pci_map_skb(sc, &re->re_src, re->re_src_skb)) {
905            safestats.st_noload++;
906            err = ENOMEM;
907            goto errout;
908        }
909    } else if (crp->crp_flags & CRYPTO_F_IOV) {
910        if (pci_map_uio(sc, &re->re_src, re->re_src_io)) {
911            safestats.st_noload++;
912            err = ENOMEM;
913            goto errout;
914        }
915    }
916    nicealign = safe_dmamap_aligned(sc, &re->re_src);
917    uniform = safe_dmamap_uniform(sc, &re->re_src);
918
919    DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
920        nicealign, uniform, re->re_src.nsegs));
921    if (re->re_src.nsegs > 1) {
922        re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
923            ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
924        for (i = 0; i < re->re_src_nsegs; i++) {
925            /* NB: no need to check if there's space */
926            pd = sc->sc_spfree;
927            if (++(sc->sc_spfree) == sc->sc_springtop)
928                sc->sc_spfree = sc->sc_spring;
929
930            KASSERT((pd->pd_flags&3) == 0 ||
931                (pd->pd_flags&3) == SAFE_PD_DONE,
932                ("bogus source particle descriptor; flags %x",
933                pd->pd_flags));
934            pd->pd_addr = re->re_src_segs[i].ds_addr;
935            pd->pd_size = re->re_src_segs[i].ds_len;
936            pd->pd_flags = SAFE_PD_READY;
937        }
938        cmd0 |= SAFE_SA_CMD0_IGATHER;
939    } else {
940        /*
941         * No need for gather, reference the operand directly.
942         */
943        re->re_desc.d_src = re->re_src_segs[0].ds_addr;
944    }
945
946    if (enccrd == NULL && maccrd != NULL) {
947        /*
948         * Hash op; no destination needed.
949         */
950    } else {
951        if (crp->crp_flags & (CRYPTO_F_IOV|CRYPTO_F_SKBUF)) {
952            if (!nicealign) {
953                safestats.st_iovmisaligned++;
954                err = EINVAL;
955                goto errout;
956            }
957            if (uniform != 1) {
958                device_printf(sc->sc_dev, "!uniform source\n");
959                if (!uniform) {
960                    /*
961                     * There's no way to handle the DMA
962                     * requirements with this uio. We
963                     * could create a separate DMA area for
964                     * the result and then copy it back,
965                     * but for now we just bail and return
966                     * an error. Note that uio requests
967                     * > SAFE_MAX_DSIZE are handled because
968                     * the DMA map and segment list for the
969                     * destination wil result in a
970                     * destination particle list that does
971                     * the necessary scatter DMA.
972                     */
973                    safestats.st_iovnotuniform++;
974                    err = EINVAL;
975                    goto errout;
976                }
977            } else
978                re->re_dst = re->re_src;
979        } else {
980            safestats.st_badflags++;
981            err = EINVAL;
982            goto errout;
983        }
984
985        if (re->re_dst.nsegs > 1) {
986            re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
987                ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
988            for (i = 0; i < re->re_dst_nsegs; i++) {
989                pd = sc->sc_dpfree;
990                KASSERT((pd->pd_flags&3) == 0 ||
991                    (pd->pd_flags&3) == SAFE_PD_DONE,
992                    ("bogus dest particle descriptor; flags %x",
993                        pd->pd_flags));
994                if (++(sc->sc_dpfree) == sc->sc_dpringtop)
995                    sc->sc_dpfree = sc->sc_dpring;
996                pd->pd_addr = re->re_dst_segs[i].ds_addr;
997                pd->pd_flags = SAFE_PD_READY;
998            }
999            cmd0 |= SAFE_SA_CMD0_OSCATTER;
1000        } else {
1001            /*
1002             * No need for scatter, reference the operand directly.
1003             */
1004            re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
1005        }
1006    }
1007
1008    /*
1009     * All done with setup; fillin the SA command words
1010     * and the packet engine descriptor. The operation
1011     * is now ready for submission to the hardware.
1012     */
1013    sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
1014    sa->sa_cmd1 = cmd1
1015            | (coffset << SAFE_SA_CMD1_OFFSET_S)
1016            | SAFE_SA_CMD1_SAREV1 /* Rev 1 SA data structure */
1017            | SAFE_SA_CMD1_SRPCI
1018            ;
1019    /*
1020     * NB: the order of writes is important here. In case the
1021     * chip is scanning the ring because of an outstanding request
1022     * it might nab this one too. In that case we need to make
1023     * sure the setup is complete before we write the length
1024     * field of the descriptor as it signals the descriptor is
1025     * ready for processing.
1026     */
1027    re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
1028    if (maccrd)
1029        re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
1030    wmb();
1031    re->re_desc.d_len = oplen
1032              | SAFE_PE_LEN_READY
1033              | (bypass << SAFE_PE_LEN_BYPASS_S)
1034              ;
1035
1036    safestats.st_ipackets++;
1037    safestats.st_ibytes += oplen;
1038
1039    if (++(sc->sc_front) == sc->sc_ringtop)
1040        sc->sc_front = sc->sc_ring;
1041
1042    /* XXX honor batching */
1043    safe_feed(sc, re);
1044    spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1045    return (0);
1046
1047errout:
1048    if (re->re_src.map != re->re_dst.map)
1049        pci_unmap_operand(sc, &re->re_dst);
1050    if (re->re_src.map)
1051        pci_unmap_operand(sc, &re->re_src);
1052    spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1053    if (err != ERESTART) {
1054        crp->crp_etype = err;
1055        crypto_done(crp);
1056    } else {
1057        sc->sc_needwakeup |= CRYPTO_SYMQ;
1058    }
1059    return (err);
1060}
1061
1062static void
1063safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
1064{
1065    struct cryptop *crp = (struct cryptop *)re->re_crp;
1066    struct cryptodesc *crd;
1067
1068    DPRINTF(("%s()\n", __FUNCTION__));
1069
1070    safestats.st_opackets++;
1071    safestats.st_obytes += re->re_dst.mapsize;
1072
1073    if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
1074        device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n",
1075            re->re_desc.d_csr,
1076            re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
1077        safestats.st_peoperr++;
1078        crp->crp_etype = EIO; /* something more meaningful? */
1079    }
1080
1081    if (re->re_dst.map != NULL && re->re_dst.map != re->re_src.map)
1082        pci_unmap_operand(sc, &re->re_dst);
1083    pci_unmap_operand(sc, &re->re_src);
1084
1085    /*
1086     * If result was written to a differet mbuf chain, swap
1087     * it in as the return value and reclaim the original.
1088     */
1089    if ((crp->crp_flags & CRYPTO_F_SKBUF) && re->re_src_skb != re->re_dst_skb) {
1090        device_printf(sc->sc_dev, "no CRYPTO_F_SKBUF swapping support\n");
1091        /* kfree_skb(skb) */
1092        /* crp->crp_buf = (caddr_t)re->re_dst_skb */
1093        return;
1094    }
1095
1096    if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
1097        /* copy out ICV result */
1098        for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1099            if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
1100                crd->crd_alg == CRYPTO_SHA1_HMAC ||
1101                crd->crd_alg == CRYPTO_NULL_HMAC))
1102                continue;
1103            if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
1104                /*
1105                 * SHA-1 ICV's are byte-swapped; fix 'em up
1106                 * before copy them to their destination.
1107                 */
1108                re->re_sastate.sa_saved_indigest[0] =
1109                    cpu_to_be32(re->re_sastate.sa_saved_indigest[0]);
1110                re->re_sastate.sa_saved_indigest[1] =
1111                    cpu_to_be32(re->re_sastate.sa_saved_indigest[1]);
1112                re->re_sastate.sa_saved_indigest[2] =
1113                    cpu_to_be32(re->re_sastate.sa_saved_indigest[2]);
1114            } else {
1115                re->re_sastate.sa_saved_indigest[0] =
1116                    cpu_to_le32(re->re_sastate.sa_saved_indigest[0]);
1117                re->re_sastate.sa_saved_indigest[1] =
1118                    cpu_to_le32(re->re_sastate.sa_saved_indigest[1]);
1119                re->re_sastate.sa_saved_indigest[2] =
1120                    cpu_to_le32(re->re_sastate.sa_saved_indigest[2]);
1121            }
1122            crypto_copyback(crp->crp_flags, crp->crp_buf,
1123                crd->crd_inject,
1124                sc->sc_sessions[re->re_sesn].ses_mlen,
1125                (caddr_t)re->re_sastate.sa_saved_indigest);
1126            break;
1127        }
1128    }
1129    crypto_done(crp);
1130}
1131
1132
1133#if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
1134#define SAFE_RNG_MAXWAIT 1000
1135
1136static void
1137safe_rng_init(struct safe_softc *sc)
1138{
1139    u_int32_t w, v;
1140    int i;
1141
1142    DPRINTF(("%s()\n", __FUNCTION__));
1143
1144    WRITE_REG(sc, SAFE_RNG_CTRL, 0);
1145    /* use default value according to the manual */
1146    WRITE_REG(sc, SAFE_RNG_CNFG, 0x834); /* magic from SafeNet */
1147    WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1148
1149    /*
1150     * There is a bug in rev 1.0 of the 1140 that when the RNG
1151     * is brought out of reset the ready status flag does not
1152     * work until the RNG has finished its internal initialization.
1153     *
1154     * So in order to determine the device is through its
1155     * initialization we must read the data register, using the
1156     * status reg in the read in case it is initialized. Then read
1157     * the data register until it changes from the first read.
1158     * Once it changes read the data register until it changes
1159     * again. At this time the RNG is considered initialized.
1160     * This could take between 750ms - 1000ms in time.
1161     */
1162    i = 0;
1163    w = READ_REG(sc, SAFE_RNG_OUT);
1164    do {
1165        v = READ_REG(sc, SAFE_RNG_OUT);
1166        if (v != w) {
1167            w = v;
1168            break;
1169        }
1170        DELAY(10);
1171    } while (++i < SAFE_RNG_MAXWAIT);
1172
1173    /* Wait Until data changes again */
1174    i = 0;
1175    do {
1176        v = READ_REG(sc, SAFE_RNG_OUT);
1177        if (v != w)
1178            break;
1179        DELAY(10);
1180    } while (++i < SAFE_RNG_MAXWAIT);
1181}
1182
1183static __inline void
1184safe_rng_disable_short_cycle(struct safe_softc *sc)
1185{
1186    DPRINTF(("%s()\n", __FUNCTION__));
1187
1188    WRITE_REG(sc, SAFE_RNG_CTRL,
1189        READ_REG(sc, SAFE_RNG_CTRL) &~ SAFE_RNG_CTRL_SHORTEN);
1190}
1191
1192static __inline void
1193safe_rng_enable_short_cycle(struct safe_softc *sc)
1194{
1195    DPRINTF(("%s()\n", __FUNCTION__));
1196
1197    WRITE_REG(sc, SAFE_RNG_CTRL,
1198        READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
1199}
1200
1201static __inline u_int32_t
1202safe_rng_read(struct safe_softc *sc)
1203{
1204    int i;
1205
1206    i = 0;
1207    while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
1208        ;
1209    return READ_REG(sc, SAFE_RNG_OUT);
1210}
1211
1212static int
1213safe_read_random(void *arg, u_int32_t *buf, int maxwords)
1214{
1215    struct safe_softc *sc = (struct safe_softc *) arg;
1216    int i, rc;
1217
1218    DPRINTF(("%s()\n", __FUNCTION__));
1219    
1220    safestats.st_rng++;
1221    /*
1222     * Fetch the next block of data.
1223     */
1224    if (maxwords > safe_rngbufsize)
1225        maxwords = safe_rngbufsize;
1226    if (maxwords > SAFE_RNG_MAXBUFSIZ)
1227        maxwords = SAFE_RNG_MAXBUFSIZ;
1228retry:
1229    /* read as much as we can */
1230    for (rc = 0; rc < maxwords; rc++) {
1231        if (READ_REG(sc, SAFE_RNG_STAT) != 0)
1232            break;
1233        buf[rc] = READ_REG(sc, SAFE_RNG_OUT);
1234    }
1235    if (rc == 0)
1236        return 0;
1237    /*
1238     * Check the comparator alarm count and reset the h/w if
1239     * it exceeds our threshold. This guards against the
1240     * hardware oscillators resonating with external signals.
1241     */
1242    if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
1243        u_int32_t freq_inc, w;
1244
1245        DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
1246            (unsigned)READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
1247        safestats.st_rngalarm++;
1248        safe_rng_enable_short_cycle(sc);
1249        freq_inc = 18;
1250        for (i = 0; i < 64; i++) {
1251            w = READ_REG(sc, SAFE_RNG_CNFG);
1252            freq_inc = ((w + freq_inc) & 0x3fL);
1253            w = ((w & ~0x3fL) | freq_inc);
1254            WRITE_REG(sc, SAFE_RNG_CNFG, w);
1255
1256            WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1257
1258            (void) safe_rng_read(sc);
1259            DELAY(25);
1260
1261            if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
1262                safe_rng_disable_short_cycle(sc);
1263                goto retry;
1264            }
1265            freq_inc = 1;
1266        }
1267        safe_rng_disable_short_cycle(sc);
1268    } else
1269        WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1270
1271    return(rc);
1272}
1273#endif /* defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG) */
1274
1275
1276/*
1277 * Resets the board. Values in the regesters are left as is
1278 * from the reset (i.e. initial values are assigned elsewhere).
1279 */
1280static void
1281safe_reset_board(struct safe_softc *sc)
1282{
1283    u_int32_t v;
1284    /*
1285     * Reset the device. The manual says no delay
1286     * is needed between marking and clearing reset.
1287     */
1288    DPRINTF(("%s()\n", __FUNCTION__));
1289
1290    v = READ_REG(sc, SAFE_PE_DMACFG) &~
1291        (SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
1292         SAFE_PE_DMACFG_SGRESET);
1293    WRITE_REG(sc, SAFE_PE_DMACFG, v
1294                    | SAFE_PE_DMACFG_PERESET
1295                    | SAFE_PE_DMACFG_PDRRESET
1296                    | SAFE_PE_DMACFG_SGRESET);
1297    WRITE_REG(sc, SAFE_PE_DMACFG, v);
1298}
1299
1300/*
1301 * Initialize registers we need to touch only once.
1302 */
1303static void
1304safe_init_board(struct safe_softc *sc)
1305{
1306    u_int32_t v, dwords;
1307
1308    DPRINTF(("%s()\n", __FUNCTION__));
1309
1310    v = READ_REG(sc, SAFE_PE_DMACFG);
1311    v &=~ ( SAFE_PE_DMACFG_PEMODE
1312            | SAFE_PE_DMACFG_FSENA /* failsafe enable */
1313            | SAFE_PE_DMACFG_GPRPCI /* gather ring on PCI */
1314            | SAFE_PE_DMACFG_SPRPCI /* scatter ring on PCI */
1315            | SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */
1316            | SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */
1317            | SAFE_PE_DMACFG_ESSA /* endian-swap SA's */
1318            | SAFE_PE_DMACFG_ESPACKET /* swap the packet data */
1319          );
1320    v |= SAFE_PE_DMACFG_FSENA /* failsafe enable */
1321      | SAFE_PE_DMACFG_GPRPCI /* gather ring on PCI */
1322      | SAFE_PE_DMACFG_SPRPCI /* scatter ring on PCI */
1323      | SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */
1324      | SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */
1325      | SAFE_PE_DMACFG_ESSA /* endian-swap SA's */
1326#if 0
1327      | SAFE_PE_DMACFG_ESPACKET /* swap the packet data */
1328#endif
1329      ;
1330    WRITE_REG(sc, SAFE_PE_DMACFG, v);
1331
1332#ifdef __BIG_ENDIAN
1333    /* tell the safenet that we are 4321 and not 1234 */
1334    WRITE_REG(sc, SAFE_ENDIAN, 0xe4e41b1b);
1335#endif
1336
1337    if (sc->sc_chiprev == SAFE_REV(1,0)) {
1338        /*
1339         * Avoid large PCI DMA transfers. Rev 1.0 has a bug where
1340         * "target mode transfers" done while the chip is DMA'ing
1341         * >1020 bytes cause the hardware to lockup. To avoid this
1342         * we reduce the max PCI transfer size and use small source
1343         * particle descriptors (<= 256 bytes).
1344         */
1345        WRITE_REG(sc, SAFE_DMA_CFG, 256);
1346        device_printf(sc->sc_dev,
1347            "Reduce max DMA size to %u words for rev %u.%u WAR\n",
1348            (unsigned) ((READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff),
1349            (unsigned) SAFE_REV_MAJ(sc->sc_chiprev),
1350            (unsigned) SAFE_REV_MIN(sc->sc_chiprev));
1351        sc->sc_max_dsize = 256;
1352    } else {
1353        sc->sc_max_dsize = SAFE_MAX_DSIZE;
1354    }
1355
1356    /* NB: operands+results are overlaid */
1357    WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
1358    WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
1359    /*
1360     * Configure ring entry size and number of items in the ring.
1361     */
1362    KASSERT((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
1363        ("PE ring entry not 32-bit aligned!"));
1364    dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
1365    WRITE_REG(sc, SAFE_PE_RINGCFG,
1366        (dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
1367    WRITE_REG(sc, SAFE_PE_RINGPOLL, 0); /* disable polling */
1368
1369    WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
1370    WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
1371    WRITE_REG(sc, SAFE_PE_PARTSIZE,
1372        (SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
1373    /*
1374     * NB: destination particles are fixed size. We use
1375     * an mbuf cluster and require all results go to
1376     * clusters or smaller.
1377     */
1378    WRITE_REG(sc, SAFE_PE_PARTCFG, sc->sc_max_dsize);
1379
1380    /* it's now safe to enable PE mode, do it */
1381    WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
1382
1383    /*
1384     * Configure hardware to use level-triggered interrupts and
1385     * to interrupt after each descriptor is processed.
1386     */
1387    WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
1388    WRITE_REG(sc, SAFE_HI_CLR, 0xffffffff);
1389    WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
1390    WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
1391}
1392
1393
1394/*
1395 * Clean up after a chip crash.
1396 * It is assumed that the caller in splimp()
1397 */
1398static void
1399safe_cleanchip(struct safe_softc *sc)
1400{
1401    DPRINTF(("%s()\n", __FUNCTION__));
1402
1403    if (sc->sc_nqchip != 0) {
1404        struct safe_ringentry *re = sc->sc_back;
1405
1406        while (re != sc->sc_front) {
1407            if (re->re_desc.d_csr != 0)
1408                safe_free_entry(sc, re);
1409            if (++re == sc->sc_ringtop)
1410                re = sc->sc_ring;
1411        }
1412        sc->sc_back = re;
1413        sc->sc_nqchip = 0;
1414    }
1415}
1416
1417/*
1418 * free a safe_q
1419 * It is assumed that the caller is within splimp().
1420 */
1421static int
1422safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
1423{
1424    struct cryptop *crp;
1425
1426    DPRINTF(("%s()\n", __FUNCTION__));
1427
1428    /*
1429     * Free header MCR
1430     */
1431    if ((re->re_dst_skb != NULL) && (re->re_src_skb != re->re_dst_skb))
1432#ifdef NOTYET
1433        m_freem(re->re_dst_m);
1434#else
1435        printk("%s,%d: SKB not supported\n", __FILE__, __LINE__);
1436#endif
1437
1438    crp = (struct cryptop *)re->re_crp;
1439    
1440    re->re_desc.d_csr = 0;
1441    
1442    crp->crp_etype = EFAULT;
1443    crypto_done(crp);
1444    return(0);
1445}
1446
1447/*
1448 * Routine to reset the chip and clean up.
1449 * It is assumed that the caller is in splimp()
1450 */
1451static void
1452safe_totalreset(struct safe_softc *sc)
1453{
1454    DPRINTF(("%s()\n", __FUNCTION__));
1455
1456    safe_reset_board(sc);
1457    safe_init_board(sc);
1458    safe_cleanchip(sc);
1459}
1460
1461/*
1462 * Is the operand suitable aligned for direct DMA. Each
1463 * segment must be aligned on a 32-bit boundary and all
1464 * but the last segment must be a multiple of 4 bytes.
1465 */
1466static int
1467safe_dmamap_aligned(struct safe_softc *sc, const struct safe_operand *op)
1468{
1469    int i;
1470
1471    DPRINTF(("%s()\n", __FUNCTION__));
1472
1473    for (i = 0; i < op->nsegs; i++) {
1474        if (op->segs[i].ds_addr & 3)
1475            return (0);
1476        if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3))
1477            return (0);
1478    }
1479    return (1);
1480}
1481
1482/*
1483 * Is the operand suitable for direct DMA as the destination
1484 * of an operation. The hardware requires that each ``particle''
1485 * but the last in an operation result have the same size. We
1486 * fix that size at SAFE_MAX_DSIZE bytes. This routine returns
1487 * 0 if some segment is not a multiple of of this size, 1 if all
1488 * segments are exactly this size, or 2 if segments are at worst
1489 * a multple of this size.
1490 */
1491static int
1492safe_dmamap_uniform(struct safe_softc *sc, const struct safe_operand *op)
1493{
1494    int result = 1;
1495
1496    DPRINTF(("%s()\n", __FUNCTION__));
1497
1498    if (op->nsegs > 0) {
1499        int i;
1500
1501        for (i = 0; i < op->nsegs-1; i++) {
1502            if (op->segs[i].ds_len % sc->sc_max_dsize)
1503                return (0);
1504            if (op->segs[i].ds_len != sc->sc_max_dsize)
1505                result = 2;
1506        }
1507    }
1508    return (result);
1509}
1510
1511static int
1512safe_kprocess(device_t dev, struct cryptkop *krp, int hint)
1513{
1514    struct safe_softc *sc = device_get_softc(dev);
1515    struct safe_pkq *q;
1516    unsigned long flags;
1517
1518    DPRINTF(("%s()\n", __FUNCTION__));
1519
1520    if (sc == NULL) {
1521        krp->krp_status = EINVAL;
1522        goto err;
1523    }
1524
1525    if (krp->krp_op != CRK_MOD_EXP) {
1526        krp->krp_status = EOPNOTSUPP;
1527        goto err;
1528    }
1529
1530    q = (struct safe_pkq *) kmalloc(sizeof(*q), GFP_KERNEL);
1531    if (q == NULL) {
1532        krp->krp_status = ENOMEM;
1533        goto err;
1534    }
1535    memset(q, 0, sizeof(*q));
1536    q->pkq_krp = krp;
1537    INIT_LIST_HEAD(&q->pkq_list);
1538
1539    spin_lock_irqsave(&sc->sc_pkmtx, flags);
1540    list_add_tail(&q->pkq_list, &sc->sc_pkq);
1541    safe_kfeed(sc);
1542    spin_unlock_irqrestore(&sc->sc_pkmtx, flags);
1543    return (0);
1544
1545err:
1546    crypto_kdone(krp);
1547    return (0);
1548}
1549
1550#define SAFE_CRK_PARAM_BASE 0
1551#define SAFE_CRK_PARAM_EXP 1
1552#define SAFE_CRK_PARAM_MOD 2
1553
1554static int
1555safe_kstart(struct safe_softc *sc)
1556{
1557    struct cryptkop *krp = sc->sc_pkq_cur->pkq_krp;
1558    int exp_bits, mod_bits, base_bits;
1559    u_int32_t op, a_off, b_off, c_off, d_off;
1560
1561    DPRINTF(("%s()\n", __FUNCTION__));
1562
1563    if (krp->krp_iparams < 3 || krp->krp_oparams != 1) {
1564        krp->krp_status = EINVAL;
1565        return (1);
1566    }
1567
1568    base_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_BASE]);
1569    if (base_bits > 2048)
1570        goto too_big;
1571    if (base_bits <= 0) /* 5. base not zero */
1572        goto too_small;
1573
1574    exp_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_EXP]);
1575    if (exp_bits > 2048)
1576        goto too_big;
1577    if (exp_bits <= 0) /* 1. exponent word length > 0 */
1578        goto too_small; /* 4. exponent not zero */
1579
1580    mod_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_MOD]);
1581    if (mod_bits > 2048)
1582        goto too_big;
1583    if (mod_bits <= 32) /* 2. modulus word length > 1 */
1584        goto too_small; /* 8. MSW of modulus != zero */
1585    if (mod_bits < exp_bits) /* 3 modulus len >= exponent len */
1586        goto too_small;
1587    if ((krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p[0] & 1) == 0)
1588        goto bad_domain; /* 6. modulus is odd */
1589    if (mod_bits > krp->krp_param[krp->krp_iparams].crp_nbits)
1590        goto too_small; /* make sure result will fit */
1591
1592    /* 7. modulus > base */
1593    if (mod_bits < base_bits)
1594        goto too_small;
1595    if (mod_bits == base_bits) {
1596        u_int8_t *basep, *modp;
1597        int i;
1598
1599        basep = krp->krp_param[SAFE_CRK_PARAM_BASE].crp_p +
1600            ((base_bits + 7) / 8) - 1;
1601        modp = krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p +
1602            ((mod_bits + 7) / 8) - 1;
1603        
1604        for (i = 0; i < (mod_bits + 7) / 8; i++, basep--, modp--) {
1605            if (*modp < *basep)
1606                goto too_small;
1607            if (*modp > *basep)
1608                break;
1609        }
1610    }
1611
1612    /* And on the 9th step, he rested. */
1613
1614    WRITE_REG(sc, SAFE_PK_A_LEN, (exp_bits + 31) / 32);
1615    WRITE_REG(sc, SAFE_PK_B_LEN, (mod_bits + 31) / 32);
1616    if (mod_bits > 1024) {
1617        op = SAFE_PK_FUNC_EXP4;
1618        a_off = 0x000;
1619        b_off = 0x100;
1620        c_off = 0x200;
1621        d_off = 0x300;
1622    } else {
1623        op = SAFE_PK_FUNC_EXP16;
1624        a_off = 0x000;
1625        b_off = 0x080;
1626        c_off = 0x100;
1627        d_off = 0x180;
1628    }
1629    sc->sc_pk_reslen = b_off - a_off;
1630    sc->sc_pk_resoff = d_off;
1631
1632    /* A is exponent, B is modulus, C is base, D is result */
1633    safe_kload_reg(sc, a_off, b_off - a_off,
1634        &krp->krp_param[SAFE_CRK_PARAM_EXP]);
1635    WRITE_REG(sc, SAFE_PK_A_ADDR, a_off >> 2);
1636    safe_kload_reg(sc, b_off, b_off - a_off,
1637        &krp->krp_param[SAFE_CRK_PARAM_MOD]);
1638    WRITE_REG(sc, SAFE_PK_B_ADDR, b_off >> 2);
1639    safe_kload_reg(sc, c_off, b_off - a_off,
1640        &krp->krp_param[SAFE_CRK_PARAM_BASE]);
1641    WRITE_REG(sc, SAFE_PK_C_ADDR, c_off >> 2);
1642    WRITE_REG(sc, SAFE_PK_D_ADDR, d_off >> 2);
1643
1644    WRITE_REG(sc, SAFE_PK_FUNC, op | SAFE_PK_FUNC_RUN);
1645
1646    return (0);
1647
1648too_big:
1649    krp->krp_status = E2BIG;
1650    return (1);
1651too_small:
1652    krp->krp_status = ERANGE;
1653    return (1);
1654bad_domain:
1655    krp->krp_status = EDOM;
1656    return (1);
1657}
1658
1659static int
1660safe_ksigbits(struct safe_softc *sc, struct crparam *cr)
1661{
1662    u_int plen = (cr->crp_nbits + 7) / 8;
1663    int i, sig = plen * 8;
1664    u_int8_t c, *p = cr->crp_p;
1665
1666    DPRINTF(("%s()\n", __FUNCTION__));
1667
1668    for (i = plen - 1; i >= 0; i--) {
1669        c = p[i];
1670        if (c != 0) {
1671            while ((c & 0x80) == 0) {
1672                sig--;
1673                c <<= 1;
1674            }
1675            break;
1676        }
1677        sig -= 8;
1678    }
1679    return (sig);
1680}
1681
1682static void
1683safe_kfeed(struct safe_softc *sc)
1684{
1685    struct safe_pkq *q, *tmp;
1686
1687    DPRINTF(("%s()\n", __FUNCTION__));
1688
1689    if (list_empty(&sc->sc_pkq) && sc->sc_pkq_cur == NULL)
1690        return;
1691    if (sc->sc_pkq_cur != NULL)
1692        return;
1693    list_for_each_entry_safe(q, tmp, &sc->sc_pkq, pkq_list) {
1694        sc->sc_pkq_cur = q;
1695        list_del(&q->pkq_list);
1696        if (safe_kstart(sc) != 0) {
1697            crypto_kdone(q->pkq_krp);
1698            kfree(q);
1699            sc->sc_pkq_cur = NULL;
1700        } else {
1701            /* op started, start polling */
1702            mod_timer(&sc->sc_pkto, jiffies + 1);
1703            break;
1704        }
1705    }
1706}
1707
1708static void
1709safe_kpoll(unsigned long arg)
1710{
1711    struct safe_softc *sc = NULL;
1712    struct safe_pkq *q;
1713    struct crparam *res;
1714    int i;
1715    u_int32_t buf[64];
1716    unsigned long flags;
1717
1718    DPRINTF(("%s()\n", __FUNCTION__));
1719
1720    if (arg >= SAFE_MAX_CHIPS)
1721        return;
1722    sc = safe_chip_idx[arg];
1723    if (!sc) {
1724        DPRINTF(("%s() - bad callback\n", __FUNCTION__));
1725        return;
1726    }
1727
1728    spin_lock_irqsave(&sc->sc_pkmtx, flags);
1729    if (sc->sc_pkq_cur == NULL)
1730        goto out;
1731    if (READ_REG(sc, SAFE_PK_FUNC) & SAFE_PK_FUNC_RUN) {
1732        /* still running, check back later */
1733        mod_timer(&sc->sc_pkto, jiffies + 1);
1734        goto out;
1735    }
1736
1737    q = sc->sc_pkq_cur;
1738    res = &q->pkq_krp->krp_param[q->pkq_krp->krp_iparams];
1739    bzero(buf, sizeof(buf));
1740    bzero(res->crp_p, (res->crp_nbits + 7) / 8);
1741    for (i = 0; i < sc->sc_pk_reslen >> 2; i++)
1742        buf[i] = le32_to_cpu(READ_REG(sc, SAFE_PK_RAM_START +
1743            sc->sc_pk_resoff + (i << 2)));
1744    bcopy(buf, res->crp_p, (res->crp_nbits + 7) / 8);
1745    /*
1746     * reduce the bits that need copying if possible
1747     */
1748    res->crp_nbits = min(res->crp_nbits,sc->sc_pk_reslen * 8);
1749    res->crp_nbits = safe_ksigbits(sc, res);
1750
1751    for (i = SAFE_PK_RAM_START; i < SAFE_PK_RAM_END; i += 4)
1752        WRITE_REG(sc, i, 0);
1753
1754    crypto_kdone(q->pkq_krp);
1755    kfree(q);
1756    sc->sc_pkq_cur = NULL;
1757
1758    safe_kfeed(sc);
1759out:
1760    spin_unlock_irqrestore(&sc->sc_pkmtx, flags);
1761}
1762
1763static void
1764safe_kload_reg(struct safe_softc *sc, u_int32_t off, u_int32_t len,
1765    struct crparam *n)
1766{
1767    u_int32_t buf[64], i;
1768
1769    DPRINTF(("%s()\n", __FUNCTION__));
1770
1771    bzero(buf, sizeof(buf));
1772    bcopy(n->crp_p, buf, (n->crp_nbits + 7) / 8);
1773
1774    for (i = 0; i < len >> 2; i++)
1775        WRITE_REG(sc, SAFE_PK_RAM_START + off + (i << 2),
1776            cpu_to_le32(buf[i]));
1777}
1778
1779#ifdef SAFE_DEBUG
1780static void
1781safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
1782{
1783    printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n"
1784        , tag
1785        , READ_REG(sc, SAFE_DMA_ENDIAN)
1786        , READ_REG(sc, SAFE_DMA_SRCADDR)
1787        , READ_REG(sc, SAFE_DMA_DSTADDR)
1788        , READ_REG(sc, SAFE_DMA_STAT)
1789    );
1790}
1791
1792static void
1793safe_dump_intrstate(struct safe_softc *sc, const char *tag)
1794{
1795    printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n"
1796        , tag
1797        , READ_REG(sc, SAFE_HI_CFG)
1798        , READ_REG(sc, SAFE_HI_MASK)
1799        , READ_REG(sc, SAFE_HI_DESC_CNT)
1800        , READ_REG(sc, SAFE_HU_STAT)
1801        , READ_REG(sc, SAFE_HM_STAT)
1802    );
1803}
1804
1805static void
1806safe_dump_ringstate(struct safe_softc *sc, const char *tag)
1807{
1808    u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
1809
1810    /* NB: assume caller has lock on ring */
1811    printf("%s: ERNGSTAT %x (next %u) back %lu front %lu\n",
1812        tag,
1813        estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
1814        (unsigned long)(sc->sc_back - sc->sc_ring),
1815        (unsigned long)(sc->sc_front - sc->sc_ring));
1816}
1817
1818static void
1819safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
1820{
1821    int ix, nsegs;
1822
1823    ix = re - sc->sc_ring;
1824    printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n"
1825        , tag
1826        , re, ix
1827        , re->re_desc.d_csr
1828        , re->re_desc.d_src
1829        , re->re_desc.d_dst
1830        , re->re_desc.d_sa
1831        , re->re_desc.d_len
1832    );
1833    if (re->re_src.nsegs > 1) {
1834        ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
1835            sizeof(struct safe_pdesc);
1836        for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {
1837            printf(" spd[%u] %p: %p size %u flags %x"
1838                , ix, &sc->sc_spring[ix]
1839                , (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr
1840                , sc->sc_spring[ix].pd_size
1841                , sc->sc_spring[ix].pd_flags
1842            );
1843            if (sc->sc_spring[ix].pd_size == 0)
1844                printf(" (zero!)");
1845            printf("\n");
1846            if (++ix == SAFE_TOTAL_SPART)
1847                ix = 0;
1848        }
1849    }
1850    if (re->re_dst.nsegs > 1) {
1851        ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
1852            sizeof(struct safe_pdesc);
1853        for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {
1854            printf(" dpd[%u] %p: %p flags %x\n"
1855                , ix, &sc->sc_dpring[ix]
1856                , (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr
1857                , sc->sc_dpring[ix].pd_flags
1858            );
1859            if (++ix == SAFE_TOTAL_DPART)
1860                ix = 0;
1861        }
1862    }
1863    printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
1864        re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
1865    printf("sa: key %x %x %x %x %x %x %x %x\n"
1866        , re->re_sa.sa_key[0]
1867        , re->re_sa.sa_key[1]
1868        , re->re_sa.sa_key[2]
1869        , re->re_sa.sa_key[3]
1870        , re->re_sa.sa_key[4]
1871        , re->re_sa.sa_key[5]
1872        , re->re_sa.sa_key[6]
1873        , re->re_sa.sa_key[7]
1874    );
1875    printf("sa: indigest %x %x %x %x %x\n"
1876        , re->re_sa.sa_indigest[0]
1877        , re->re_sa.sa_indigest[1]
1878        , re->re_sa.sa_indigest[2]
1879        , re->re_sa.sa_indigest[3]
1880        , re->re_sa.sa_indigest[4]
1881    );
1882    printf("sa: outdigest %x %x %x %x %x\n"
1883        , re->re_sa.sa_outdigest[0]
1884        , re->re_sa.sa_outdigest[1]
1885        , re->re_sa.sa_outdigest[2]
1886        , re->re_sa.sa_outdigest[3]
1887        , re->re_sa.sa_outdigest[4]
1888    );
1889    printf("sr: iv %x %x %x %x\n"
1890        , re->re_sastate.sa_saved_iv[0]
1891        , re->re_sastate.sa_saved_iv[1]
1892        , re->re_sastate.sa_saved_iv[2]
1893        , re->re_sastate.sa_saved_iv[3]
1894    );
1895    printf("sr: hashbc %u indigest %x %x %x %x %x\n"
1896        , re->re_sastate.sa_saved_hashbc
1897        , re->re_sastate.sa_saved_indigest[0]
1898        , re->re_sastate.sa_saved_indigest[1]
1899        , re->re_sastate.sa_saved_indigest[2]
1900        , re->re_sastate.sa_saved_indigest[3]
1901        , re->re_sastate.sa_saved_indigest[4]
1902    );
1903}
1904
1905static void
1906safe_dump_ring(struct safe_softc *sc, const char *tag)
1907{
1908    unsigned long flags;
1909
1910    spin_lock_irqsave(&sc->sc_ringmtx, flags);
1911    printf("\nSafeNet Ring State:\n");
1912    safe_dump_intrstate(sc, tag);
1913    safe_dump_dmastatus(sc, tag);
1914    safe_dump_ringstate(sc, tag);
1915    if (sc->sc_nqchip) {
1916        struct safe_ringentry *re = sc->sc_back;
1917        do {
1918            safe_dump_request(sc, tag, re);
1919            if (++re == sc->sc_ringtop)
1920                re = sc->sc_ring;
1921        } while (re != sc->sc_front);
1922    }
1923    spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1924}
1925#endif /* SAFE_DEBUG */
1926
1927
1928static int safe_probe(struct pci_dev *dev, const struct pci_device_id *ent)
1929{
1930    struct safe_softc *sc = NULL;
1931    u32 mem_start, mem_len, cmd;
1932    int i, rc, devinfo;
1933    dma_addr_t raddr;
1934    static int num_chips = 0;
1935
1936    DPRINTF(("%s()\n", __FUNCTION__));
1937
1938    if (pci_enable_device(dev) < 0)
1939        return(-ENODEV);
1940
1941    if (!dev->irq) {
1942        printk("safe: found device with no IRQ assigned. check BIOS settings!");
1943        pci_disable_device(dev);
1944        return(-ENODEV);
1945    }
1946
1947    if (pci_set_mwi(dev)) {
1948        printk("safe: pci_set_mwi failed!");
1949        return(-ENODEV);
1950    }
1951
1952    sc = (struct safe_softc *) kmalloc(sizeof(*sc), GFP_KERNEL);
1953    if (!sc)
1954        return(-ENOMEM);
1955    memset(sc, 0, sizeof(*sc));
1956
1957    softc_device_init(sc, "safe", num_chips, safe_methods);
1958
1959    sc->sc_irq = -1;
1960    sc->sc_cid = -1;
1961    sc->sc_pcidev = dev;
1962    if (num_chips < SAFE_MAX_CHIPS) {
1963        safe_chip_idx[device_get_unit(sc->sc_dev)] = sc;
1964        num_chips++;
1965    }
1966
1967    INIT_LIST_HEAD(&sc->sc_pkq);
1968    spin_lock_init(&sc->sc_pkmtx);
1969
1970    pci_set_drvdata(sc->sc_pcidev, sc);
1971
1972    /* we read its hardware registers as memory */
1973    mem_start = pci_resource_start(sc->sc_pcidev, 0);
1974    mem_len = pci_resource_len(sc->sc_pcidev, 0);
1975
1976    sc->sc_base_addr = (ocf_iomem_t) ioremap(mem_start, mem_len);
1977    if (!sc->sc_base_addr) {
1978        device_printf(sc->sc_dev, "failed to ioremap 0x%x-0x%x\n",
1979                mem_start, mem_start + mem_len - 1);
1980        goto out;
1981    }
1982
1983    /* fix up the bus size */
1984    if (pci_set_dma_mask(sc->sc_pcidev, DMA_32BIT_MASK)) {
1985        device_printf(sc->sc_dev, "No usable DMA configuration, aborting.\n");
1986        goto out;
1987    }
1988    if (pci_set_consistent_dma_mask(sc->sc_pcidev, DMA_32BIT_MASK)) {
1989        device_printf(sc->sc_dev, "No usable consistent DMA configuration, aborting.\n");
1990        goto out;
1991    }
1992
1993    pci_set_master(sc->sc_pcidev);
1994
1995    pci_read_config_dword(sc->sc_pcidev, PCI_COMMAND, &cmd);
1996
1997    if (!(cmd & PCI_COMMAND_MEMORY)) {
1998        device_printf(sc->sc_dev, "failed to enable memory mapping\n");
1999        goto out;
2000    }
2001
2002    if (!(cmd & PCI_COMMAND_MASTER)) {
2003        device_printf(sc->sc_dev, "failed to enable bus mastering\n");
2004        goto out;
2005    }
2006
2007    rc = request_irq(dev->irq, safe_intr, IRQF_SHARED, "safe", sc);
2008    if (rc) {
2009        device_printf(sc->sc_dev, "failed to hook irq %d\n", sc->sc_irq);
2010        goto out;
2011    }
2012    sc->sc_irq = dev->irq;
2013
2014    sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
2015            (SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
2016
2017    /*
2018     * Allocate packet engine descriptors.
2019     */
2020    sc->sc_ringalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2021            SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2022            &sc->sc_ringalloc.dma_paddr);
2023    if (!sc->sc_ringalloc.dma_vaddr) {
2024        device_printf(sc->sc_dev, "cannot allocate PE descriptor ring\n");
2025        goto out;
2026    }
2027
2028    /*
2029     * Hookup the static portion of all our data structures.
2030     */
2031    sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
2032    sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
2033    sc->sc_front = sc->sc_ring;
2034    sc->sc_back = sc->sc_ring;
2035    raddr = sc->sc_ringalloc.dma_paddr;
2036    bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
2037    for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
2038        struct safe_ringentry *re = &sc->sc_ring[i];
2039
2040        re->re_desc.d_sa = raddr +
2041            offsetof(struct safe_ringentry, re_sa);
2042        re->re_sa.sa_staterec = raddr +
2043            offsetof(struct safe_ringentry, re_sastate);
2044
2045        raddr += sizeof (struct safe_ringentry);
2046    }
2047    spin_lock_init(&sc->sc_ringmtx);
2048
2049    /*
2050     * Allocate scatter and gather particle descriptors.
2051     */
2052    sc->sc_spalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2053            SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
2054            &sc->sc_spalloc.dma_paddr);
2055    if (!sc->sc_spalloc.dma_vaddr) {
2056        device_printf(sc->sc_dev, "cannot allocate source particle descriptor ring\n");
2057        goto out;
2058    }
2059    sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
2060    sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
2061    sc->sc_spfree = sc->sc_spring;
2062    bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
2063
2064    sc->sc_dpalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2065            SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2066            &sc->sc_dpalloc.dma_paddr);
2067    if (!sc->sc_dpalloc.dma_vaddr) {
2068        device_printf(sc->sc_dev, "cannot allocate destination particle descriptor ring\n");
2069        goto out;
2070    }
2071    sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
2072    sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
2073    sc->sc_dpfree = sc->sc_dpring;
2074    bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
2075
2076    sc->sc_cid = crypto_get_driverid(softc_get_device(sc), CRYPTOCAP_F_HARDWARE);
2077    if (sc->sc_cid < 0) {
2078        device_printf(sc->sc_dev, "could not get crypto driver id\n");
2079        goto out;
2080    }
2081
2082    printf("%s:", device_get_nameunit(sc->sc_dev));
2083
2084    devinfo = READ_REG(sc, SAFE_DEVINFO);
2085    if (devinfo & SAFE_DEVINFO_RNG) {
2086        sc->sc_flags |= SAFE_FLAGS_RNG;
2087        printf(" rng");
2088    }
2089    if (devinfo & SAFE_DEVINFO_PKEY) {
2090        printf(" key");
2091        sc->sc_flags |= SAFE_FLAGS_KEY;
2092        crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
2093#if 0
2094        crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
2095#endif
2096        init_timer(&sc->sc_pkto);
2097        sc->sc_pkto.function = safe_kpoll;
2098        sc->sc_pkto.data = (unsigned long) device_get_unit(sc->sc_dev);
2099    }
2100    if (devinfo & SAFE_DEVINFO_DES) {
2101        printf(" des/3des");
2102        crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
2103        crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
2104    }
2105    if (devinfo & SAFE_DEVINFO_AES) {
2106        printf(" aes");
2107        crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
2108    }
2109    if (devinfo & SAFE_DEVINFO_MD5) {
2110        printf(" md5");
2111        crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
2112    }
2113    if (devinfo & SAFE_DEVINFO_SHA1) {
2114        printf(" sha1");
2115        crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
2116    }
2117    printf(" null");
2118    crypto_register(sc->sc_cid, CRYPTO_NULL_CBC, 0, 0);
2119    crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0);
2120    /* XXX other supported algorithms */
2121    printf("\n");
2122
2123    safe_reset_board(sc); /* reset h/w */
2124    safe_init_board(sc); /* init h/w */
2125
2126#if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
2127    if (sc->sc_flags & SAFE_FLAGS_RNG) {
2128        safe_rng_init(sc);
2129        crypto_rregister(sc->sc_cid, safe_read_random, sc);
2130    }
2131#endif /* SAFE_NO_RNG */
2132
2133    return (0);
2134
2135out:
2136    if (sc->sc_cid >= 0)
2137        crypto_unregister_all(sc->sc_cid);
2138    if (sc->sc_irq != -1)
2139        free_irq(sc->sc_irq, sc);
2140    if (sc->sc_ringalloc.dma_vaddr)
2141        pci_free_consistent(sc->sc_pcidev,
2142                SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2143                sc->sc_ringalloc.dma_vaddr, sc->sc_ringalloc.dma_paddr);
2144    if (sc->sc_spalloc.dma_vaddr)
2145        pci_free_consistent(sc->sc_pcidev,
2146                SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2147                sc->sc_spalloc.dma_vaddr, sc->sc_spalloc.dma_paddr);
2148    if (sc->sc_dpalloc.dma_vaddr)
2149        pci_free_consistent(sc->sc_pcidev,
2150                SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2151                sc->sc_dpalloc.dma_vaddr, sc->sc_dpalloc.dma_paddr);
2152    kfree(sc);
2153    return(-ENODEV);
2154}
2155
2156static void safe_remove(struct pci_dev *dev)
2157{
2158    struct safe_softc *sc = pci_get_drvdata(dev);
2159
2160    DPRINTF(("%s()\n", __FUNCTION__));
2161
2162    /* XXX wait/abort active ops */
2163
2164    WRITE_REG(sc, SAFE_HI_MASK, 0); /* disable interrupts */
2165
2166    del_timer_sync(&sc->sc_pkto);
2167
2168    crypto_unregister_all(sc->sc_cid);
2169
2170    safe_cleanchip(sc);
2171
2172    if (sc->sc_irq != -1)
2173        free_irq(sc->sc_irq, sc);
2174    if (sc->sc_ringalloc.dma_vaddr)
2175        pci_free_consistent(sc->sc_pcidev,
2176                SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2177                sc->sc_ringalloc.dma_vaddr, sc->sc_ringalloc.dma_paddr);
2178    if (sc->sc_spalloc.dma_vaddr)
2179        pci_free_consistent(sc->sc_pcidev,
2180                SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2181                sc->sc_spalloc.dma_vaddr, sc->sc_spalloc.dma_paddr);
2182    if (sc->sc_dpalloc.dma_vaddr)
2183        pci_free_consistent(sc->sc_pcidev,
2184                SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2185                sc->sc_dpalloc.dma_vaddr, sc->sc_dpalloc.dma_paddr);
2186    sc->sc_irq = -1;
2187    sc->sc_ringalloc.dma_vaddr = NULL;
2188    sc->sc_spalloc.dma_vaddr = NULL;
2189    sc->sc_dpalloc.dma_vaddr = NULL;
2190}
2191
2192static struct pci_device_id safe_pci_tbl[] = {
2193    { PCI_VENDOR_SAFENET, PCI_PRODUCT_SAFEXCEL,
2194      PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
2195    { },
2196};
2197MODULE_DEVICE_TABLE(pci, safe_pci_tbl);
2198
2199static struct pci_driver safe_driver = {
2200    .name = "safe",
2201    .id_table = safe_pci_tbl,
2202    .probe = safe_probe,
2203    .remove = safe_remove,
2204    /* add PM stuff here one day */
2205};
2206
2207static int __init safe_init (void)
2208{
2209    struct safe_softc *sc = NULL;
2210    int rc;
2211
2212    DPRINTF(("%s(%p)\n", __FUNCTION__, safe_init));
2213
2214    rc = pci_register_driver(&safe_driver);
2215    pci_register_driver_compat(&safe_driver, rc);
2216
2217    return rc;
2218}
2219
2220static void __exit safe_exit (void)
2221{
2222    pci_unregister_driver(&safe_driver);
2223}
2224
2225module_init(safe_init);
2226module_exit(safe_exit);
2227
2228MODULE_LICENSE("BSD");
2229MODULE_AUTHOR("David McCullough <david_mccullough@mcafee.com>");
2230MODULE_DESCRIPTION("OCF driver for safenet PCI crypto devices");
2231

Archive Download this file

Branches:
backfire
history
master
release_2010-11-17
release_2010-12-14
release_2011-02-23
release_2011-05-28
release_2011-08-27
release_2011-11-13
release_2012-03-18
release_2012-04-09
release_2012-10-11
uclibc_nptl



interactive