Root/package/broadcom-wl/src/driver/hnddma.c

1/*
2 * Generic Broadcom Home Networking Division (HND) DMA module.
3 * This supports the following chips: BCM42xx, 44xx, 47xx .
4 *
5 * Copyright 2007, Broadcom Corporation
6 * All Rights Reserved.
7 *
8 * THIS SOFTWARE IS OFFERED "AS IS", AND BROADCOM GRANTS NO WARRANTIES OF ANY
9 * KIND, EXPRESS OR IMPLIED, BY STATUTE, COMMUNICATION OR OTHERWISE. BROADCOM
10 * SPECIFICALLY DISCLAIMS ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS
11 * FOR A SPECIFIC PURPOSE OR NONINFRINGEMENT CONCERNING THIS SOFTWARE.
12 *
13 */
14
15#include <typedefs.h>
16#include <bcmdefs.h>
17#include <osl.h>
18#include "linux_osl.h"
19#include <bcmendian.h>
20#include <sbconfig.h>
21#include "bcmutils.h"
22#include <bcmdevs.h>
23#include <sbutils.h>
24
25#include "sbhnddma.h"
26#include "hnddma.h"
27
28/* debug/trace */
29#ifdef BCMDBG
30#define DMA_ERROR(args) if (!(*di->msg_level & 1)); else printf args
31#define DMA_TRACE(args) if (!(*di->msg_level & 2)); else printf args
32#else
33#define DMA_ERROR(args)
34#define DMA_TRACE(args)
35#endif
36
37/* default dma message level (if input msg_level pointer is null in dma_attach()) */
38static uint dma_msg_level = 0;
39
40#define MAXNAMEL 8 /* 8 char names */
41
42#define DI_INFO(dmah) (dma_info_t *)dmah
43typedef struct osl_dmainfo osldma_t;
44
45/* dma engine software state */
46typedef struct dma_info
47{
48  struct hnddma_pub hnddma; /* exported structure, don't use hnddma_t,
49                 * which could be const
50                 */
51  uint *msg_level; /* message level pointer */
52  char name[MAXNAMEL]; /* callers name for diag msgs */
53
54  void *osh; /* os handle */
55  sb_t *sbh; /* sb handle */
56
57  bool dma64; /* dma64 enabled */
58  bool addrext; /* this dma engine supports DmaExtendedAddrChanges */
59
60  dma32regs_t *d32txregs; /* 32 bits dma tx engine registers */
61  dma32regs_t *d32rxregs; /* 32 bits dma rx engine registers */
62  dma64regs_t *d64txregs; /* 64 bits dma tx engine registers */
63  dma64regs_t *d64rxregs; /* 64 bits dma rx engine registers */
64
65  uint32 dma64align; /* either 8k or 4k depends on number of dd */
66  dma32dd_t *txd32; /* pointer to dma32 tx descriptor ring */
67  dma64dd_t *txd64; /* pointer to dma64 tx descriptor ring */
68  uint ntxd; /* # tx descriptors tunable */
69  uint txin; /* index of next descriptor to reclaim */
70  uint txout; /* index of next descriptor to post */
71  void **txp; /* pointer to parallel array of pointers to packets */
72  osldma_t *tx_dmah; /* DMA TX descriptor ring handle */
73  osldma_t **txp_dmah; /* DMA TX packet data handle */
74  ulong txdpa; /* physical address of descriptor ring */
75  uint txdalign; /* #bytes added to alloc'd mem to align txd */
76  uint txdalloc; /* #bytes allocated for the ring */
77
78  dma32dd_t *rxd32; /* pointer to dma32 rx descriptor ring */
79  dma64dd_t *rxd64; /* pointer to dma64 rx descriptor ring */
80  uint nrxd; /* # rx descriptors tunable */
81  uint rxin; /* index of next descriptor to reclaim */
82  uint rxout; /* index of next descriptor to post */
83  void **rxp; /* pointer to parallel array of pointers to packets */
84  osldma_t *rx_dmah; /* DMA RX descriptor ring handle */
85  osldma_t **rxp_dmah; /* DMA RX packet data handle */
86  ulong rxdpa; /* physical address of descriptor ring */
87  uint rxdalign; /* #bytes added to alloc'd mem to align rxd */
88  uint rxdalloc; /* #bytes allocated for the ring */
89
90  /* tunables */
91  uint rxbufsize; /* rx buffer size in bytes,
92                   not including the extra headroom
93                 */
94  uint nrxpost; /* # rx buffers to keep posted */
95  uint rxoffset; /* rxcontrol offset */
96  uint ddoffsetlow; /* add to get dma address of descriptor ring, low 32 bits */
97  uint ddoffsethigh; /* high 32 bits */
98  uint dataoffsetlow; /* add to get dma address of data buffer, low 32 bits */
99  uint dataoffsethigh; /* high 32 bits */
100} dma_info_t;
101
102#ifdef BCMDMA64
103#define DMA64_ENAB(di) ((di)->dma64)
104#define DMA64_CAP TRUE
105#else
106#define DMA64_ENAB(di) (0)
107#define DMA64_CAP FALSE
108#endif
109
110/* descriptor bumping macros */
111#define XXD(x, n) ((x) & ((n) - 1)) /* faster than %, but n must be power of 2 */
112#define TXD(x) XXD((x), di->ntxd)
113#define RXD(x) XXD((x), di->nrxd)
114#define NEXTTXD(i) TXD(i + 1)
115#define PREVTXD(i) TXD(i - 1)
116#define NEXTRXD(i) RXD(i + 1)
117#define NTXDACTIVE(h, t) TXD(t - h)
118#define NRXDACTIVE(h, t) RXD(t - h)
119
120/* macros to convert between byte offsets and indexes */
121#define B2I(bytes, type) ((bytes) / sizeof(type))
122#define I2B(index, type) ((index) * sizeof(type))
123
124#define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */
125#define PCI32ADDR_HIGH_SHIFT 30 /* address[31:30] */
126
127
128/* common prototypes */
129static bool _dma_isaddrext (dma_info_t * di);
130static bool _dma_alloc (dma_info_t * di, uint direction);
131static void _dma_detach (dma_info_t * di);
132static void _dma_ddtable_init (dma_info_t * di, uint direction, ulong pa);
133static void _dma_rxinit (dma_info_t * di);
134static void *_dma_rx (dma_info_t * di);
135static void _dma_rxfill (dma_info_t * di);
136static void _dma_rxreclaim (dma_info_t * di);
137static void _dma_rxenable (dma_info_t * di);
138static void *_dma_getnextrxp (dma_info_t * di, bool forceall);
139
140static void _dma_txblock (dma_info_t * di);
141static void _dma_txunblock (dma_info_t * di);
142static uint _dma_txactive (dma_info_t * di);
143
144static void *_dma_peeknexttxp (dma_info_t * di);
145static uintptr _dma_getvar (dma_info_t * di, const char *name);
146static void _dma_counterreset (dma_info_t * di);
147static void _dma_fifoloopbackenable (dma_info_t * di);
148
149/* ** 32 bit DMA prototypes */
150static bool dma32_alloc (dma_info_t * di, uint direction);
151static bool dma32_txreset (dma_info_t * di);
152static bool dma32_rxreset (dma_info_t * di);
153static bool dma32_txsuspendedidle (dma_info_t * di);
154static int dma32_txfast (dma_info_t * di, void *p0, bool commit);
155static void *dma32_getnexttxp (dma_info_t * di, bool forceall);
156static void *dma32_getnextrxp (dma_info_t * di, bool forceall);
157static void dma32_txrotate (dma_info_t * di);
158static bool dma32_rxidle (dma_info_t * di);
159static void dma32_txinit (dma_info_t * di);
160static bool dma32_txenabled (dma_info_t * di);
161static void dma32_txsuspend (dma_info_t * di);
162static void dma32_txresume (dma_info_t * di);
163static bool dma32_txsuspended (dma_info_t * di);
164static void dma32_txreclaim (dma_info_t * di, bool forceall);
165static bool dma32_txstopped (dma_info_t * di);
166static bool dma32_rxstopped (dma_info_t * di);
167static bool dma32_rxenabled (dma_info_t * di);
168static bool _dma32_addrext (osl_t * osh, dma32regs_t * dma32regs);
169
170/* ** 64 bit DMA prototypes and stubs */
171#ifdef BCMDMA64
172static bool dma64_alloc (dma_info_t * di, uint direction);
173static bool dma64_txreset (dma_info_t * di);
174static bool dma64_rxreset (dma_info_t * di);
175static bool dma64_txsuspendedidle (dma_info_t * di);
176static int dma64_txfast (dma_info_t * di, void *p0, bool commit);
177static void *dma64_getnexttxp (dma_info_t * di, bool forceall);
178static void *dma64_getnextrxp (dma_info_t * di, bool forceall);
179static void dma64_txrotate (dma_info_t * di);
180
181static bool dma64_rxidle (dma_info_t * di);
182static void dma64_txinit (dma_info_t * di);
183static bool dma64_txenabled (dma_info_t * di);
184static void dma64_txsuspend (dma_info_t * di);
185static void dma64_txresume (dma_info_t * di);
186static bool dma64_txsuspended (dma_info_t * di);
187static void dma64_txreclaim (dma_info_t * di, bool forceall);
188static bool dma64_txstopped (dma_info_t * di);
189static bool dma64_rxstopped (dma_info_t * di);
190static bool dma64_rxenabled (dma_info_t * di);
191static bool _dma64_addrext (osl_t * osh, dma64regs_t * dma64regs);
192
193#else
194static bool
195dma64_alloc (dma_info_t * di, uint direction)
196{
197  return FALSE;
198}
199static bool
200dma64_txreset (dma_info_t * di)
201{
202  return FALSE;
203}
204static bool
205dma64_rxreset (dma_info_t * di)
206{
207  return FALSE;
208}
209static bool
210dma64_txsuspendedidle (dma_info_t * di)
211{
212  return FALSE;
213}
214static int
215dma64_txfast (dma_info_t * di, void *p0, bool commit)
216{
217  return 0;
218}
219static void *
220dma64_getnexttxp (dma_info_t * di, bool forceall)
221{
222  return NULL;
223}
224static void *
225dma64_getnextrxp (dma_info_t * di, bool forceall)
226{
227  return NULL;
228}
229static void
230dma64_txrotate (dma_info_t * di)
231{
232  return;
233}
234
235static bool
236dma64_rxidle (dma_info_t * di)
237{
238  return FALSE;
239}
240static void
241dma64_txinit (dma_info_t * di)
242{
243  return;
244}
245static bool
246dma64_txenabled (dma_info_t * di)
247{
248  return FALSE;
249}
250static void
251dma64_txsuspend (dma_info_t * di)
252{
253  return;
254}
255static void
256dma64_txresume (dma_info_t * di)
257{
258  return;
259}
260static bool
261dma64_txsuspended (dma_info_t * di)
262{
263  return FALSE;
264}
265static void
266dma64_txreclaim (dma_info_t * di, bool forceall)
267{
268  return;
269}
270static bool
271dma64_txstopped (dma_info_t * di)
272{
273  return FALSE;
274}
275static bool
276dma64_rxstopped (dma_info_t * di)
277{
278  return FALSE;
279}
280static bool
281dma64_rxenabled (dma_info_t * di)
282{
283  return FALSE;
284}
285static bool
286_dma64_addrext (osl_t * osh, dma64regs_t * dma64regs)
287{
288  return FALSE;
289}
290
291#endif /* BCMDMA64 */
292
293#ifdef BCMDBG
294static void dma32_dumpring (dma_info_t * di, struct bcmstrbuf *b,
295                dma32dd_t * ring, uint start, uint end,
296                uint max_num);
297static void dma32_dump (dma_info_t * di, struct bcmstrbuf *b, bool dumpring);
298static void dma32_dumptx (dma_info_t * di, struct bcmstrbuf *b,
299              bool dumpring);
300static void dma32_dumprx (dma_info_t * di, struct bcmstrbuf *b,
301              bool dumpring);
302
303static void dma64_dumpring (dma_info_t * di, struct bcmstrbuf *b,
304                dma64dd_t * ring, uint start, uint end,
305                uint max_num);
306static void dma64_dump (dma_info_t * di, struct bcmstrbuf *b, bool dumpring);
307static void dma64_dumptx (dma_info_t * di, struct bcmstrbuf *b,
308              bool dumpring);
309static void dma64_dumprx (dma_info_t * di, struct bcmstrbuf *b,
310              bool dumpring);
311#endif
312
313
314static di_fcn_t dma64proc = {
315  (di_detach_t) _dma_detach,
316  (di_txinit_t) dma64_txinit,
317  (di_txreset_t) dma64_txreset,
318  (di_txenabled_t) dma64_txenabled,
319  (di_txsuspend_t) dma64_txsuspend,
320  (di_txresume_t) dma64_txresume,
321  (di_txsuspended_t) dma64_txsuspended,
322  (di_txsuspendedidle_t) dma64_txsuspendedidle,
323  (di_txfast_t) dma64_txfast,
324  (di_txstopped_t) dma64_txstopped,
325  (di_txreclaim_t) dma64_txreclaim,
326  (di_getnexttxp_t) dma64_getnexttxp,
327  (di_peeknexttxp_t) _dma_peeknexttxp,
328  (di_txblock_t) _dma_txblock,
329  (di_txunblock_t) _dma_txunblock,
330  (di_txactive_t) _dma_txactive,
331  (di_txrotate_t) dma64_txrotate,
332
333  (di_rxinit_t) _dma_rxinit,
334  (di_rxreset_t) dma64_rxreset,
335  (di_rxidle_t) dma64_rxidle,
336  (di_rxstopped_t) dma64_rxstopped,
337  (di_rxenable_t) _dma_rxenable,
338  (di_rxenabled_t) dma64_rxenabled,
339  (di_rx_t) _dma_rx,
340  (di_rxfill_t) _dma_rxfill,
341  (di_rxreclaim_t) _dma_rxreclaim,
342  (di_getnextrxp_t) _dma_getnextrxp,
343
344  (di_fifoloopbackenable_t) _dma_fifoloopbackenable,
345  (di_getvar_t) _dma_getvar,
346  (di_counterreset_t) _dma_counterreset,
347
348#ifdef BCMDBG
349  (di_dump_t) dma64_dump,
350  (di_dumptx_t) dma64_dumptx,
351  (di_dumprx_t) dma64_dumprx,
352#else
353  NULL,
354  NULL,
355  NULL,
356#endif
357  34
358};
359
360static di_fcn_t dma32proc = {
361  (di_detach_t) _dma_detach,
362  (di_txinit_t) dma32_txinit,
363  (di_txreset_t) dma32_txreset,
364  (di_txenabled_t) dma32_txenabled,
365  (di_txsuspend_t) dma32_txsuspend,
366  (di_txresume_t) dma32_txresume,
367  (di_txsuspended_t) dma32_txsuspended,
368  (di_txsuspendedidle_t) dma32_txsuspendedidle,
369  (di_txfast_t) dma32_txfast,
370  (di_txstopped_t) dma32_txstopped,
371  (di_txreclaim_t) dma32_txreclaim,
372  (di_getnexttxp_t) dma32_getnexttxp,
373  (di_peeknexttxp_t) _dma_peeknexttxp,
374  (di_txblock_t) _dma_txblock,
375  (di_txunblock_t) _dma_txunblock,
376  (di_txactive_t) _dma_txactive,
377  (di_txrotate_t) dma32_txrotate,
378
379  (di_rxinit_t) _dma_rxinit,
380  (di_rxreset_t) dma32_rxreset,
381  (di_rxidle_t) dma32_rxidle,
382  (di_rxstopped_t) dma32_rxstopped,
383  (di_rxenable_t) _dma_rxenable,
384  (di_rxenabled_t) dma32_rxenabled,
385  (di_rx_t) _dma_rx,
386  (di_rxfill_t) _dma_rxfill,
387  (di_rxreclaim_t) _dma_rxreclaim,
388  (di_getnextrxp_t) _dma_getnextrxp,
389
390  (di_fifoloopbackenable_t) _dma_fifoloopbackenable,
391  (di_getvar_t) _dma_getvar,
392  (di_counterreset_t) _dma_counterreset,
393
394#ifdef BCMDBG
395  (di_dump_t) dma32_dump,
396  (di_dumptx_t) dma32_dumptx,
397  (di_dumprx_t) dma32_dumprx,
398#else
399  NULL,
400  NULL,
401  NULL,
402#endif
403  34
404};
405
406hnddma_t *
407dma_attach (osl_t * osh, char *name, sb_t * sbh, void *dmaregstx,
408        void *dmaregsrx, uint ntxd, uint nrxd, uint rxbufsize,
409        uint nrxpost, uint rxoffset, uint * msg_level)
410{
411  dma_info_t *di;
412  uint size;
413
414  /* allocate private info structure */
415  if ((di = MALLOC (osh, sizeof (dma_info_t))) == NULL)
416    {
417#ifdef BCMDBG
418      printf ("dma_attach: out of memory, malloced %d bytes\n",
419          MALLOCED (osh));
420#endif
421      return (NULL);
422    }
423  bzero ((char *) di, sizeof (dma_info_t));
424
425  di->msg_level = msg_level ? msg_level : &dma_msg_level;
426
427  /* old chips w/o sb is no longer supported */
428  ASSERT (sbh != NULL);
429
430  di->dma64 = ((sb_coreflagshi (sbh, 0, 0) & SBTMH_DMA64) == SBTMH_DMA64);
431
432#ifndef BCMDMA64
433  if (di->dma64)
434    {
435      DMA_ERROR (("dma_attach: driver doesn't have the capability to support "
436          "64 bits DMA\n"));
437      goto fail;
438    }
439#endif
440
441  /* check arguments */
442  ASSERT (ISPOWEROF2 (ntxd));
443  ASSERT (ISPOWEROF2 (nrxd));
444  if (nrxd == 0)
445    ASSERT (dmaregsrx == NULL);
446  if (ntxd == 0)
447    ASSERT (dmaregstx == NULL);
448
449
450  /* init dma reg pointer */
451  if (di->dma64)
452    {
453      ASSERT (ntxd <= D64MAXDD);
454      ASSERT (nrxd <= D64MAXDD);
455      di->d64txregs = (dma64regs_t *) dmaregstx;
456      di->d64rxregs = (dma64regs_t *) dmaregsrx;
457
458      di->dma64align = D64RINGALIGN;
459      if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2))
460    {
461      /* for smaller dd table, HW relax the alignment requirement */
462      di->dma64align = D64RINGALIGN / 2;
463    }
464    }
465  else
466    {
467      ASSERT (ntxd <= D32MAXDD);
468      ASSERT (nrxd <= D32MAXDD);
469      di->d32txregs = (dma32regs_t *) dmaregstx;
470      di->d32rxregs = (dma32regs_t *) dmaregsrx;
471    }
472
473  DMA_TRACE (("%s: dma_attach: %s osh %p ntxd %d nrxd %d rxbufsize %d nrxpost %d " "rxoffset %d dmaregstx %p dmaregsrx %p\n", name, (di->dma64 ? "DMA64" : "DMA32"), osh, ntxd, nrxd, rxbufsize, nrxpost, rxoffset, dmaregstx, dmaregsrx));
474
475  /* make a private copy of our callers name */
476  strncpy (di->name, name, MAXNAMEL);
477  di->name[MAXNAMEL - 1] = '\0';
478
479  di->osh = osh;
480  di->sbh = sbh;
481
482  /* save tunables */
483  di->ntxd = ntxd;
484  di->nrxd = nrxd;
485
486  /* the actual dma size doesn't include the extra headroom */
487  if (rxbufsize > BCMEXTRAHDROOM)
488    di->rxbufsize = rxbufsize - BCMEXTRAHDROOM;
489  else
490    di->rxbufsize = rxbufsize;
491
492  di->nrxpost = nrxpost;
493  di->rxoffset = rxoffset;
494
495  /*
496   * figure out the DMA physical address offset for dd and data
497   * for old chips w/o sb, use zero
498   * for new chips w sb,
499   * PCI/PCIE: they map silicon backplace address to zero based memory, need offset
500   * Other bus: use zero
501   * SB_BUS BIGENDIAN kludge: use sdram swapped region for data buffer, not descriptor
502   */
503  di->ddoffsetlow = 0;
504  di->dataoffsetlow = 0;
505  /* for pci bus, add offset */
506  if (sbh->bustype == PCI_BUS)
507    {
508      if ((sbh->buscoretype == SB_PCIE) && di->dma64)
509    {
510      /* pcie with DMA64 */
511      di->ddoffsetlow = 0;
512      di->ddoffsethigh = SB_PCIE_DMA_H32;
513    }
514      else
515    {
516      /* pci(DMA32/DMA64) or pcie with DMA32 */
517      di->ddoffsetlow = SB_PCI_DMA;
518      di->ddoffsethigh = 0;
519    }
520      di->dataoffsetlow = di->ddoffsetlow;
521      di->dataoffsethigh = di->ddoffsethigh;
522    }
523
524#if defined(__mips__) && defined(IL_BIGENDIAN)
525  di->dataoffsetlow = di->dataoffsetlow + SB_SDRAM_SWAPPED;
526#endif
527
528  di->addrext = _dma_isaddrext (di);
529
530  /* allocate tx packet pointer vector */
531  if (ntxd)
532    {
533      size = ntxd * sizeof (void *);
534      if ((di->txp = MALLOC (osh, size)) == NULL)
535    {
536      DMA_ERROR (("%s: dma_attach: out of tx memory, malloced %d bytes\n",
537              di->name, MALLOCED (osh)));
538      goto fail;
539    }
540      bzero ((char *) di->txp, size);
541    }
542
543  /* allocate rx packet pointer vector */
544  if (nrxd)
545    {
546      size = nrxd * sizeof (void *);
547      if ((di->rxp = MALLOC (osh, size)) == NULL)
548    {
549      DMA_ERROR (("%s: dma_attach: out of rx memory, malloced %d bytes\n",
550              di->name, MALLOCED (osh)));
551      goto fail;
552    }
553      bzero ((char *) di->rxp, size);
554    }
555
556  /* allocate transmit descriptor ring, only need ntxd descriptors but it must be aligned */
557  if (ntxd)
558    {
559      if (!_dma_alloc (di, DMA_TX))
560    goto fail;
561    }
562
563  /* allocate receive descriptor ring, only need nrxd descriptors but it must be aligned */
564  if (nrxd)
565    {
566      if (!_dma_alloc (di, DMA_RX))
567    goto fail;
568    }
569
570  if ((di->ddoffsetlow == SB_PCI_DMA) && (di->txdpa > SB_PCI_DMA_SZ)
571      && !di->addrext)
572    {
573      DMA_ERROR (("%s: dma_attach: txdpa 0x%lx: addrext not supported\n",
574          di->name, di->txdpa));
575      goto fail;
576    }
577  if ((di->ddoffsetlow == SB_PCI_DMA) && (di->rxdpa > SB_PCI_DMA_SZ)
578      && !di->addrext)
579    {
580      DMA_ERROR (("%s: dma_attach: rxdpa 0x%lx: addrext not supported\n",
581          di->name, di->rxdpa));
582      goto fail;
583    }
584
585  DMA_TRACE (("ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh " "0x%x addrext %d\n", di->ddoffsetlow, di->ddoffsethigh, di->dataoffsetlow, di->dataoffsethigh, di->addrext));
586
587  /* allocate tx packet pointer vector and DMA mapping vectors */
588  if (ntxd)
589    {
590
591      size = ntxd * sizeof (osldma_t **);
592      if ((di->txp_dmah = (osldma_t **) MALLOC (osh, size)) == NULL)
593    goto fail;
594      bzero ((char *) di->txp_dmah, size);
595    }
596  else
597    di->txp_dmah = NULL;
598
599  /* allocate rx packet pointer vector and DMA mapping vectors */
600  if (nrxd)
601    {
602
603      size = nrxd * sizeof (osldma_t **);
604      if ((di->rxp_dmah = (osldma_t **) MALLOC (osh, size)) == NULL)
605    goto fail;
606      bzero ((char *) di->rxp_dmah, size);
607
608    }
609  else
610    di->rxp_dmah = NULL;
611
612  /* initialize opsvec of function pointers */
613  di->hnddma.di_fn = DMA64_ENAB (di) ? dma64proc : dma32proc;
614
615  return ((hnddma_t *) di);
616
617fail:
618  _dma_detach (di);
619  return (NULL);
620}
621
622/* init the tx or rx descriptor */
623static INLINE void
624dma32_dd_upd (dma_info_t * di, dma32dd_t * ddring, ulong pa, uint outidx,
625          uint32 * flags, uint32 bufcount)
626{
627  /* dma32 uses 32 bits control to fit both flags and bufcounter */
628  *flags = *flags | (bufcount & CTRL_BC_MASK);
629
630  if ((di->dataoffsetlow != SB_PCI_DMA) || !(pa & PCI32ADDR_HIGH))
631    {
632      W_SM (&ddring[outidx].addr, BUS_SWAP32 (pa + di->dataoffsetlow));
633      W_SM (&ddring[outidx].ctrl, BUS_SWAP32 (*flags));
634    }
635  else
636    {
637      /* address extension */
638      uint32 ae;
639      ASSERT (di->addrext);
640      ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
641      pa &= ~PCI32ADDR_HIGH;
642
643      *flags |= (ae << CTRL_AE_SHIFT);
644      W_SM (&ddring[outidx].addr, BUS_SWAP32 (pa + di->dataoffsetlow));
645      W_SM (&ddring[outidx].ctrl, BUS_SWAP32 (*flags));
646    }
647}
648
649static INLINE void
650dma64_dd_upd (dma_info_t * di, dma64dd_t * ddring, ulong pa, uint outidx,
651          uint32 * flags, uint32 bufcount)
652{
653  uint32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;
654
655  /* PCI bus with big(>1G) physical address, use address extension */
656  if ((di->dataoffsetlow != SB_PCI_DMA) || !(pa & PCI32ADDR_HIGH))
657    {
658      W_SM (&ddring[outidx].addrlow, BUS_SWAP32 (pa + di->dataoffsetlow));
659      W_SM (&ddring[outidx].addrhigh, BUS_SWAP32 (0 + di->dataoffsethigh));
660      W_SM (&ddring[outidx].ctrl1, BUS_SWAP32 (*flags));
661      W_SM (&ddring[outidx].ctrl2, BUS_SWAP32 (ctrl2));
662    }
663  else
664    {
665      /* address extension */
666      uint32 ae;
667      ASSERT (di->addrext);
668
669      ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
670      pa &= ~PCI32ADDR_HIGH;
671
672      ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;
673      W_SM (&ddring[outidx].addrlow, BUS_SWAP32 (pa + di->dataoffsetlow));
674      W_SM (&ddring[outidx].addrhigh, BUS_SWAP32 (0 + di->dataoffsethigh));
675      W_SM (&ddring[outidx].ctrl1, BUS_SWAP32 (*flags));
676      W_SM (&ddring[outidx].ctrl2, BUS_SWAP32 (ctrl2));
677    }
678}
679
680static bool
681_dma32_addrext (osl_t * osh, dma32regs_t * dma32regs)
682{
683  uint32 w;
684
685  OR_REG (osh, &dma32regs->control, XC_AE);
686  w = R_REG (osh, &dma32regs->control);
687  AND_REG (osh, &dma32regs->control, ~XC_AE);
688  return ((w & XC_AE) == XC_AE);
689}
690
691static bool
692_dma_alloc (dma_info_t * di, uint direction)
693{
694  if (DMA64_ENAB (di))
695    {
696      return dma64_alloc (di, direction);
697    }
698  else
699    {
700      return dma32_alloc (di, direction);
701    }
702}
703
704/* !! may be called with core in reset */
705static void
706_dma_detach (dma_info_t * di)
707{
708  if (di == NULL)
709    return;
710
711  DMA_TRACE (("%s: dma_detach\n", di->name));
712
713  /* shouldn't be here if descriptors are unreclaimed */
714  ASSERT (di->txin == di->txout);
715  ASSERT (di->rxin == di->rxout);
716
717  /* free dma descriptor rings */
718  if (DMA64_ENAB (di))
719    {
720      if (di->txd64)
721    DMA_FREE_CONSISTENT (di->osh,
722                 ((int8 *) (uintptr) di->txd64 - di->txdalign),
723                 di->txdalloc, (di->txdpa - di->txdalign),
724                 &di->tx_dmah);
725      if (di->rxd64)
726    DMA_FREE_CONSISTENT (di->osh,
727                 ((int8 *) (uintptr) di->rxd64 - di->rxdalign),
728                 di->rxdalloc, (di->rxdpa - di->rxdalign),
729                 &di->rx_dmah);
730    }
731  else
732    {
733      if (di->txd32)
734    DMA_FREE_CONSISTENT (di->osh,
735                 ((int8 *) (uintptr) di->txd32 - di->txdalign),
736                 di->txdalloc, (di->txdpa - di->txdalign),
737                 &di->tx_dmah);
738      if (di->rxd32)
739    DMA_FREE_CONSISTENT (di->osh,
740                 ((int8 *) (uintptr) di->rxd32 - di->rxdalign),
741                 di->rxdalloc, (di->rxdpa - di->rxdalign),
742                 &di->rx_dmah);
743    }
744
745  /* free packet pointer vectors */
746  if (di->txp)
747    MFREE (di->osh, (void *) di->txp, (di->ntxd * sizeof (void *)));
748  if (di->rxp)
749    MFREE (di->osh, (void *) di->rxp, (di->nrxd * sizeof (void *)));
750
751  /* free tx packet DMA handles */
752  if (di->txp_dmah)
753    MFREE (di->osh, (void *) di->txp_dmah, di->ntxd * sizeof (osldma_t **));
754
755  /* free rx packet DMA handles */
756  if (di->rxp_dmah)
757    MFREE (di->osh, (void *) di->rxp_dmah, di->nrxd * sizeof (osldma_t **));
758
759  /* free our private info structure */
760  MFREE (di->osh, (void *) di, sizeof (dma_info_t));
761
762}
763
764/* return TRUE if this dma engine supports DmaExtendedAddrChanges, otherwise FALSE */
765static bool
766_dma_isaddrext (dma_info_t * di)
767{
768  if (DMA64_ENAB (di))
769    {
770      /* DMA64 supports full 32 bits or 64 bits. AE is always valid */
771
772      /* not all tx or rx channel are available */
773      if (di->d64txregs != NULL)
774    {
775      if (!_dma64_addrext (di->osh, di->d64txregs))
776        {
777          DMA_ERROR (("%s: _dma_isaddrext: DMA64 tx doesn't have AE set\n", di->name));
778          ASSERT (0);
779        }
780      return TRUE;
781    }
782      else if (di->d64rxregs != NULL)
783    {
784      if (!_dma64_addrext (di->osh, di->d64rxregs))
785        {
786          DMA_ERROR (("%s: _dma_isaddrext: DMA64 rx doesn't have AE set\n", di->name));
787          ASSERT (0);
788        }
789      return TRUE;
790    }
791      return FALSE;
792    }
793  else if (di->d32txregs)
794    return (_dma32_addrext (di->osh, di->d32txregs));
795  else if (di->d32rxregs)
796    return (_dma32_addrext (di->osh, di->d32rxregs));
797  return FALSE;
798}
799
800/* initialize descriptor table base address */
801static void
802_dma_ddtable_init (dma_info_t * di, uint direction, ulong pa)
803{
804  if (DMA64_ENAB (di))
805    {
806
807      if ((di->ddoffsetlow != SB_PCI_DMA) || !(pa & PCI32ADDR_HIGH))
808    {
809      if (direction == DMA_TX)
810        {
811          W_REG (di->osh, &di->d64txregs->addrlow,
812             (pa + di->ddoffsetlow));
813          W_REG (di->osh, &di->d64txregs->addrhigh, di->ddoffsethigh);
814        }
815      else
816        {
817          W_REG (di->osh, &di->d64rxregs->addrlow,
818             (pa + di->ddoffsetlow));
819          W_REG (di->osh, &di->d64rxregs->addrhigh, di->ddoffsethigh);
820        }
821    }
822      else
823    {
824      /* DMA64 32bits address extension */
825      uint32 ae;
826      ASSERT (di->addrext);
827
828      /* shift the high bit(s) from pa to ae */
829      ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
830      pa &= ~PCI32ADDR_HIGH;
831
832      if (direction == DMA_TX)
833        {
834          W_REG (di->osh, &di->d64txregs->addrlow,
835             (pa + di->ddoffsetlow));
836          W_REG (di->osh, &di->d64txregs->addrhigh, di->ddoffsethigh);
837          SET_REG (di->osh, &di->d64txregs->control, D64_XC_AE,
838               (ae << D64_XC_AE_SHIFT));
839        }
840      else
841        {
842          W_REG (di->osh, &di->d64rxregs->addrlow,
843             (pa + di->ddoffsetlow));
844          W_REG (di->osh, &di->d64rxregs->addrhigh, di->ddoffsethigh);
845          SET_REG (di->osh, &di->d64rxregs->control, D64_RC_AE,
846               (ae << D64_RC_AE_SHIFT));
847        }
848    }
849
850    }
851  else
852    {
853      if ((di->ddoffsetlow != SB_PCI_DMA) || !(pa & PCI32ADDR_HIGH))
854    {
855      if (direction == DMA_TX)
856        W_REG (di->osh, &di->d32txregs->addr, (pa + di->ddoffsetlow));
857      else
858        W_REG (di->osh, &di->d32rxregs->addr, (pa + di->ddoffsetlow));
859    }
860      else
861    {
862      /* dma32 address extension */
863      uint32 ae;
864      ASSERT (di->addrext);
865
866      /* shift the high bit(s) from pa to ae */
867      ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
868      pa &= ~PCI32ADDR_HIGH;
869
870      if (direction == DMA_TX)
871        {
872          W_REG (di->osh, &di->d32txregs->addr, (pa + di->ddoffsetlow));
873          SET_REG (di->osh, &di->d32txregs->control, XC_AE,
874               ae << XC_AE_SHIFT);
875        }
876      else
877        {
878          W_REG (di->osh, &di->d32rxregs->addr, (pa + di->ddoffsetlow));
879          SET_REG (di->osh, &di->d32rxregs->control, RC_AE,
880               ae << RC_AE_SHIFT);
881        }
882    }
883    }
884}
885
886static void
887_dma_fifoloopbackenable (dma_info_t * di)
888{
889  DMA_TRACE (("%s: dma_fifoloopbackenable\n", di->name));
890  if (DMA64_ENAB (di))
891    OR_REG (di->osh, &di->d64txregs->control, D64_XC_LE);
892  else
893    OR_REG (di->osh, &di->d32txregs->control, XC_LE);
894}
895
896static void
897_dma_rxinit (dma_info_t * di)
898{
899  DMA_TRACE (("%s: dma_rxinit\n", di->name));
900
901  if (di->nrxd == 0)
902    return;
903
904  di->rxin = di->rxout = 0;
905
906  /* clear rx descriptor ring */
907  if (DMA64_ENAB (di))
908    BZERO_SM ((void *) (uintptr) di->rxd64, (di->nrxd * sizeof (dma64dd_t)));
909  else
910    BZERO_SM ((void *) (uintptr) di->rxd32, (di->nrxd * sizeof (dma32dd_t)));
911
912  _dma_rxenable (di);
913  _dma_ddtable_init (di, DMA_RX, di->rxdpa);
914}
915
916static void
917_dma_rxenable (dma_info_t * di)
918{
919  DMA_TRACE (("%s: dma_rxenable\n", di->name));
920
921  if (DMA64_ENAB (di))
922    W_REG (di->osh, &di->d64rxregs->control,
923       ((di->rxoffset << D64_RC_RO_SHIFT) | D64_RC_RE));
924  else
925    W_REG (di->osh, &di->d32rxregs->control,
926       ((di->rxoffset << RC_RO_SHIFT) | RC_RE));
927}
928
929/* !! rx entry routine, returns a pointer to the next frame received,
930 * or NULL if there are no more
931 */
932static void *
933_dma_rx (dma_info_t * di)
934{
935  void *p;
936  uint len;
937  int skiplen = 0;
938
939  while ((p = _dma_getnextrxp (di, FALSE)))
940    {
941      /* skip giant packets which span multiple rx descriptors */
942      if (skiplen > 0)
943    {
944      skiplen -= di->rxbufsize;
945      if (skiplen < 0)
946        skiplen = 0;
947      PKTFREE (di->osh, p, FALSE);
948      continue;
949    }
950
951      len = ltoh16 (*(uint16 *) (PKTDATA (di->osh, p)));
952      DMA_TRACE (("%s: dma_rx len %d\n", di->name, len));
953
954      /* bad frame length check */
955      if (len > (di->rxbufsize - di->rxoffset))
956    {
957      DMA_ERROR (("%s: dma_rx: bad frame length (%d)\n", di->name, len));
958      if (len > 0)
959        skiplen = len - (di->rxbufsize - di->rxoffset);
960      PKTFREE (di->osh, p, FALSE);
961      di->hnddma.rxgiants++;
962      continue;
963    }
964
965      /* set actual length */
966      PKTSETLEN (di->osh, p, (di->rxoffset + len));
967
968      break;
969    }
970
971  return (p);
972}
973
974/* post receive buffers */
975static void
976_dma_rxfill (dma_info_t * di)
977{
978  void *p;
979  uint rxin, rxout;
980  uint32 flags = 0;
981  uint n;
982  uint i;
983  uint32 pa;
984  uint extra_offset = 0;
985
986  /*
987   * Determine how many receive buffers we're lacking
988   * from the full complement, allocate, initialize,
989   * and post them, then update the chip rx lastdscr.
990   */
991
992  rxin = di->rxin;
993  rxout = di->rxout;
994
995  n = di->nrxpost - NRXDACTIVE (rxin, rxout);
996
997  DMA_TRACE (("%s: dma_rxfill: post %d\n", di->name, n));
998
999  if (di->rxbufsize > BCMEXTRAHDROOM)
1000    extra_offset = BCMEXTRAHDROOM;
1001
1002  for (i = 0; i < n; i++)
1003    {
1004      /* the di->rxbufsize doesn't include the extra headroom, we need to add it to the
1005         size to be allocated
1006       */
1007      if ((p = PKTGET (di->osh, di->rxbufsize + extra_offset, FALSE)) == NULL)
1008    {
1009      DMA_ERROR (("%s: dma_rxfill: out of rxbufs\n", di->name));
1010      di->hnddma.rxnobuf++;
1011      break;
1012    }
1013      /* reserve an extra headroom, if applicable */
1014      if (extra_offset)
1015    PKTPULL (di->osh, p, extra_offset);
1016
1017      /* Do a cached write instead of uncached write since DMA_MAP
1018       * will flush the cache.
1019       */
1020      *(uint32 *) (PKTDATA (di->osh, p)) = 0;
1021
1022      pa = (uint32) DMA_MAP (di->osh, PKTDATA (di->osh, p),
1023                 di->rxbufsize, DMA_RX, p, &di->rxp_dmah[rxout]);
1024
1025      ASSERT (ISALIGNED (pa, 4));
1026
1027      /* save the free packet pointer */
1028      ASSERT (di->rxp[rxout] == NULL);
1029      di->rxp[rxout] = p;
1030
1031      /* reset flags for each descriptor */
1032      flags = 0;
1033      if (DMA64_ENAB (di))
1034    {
1035      if (rxout == (di->nrxd - 1))
1036        flags = D64_CTRL1_EOT;
1037
1038      dma64_dd_upd (di, di->rxd64, pa, rxout, &flags, di->rxbufsize);
1039    }
1040      else
1041    {
1042      if (rxout == (di->nrxd - 1))
1043        flags = CTRL_EOT;
1044
1045      dma32_dd_upd (di, di->rxd32, pa, rxout, &flags, di->rxbufsize);
1046    }
1047      rxout = NEXTRXD (rxout);
1048    }
1049
1050  di->rxout = rxout;
1051
1052  /* update the chip lastdscr pointer */
1053  if (DMA64_ENAB (di))
1054    {
1055      W_REG (di->osh, &di->d64rxregs->ptr, I2B (rxout, dma64dd_t));
1056    }
1057  else
1058    {
1059      W_REG (di->osh, &di->d32rxregs->ptr, I2B (rxout, dma32dd_t));
1060    }
1061}
1062
1063/* like getnexttxp but no reclaim */
1064static void *
1065_dma_peeknexttxp (dma_info_t * di)
1066{
1067  uint end, i;
1068
1069  if (di->ntxd == 0)
1070    return (NULL);
1071
1072  if (DMA64_ENAB (di))
1073    {
1074      end =
1075    B2I (R_REG (di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK,
1076         dma64dd_t);
1077    }
1078  else
1079    {
1080      end =
1081    B2I (R_REG (di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t);
1082    }
1083
1084  for (i = di->txin; i != end; i = NEXTTXD (i))
1085    if (di->txp[i])
1086      return (di->txp[i]);
1087
1088  return (NULL);
1089}
1090
1091static void
1092_dma_rxreclaim (dma_info_t * di)
1093{
1094  void *p;
1095
1096  /* "unused local" warning suppression for OSLs that
1097   * define PKTFREE() without using the di->osh arg
1098   */
1099  di = di;
1100
1101  DMA_TRACE (("%s: dma_rxreclaim\n", di->name));
1102
1103  while ((p = _dma_getnextrxp (di, TRUE)))
1104    PKTFREE (di->osh, p, FALSE);
1105}
1106
1107static void *
1108_dma_getnextrxp (dma_info_t * di, bool forceall)
1109{
1110  if (di->nrxd == 0)
1111    return (NULL);
1112
1113  if (DMA64_ENAB (di))
1114    {
1115      return dma64_getnextrxp (di, forceall);
1116    }
1117  else
1118    {
1119      return dma32_getnextrxp (di, forceall);
1120    }
1121}
1122
1123static void
1124_dma_txblock (dma_info_t * di)
1125{
1126  di->hnddma.txavail = 0;
1127}
1128
1129static void
1130_dma_txunblock (dma_info_t * di)
1131{
1132  di->hnddma.txavail = di->ntxd - NTXDACTIVE (di->txin, di->txout) - 1;
1133}
1134
1135static uint
1136_dma_txactive (dma_info_t * di)
1137{
1138  return (NTXDACTIVE (di->txin, di->txout));
1139}
1140
1141static void
1142_dma_counterreset (dma_info_t * di)
1143{
1144  /* reset all software counter */
1145  di->hnddma.rxgiants = 0;
1146  di->hnddma.rxnobuf = 0;
1147  di->hnddma.txnobuf = 0;
1148}
1149
1150/* get the address of the var in order to change later */
1151static uintptr
1152_dma_getvar (dma_info_t * di, const char *name)
1153{
1154  if (!strcmp (name, "&txavail"))
1155    return ((uintptr) & (di->hnddma.txavail));
1156  else
1157    {
1158      ASSERT (0);
1159    }
1160  return (0);
1161}
1162
1163void
1164dma_txpioloopback (osl_t * osh, dma32regs_t * regs)
1165{
1166  OR_REG (osh, &regs->control, XC_LE);
1167}
1168
1169#ifdef BCMDBG
1170static void
1171dma32_dumpring (dma_info_t * di, struct bcmstrbuf *b, dma32dd_t * ring,
1172        uint start, uint end, uint max_num)
1173{
1174  uint i;
1175
1176  for (i = start; i != end; i = XXD ((i + 1), max_num))
1177    {
1178      /* in the format of high->low 8 bytes */
1179      bcm_bprintf (b, "ring index %d: 0x%x %x\n", i, ring[i].addr,
1180           ring[i].ctrl);
1181    }
1182}
1183
1184static void
1185dma32_dumptx (dma_info_t * di, struct bcmstrbuf *b, bool dumpring)
1186{
1187  if (di->ntxd == 0)
1188    return;
1189
1190  bcm_bprintf (b, "DMA32: txd32 %p txdpa 0x%lx txp %p txin %d txout %d "
1191           "txavail %d\n", di->txd32, di->txdpa, di->txp, di->txin,
1192           di->txout, di->hnddma.txavail);
1193
1194  bcm_bprintf (b, "xmtcontrol 0x%x xmtaddr 0x%x xmtptr 0x%x xmtstatus 0x%x\n",
1195           R_REG (di->osh, &di->d32txregs->control),
1196           R_REG (di->osh, &di->d32txregs->addr),
1197           R_REG (di->osh, &di->d32txregs->ptr),
1198           R_REG (di->osh, &di->d32txregs->status));
1199
1200  if (dumpring && di->txd32)
1201    dma32_dumpring (di, b, di->txd32, di->txin, di->txout, di->ntxd);
1202}
1203
1204static void
1205dma32_dumprx (dma_info_t * di, struct bcmstrbuf *b, bool dumpring)
1206{
1207  if (di->nrxd == 0)
1208    return;
1209
1210  bcm_bprintf (b, "DMA32: rxd32 %p rxdpa 0x%lx rxp %p rxin %d rxout %d\n",
1211           di->rxd32, di->rxdpa, di->rxp, di->rxin, di->rxout);
1212
1213  bcm_bprintf (b, "rcvcontrol 0x%x rcvaddr 0x%x rcvptr 0x%x rcvstatus 0x%x\n",
1214           R_REG (di->osh, &di->d32rxregs->control),
1215           R_REG (di->osh, &di->d32rxregs->addr),
1216           R_REG (di->osh, &di->d32rxregs->ptr),
1217           R_REG (di->osh, &di->d32rxregs->status));
1218  if (di->rxd32 && dumpring)
1219    dma32_dumpring (di, b, di->rxd32, di->rxin, di->rxout, di->nrxd);
1220}
1221
1222static void
1223dma32_dump (dma_info_t * di, struct bcmstrbuf *b, bool dumpring)
1224{
1225  dma32_dumptx (di, b, dumpring);
1226  dma32_dumprx (di, b, dumpring);
1227}
1228
1229static void
1230dma64_dumpring (dma_info_t * di, struct bcmstrbuf *b, dma64dd_t * ring,
1231        uint start, uint end, uint max_num)
1232{
1233  uint i;
1234
1235  for (i = start; i != end; i = XXD ((i + 1), max_num))
1236    {
1237      /* in the format of high->low 16 bytes */
1238      bcm_bprintf (b, "ring index %d: 0x%x %x %x %x\n",
1239           i, ring[i].addrhigh, ring[i].addrlow, ring[i].ctrl2,
1240           ring[i].ctrl1);
1241    }
1242}
1243
1244static void
1245dma64_dumptx (dma_info_t * di, struct bcmstrbuf *b, bool dumpring)
1246{
1247  if (di->ntxd == 0)
1248    return;
1249
1250  bcm_bprintf (b, "DMA64: txd64 %p txdpa 0x%lx txp %p txin %d txout %d "
1251           "txavail %d\n", di->txd64, di->txdpa, di->txp, di->txin,
1252           di->txout, di->hnddma.txavail);
1253
1254  bcm_bprintf (b, "xmtcontrol 0x%x xmtaddrlow 0x%x xmtaddrhigh 0x%x "
1255           "xmtptr 0x%x xmtstatus0 0x%x xmtstatus1 0x%x\n",
1256           R_REG (di->osh, &di->d64txregs->control),
1257           R_REG (di->osh, &di->d64txregs->addrlow),
1258           R_REG (di->osh, &di->d64txregs->addrhigh),
1259           R_REG (di->osh, &di->d64txregs->ptr),
1260           R_REG (di->osh, &di->d64txregs->status0),
1261           R_REG (di->osh, &di->d64txregs->status1));
1262
1263  if (dumpring && di->txd64)
1264    {
1265      dma64_dumpring (di, b, di->txd64, di->txin, di->txout, di->ntxd);
1266    }
1267}
1268
1269static void
1270dma64_dumprx (dma_info_t * di, struct bcmstrbuf *b, bool dumpring)
1271{
1272  if (di->nrxd == 0)
1273    return;
1274
1275  bcm_bprintf (b, "DMA64: rxd64 %p rxdpa 0x%lx rxp %p rxin %d rxout %d\n",
1276           di->rxd64, di->rxdpa, di->rxp, di->rxin, di->rxout);
1277
1278  bcm_bprintf (b, "rcvcontrol 0x%x rcvaddrlow 0x%x rcvaddrhigh 0x%x rcvptr "
1279           "0x%x rcvstatus0 0x%x rcvstatus1 0x%x\n",
1280           R_REG (di->osh, &di->d64rxregs->control),
1281           R_REG (di->osh, &di->d64rxregs->addrlow),
1282           R_REG (di->osh, &di->d64rxregs->addrhigh),
1283           R_REG (di->osh, &di->d64rxregs->ptr),
1284           R_REG (di->osh, &di->d64rxregs->status0),
1285           R_REG (di->osh, &di->d64rxregs->status1));
1286  if (di->rxd64 && dumpring)
1287    {
1288      dma64_dumpring (di, b, di->rxd64, di->rxin, di->rxout, di->nrxd);
1289    }
1290}
1291
1292static void
1293dma64_dump (dma_info_t * di, struct bcmstrbuf *b, bool dumpring)
1294{
1295  dma64_dumptx (di, b, dumpring);
1296  dma64_dumprx (di, b, dumpring);
1297}
1298
1299#endif /* BCMDBG */
1300
1301
1302/* 32 bits DMA functions */
1303static void
1304dma32_txinit (dma_info_t * di)
1305{
1306  DMA_TRACE (("%s: dma_txinit\n", di->name));
1307
1308  if (di->ntxd == 0)
1309    return;
1310
1311  di->txin = di->txout = 0;
1312  di->hnddma.txavail = di->ntxd - 1;
1313
1314  /* clear tx descriptor ring */
1315  BZERO_SM ((void *) (uintptr) di->txd32, (di->ntxd * sizeof (dma32dd_t)));
1316  W_REG (di->osh, &di->d32txregs->control, XC_XE);
1317  _dma_ddtable_init (di, DMA_TX, di->txdpa);
1318}
1319
1320static bool
1321dma32_txenabled (dma_info_t * di)
1322{
1323  uint32 xc;
1324
1325  /* If the chip is dead, it is not enabled :-) */
1326  xc = R_REG (di->osh, &di->d32txregs->control);
1327  return ((xc != 0xffffffff) && (xc & XC_XE));
1328}
1329
1330static void
1331dma32_txsuspend (dma_info_t * di)
1332{
1333  DMA_TRACE (("%s: dma_txsuspend\n", di->name));
1334
1335  if (di->ntxd == 0)
1336    return;
1337
1338  OR_REG (di->osh, &di->d32txregs->control, XC_SE);
1339}
1340
1341static void
1342dma32_txresume (dma_info_t * di)
1343{
1344  DMA_TRACE (("%s: dma_txresume\n", di->name));
1345
1346  if (di->ntxd == 0)
1347    return;
1348
1349  AND_REG (di->osh, &di->d32txregs->control, ~XC_SE);
1350}
1351
1352static bool
1353dma32_txsuspended (dma_info_t * di)
1354{
1355  return (di->ntxd == 0)
1356    || ((R_REG (di->osh, &di->d32txregs->control) & XC_SE) == XC_SE);
1357}
1358
1359static void
1360dma32_txreclaim (dma_info_t * di, bool forceall)
1361{
1362  void *p;
1363
1364  DMA_TRACE (("%s: dma_txreclaim %s\n", di->name, forceall ? "all" : ""));
1365
1366  while ((p = dma32_getnexttxp (di, forceall)))
1367    PKTFREE (di->osh, p, TRUE);
1368}
1369
1370static bool
1371dma32_txstopped (dma_info_t * di)
1372{
1373  return ((R_REG (di->osh, &di->d32txregs->status) & XS_XS_MASK) ==
1374      XS_XS_STOPPED);
1375}
1376
1377static bool
1378dma32_rxstopped (dma_info_t * di)
1379{
1380  return ((R_REG (di->osh, &di->d32rxregs->status) & RS_RS_MASK) ==
1381      RS_RS_STOPPED);
1382}
1383
1384static bool
1385dma32_alloc (dma_info_t * di, uint direction)
1386{
1387  uint size;
1388  uint ddlen;
1389  void *va;
1390
1391  ddlen = sizeof (dma32dd_t);
1392
1393  size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
1394
1395  if (!ISALIGNED (DMA_CONSISTENT_ALIGN, D32RINGALIGN))
1396    size += D32RINGALIGN;
1397
1398
1399  if (direction == DMA_TX)
1400    {
1401      if ((va =
1402       DMA_ALLOC_CONSISTENT (di->osh, size, &di->txdpa,
1403                 &di->tx_dmah)) == NULL)
1404    {
1405      DMA_ERROR (("%s: dma_attach: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
1406              di->name));
1407      return FALSE;
1408    }
1409
1410      di->txd32 = (dma32dd_t *) ROUNDUP ((uintptr) va, D32RINGALIGN);
1411      di->txdalign = (uint) ((int8 *) (uintptr) di->txd32 - (int8 *) va);
1412      di->txdpa += di->txdalign;
1413      di->txdalloc = size;
1414      ASSERT (ISALIGNED ((uintptr) di->txd32, D32RINGALIGN));
1415    }
1416  else
1417    {
1418      if ((va =
1419       DMA_ALLOC_CONSISTENT (di->osh, size, &di->rxdpa,
1420                 &di->rx_dmah)) == NULL)
1421    {
1422      DMA_ERROR (("%s: dma_attach: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
1423              di->name));
1424      return FALSE;
1425    }
1426      di->rxd32 = (dma32dd_t *) ROUNDUP ((uintptr) va, D32RINGALIGN);
1427      di->rxdalign = (uint) ((int8 *) (uintptr) di->rxd32 - (int8 *) va);
1428      di->rxdpa += di->rxdalign;
1429      di->rxdalloc = size;
1430      ASSERT (ISALIGNED ((uintptr) di->rxd32, D32RINGALIGN));
1431    }
1432
1433  return TRUE;
1434}
1435
1436static bool
1437dma32_txreset (dma_info_t * di)
1438{
1439  uint32 status;
1440
1441  if (di->ntxd == 0)
1442    return TRUE;
1443
1444  /* suspend tx DMA first */
1445  W_REG (di->osh, &di->d32txregs->control, XC_SE);
1446  SPINWAIT (((status = (R_REG (di->osh, &di->d32txregs->status) & XS_XS_MASK))
1447         != XS_XS_DISABLED) &&
1448        (status != XS_XS_IDLE) && (status != XS_XS_STOPPED), (10000));
1449
1450  W_REG (di->osh, &di->d32txregs->control, 0);
1451  SPINWAIT (((status = (R_REG (di->osh,
1452                   &di->d32txregs->status) & XS_XS_MASK)) !=
1453         XS_XS_DISABLED), 10000);
1454
1455  /* wait for the last transaction to complete */
1456  OSL_DELAY (300);
1457
1458  return (status == XS_XS_DISABLED);
1459}
1460
1461static bool
1462dma32_rxidle (dma_info_t * di)
1463{
1464  DMA_TRACE (("%s: dma_rxidle\n", di->name));
1465
1466  if (di->nrxd == 0)
1467    return TRUE;
1468
1469  return ((R_REG (di->osh, &di->d32rxregs->status) & RS_CD_MASK) ==
1470      R_REG (di->osh, &di->d32rxregs->ptr));
1471}
1472
1473static bool
1474dma32_rxreset (dma_info_t * di)
1475{
1476  uint32 status;
1477
1478  if (di->nrxd == 0)
1479    return TRUE;
1480
1481  W_REG (di->osh, &di->d32rxregs->control, 0);
1482  SPINWAIT (((status = (R_REG (di->osh,
1483                   &di->d32rxregs->status) & RS_RS_MASK)) !=
1484         RS_RS_DISABLED), 10000);
1485
1486  return (status == RS_RS_DISABLED);
1487}
1488
1489static bool
1490dma32_rxenabled (dma_info_t * di)
1491{
1492  uint32 rc;
1493
1494  rc = R_REG (di->osh, &di->d32rxregs->control);
1495  return ((rc != 0xffffffff) && (rc & RC_RE));
1496}
1497
1498static bool
1499dma32_txsuspendedidle (dma_info_t * di)
1500{
1501  if (di->ntxd == 0)
1502    return TRUE;
1503
1504  if (!(R_REG (di->osh, &di->d32txregs->control) & XC_SE))
1505    return 0;
1506
1507  if ((R_REG (di->osh, &di->d32txregs->status) & XS_XS_MASK) != XS_XS_IDLE)
1508    return 0;
1509
1510  OSL_DELAY (2);
1511  return ((R_REG (di->osh, &di->d32txregs->status) & XS_XS_MASK) ==
1512      XS_XS_IDLE);
1513}
1514
1515/* !! tx entry routine
1516 * supports full 32bit dma engine buffer addressing so
1517 * dma buffers can cross 4 Kbyte page boundaries.
1518 */
1519static int
1520dma32_txfast (dma_info_t * di, void *p0, bool commit)
1521{
1522  void *p, *next;
1523  uchar *data;
1524  uint len;
1525  uint txout;
1526  uint32 flags = 0;
1527  uint32 pa;
1528
1529  DMA_TRACE (("%s: dma_txfast\n", di->name));
1530
1531  txout = di->txout;
1532
1533  /*
1534   * Walk the chain of packet buffers
1535   * allocating and initializing transmit descriptor entries.
1536   */
1537  for (p = p0; p; p = next)
1538    {
1539      data = PKTDATA (di->osh, p);
1540      len = PKTLEN (di->osh, p);
1541      next = PKTNEXT (di->osh, p);
1542
1543      /* return nonzero if out of tx descriptors */
1544      if (NEXTTXD (txout) == di->txin)
1545    goto outoftxd;
1546
1547      if (len == 0)
1548    continue;
1549
1550      /* get physical address of buffer start */
1551      pa =
1552    (uint32) DMA_MAP (di->osh, data, len, DMA_TX, p,
1553              &di->txp_dmah[txout]);
1554
1555      flags = 0;
1556      if (p == p0)
1557    flags |= CTRL_SOF;
1558      if (next == NULL)
1559    flags |= (CTRL_IOC | CTRL_EOF);
1560      if (txout == (di->ntxd - 1))
1561    flags |= CTRL_EOT;
1562
1563      dma32_dd_upd (di, di->txd32, pa, txout, &flags, len);
1564      ASSERT (di->txp[txout] == NULL);
1565
1566      txout = NEXTTXD (txout);
1567    }
1568
1569  /* if last txd eof not set, fix it */
1570  if (!(flags & CTRL_EOF))
1571    W_SM (&di->txd32[PREVTXD (txout)].ctrl,
1572      BUS_SWAP32 (flags | CTRL_IOC | CTRL_EOF));
1573
1574  /* save the packet */
1575  di->txp[PREVTXD (txout)] = p0;
1576
1577  /* bump the tx descriptor index */
1578  di->txout = txout;
1579
1580  /* kick the chip */
1581  if (commit)
1582    W_REG (di->osh, &di->d32txregs->ptr, I2B (txout, dma32dd_t));
1583
1584  /* tx flow control */
1585  di->hnddma.txavail = di->ntxd - NTXDACTIVE (di->txin, di->txout) - 1;
1586
1587  return (0);
1588
1589outoftxd:
1590  DMA_ERROR (("%s: dma_txfast: out of txds\n", di->name));
1591  PKTFREE (di->osh, p0, TRUE);
1592  di->hnddma.txavail = 0;
1593  di->hnddma.txnobuf++;
1594  return (-1);
1595}
1596
1597/*
1598 * Reclaim next completed txd (txds if using chained buffers) and
1599 * return associated packet.
1600 * If 'force' is true, reclaim txd(s) and return associated packet
1601 * regardless of the value of the hardware "curr" pointer.
1602 */
1603static void *
1604dma32_getnexttxp (dma_info_t * di, bool forceall)
1605{
1606  uint start, end, i;
1607  void *txp;
1608
1609  DMA_TRACE (("%s: dma_getnexttxp %s\n", di->name, forceall ? "all" : ""));
1610
1611  if (di->ntxd == 0)
1612    return (NULL);
1613
1614  txp = NULL;
1615
1616  start = di->txin;
1617  if (forceall)
1618    end = di->txout;
1619  else
1620    end =
1621      B2I (R_REG (di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t);
1622
1623  if ((start == 0) && (end > di->txout))
1624    goto bogus;
1625
1626  for (i = start; i != end && !txp; i = NEXTTXD (i))
1627    {
1628      DMA_UNMAP (di->osh,
1629         (BUS_SWAP32 (R_SM (&di->txd32[i].addr)) - di->dataoffsetlow),
1630         (BUS_SWAP32 (R_SM (&di->txd32[i].ctrl)) & CTRL_BC_MASK),
1631         DMA_TX, di->txp[i], &di->txp_dmah[i]);
1632
1633      W_SM (&di->txd32[i].addr, 0xdeadbeef);
1634      txp = di->txp[i];
1635      di->txp[i] = NULL;
1636    }
1637
1638  di->txin = i;
1639
1640  /* tx flow control */
1641  di->hnddma.txavail = di->ntxd - NTXDACTIVE (di->txin, di->txout) - 1;
1642
1643  return (txp);
1644
1645bogus:
1646/*
1647    DMA_ERROR(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n",
1648        start, end, di->txout, forceall));
1649*/
1650  return (NULL);
1651}
1652
1653static void *
1654dma32_getnextrxp (dma_info_t * di, bool forceall)
1655{
1656  uint i;
1657  void *rxp;
1658
1659  /* if forcing, dma engine must be disabled */
1660  ASSERT (!forceall || !dma32_rxenabled (di));
1661
1662  i = di->rxin;
1663
1664  /* return if no packets posted */
1665  if (i == di->rxout)
1666    return (NULL);
1667
1668  /* ignore curr if forceall */
1669  if (!forceall
1670      && (i ==
1671      B2I (R_REG (di->osh, &di->d32rxregs->status) & RS_CD_MASK,
1672           dma32dd_t)))
1673    return (NULL);
1674
1675  /* get the packet pointer that corresponds to the rx descriptor */
1676  rxp = di->rxp[i];
1677  ASSERT (rxp);
1678  di->rxp[i] = NULL;
1679
1680  /* clear this packet from the descriptor ring */
1681  DMA_UNMAP (di->osh,
1682         (BUS_SWAP32 (R_SM (&di->rxd32[i].addr)) - di->dataoffsetlow),
1683         di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]);
1684
1685  W_SM (&di->rxd32[i].addr, 0xdeadbeef);
1686
1687  di->rxin = NEXTRXD (i);
1688
1689  return (rxp);
1690}
1691
1692/*
1693 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
1694 */
1695static void
1696dma32_txrotate (dma_info_t * di)
1697{
1698  uint ad;
1699  uint nactive;
1700  uint rot;
1701  uint old, new;
1702  uint32 w;
1703  uint first, last;
1704
1705  ASSERT (dma32_txsuspendedidle (di));
1706
1707  nactive = _dma_txactive (di);
1708  ad =
1709    B2I (((R_REG (di->osh, &di->d32txregs->status) & XS_AD_MASK) >>
1710      XS_AD_SHIFT), dma32dd_t);
1711  rot = TXD (ad - di->txin);
1712
1713  ASSERT (rot < di->ntxd);
1714
1715  /* full-ring case is a lot harder - don't worry about this */
1716  if (rot >= (di->ntxd - nactive))
1717    {
1718      DMA_ERROR (("%s: dma_txrotate: ring full - punt\n", di->name));
1719      return;
1720    }
1721
1722  first = di->txin;
1723  last = PREVTXD (di->txout);
1724
1725  /* move entries starting at last and moving backwards to first */
1726  for (old = last; old != PREVTXD (first); old = PREVTXD (old))
1727    {
1728      new = TXD (old + rot);
1729
1730      /*
1731       * Move the tx dma descriptor.
1732       * EOT is set only in the last entry in the ring.
1733       */
1734      w = BUS_SWAP32 (R_SM (&di->txd32[old].ctrl)) & ~CTRL_EOT;
1735      if (new == (di->ntxd - 1))
1736    w |= CTRL_EOT;
1737      W_SM (&di->txd32[new].ctrl, BUS_SWAP32 (w));
1738      W_SM (&di->txd32[new].addr, R_SM (&di->txd32[old].addr));
1739
1740      /* zap the old tx dma descriptor address field */
1741      W_SM (&di->txd32[old].addr, BUS_SWAP32 (0xdeadbeef));
1742
1743      /* move the corresponding txp[] entry */
1744      ASSERT (di->txp[new] == NULL);
1745      di->txp[new] = di->txp[old];
1746      di->txp[old] = NULL;
1747    }
1748
1749  /* update txin and txout */
1750  di->txin = ad;
1751  di->txout = TXD (di->txout + rot);
1752  di->hnddma.txavail = di->ntxd - NTXDACTIVE (di->txin, di->txout) - 1;
1753
1754  /* kick the chip */
1755  W_REG (di->osh, &di->d32txregs->ptr, I2B (di->txout, dma32dd_t));
1756}
1757
1758/* 64 bits DMA functions */
1759
1760#ifdef BCMDMA64
1761static void
1762dma64_txinit (dma_info_t * di)
1763{
1764  DMA_TRACE (("%s: dma_txinit\n", di->name));
1765
1766  if (di->ntxd == 0)
1767    return;
1768
1769  di->txin = di->txout = 0;
1770  di->hnddma.txavail = di->ntxd - 1;
1771
1772  /* clear tx descriptor ring */
1773  BZERO_SM ((void *) (uintptr) di->txd64, (di->ntxd * sizeof (dma64dd_t)));
1774  W_REG (di->osh, &di->d64txregs->control, D64_XC_XE);
1775  _dma_ddtable_init (di, DMA_TX, di->txdpa);
1776}
1777
1778static bool
1779dma64_txenabled (dma_info_t * di)
1780{
1781  uint32 xc;
1782
1783  /* If the chip is dead, it is not enabled :-) */
1784  xc = R_REG (di->osh, &di->d64txregs->control);
1785  return ((xc != 0xffffffff) && (xc & D64_XC_XE));
1786}
1787
1788static void
1789dma64_txsuspend (dma_info_t * di)
1790{
1791  DMA_TRACE (("%s: dma_txsuspend\n", di->name));
1792
1793  if (di->ntxd == 0)
1794    return;
1795
1796  OR_REG (di->osh, &di->d64txregs->control, D64_XC_SE);
1797}
1798
1799static void
1800dma64_txresume (dma_info_t * di)
1801{
1802  DMA_TRACE (("%s: dma_txresume\n", di->name));
1803
1804  if (di->ntxd == 0)
1805    return;
1806
1807  AND_REG (di->osh, &di->d64txregs->control, ~D64_XC_SE);
1808}
1809
1810static bool
1811dma64_txsuspended (dma_info_t * di)
1812{
1813  return (di->ntxd == 0)
1814    || ((R_REG (di->osh, &di->d64txregs->control) & D64_XC_SE) == D64_XC_SE);
1815}
1816
1817static void
1818dma64_txreclaim (dma_info_t * di, bool forceall)
1819{
1820  void *p;
1821
1822  DMA_TRACE (("%s: dma_txreclaim %s\n", di->name, forceall ? "all" : ""));
1823
1824  while ((p = dma64_getnexttxp (di, forceall)))
1825    PKTFREE (di->osh, p, TRUE);
1826}
1827
1828static bool
1829dma64_txstopped (dma_info_t * di)
1830{
1831  return ((R_REG (di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) ==
1832      D64_XS0_XS_STOPPED);
1833}
1834
1835static bool
1836dma64_rxstopped (dma_info_t * di)
1837{
1838  return ((R_REG (di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK) ==
1839      D64_RS0_RS_STOPPED);
1840}
1841
1842static bool
1843dma64_alloc (dma_info_t * di, uint direction)
1844{
1845  uint size;
1846  uint ddlen;
1847  uint32 alignbytes;
1848  void *va;
1849
1850  ddlen = sizeof (dma64dd_t);
1851
1852  size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
1853
1854  alignbytes = di->dma64align;
1855
1856  if (!ISALIGNED (DMA_CONSISTENT_ALIGN, alignbytes))
1857    size += alignbytes;
1858
1859  if (direction == DMA_TX)
1860    {
1861      if ((va =
1862       DMA_ALLOC_CONSISTENT (di->osh, size, &di->txdpa,
1863                 &di->tx_dmah)) == NULL)
1864    {
1865      DMA_ERROR (("%s: dma_attach: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
1866              di->name));
1867      return FALSE;
1868    }
1869
1870      di->txd64 = (dma64dd_t *) ROUNDUP ((uintptr) va, alignbytes);
1871      di->txdalign = (uint) ((int8 *) (uintptr) di->txd64 - (int8 *) va);
1872      di->txdpa += di->txdalign;
1873      di->txdalloc = size;
1874      ASSERT (ISALIGNED ((uintptr) di->txd64, alignbytes));
1875    }
1876  else
1877    {
1878      if ((va =
1879       DMA_ALLOC_CONSISTENT (di->osh, size, &di->rxdpa,
1880                 &di->rx_dmah)) == NULL)
1881    {
1882      DMA_ERROR (("%s: dma_attach: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
1883              di->name));
1884      return FALSE;
1885    }
1886      di->rxd64 = (dma64dd_t *) ROUNDUP ((uintptr) va, alignbytes);
1887      di->rxdalign = (uint) ((int8 *) (uintptr) di->rxd64 - (int8 *) va);
1888      di->rxdpa += di->rxdalign;
1889      di->rxdalloc = size;
1890      ASSERT (ISALIGNED ((uintptr) di->rxd64, alignbytes));
1891    }
1892
1893  return TRUE;
1894}
1895
1896static bool
1897dma64_txreset (dma_info_t * di)
1898{
1899  uint32 status;
1900
1901  if (di->ntxd == 0)
1902    return TRUE;
1903
1904  /* suspend tx DMA first */
1905  W_REG (di->osh, &di->d64txregs->control, D64_XC_SE);
1906  SPINWAIT (((status =
1907          (R_REG (di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) !=
1908         D64_XS0_XS_DISABLED) && (status != D64_XS0_XS_IDLE)
1909        && (status != D64_XS0_XS_STOPPED), 10000);
1910
1911  W_REG (di->osh, &di->d64txregs->control, 0);
1912  SPINWAIT (((status =
1913          (R_REG (di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) !=
1914         D64_XS0_XS_DISABLED), 10000);
1915
1916  /* wait for the last transaction to complete */
1917  OSL_DELAY (300);
1918
1919  return (status == D64_XS0_XS_DISABLED);
1920}
1921
1922static bool
1923dma64_rxidle (dma_info_t * di)
1924{
1925  DMA_TRACE (("%s: dma_rxidle\n", di->name));
1926
1927  if (di->nrxd == 0)
1928    return TRUE;
1929
1930  return ((R_REG (di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) ==
1931      R_REG (di->osh, &di->d64rxregs->ptr));
1932}
1933
1934static bool
1935dma64_rxreset (dma_info_t * di)
1936{
1937  uint32 status;
1938
1939  if (di->nrxd == 0)
1940    return TRUE;
1941
1942  W_REG (di->osh, &di->d64rxregs->control, 0);
1943  SPINWAIT (((status =
1944          (R_REG (di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK)) !=
1945         D64_RS0_RS_DISABLED), 10000);
1946
1947  return (status == D64_RS0_RS_DISABLED);
1948}
1949
1950static bool
1951dma64_rxenabled (dma_info_t * di)
1952{
1953  uint32 rc;
1954
1955  rc = R_REG (di->osh, &di->d64rxregs->control);
1956  return ((rc != 0xffffffff) && (rc & D64_RC_RE));
1957}
1958
1959static bool
1960dma64_txsuspendedidle (dma_info_t * di)
1961{
1962
1963  if (di->ntxd == 0)
1964    return TRUE;
1965
1966  if (!(R_REG (di->osh, &di->d64txregs->control) & D64_XC_SE))
1967    return 0;
1968
1969  if ((R_REG (di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) ==
1970      D64_XS0_XS_IDLE)
1971    return 1;
1972
1973  return 0;
1974}
1975
1976
1977/* !! tx entry routine */
1978static int
1979dma64_txfast (dma_info_t * di, void *p0, bool commit)
1980{
1981  void *p, *next;
1982  uchar *data;
1983  uint len;
1984  uint txout;
1985  uint32 flags = 0;
1986  uint32 pa;
1987
1988  DMA_TRACE (("%s: dma_txfast\n", di->name));
1989
1990  txout = di->txout;
1991
1992  /*
1993   * Walk the chain of packet buffers
1994   * allocating and initializing transmit descriptor entries.
1995   */
1996  for (p = p0; p; p = next)
1997    {
1998      data = PKTDATA (di->osh, p);
1999      len = PKTLEN (di->osh, p);
2000      next = PKTNEXT (di->osh, p);
2001
2002      /* return nonzero if out of tx descriptors */
2003      if (NEXTTXD (txout) == di->txin)
2004    goto outoftxd;
2005
2006      if (len == 0)
2007    continue;
2008
2009      /* get physical address of buffer start */
2010      pa =
2011    (uint32) DMA_MAP (di->osh, data, len, DMA_TX, p,
2012              &di->txp_dmah[txout]);
2013
2014      flags = 0;
2015      if (p == p0)
2016    flags |= D64_CTRL1_SOF;
2017      if (next == NULL)
2018    flags |= (D64_CTRL1_IOC | D64_CTRL1_EOF);
2019      if (txout == (di->ntxd - 1))
2020    flags |= D64_CTRL1_EOT;
2021
2022      dma64_dd_upd (di, di->txd64, pa, txout, &flags, len);
2023      ASSERT (di->txp[txout] == NULL);
2024
2025      txout = NEXTTXD (txout);
2026    }
2027
2028  /* if last txd eof not set, fix it */
2029  if (!(flags & D64_CTRL1_EOF))
2030    W_SM (&di->txd64[PREVTXD (txout)].ctrl1,
2031      BUS_SWAP32 (flags | D64_CTRL1_IOC | D64_CTRL1_EOF));
2032
2033  /* save the packet */
2034  di->txp[PREVTXD (txout)] = p0;
2035
2036  /* bump the tx descriptor index */
2037  di->txout = txout;
2038
2039  /* kick the chip */
2040  if (commit)
2041    W_REG (di->osh, &di->d64txregs->ptr, I2B (txout, dma64dd_t));
2042
2043  /* tx flow control */
2044  di->hnddma.txavail = di->ntxd - NTXDACTIVE (di->txin, di->txout) - 1;
2045
2046  return (0);
2047
2048outoftxd:
2049  DMA_ERROR (("%s: dma_txfast: out of txds\n", di->name));
2050  PKTFREE (di->osh, p0, TRUE);
2051  di->hnddma.txavail = 0;
2052  di->hnddma.txnobuf++;
2053  return (-1);
2054}
2055
2056/*
2057 * Reclaim next completed txd (txds if using chained buffers) and
2058 * return associated packet.
2059 * If 'force' is true, reclaim txd(s) and return associated packet
2060 * regardless of the value of the hardware "curr" pointer.
2061 */
2062static void *
2063dma64_getnexttxp (dma_info_t * di, bool forceall)
2064{
2065  uint start, end, i;
2066  void *txp;
2067
2068  DMA_TRACE (("%s: dma_getnexttxp %s\n", di->name, forceall ? "all" : ""));
2069
2070  if (di->ntxd == 0)
2071    return (NULL);
2072
2073  txp = NULL;
2074
2075  start = di->txin;
2076  if (forceall)
2077    end = di->txout;
2078  else
2079    end =
2080      B2I (R_REG (di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK,
2081       dma64dd_t);
2082
2083  if ((start == 0) && (end > di->txout))
2084    goto bogus;
2085
2086  for (i = start; i != end && !txp; i = NEXTTXD (i))
2087    {
2088      DMA_UNMAP (di->osh,
2089         (BUS_SWAP32 (R_SM (&di->txd64[i].addrlow)) -
2090          di->dataoffsetlow),
2091         (BUS_SWAP32 (R_SM (&di->txd64[i].ctrl2)) &
2092          D64_CTRL2_BC_MASK), DMA_TX, di->txp[i], &di->txp_dmah[i]);
2093
2094      W_SM (&di->txd64[i].addrlow, 0xdeadbeef);
2095      W_SM (&di->txd64[i].addrhigh, 0xdeadbeef);
2096
2097      txp = di->txp[i];
2098      di->txp[i] = NULL;
2099    }
2100
2101  di->txin = i;
2102
2103  /* tx flow control */
2104  di->hnddma.txavail = di->ntxd - NTXDACTIVE (di->txin, di->txout) - 1;
2105
2106  return (txp);
2107
2108bogus:
2109/*
2110    DMA_ERROR(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n",
2111        start, end, di->txout, forceall));
2112*/
2113  return (NULL);
2114}
2115
2116static void *
2117dma64_getnextrxp (dma_info_t * di, bool forceall)
2118{
2119  uint i;
2120  void *rxp;
2121
2122  /* if forcing, dma engine must be disabled */
2123  ASSERT (!forceall || !dma64_rxenabled (di));
2124
2125  i = di->rxin;
2126
2127  /* return if no packets posted */
2128  if (i == di->rxout)
2129    return (NULL);
2130
2131  /* ignore curr if forceall */
2132  if (!forceall &&
2133      (i ==
2134       B2I (R_REG (di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK,
2135        dma64dd_t)))
2136    return (NULL);
2137
2138  /* get the packet pointer that corresponds to the rx descriptor */
2139  rxp = di->rxp[i];
2140  ASSERT (rxp);
2141  di->rxp[i] = NULL;
2142
2143  /* clear this packet from the descriptor ring */
2144  DMA_UNMAP (di->osh,
2145         (BUS_SWAP32 (R_SM (&di->rxd64[i].addrlow)) - di->dataoffsetlow),
2146         di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]);
2147
2148  W_SM (&di->rxd64[i].addrlow, 0xdeadbeef);
2149  W_SM (&di->rxd64[i].addrhigh, 0xdeadbeef);
2150
2151  di->rxin = NEXTRXD (i);
2152
2153  return (rxp);
2154}
2155
2156static bool
2157_dma64_addrext (osl_t * osh, dma64regs_t * dma64regs)
2158{
2159  uint32 w;
2160  OR_REG (osh, &dma64regs->control, D64_XC_AE);
2161  w = R_REG (osh, &dma64regs->control);
2162  AND_REG (osh, &dma64regs->control, ~D64_XC_AE);
2163  return ((w & D64_XC_AE) == D64_XC_AE);
2164}
2165
2166/*
2167 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
2168 */
2169static void
2170dma64_txrotate (dma_info_t * di)
2171{
2172  uint ad;
2173  uint nactive;
2174  uint rot;
2175  uint old, new;
2176  uint32 w;
2177  uint first, last;
2178
2179  ASSERT (dma64_txsuspendedidle (di));
2180
2181  nactive = _dma_txactive (di);
2182  ad =
2183    B2I ((R_REG (di->osh, &di->d64txregs->status1) & D64_XS1_AD_MASK),
2184     dma64dd_t);
2185  rot = TXD (ad - di->txin);
2186
2187  ASSERT (rot < di->ntxd);
2188
2189  /* full-ring case is a lot harder - don't worry about this */
2190  if (rot >= (di->ntxd - nactive))
2191    {
2192      DMA_ERROR (("%s: dma_txrotate: ring full - punt\n", di->name));
2193      return;
2194    }
2195
2196  first = di->txin;
2197  last = PREVTXD (di->txout);
2198
2199  /* move entries starting at last and moving backwards to first */
2200  for (old = last; old != PREVTXD (first); old = PREVTXD (old))
2201    {
2202      new = TXD (old + rot);
2203
2204      /*
2205       * Move the tx dma descriptor.
2206       * EOT is set only in the last entry in the ring.
2207       */
2208      w = BUS_SWAP32 (R_SM (&di->txd64[old].ctrl1)) & ~D64_CTRL1_EOT;
2209      if (new == (di->ntxd - 1))
2210    w |= D64_CTRL1_EOT;
2211      W_SM (&di->txd64[new].ctrl1, BUS_SWAP32 (w));
2212
2213      w = BUS_SWAP32 (R_SM (&di->txd64[old].ctrl2));
2214      W_SM (&di->txd64[new].ctrl2, BUS_SWAP32 (w));
2215
2216      W_SM (&di->txd64[new].addrlow, R_SM (&di->txd64[old].addrlow));
2217      W_SM (&di->txd64[new].addrhigh, R_SM (&di->txd64[old].addrhigh));
2218
2219      /* zap the old tx dma descriptor address field */
2220      W_SM (&di->txd64[old].addrlow, BUS_SWAP32 (0xdeadbeef));
2221      W_SM (&di->txd64[old].addrhigh, BUS_SWAP32 (0xdeadbeef));
2222
2223      /* move the corresponding txp[] entry */
2224      ASSERT (di->txp[new] == NULL);
2225      di->txp[new] = di->txp[old];
2226      di->txp[old] = NULL;
2227    }
2228
2229  /* update txin and txout */
2230  di->txin = ad;
2231  di->txout = TXD (di->txout + rot);
2232  di->hnddma.txavail = di->ntxd - NTXDACTIVE (di->txin, di->txout) - 1;
2233
2234  /* kick the chip */
2235  W_REG (di->osh, &di->d64txregs->ptr, I2B (di->txout, dma64dd_t));
2236}
2237
2238#endif /* BCMDMA64 */
2239
2240uint
2241dma_addrwidth (sb_t * sbh, void *dmaregs)
2242{
2243  dma32regs_t *dma32regs;
2244  osl_t *osh;
2245
2246  osh = sb_osh (sbh);
2247
2248  if (DMA64_CAP)
2249    {
2250      /* DMA engine is 64-bit capable */
2251      if (((sb_coreflagshi (sbh, 0, 0) & SBTMH_DMA64) == SBTMH_DMA64))
2252    {
2253      /* backplane are 64 bits capable */
2254      if (sb_backplane64 (sbh))
2255        /* If bus is System Backplane or PCIE then we can access 64-bits */
2256        if ((BUSTYPE (sbh->bustype) == SB_BUS) ||
2257        ((BUSTYPE (sbh->bustype) == PCI_BUS) &&
2258         sbh->buscoretype == SB_PCIE))
2259          return (DMADDRWIDTH_64);
2260
2261      /* DMA64 is always 32 bits capable, AE is always TRUE */
2262#ifdef BCMDMA64
2263      ASSERT (_dma64_addrext (osh, (dma64regs_t *) dmaregs));
2264#endif
2265      return (DMADDRWIDTH_32);
2266    }
2267    }
2268
2269  /* Start checking for 32-bit / 30-bit addressing */
2270  dma32regs = (dma32regs_t *) dmaregs;
2271
2272  /* For System Backplane, PCIE bus or addrext feature, 32-bits ok */
2273  if ((BUSTYPE (sbh->bustype) == SB_BUS) ||
2274      ((BUSTYPE (sbh->bustype) == PCI_BUS) && sbh->buscoretype == SB_PCIE) ||
2275      (_dma32_addrext (osh, dma32regs)))
2276    return (DMADDRWIDTH_32);
2277
2278  /* Fallthru */
2279  return (DMADDRWIDTH_30);
2280}
2281

Archive Download this file

Branches:
backfire
history
master
release_2010-11-17
release_2010-12-14
release_2011-02-23
release_2011-05-28
release_2011-08-27
release_2011-11-13
release_2012-03-18
release_2012-04-09
release_2012-10-11
uclibc_nptl



interactive