Root/drivers/xen/grant-table.c

1/******************************************************************************
2 * grant_table.c
3 *
4 * Granting foreign access to our memory reservation.
5 *
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
31 * IN THE SOFTWARE.
32 */
33
34#include <linux/module.h>
35#include <linux/sched.h>
36#include <linux/mm.h>
37#include <linux/slab.h>
38#include <linux/vmalloc.h>
39#include <linux/uaccess.h>
40#include <linux/io.h>
41#include <linux/hardirq.h>
42
43#include <xen/xen.h>
44#include <xen/interface/xen.h>
45#include <xen/page.h>
46#include <xen/grant_table.h>
47#include <xen/interface/memory.h>
48#include <xen/hvc-console.h>
49#include <asm/xen/hypercall.h>
50
51#include <asm/pgtable.h>
52#include <asm/sync_bitops.h>
53
54/* External tools reserve first few grant table entries. */
55#define NR_RESERVED_ENTRIES 8
56#define GNTTAB_LIST_END 0xffffffff
57#define GREFS_PER_GRANT_FRAME \
58(grant_table_version == 1 ? \
59(PAGE_SIZE / sizeof(struct grant_entry_v1)) : \
60(PAGE_SIZE / sizeof(union grant_entry_v2)))
61
62static grant_ref_t **gnttab_list;
63static unsigned int nr_grant_frames;
64static unsigned int boot_max_nr_grant_frames;
65static int gnttab_free_count;
66static grant_ref_t gnttab_free_head;
67static DEFINE_SPINLOCK(gnttab_list_lock);
68unsigned long xen_hvm_resume_frames;
69EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
70
71static union {
72    struct grant_entry_v1 *v1;
73    union grant_entry_v2 *v2;
74    void *addr;
75} gnttab_shared;
76
77/*This is a structure of function pointers for grant table*/
78struct gnttab_ops {
79    /*
80     * Mapping a list of frames for storing grant entries. Frames parameter
81     * is used to store grant table address when grant table being setup,
82     * nr_gframes is the number of frames to map grant table. Returning
83     * GNTST_okay means success and negative value means failure.
84     */
85    int (*map_frames)(unsigned long *frames, unsigned int nr_gframes);
86    /*
87     * Release a list of frames which are mapped in map_frames for grant
88     * entry status.
89     */
90    void (*unmap_frames)(void);
91    /*
92     * Introducing a valid entry into the grant table, granting the frame of
93     * this grant entry to domain for accessing or transfering. Ref
94     * parameter is reference of this introduced grant entry, domid is id of
95     * granted domain, frame is the page frame to be granted, and flags is
96     * status of the grant entry to be updated.
97     */
98    void (*update_entry)(grant_ref_t ref, domid_t domid,
99                 unsigned long frame, unsigned flags);
100    /*
101     * Stop granting a grant entry to domain for accessing. Ref parameter is
102     * reference of a grant entry whose grant access will be stopped,
103     * readonly is not in use in this function. If the grant entry is
104     * currently mapped for reading or writing, just return failure(==0)
105     * directly and don't tear down the grant access. Otherwise, stop grant
106     * access for this entry and return success(==1).
107     */
108    int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
109    /*
110     * Stop granting a grant entry to domain for transfer. Ref parameter is
111     * reference of a grant entry whose grant transfer will be stopped. If
112     * tranfer has not started, just reclaim the grant entry and return
113     * failure(==0). Otherwise, wait for the transfer to complete and then
114     * return the frame.
115     */
116    unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
117    /*
118     * Query the status of a grant entry. Ref parameter is reference of
119     * queried grant entry, return value is the status of queried entry.
120     * Detailed status(writing/reading) can be gotten from the return value
121     * by bit operations.
122     */
123    int (*query_foreign_access)(grant_ref_t ref);
124    /*
125     * Grant a domain to access a range of bytes within the page referred by
126     * an available grant entry. Ref parameter is reference of a grant entry
127     * which will be sub-page accessed, domid is id of grantee domain, frame
128     * is frame address of subpage grant, flags is grant type and flag
129     * information, page_off is offset of the range of bytes, and length is
130     * length of bytes to be accessed.
131     */
132    void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
133                     unsigned long frame, int flags,
134                     unsigned page_off, unsigned length);
135    /*
136     * Redirect an available grant entry on domain A to another grant
137     * reference of domain B, then allow domain C to use grant reference
138     * of domain B transitively. Ref parameter is an available grant entry
139     * reference on domain A, domid is id of domain C which accesses grant
140     * entry transitively, flags is grant type and flag information,
141     * trans_domid is id of domain B whose grant entry is finally accessed
142     * transitively, trans_gref is grant entry transitive reference of
143     * domain B.
144     */
145    void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
146                   domid_t trans_domid, grant_ref_t trans_gref);
147};
148
149static struct gnttab_ops *gnttab_interface;
150
151/*This reflects status of grant entries, so act as a global value*/
152static grant_status_t *grstatus;
153
154static int grant_table_version;
155
156static struct gnttab_free_callback *gnttab_free_callback_list;
157
158static int gnttab_expand(unsigned int req_entries);
159
160#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
161#define SPP (PAGE_SIZE / sizeof(grant_status_t))
162
163static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
164{
165    return &gnttab_list[(entry) / RPP][(entry) % RPP];
166}
167/* This can be used as an l-value */
168#define gnttab_entry(entry) (*__gnttab_entry(entry))
169
170static int get_free_entries(unsigned count)
171{
172    unsigned long flags;
173    int ref, rc = 0;
174    grant_ref_t head;
175
176    spin_lock_irqsave(&gnttab_list_lock, flags);
177
178    if ((gnttab_free_count < count) &&
179        ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
180        spin_unlock_irqrestore(&gnttab_list_lock, flags);
181        return rc;
182    }
183
184    ref = head = gnttab_free_head;
185    gnttab_free_count -= count;
186    while (count-- > 1)
187        head = gnttab_entry(head);
188    gnttab_free_head = gnttab_entry(head);
189    gnttab_entry(head) = GNTTAB_LIST_END;
190
191    spin_unlock_irqrestore(&gnttab_list_lock, flags);
192
193    return ref;
194}
195
196static void do_free_callbacks(void)
197{
198    struct gnttab_free_callback *callback, *next;
199
200    callback = gnttab_free_callback_list;
201    gnttab_free_callback_list = NULL;
202
203    while (callback != NULL) {
204        next = callback->next;
205        if (gnttab_free_count >= callback->count) {
206            callback->next = NULL;
207            callback->fn(callback->arg);
208        } else {
209            callback->next = gnttab_free_callback_list;
210            gnttab_free_callback_list = callback;
211        }
212        callback = next;
213    }
214}
215
216static inline void check_free_callbacks(void)
217{
218    if (unlikely(gnttab_free_callback_list))
219        do_free_callbacks();
220}
221
222static void put_free_entry(grant_ref_t ref)
223{
224    unsigned long flags;
225    spin_lock_irqsave(&gnttab_list_lock, flags);
226    gnttab_entry(ref) = gnttab_free_head;
227    gnttab_free_head = ref;
228    gnttab_free_count++;
229    check_free_callbacks();
230    spin_unlock_irqrestore(&gnttab_list_lock, flags);
231}
232
233/*
234 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
235 * Introducing a valid entry into the grant table:
236 * 1. Write ent->domid.
237 * 2. Write ent->frame:
238 * GTF_permit_access: Frame to which access is permitted.
239 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
240 * frame, or zero if none.
241 * 3. Write memory barrier (WMB).
242 * 4. Write ent->flags, inc. valid type.
243 */
244static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
245                   unsigned long frame, unsigned flags)
246{
247    gnttab_shared.v1[ref].domid = domid;
248    gnttab_shared.v1[ref].frame = frame;
249    wmb();
250    gnttab_shared.v1[ref].flags = flags;
251}
252
253static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
254                   unsigned long frame, unsigned flags)
255{
256    gnttab_shared.v2[ref].hdr.domid = domid;
257    gnttab_shared.v2[ref].full_page.frame = frame;
258    wmb();
259    gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
260}
261
262/*
263 * Public grant-issuing interface functions
264 */
265void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
266                     unsigned long frame, int readonly)
267{
268    gnttab_interface->update_entry(ref, domid, frame,
269               GTF_permit_access | (readonly ? GTF_readonly : 0));
270}
271EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
272
273int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
274                int readonly)
275{
276    int ref;
277
278    ref = get_free_entries(1);
279    if (unlikely(ref < 0))
280        return -ENOSPC;
281
282    gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
283
284    return ref;
285}
286EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
287
288void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
289                    unsigned long frame, int flags,
290                    unsigned page_off,
291                    unsigned length)
292{
293    gnttab_shared.v2[ref].sub_page.frame = frame;
294    gnttab_shared.v2[ref].sub_page.page_off = page_off;
295    gnttab_shared.v2[ref].sub_page.length = length;
296    gnttab_shared.v2[ref].hdr.domid = domid;
297    wmb();
298    gnttab_shared.v2[ref].hdr.flags =
299                GTF_permit_access | GTF_sub_page | flags;
300}
301
302int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
303                        unsigned long frame, int flags,
304                        unsigned page_off,
305                        unsigned length)
306{
307    if (flags & (GTF_accept_transfer | GTF_reading |
308             GTF_writing | GTF_transitive))
309        return -EPERM;
310
311    if (gnttab_interface->update_subpage_entry == NULL)
312        return -ENOSYS;
313
314    gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
315                           page_off, length);
316
317    return 0;
318}
319EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
320
321int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
322                    int flags, unsigned page_off,
323                    unsigned length)
324{
325    int ref, rc;
326
327    ref = get_free_entries(1);
328    if (unlikely(ref < 0))
329        return -ENOSPC;
330
331    rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
332                             page_off, length);
333    if (rc < 0) {
334        put_free_entry(ref);
335        return rc;
336    }
337
338    return ref;
339}
340EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
341
342bool gnttab_subpage_grants_available(void)
343{
344    return gnttab_interface->update_subpage_entry != NULL;
345}
346EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
347
348void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
349                  int flags, domid_t trans_domid,
350                  grant_ref_t trans_gref)
351{
352    gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
353    gnttab_shared.v2[ref].transitive.gref = trans_gref;
354    gnttab_shared.v2[ref].hdr.domid = domid;
355    wmb();
356    gnttab_shared.v2[ref].hdr.flags =
357                GTF_permit_access | GTF_transitive | flags;
358}
359
360int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
361                      int flags, domid_t trans_domid,
362                      grant_ref_t trans_gref)
363{
364    if (flags & (GTF_accept_transfer | GTF_reading |
365             GTF_writing | GTF_sub_page))
366        return -EPERM;
367
368    if (gnttab_interface->update_trans_entry == NULL)
369        return -ENOSYS;
370
371    gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
372                         trans_gref);
373
374    return 0;
375}
376EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
377
378int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
379                      domid_t trans_domid,
380                      grant_ref_t trans_gref)
381{
382    int ref, rc;
383
384    ref = get_free_entries(1);
385    if (unlikely(ref < 0))
386        return -ENOSPC;
387
388    rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
389                           trans_domid, trans_gref);
390    if (rc < 0) {
391        put_free_entry(ref);
392        return rc;
393    }
394
395    return ref;
396}
397EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
398
399bool gnttab_trans_grants_available(void)
400{
401    return gnttab_interface->update_trans_entry != NULL;
402}
403EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
404
405static int gnttab_query_foreign_access_v1(grant_ref_t ref)
406{
407    return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
408}
409
410static int gnttab_query_foreign_access_v2(grant_ref_t ref)
411{
412    return grstatus[ref] & (GTF_reading|GTF_writing);
413}
414
415int gnttab_query_foreign_access(grant_ref_t ref)
416{
417    return gnttab_interface->query_foreign_access(ref);
418}
419EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
420
421static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
422{
423    u16 flags, nflags;
424    u16 *pflags;
425
426    pflags = &gnttab_shared.v1[ref].flags;
427    nflags = *pflags;
428    do {
429        flags = nflags;
430        if (flags & (GTF_reading|GTF_writing))
431            return 0;
432    } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
433
434    return 1;
435}
436
437static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
438{
439    gnttab_shared.v2[ref].hdr.flags = 0;
440    mb();
441    if (grstatus[ref] & (GTF_reading|GTF_writing)) {
442        return 0;
443    } else {
444        /* The read of grstatus needs to have acquire
445        semantics. On x86, reads already have
446        that, and we just need to protect against
447        compiler reorderings. On other
448        architectures we may need a full
449        barrier. */
450#ifdef CONFIG_X86
451        barrier();
452#else
453        mb();
454#endif
455    }
456
457    return 1;
458}
459
460static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
461{
462    return gnttab_interface->end_foreign_access_ref(ref, readonly);
463}
464
465int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
466{
467    if (_gnttab_end_foreign_access_ref(ref, readonly))
468        return 1;
469    pr_warn("WARNING: g.e. %#x still in use!\n", ref);
470    return 0;
471}
472EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
473
474struct deferred_entry {
475    struct list_head list;
476    grant_ref_t ref;
477    bool ro;
478    uint16_t warn_delay;
479    struct page *page;
480};
481static LIST_HEAD(deferred_list);
482static void gnttab_handle_deferred(unsigned long);
483static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0);
484
485static void gnttab_handle_deferred(unsigned long unused)
486{
487    unsigned int nr = 10;
488    struct deferred_entry *first = NULL;
489    unsigned long flags;
490
491    spin_lock_irqsave(&gnttab_list_lock, flags);
492    while (nr--) {
493        struct deferred_entry *entry
494            = list_first_entry(&deferred_list,
495                       struct deferred_entry, list);
496
497        if (entry == first)
498            break;
499        list_del(&entry->list);
500        spin_unlock_irqrestore(&gnttab_list_lock, flags);
501        if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
502            put_free_entry(entry->ref);
503            if (entry->page) {
504                pr_debug("freeing g.e. %#x (pfn %#lx)\n",
505                     entry->ref, page_to_pfn(entry->page));
506                __free_page(entry->page);
507            } else
508                pr_info("freeing g.e. %#x\n", entry->ref);
509            kfree(entry);
510            entry = NULL;
511        } else {
512            if (!--entry->warn_delay)
513                pr_info("g.e. %#x still pending\n",
514                    entry->ref);
515            if (!first)
516                first = entry;
517        }
518        spin_lock_irqsave(&gnttab_list_lock, flags);
519        if (entry)
520            list_add_tail(&entry->list, &deferred_list);
521        else if (list_empty(&deferred_list))
522            break;
523    }
524    if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
525        deferred_timer.expires = jiffies + HZ;
526        add_timer(&deferred_timer);
527    }
528    spin_unlock_irqrestore(&gnttab_list_lock, flags);
529}
530
531static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
532                struct page *page)
533{
534    struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
535    const char *what = KERN_WARNING "leaking";
536
537    if (entry) {
538        unsigned long flags;
539
540        entry->ref = ref;
541        entry->ro = readonly;
542        entry->page = page;
543        entry->warn_delay = 60;
544        spin_lock_irqsave(&gnttab_list_lock, flags);
545        list_add_tail(&entry->list, &deferred_list);
546        if (!timer_pending(&deferred_timer)) {
547            deferred_timer.expires = jiffies + HZ;
548            add_timer(&deferred_timer);
549        }
550        spin_unlock_irqrestore(&gnttab_list_lock, flags);
551        what = KERN_DEBUG "deferring";
552    }
553    printk("%s g.e. %#x (pfn %#lx)\n",
554           what, ref, page ? page_to_pfn(page) : -1);
555}
556
557void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
558                   unsigned long page)
559{
560    if (gnttab_end_foreign_access_ref(ref, readonly)) {
561        put_free_entry(ref);
562        if (page != 0)
563            free_page(page);
564    } else
565        gnttab_add_deferred(ref, readonly,
566                    page ? virt_to_page(page) : NULL);
567}
568EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
569
570int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
571{
572    int ref;
573
574    ref = get_free_entries(1);
575    if (unlikely(ref < 0))
576        return -ENOSPC;
577    gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
578
579    return ref;
580}
581EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
582
583void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
584                       unsigned long pfn)
585{
586    gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
587}
588EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
589
590static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
591{
592    unsigned long frame;
593    u16 flags;
594    u16 *pflags;
595
596    pflags = &gnttab_shared.v1[ref].flags;
597
598    /*
599     * If a transfer is not even yet started, try to reclaim the grant
600     * reference and return failure (== 0).
601     */
602    while (!((flags = *pflags) & GTF_transfer_committed)) {
603        if (sync_cmpxchg(pflags, flags, 0) == flags)
604            return 0;
605        cpu_relax();
606    }
607
608    /* If a transfer is in progress then wait until it is completed. */
609    while (!(flags & GTF_transfer_completed)) {
610        flags = *pflags;
611        cpu_relax();
612    }
613
614    rmb(); /* Read the frame number /after/ reading completion status. */
615    frame = gnttab_shared.v1[ref].frame;
616    BUG_ON(frame == 0);
617
618    return frame;
619}
620
621static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
622{
623    unsigned long frame;
624    u16 flags;
625    u16 *pflags;
626
627    pflags = &gnttab_shared.v2[ref].hdr.flags;
628
629    /*
630     * If a transfer is not even yet started, try to reclaim the grant
631     * reference and return failure (== 0).
632     */
633    while (!((flags = *pflags) & GTF_transfer_committed)) {
634        if (sync_cmpxchg(pflags, flags, 0) == flags)
635            return 0;
636        cpu_relax();
637    }
638
639    /* If a transfer is in progress then wait until it is completed. */
640    while (!(flags & GTF_transfer_completed)) {
641        flags = *pflags;
642        cpu_relax();
643    }
644
645    rmb(); /* Read the frame number /after/ reading completion status. */
646    frame = gnttab_shared.v2[ref].full_page.frame;
647    BUG_ON(frame == 0);
648
649    return frame;
650}
651
652unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
653{
654    return gnttab_interface->end_foreign_transfer_ref(ref);
655}
656EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
657
658unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
659{
660    unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
661    put_free_entry(ref);
662    return frame;
663}
664EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
665
666void gnttab_free_grant_reference(grant_ref_t ref)
667{
668    put_free_entry(ref);
669}
670EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
671
672void gnttab_free_grant_references(grant_ref_t head)
673{
674    grant_ref_t ref;
675    unsigned long flags;
676    int count = 1;
677    if (head == GNTTAB_LIST_END)
678        return;
679    spin_lock_irqsave(&gnttab_list_lock, flags);
680    ref = head;
681    while (gnttab_entry(ref) != GNTTAB_LIST_END) {
682        ref = gnttab_entry(ref);
683        count++;
684    }
685    gnttab_entry(ref) = gnttab_free_head;
686    gnttab_free_head = head;
687    gnttab_free_count += count;
688    check_free_callbacks();
689    spin_unlock_irqrestore(&gnttab_list_lock, flags);
690}
691EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
692
693int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
694{
695    int h = get_free_entries(count);
696
697    if (h < 0)
698        return -ENOSPC;
699
700    *head = h;
701
702    return 0;
703}
704EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
705
706int gnttab_empty_grant_references(const grant_ref_t *private_head)
707{
708    return (*private_head == GNTTAB_LIST_END);
709}
710EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
711
712int gnttab_claim_grant_reference(grant_ref_t *private_head)
713{
714    grant_ref_t g = *private_head;
715    if (unlikely(g == GNTTAB_LIST_END))
716        return -ENOSPC;
717    *private_head = gnttab_entry(g);
718    return g;
719}
720EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
721
722void gnttab_release_grant_reference(grant_ref_t *private_head,
723                    grant_ref_t release)
724{
725    gnttab_entry(release) = *private_head;
726    *private_head = release;
727}
728EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
729
730void gnttab_request_free_callback(struct gnttab_free_callback *callback,
731                  void (*fn)(void *), void *arg, u16 count)
732{
733    unsigned long flags;
734    spin_lock_irqsave(&gnttab_list_lock, flags);
735    if (callback->next)
736        goto out;
737    callback->fn = fn;
738    callback->arg = arg;
739    callback->count = count;
740    callback->next = gnttab_free_callback_list;
741    gnttab_free_callback_list = callback;
742    check_free_callbacks();
743out:
744    spin_unlock_irqrestore(&gnttab_list_lock, flags);
745}
746EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
747
748void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
749{
750    struct gnttab_free_callback **pcb;
751    unsigned long flags;
752
753    spin_lock_irqsave(&gnttab_list_lock, flags);
754    for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
755        if (*pcb == callback) {
756            *pcb = callback->next;
757            break;
758        }
759    }
760    spin_unlock_irqrestore(&gnttab_list_lock, flags);
761}
762EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
763
764static int grow_gnttab_list(unsigned int more_frames)
765{
766    unsigned int new_nr_grant_frames, extra_entries, i;
767    unsigned int nr_glist_frames, new_nr_glist_frames;
768
769    new_nr_grant_frames = nr_grant_frames + more_frames;
770    extra_entries = more_frames * GREFS_PER_GRANT_FRAME;
771
772    nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
773    new_nr_glist_frames =
774        (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
775    for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
776        gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
777        if (!gnttab_list[i])
778            goto grow_nomem;
779    }
780
781
782    for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
783         i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
784        gnttab_entry(i) = i + 1;
785
786    gnttab_entry(i) = gnttab_free_head;
787    gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
788    gnttab_free_count += extra_entries;
789
790    nr_grant_frames = new_nr_grant_frames;
791
792    check_free_callbacks();
793
794    return 0;
795
796grow_nomem:
797    for ( ; i >= nr_glist_frames; i--)
798        free_page((unsigned long) gnttab_list[i]);
799    return -ENOMEM;
800}
801
802static unsigned int __max_nr_grant_frames(void)
803{
804    struct gnttab_query_size query;
805    int rc;
806
807    query.dom = DOMID_SELF;
808
809    rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
810    if ((rc < 0) || (query.status != GNTST_okay))
811        return 4; /* Legacy max supported number of frames */
812
813    return query.max_nr_frames;
814}
815
816unsigned int gnttab_max_grant_frames(void)
817{
818    unsigned int xen_max = __max_nr_grant_frames();
819
820    if (xen_max > boot_max_nr_grant_frames)
821        return boot_max_nr_grant_frames;
822    return xen_max;
823}
824EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
825
826int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
827            struct gnttab_map_grant_ref *kmap_ops,
828            struct page **pages, unsigned int count)
829{
830    int i, ret;
831    bool lazy = false;
832    pte_t *pte;
833    unsigned long mfn;
834
835    ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
836    if (ret)
837        return ret;
838
839    if (xen_feature(XENFEAT_auto_translated_physmap))
840        return ret;
841
842    if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
843        arch_enter_lazy_mmu_mode();
844        lazy = true;
845    }
846
847    for (i = 0; i < count; i++) {
848        /* Do not add to override if the map failed. */
849        if (map_ops[i].status)
850            continue;
851
852        if (map_ops[i].flags & GNTMAP_contains_pte) {
853            pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
854                (map_ops[i].host_addr & ~PAGE_MASK));
855            mfn = pte_mfn(*pte);
856        } else {
857            mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
858        }
859        ret = m2p_add_override(mfn, pages[i], kmap_ops ?
860                       &kmap_ops[i] : NULL);
861        if (ret)
862            return ret;
863    }
864
865    if (lazy)
866        arch_leave_lazy_mmu_mode();
867
868    return ret;
869}
870EXPORT_SYMBOL_GPL(gnttab_map_refs);
871
872int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
873              struct gnttab_map_grant_ref *kmap_ops,
874              struct page **pages, unsigned int count)
875{
876    int i, ret;
877    bool lazy = false;
878
879    ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
880    if (ret)
881        return ret;
882
883    if (xen_feature(XENFEAT_auto_translated_physmap))
884        return ret;
885
886    if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
887        arch_enter_lazy_mmu_mode();
888        lazy = true;
889    }
890
891    for (i = 0; i < count; i++) {
892        ret = m2p_remove_override(pages[i], kmap_ops ?
893                       &kmap_ops[i] : NULL);
894        if (ret)
895            return ret;
896    }
897
898    if (lazy)
899        arch_leave_lazy_mmu_mode();
900
901    return ret;
902}
903EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
904
905static unsigned nr_status_frames(unsigned nr_grant_frames)
906{
907    return (nr_grant_frames * GREFS_PER_GRANT_FRAME + SPP - 1) / SPP;
908}
909
910static int gnttab_map_frames_v1(unsigned long *frames, unsigned int nr_gframes)
911{
912    int rc;
913
914    rc = arch_gnttab_map_shared(frames, nr_gframes,
915                    gnttab_max_grant_frames(),
916                    &gnttab_shared.addr);
917    BUG_ON(rc);
918
919    return 0;
920}
921
922static void gnttab_unmap_frames_v1(void)
923{
924    arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
925}
926
927static int gnttab_map_frames_v2(unsigned long *frames, unsigned int nr_gframes)
928{
929    uint64_t *sframes;
930    unsigned int nr_sframes;
931    struct gnttab_get_status_frames getframes;
932    int rc;
933
934    nr_sframes = nr_status_frames(nr_gframes);
935
936    /* No need for kzalloc as it is initialized in following hypercall
937     * GNTTABOP_get_status_frames.
938     */
939    sframes = kmalloc(nr_sframes * sizeof(uint64_t), GFP_ATOMIC);
940    if (!sframes)
941        return -ENOMEM;
942
943    getframes.dom = DOMID_SELF;
944    getframes.nr_frames = nr_sframes;
945    set_xen_guest_handle(getframes.frame_list, sframes);
946
947    rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
948                       &getframes, 1);
949    if (rc == -ENOSYS) {
950        kfree(sframes);
951        return -ENOSYS;
952    }
953
954    BUG_ON(rc || getframes.status);
955
956    rc = arch_gnttab_map_status(sframes, nr_sframes,
957                    nr_status_frames(gnttab_max_grant_frames()),
958                    &grstatus);
959    BUG_ON(rc);
960    kfree(sframes);
961
962    rc = arch_gnttab_map_shared(frames, nr_gframes,
963                    gnttab_max_grant_frames(),
964                    &gnttab_shared.addr);
965    BUG_ON(rc);
966
967    return 0;
968}
969
970static void gnttab_unmap_frames_v2(void)
971{
972    arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
973    arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
974}
975
976static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
977{
978    struct gnttab_setup_table setup;
979    unsigned long *frames;
980    unsigned int nr_gframes = end_idx + 1;
981    int rc;
982
983    if (xen_hvm_domain()) {
984        struct xen_add_to_physmap xatp;
985        unsigned int i = end_idx;
986        rc = 0;
987        /*
988         * Loop backwards, so that the first hypercall has the largest
989         * index, ensuring that the table will grow only once.
990         */
991        do {
992            xatp.domid = DOMID_SELF;
993            xatp.idx = i;
994            xatp.space = XENMAPSPACE_grant_table;
995            xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
996            rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
997            if (rc != 0) {
998                printk(KERN_WARNING
999                        "grant table add_to_physmap failed, err=%d\n", rc);
1000                break;
1001            }
1002        } while (i-- > start_idx);
1003
1004        return rc;
1005    }
1006
1007    /* No need for kzalloc as it is initialized in following hypercall
1008     * GNTTABOP_setup_table.
1009     */
1010    frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1011    if (!frames)
1012        return -ENOMEM;
1013
1014    setup.dom = DOMID_SELF;
1015    setup.nr_frames = nr_gframes;
1016    set_xen_guest_handle(setup.frame_list, frames);
1017
1018    rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1019    if (rc == -ENOSYS) {
1020        kfree(frames);
1021        return -ENOSYS;
1022    }
1023
1024    BUG_ON(rc || setup.status);
1025
1026    rc = gnttab_interface->map_frames(frames, nr_gframes);
1027
1028    kfree(frames);
1029
1030    return rc;
1031}
1032
1033static struct gnttab_ops gnttab_v1_ops = {
1034    .map_frames = gnttab_map_frames_v1,
1035    .unmap_frames = gnttab_unmap_frames_v1,
1036    .update_entry = gnttab_update_entry_v1,
1037    .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,
1038    .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1,
1039    .query_foreign_access = gnttab_query_foreign_access_v1,
1040};
1041
1042static struct gnttab_ops gnttab_v2_ops = {
1043    .map_frames = gnttab_map_frames_v2,
1044    .unmap_frames = gnttab_unmap_frames_v2,
1045    .update_entry = gnttab_update_entry_v2,
1046    .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,
1047    .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2,
1048    .query_foreign_access = gnttab_query_foreign_access_v2,
1049    .update_subpage_entry = gnttab_update_subpage_entry_v2,
1050    .update_trans_entry = gnttab_update_trans_entry_v2,
1051};
1052
1053static void gnttab_request_version(void)
1054{
1055    int rc;
1056    struct gnttab_set_version gsv;
1057
1058    if (xen_hvm_domain())
1059        gsv.version = 1;
1060    else
1061        gsv.version = 2;
1062    rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
1063    if (rc == 0 && gsv.version == 2) {
1064        grant_table_version = 2;
1065        gnttab_interface = &gnttab_v2_ops;
1066    } else if (grant_table_version == 2) {
1067        /*
1068         * If we've already used version 2 features,
1069         * but then suddenly discover that they're not
1070         * available (e.g. migrating to an older
1071         * version of Xen), almost unbounded badness
1072         * can happen.
1073         */
1074        panic("we need grant tables version 2, but only version 1 is available");
1075    } else {
1076        grant_table_version = 1;
1077        gnttab_interface = &gnttab_v1_ops;
1078    }
1079    printk(KERN_INFO "Grant tables using version %d layout.\n",
1080        grant_table_version);
1081}
1082
1083int gnttab_resume(void)
1084{
1085    unsigned int max_nr_gframes;
1086
1087    gnttab_request_version();
1088    max_nr_gframes = gnttab_max_grant_frames();
1089    if (max_nr_gframes < nr_grant_frames)
1090        return -ENOSYS;
1091
1092    if (xen_pv_domain())
1093        return gnttab_map(0, nr_grant_frames - 1);
1094
1095    if (gnttab_shared.addr == NULL) {
1096        gnttab_shared.addr = ioremap(xen_hvm_resume_frames,
1097                        PAGE_SIZE * max_nr_gframes);
1098        if (gnttab_shared.addr == NULL) {
1099            printk(KERN_WARNING
1100                    "Failed to ioremap gnttab share frames!");
1101            return -ENOMEM;
1102        }
1103    }
1104
1105    gnttab_map(0, nr_grant_frames - 1);
1106
1107    return 0;
1108}
1109
1110int gnttab_suspend(void)
1111{
1112    gnttab_interface->unmap_frames();
1113    return 0;
1114}
1115
1116static int gnttab_expand(unsigned int req_entries)
1117{
1118    int rc;
1119    unsigned int cur, extra;
1120
1121    cur = nr_grant_frames;
1122    extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
1123         GREFS_PER_GRANT_FRAME);
1124    if (cur + extra > gnttab_max_grant_frames())
1125        return -ENOSPC;
1126
1127    rc = gnttab_map(cur, cur + extra - 1);
1128    if (rc == 0)
1129        rc = grow_gnttab_list(extra);
1130
1131    return rc;
1132}
1133
1134int gnttab_init(void)
1135{
1136    int i;
1137    unsigned int max_nr_glist_frames, nr_glist_frames;
1138    unsigned int nr_init_grefs;
1139    int ret;
1140
1141    nr_grant_frames = 1;
1142    boot_max_nr_grant_frames = __max_nr_grant_frames();
1143
1144    /* Determine the maximum number of frames required for the
1145     * grant reference free list on the current hypervisor.
1146     */
1147    max_nr_glist_frames = (boot_max_nr_grant_frames *
1148                   GREFS_PER_GRANT_FRAME / RPP);
1149
1150    gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1151                  GFP_KERNEL);
1152    if (gnttab_list == NULL)
1153        return -ENOMEM;
1154
1155    nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
1156    for (i = 0; i < nr_glist_frames; i++) {
1157        gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1158        if (gnttab_list[i] == NULL) {
1159            ret = -ENOMEM;
1160            goto ini_nomem;
1161        }
1162    }
1163
1164    if (gnttab_resume() < 0) {
1165        ret = -ENODEV;
1166        goto ini_nomem;
1167    }
1168
1169    nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
1170
1171    for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1172        gnttab_entry(i) = i + 1;
1173
1174    gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1175    gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1176    gnttab_free_head = NR_RESERVED_ENTRIES;
1177
1178    printk("Grant table initialized\n");
1179    return 0;
1180
1181 ini_nomem:
1182    for (i--; i >= 0; i--)
1183        free_page((unsigned long)gnttab_list[i]);
1184    kfree(gnttab_list);
1185    return ret;
1186}
1187EXPORT_SYMBOL_GPL(gnttab_init);
1188
1189static int __devinit __gnttab_init(void)
1190{
1191    /* Delay grant-table initialization in the PV on HVM case */
1192    if (xen_hvm_domain())
1193        return 0;
1194
1195    if (!xen_pv_domain())
1196        return -ENODEV;
1197
1198    return gnttab_init();
1199}
1200
1201core_initcall(__gnttab_init);
1202

Archive Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9



interactive