Root/block/elevator.c

1/*
2 * Block device elevator/IO-scheduler.
3 *
4 * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
5 *
6 * 30042000 Jens Axboe <axboe@kernel.dk> :
7 *
8 * Split the elevator a bit so that it is possible to choose a different
9 * one or even write a new "plug in". There are three pieces:
10 * - elevator_fn, inserts a new request in the queue list
11 * - elevator_merge_fn, decides whether a new buffer can be merged with
12 * an existing request
13 * - elevator_dequeue_fn, called when a request is taken off the active list
14 *
15 * 20082000 Dave Jones <davej@suse.de> :
16 * Removed tests for max-bomb-segments, which was breaking elvtune
17 * when run without -bN
18 *
19 * Jens:
20 * - Rework again to work with bio instead of buffer_heads
21 * - loose bi_dev comparisons, partition handling is right now
22 * - completely modularize elevator setup and teardown
23 *
24 */
25#include <linux/kernel.h>
26#include <linux/fs.h>
27#include <linux/blkdev.h>
28#include <linux/elevator.h>
29#include <linux/bio.h>
30#include <linux/module.h>
31#include <linux/slab.h>
32#include <linux/init.h>
33#include <linux/compiler.h>
34#include <linux/delay.h>
35#include <linux/blktrace_api.h>
36#include <linux/hash.h>
37#include <linux/uaccess.h>
38
39#include <trace/events/block.h>
40
41#include "blk.h"
42
43static DEFINE_SPINLOCK(elv_list_lock);
44static LIST_HEAD(elv_list);
45
46/*
47 * Merge hash stuff.
48 */
49static const int elv_hash_shift = 6;
50#define ELV_HASH_BLOCK(sec) ((sec) >> 3)
51#define ELV_HASH_FN(sec) \
52        (hash_long(ELV_HASH_BLOCK((sec)), elv_hash_shift))
53#define ELV_HASH_ENTRIES (1 << elv_hash_shift)
54#define rq_hash_key(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq))
55
56/*
57 * Query io scheduler to see if the current process issuing bio may be
58 * merged with rq.
59 */
60static int elv_iosched_allow_merge(struct request *rq, struct bio *bio)
61{
62    struct request_queue *q = rq->q;
63    struct elevator_queue *e = q->elevator;
64
65    if (e->ops->elevator_allow_merge_fn)
66        return e->ops->elevator_allow_merge_fn(q, rq, bio);
67
68    return 1;
69}
70
71/*
72 * can we safely merge with this request?
73 */
74int elv_rq_merge_ok(struct request *rq, struct bio *bio)
75{
76    if (!rq_mergeable(rq))
77        return 0;
78
79    /*
80     * Don't merge file system requests and discard requests
81     */
82    if (bio_rw_flagged(bio, BIO_RW_DISCARD) !=
83        bio_rw_flagged(rq->bio, BIO_RW_DISCARD))
84        return 0;
85
86    /*
87     * different data direction or already started, don't merge
88     */
89    if (bio_data_dir(bio) != rq_data_dir(rq))
90        return 0;
91
92    /*
93     * must be same device and not a special request
94     */
95    if (rq->rq_disk != bio->bi_bdev->bd_disk || rq->special)
96        return 0;
97
98    /*
99     * only merge integrity protected bio into ditto rq
100     */
101    if (bio_integrity(bio) != blk_integrity_rq(rq))
102        return 0;
103
104    if (!elv_iosched_allow_merge(rq, bio))
105        return 0;
106
107    return 1;
108}
109EXPORT_SYMBOL(elv_rq_merge_ok);
110
111static inline int elv_try_merge(struct request *__rq, struct bio *bio)
112{
113    int ret = ELEVATOR_NO_MERGE;
114
115    /*
116     * we can merge and sequence is ok, check if it's possible
117     */
118    if (elv_rq_merge_ok(__rq, bio)) {
119        if (blk_rq_pos(__rq) + blk_rq_sectors(__rq) == bio->bi_sector)
120            ret = ELEVATOR_BACK_MERGE;
121        else if (blk_rq_pos(__rq) - bio_sectors(bio) == bio->bi_sector)
122            ret = ELEVATOR_FRONT_MERGE;
123    }
124
125    return ret;
126}
127
128static struct elevator_type *elevator_find(const char *name)
129{
130    struct elevator_type *e;
131
132    list_for_each_entry(e, &elv_list, list) {
133        if (!strcmp(e->elevator_name, name))
134            return e;
135    }
136
137    return NULL;
138}
139
140static void elevator_put(struct elevator_type *e)
141{
142    module_put(e->elevator_owner);
143}
144
145static struct elevator_type *elevator_get(const char *name)
146{
147    struct elevator_type *e;
148
149    spin_lock(&elv_list_lock);
150
151    e = elevator_find(name);
152    if (!e) {
153        char elv[ELV_NAME_MAX + strlen("-iosched")];
154
155        spin_unlock(&elv_list_lock);
156
157        snprintf(elv, sizeof(elv), "%s-iosched", name);
158
159        request_module("%s", elv);
160        spin_lock(&elv_list_lock);
161        e = elevator_find(name);
162    }
163
164    if (e && !try_module_get(e->elevator_owner))
165        e = NULL;
166
167    spin_unlock(&elv_list_lock);
168
169    return e;
170}
171
172static void *elevator_init_queue(struct request_queue *q,
173                 struct elevator_queue *eq)
174{
175    return eq->ops->elevator_init_fn(q);
176}
177
178static void elevator_attach(struct request_queue *q, struct elevator_queue *eq,
179               void *data)
180{
181    q->elevator = eq;
182    eq->elevator_data = data;
183}
184
185static char chosen_elevator[16];
186
187static int __init elevator_setup(char *str)
188{
189    /*
190     * Be backwards-compatible with previous kernels, so users
191     * won't get the wrong elevator.
192     */
193    strncpy(chosen_elevator, str, sizeof(chosen_elevator) - 1);
194    return 1;
195}
196
197__setup("elevator=", elevator_setup);
198
199static struct kobj_type elv_ktype;
200
201static struct elevator_queue *elevator_alloc(struct request_queue *q,
202                  struct elevator_type *e)
203{
204    struct elevator_queue *eq;
205    int i;
206
207    eq = kmalloc_node(sizeof(*eq), GFP_KERNEL | __GFP_ZERO, q->node);
208    if (unlikely(!eq))
209        goto err;
210
211    eq->ops = &e->ops;
212    eq->elevator_type = e;
213    kobject_init(&eq->kobj, &elv_ktype);
214    mutex_init(&eq->sysfs_lock);
215
216    eq->hash = kmalloc_node(sizeof(struct hlist_head) * ELV_HASH_ENTRIES,
217                    GFP_KERNEL, q->node);
218    if (!eq->hash)
219        goto err;
220
221    for (i = 0; i < ELV_HASH_ENTRIES; i++)
222        INIT_HLIST_HEAD(&eq->hash[i]);
223
224    return eq;
225err:
226    kfree(eq);
227    elevator_put(e);
228    return NULL;
229}
230
231static void elevator_release(struct kobject *kobj)
232{
233    struct elevator_queue *e;
234
235    e = container_of(kobj, struct elevator_queue, kobj);
236    elevator_put(e->elevator_type);
237    kfree(e->hash);
238    kfree(e);
239}
240
241int elevator_init(struct request_queue *q, char *name)
242{
243    struct elevator_type *e = NULL;
244    struct elevator_queue *eq;
245    int ret = 0;
246    void *data;
247
248    INIT_LIST_HEAD(&q->queue_head);
249    q->last_merge = NULL;
250    q->end_sector = 0;
251    q->boundary_rq = NULL;
252
253    if (name) {
254        e = elevator_get(name);
255        if (!e)
256            return -EINVAL;
257    }
258
259    if (!e && *chosen_elevator) {
260        e = elevator_get(chosen_elevator);
261        if (!e)
262            printk(KERN_ERR "I/O scheduler %s not found\n",
263                            chosen_elevator);
264    }
265
266    if (!e) {
267        e = elevator_get(CONFIG_DEFAULT_IOSCHED);
268        if (!e) {
269            printk(KERN_ERR
270                "Default I/O scheduler not found. " \
271                "Using noop.\n");
272            e = elevator_get("noop");
273        }
274    }
275
276    eq = elevator_alloc(q, e);
277    if (!eq)
278        return -ENOMEM;
279
280    data = elevator_init_queue(q, eq);
281    if (!data) {
282        kobject_put(&eq->kobj);
283        return -ENOMEM;
284    }
285
286    elevator_attach(q, eq, data);
287    return ret;
288}
289EXPORT_SYMBOL(elevator_init);
290
291void elevator_exit(struct elevator_queue *e)
292{
293    mutex_lock(&e->sysfs_lock);
294    if (e->ops->elevator_exit_fn)
295        e->ops->elevator_exit_fn(e);
296    e->ops = NULL;
297    mutex_unlock(&e->sysfs_lock);
298
299    kobject_put(&e->kobj);
300}
301EXPORT_SYMBOL(elevator_exit);
302
303static inline void __elv_rqhash_del(struct request *rq)
304{
305    hlist_del_init(&rq->hash);
306}
307
308static void elv_rqhash_del(struct request_queue *q, struct request *rq)
309{
310    if (ELV_ON_HASH(rq))
311        __elv_rqhash_del(rq);
312}
313
314static void elv_rqhash_add(struct request_queue *q, struct request *rq)
315{
316    struct elevator_queue *e = q->elevator;
317
318    BUG_ON(ELV_ON_HASH(rq));
319    hlist_add_head(&rq->hash, &e->hash[ELV_HASH_FN(rq_hash_key(rq))]);
320}
321
322static void elv_rqhash_reposition(struct request_queue *q, struct request *rq)
323{
324    __elv_rqhash_del(rq);
325    elv_rqhash_add(q, rq);
326}
327
328static struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)
329{
330    struct elevator_queue *e = q->elevator;
331    struct hlist_head *hash_list = &e->hash[ELV_HASH_FN(offset)];
332    struct hlist_node *entry, *next;
333    struct request *rq;
334
335    hlist_for_each_entry_safe(rq, entry, next, hash_list, hash) {
336        BUG_ON(!ELV_ON_HASH(rq));
337
338        if (unlikely(!rq_mergeable(rq))) {
339            __elv_rqhash_del(rq);
340            continue;
341        }
342
343        if (rq_hash_key(rq) == offset)
344            return rq;
345    }
346
347    return NULL;
348}
349
350/*
351 * RB-tree support functions for inserting/lookup/removal of requests
352 * in a sorted RB tree.
353 */
354struct request *elv_rb_add(struct rb_root *root, struct request *rq)
355{
356    struct rb_node **p = &root->rb_node;
357    struct rb_node *parent = NULL;
358    struct request *__rq;
359
360    while (*p) {
361        parent = *p;
362        __rq = rb_entry(parent, struct request, rb_node);
363
364        if (blk_rq_pos(rq) < blk_rq_pos(__rq))
365            p = &(*p)->rb_left;
366        else if (blk_rq_pos(rq) > blk_rq_pos(__rq))
367            p = &(*p)->rb_right;
368        else
369            return __rq;
370    }
371
372    rb_link_node(&rq->rb_node, parent, p);
373    rb_insert_color(&rq->rb_node, root);
374    return NULL;
375}
376EXPORT_SYMBOL(elv_rb_add);
377
378void elv_rb_del(struct rb_root *root, struct request *rq)
379{
380    BUG_ON(RB_EMPTY_NODE(&rq->rb_node));
381    rb_erase(&rq->rb_node, root);
382    RB_CLEAR_NODE(&rq->rb_node);
383}
384EXPORT_SYMBOL(elv_rb_del);
385
386struct request *elv_rb_find(struct rb_root *root, sector_t sector)
387{
388    struct rb_node *n = root->rb_node;
389    struct request *rq;
390
391    while (n) {
392        rq = rb_entry(n, struct request, rb_node);
393
394        if (sector < blk_rq_pos(rq))
395            n = n->rb_left;
396        else if (sector > blk_rq_pos(rq))
397            n = n->rb_right;
398        else
399            return rq;
400    }
401
402    return NULL;
403}
404EXPORT_SYMBOL(elv_rb_find);
405
406/*
407 * Insert rq into dispatch queue of q. Queue lock must be held on
408 * entry. rq is sort instead into the dispatch queue. To be used by
409 * specific elevators.
410 */
411void elv_dispatch_sort(struct request_queue *q, struct request *rq)
412{
413    sector_t boundary;
414    struct list_head *entry;
415    int stop_flags;
416
417    if (q->last_merge == rq)
418        q->last_merge = NULL;
419
420    elv_rqhash_del(q, rq);
421
422    q->nr_sorted--;
423
424    boundary = q->end_sector;
425    stop_flags = REQ_SOFTBARRIER | REQ_HARDBARRIER | REQ_STARTED;
426    list_for_each_prev(entry, &q->queue_head) {
427        struct request *pos = list_entry_rq(entry);
428
429        if (blk_discard_rq(rq) != blk_discard_rq(pos))
430            break;
431        if (rq_data_dir(rq) != rq_data_dir(pos))
432            break;
433        if (pos->cmd_flags & stop_flags)
434            break;
435        if (blk_rq_pos(rq) >= boundary) {
436            if (blk_rq_pos(pos) < boundary)
437                continue;
438        } else {
439            if (blk_rq_pos(pos) >= boundary)
440                break;
441        }
442        if (blk_rq_pos(rq) >= blk_rq_pos(pos))
443            break;
444    }
445
446    list_add(&rq->queuelist, entry);
447}
448EXPORT_SYMBOL(elv_dispatch_sort);
449
450/*
451 * Insert rq into dispatch queue of q. Queue lock must be held on
452 * entry. rq is added to the back of the dispatch queue. To be used by
453 * specific elevators.
454 */
455void elv_dispatch_add_tail(struct request_queue *q, struct request *rq)
456{
457    if (q->last_merge == rq)
458        q->last_merge = NULL;
459
460    elv_rqhash_del(q, rq);
461
462    q->nr_sorted--;
463
464    q->end_sector = rq_end_sector(rq);
465    q->boundary_rq = rq;
466    list_add_tail(&rq->queuelist, &q->queue_head);
467}
468EXPORT_SYMBOL(elv_dispatch_add_tail);
469
470int elv_merge(struct request_queue *q, struct request **req, struct bio *bio)
471{
472    struct elevator_queue *e = q->elevator;
473    struct request *__rq;
474    int ret;
475
476    /*
477     * Levels of merges:
478     * nomerges: No merges at all attempted
479     * noxmerges: Only simple one-hit cache try
480     * merges: All merge tries attempted
481     */
482    if (blk_queue_nomerges(q))
483        return ELEVATOR_NO_MERGE;
484
485    /*
486     * First try one-hit cache.
487     */
488    if (q->last_merge) {
489        ret = elv_try_merge(q->last_merge, bio);
490        if (ret != ELEVATOR_NO_MERGE) {
491            *req = q->last_merge;
492            return ret;
493        }
494    }
495
496    if (blk_queue_noxmerges(q))
497        return ELEVATOR_NO_MERGE;
498
499    /*
500     * See if our hash lookup can find a potential backmerge.
501     */
502    __rq = elv_rqhash_find(q, bio->bi_sector);
503    if (__rq && elv_rq_merge_ok(__rq, bio)) {
504        *req = __rq;
505        return ELEVATOR_BACK_MERGE;
506    }
507
508    if (e->ops->elevator_merge_fn)
509        return e->ops->elevator_merge_fn(q, req, bio);
510
511    return ELEVATOR_NO_MERGE;
512}
513
514void elv_merged_request(struct request_queue *q, struct request *rq, int type)
515{
516    struct elevator_queue *e = q->elevator;
517
518    if (e->ops->elevator_merged_fn)
519        e->ops->elevator_merged_fn(q, rq, type);
520
521    if (type == ELEVATOR_BACK_MERGE)
522        elv_rqhash_reposition(q, rq);
523
524    q->last_merge = rq;
525}
526
527void elv_merge_requests(struct request_queue *q, struct request *rq,
528                 struct request *next)
529{
530    struct elevator_queue *e = q->elevator;
531
532    if (e->ops->elevator_merge_req_fn)
533        e->ops->elevator_merge_req_fn(q, rq, next);
534
535    elv_rqhash_reposition(q, rq);
536    elv_rqhash_del(q, next);
537
538    q->nr_sorted--;
539    q->last_merge = rq;
540}
541
542void elv_requeue_request(struct request_queue *q, struct request *rq)
543{
544    /*
545     * it already went through dequeue, we need to decrement the
546     * in_flight count again
547     */
548    if (blk_account_rq(rq)) {
549        q->in_flight[rq_is_sync(rq)]--;
550        if (blk_sorted_rq(rq))
551            elv_deactivate_rq(q, rq);
552    }
553
554    rq->cmd_flags &= ~REQ_STARTED;
555
556    elv_insert(q, rq, ELEVATOR_INSERT_REQUEUE);
557}
558
559void elv_drain_elevator(struct request_queue *q)
560{
561    static int printed;
562    while (q->elevator->ops->elevator_dispatch_fn(q, 1))
563        ;
564    if (q->nr_sorted == 0)
565        return;
566    if (printed++ < 10) {
567        printk(KERN_ERR "%s: forced dispatching is broken "
568               "(nr_sorted=%u), please report this\n",
569               q->elevator->elevator_type->elevator_name, q->nr_sorted);
570    }
571}
572
573/*
574 * Call with queue lock held, interrupts disabled
575 */
576void elv_quiesce_start(struct request_queue *q)
577{
578    if (!q->elevator)
579        return;
580
581    queue_flag_set(QUEUE_FLAG_ELVSWITCH, q);
582
583    /*
584     * make sure we don't have any requests in flight
585     */
586    elv_drain_elevator(q);
587    while (q->rq.elvpriv) {
588        __blk_run_queue(q);
589        spin_unlock_irq(q->queue_lock);
590        msleep(10);
591        spin_lock_irq(q->queue_lock);
592        elv_drain_elevator(q);
593    }
594}
595
596void elv_quiesce_end(struct request_queue *q)
597{
598    queue_flag_clear(QUEUE_FLAG_ELVSWITCH, q);
599}
600
601void elv_insert(struct request_queue *q, struct request *rq, int where)
602{
603    struct list_head *pos;
604    unsigned ordseq;
605    int unplug_it = 1;
606
607    trace_block_rq_insert(q, rq);
608
609    rq->q = q;
610
611    switch (where) {
612    case ELEVATOR_INSERT_FRONT:
613        rq->cmd_flags |= REQ_SOFTBARRIER;
614
615        list_add(&rq->queuelist, &q->queue_head);
616        break;
617
618    case ELEVATOR_INSERT_BACK:
619        rq->cmd_flags |= REQ_SOFTBARRIER;
620        elv_drain_elevator(q);
621        list_add_tail(&rq->queuelist, &q->queue_head);
622        /*
623         * We kick the queue here for the following reasons.
624         * - The elevator might have returned NULL previously
625         * to delay requests and returned them now. As the
626         * queue wasn't empty before this request, ll_rw_blk
627         * won't run the queue on return, resulting in hang.
628         * - Usually, back inserted requests won't be merged
629         * with anything. There's no point in delaying queue
630         * processing.
631         */
632        __blk_run_queue(q);
633        break;
634
635    case ELEVATOR_INSERT_SORT:
636        BUG_ON(!blk_fs_request(rq) && !blk_discard_rq(rq));
637        rq->cmd_flags |= REQ_SORTED;
638        q->nr_sorted++;
639        if (rq_mergeable(rq)) {
640            elv_rqhash_add(q, rq);
641            if (!q->last_merge)
642                q->last_merge = rq;
643        }
644
645        /*
646         * Some ioscheds (cfq) run q->request_fn directly, so
647         * rq cannot be accessed after calling
648         * elevator_add_req_fn.
649         */
650        q->elevator->ops->elevator_add_req_fn(q, rq);
651        break;
652
653    case ELEVATOR_INSERT_REQUEUE:
654        /*
655         * If ordered flush isn't in progress, we do front
656         * insertion; otherwise, requests should be requeued
657         * in ordseq order.
658         */
659        rq->cmd_flags |= REQ_SOFTBARRIER;
660
661        /*
662         * Most requeues happen because of a busy condition,
663         * don't force unplug of the queue for that case.
664         */
665        unplug_it = 0;
666
667        if (q->ordseq == 0) {
668            list_add(&rq->queuelist, &q->queue_head);
669            break;
670        }
671
672        ordseq = blk_ordered_req_seq(rq);
673
674        list_for_each(pos, &q->queue_head) {
675            struct request *pos_rq = list_entry_rq(pos);
676            if (ordseq <= blk_ordered_req_seq(pos_rq))
677                break;
678        }
679
680        list_add_tail(&rq->queuelist, pos);
681        break;
682
683    default:
684        printk(KERN_ERR "%s: bad insertion point %d\n",
685               __func__, where);
686        BUG();
687    }
688
689    if (unplug_it && blk_queue_plugged(q)) {
690        int nrq = q->rq.count[BLK_RW_SYNC] + q->rq.count[BLK_RW_ASYNC]
691                - queue_in_flight(q);
692
693        if (nrq >= q->unplug_thresh)
694            __generic_unplug_device(q);
695    }
696}
697
698void __elv_add_request(struct request_queue *q, struct request *rq, int where,
699               int plug)
700{
701    if (q->ordcolor)
702        rq->cmd_flags |= REQ_ORDERED_COLOR;
703
704    if (rq->cmd_flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER)) {
705        /*
706         * toggle ordered color
707         */
708        if (blk_barrier_rq(rq))
709            q->ordcolor ^= 1;
710
711        /*
712         * barriers implicitly indicate back insertion
713         */
714        if (where == ELEVATOR_INSERT_SORT)
715            where = ELEVATOR_INSERT_BACK;
716
717        /*
718         * this request is scheduling boundary, update
719         * end_sector
720         */
721        if (blk_fs_request(rq) || blk_discard_rq(rq)) {
722            q->end_sector = rq_end_sector(rq);
723            q->boundary_rq = rq;
724        }
725    } else if (!(rq->cmd_flags & REQ_ELVPRIV) &&
726            where == ELEVATOR_INSERT_SORT)
727        where = ELEVATOR_INSERT_BACK;
728
729    if (plug)
730        blk_plug_device(q);
731
732    elv_insert(q, rq, where);
733}
734EXPORT_SYMBOL(__elv_add_request);
735
736void elv_add_request(struct request_queue *q, struct request *rq, int where,
737             int plug)
738{
739    unsigned long flags;
740
741    spin_lock_irqsave(q->queue_lock, flags);
742    __elv_add_request(q, rq, where, plug);
743    spin_unlock_irqrestore(q->queue_lock, flags);
744}
745EXPORT_SYMBOL(elv_add_request);
746
747int elv_queue_empty(struct request_queue *q)
748{
749    struct elevator_queue *e = q->elevator;
750
751    if (!list_empty(&q->queue_head))
752        return 0;
753
754    if (e->ops->elevator_queue_empty_fn)
755        return e->ops->elevator_queue_empty_fn(q);
756
757    return 1;
758}
759EXPORT_SYMBOL(elv_queue_empty);
760
761struct request *elv_latter_request(struct request_queue *q, struct request *rq)
762{
763    struct elevator_queue *e = q->elevator;
764
765    if (e->ops->elevator_latter_req_fn)
766        return e->ops->elevator_latter_req_fn(q, rq);
767    return NULL;
768}
769
770struct request *elv_former_request(struct request_queue *q, struct request *rq)
771{
772    struct elevator_queue *e = q->elevator;
773
774    if (e->ops->elevator_former_req_fn)
775        return e->ops->elevator_former_req_fn(q, rq);
776    return NULL;
777}
778
779int elv_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
780{
781    struct elevator_queue *e = q->elevator;
782
783    if (e->ops->elevator_set_req_fn)
784        return e->ops->elevator_set_req_fn(q, rq, gfp_mask);
785
786    rq->elevator_private = NULL;
787    return 0;
788}
789
790void elv_put_request(struct request_queue *q, struct request *rq)
791{
792    struct elevator_queue *e = q->elevator;
793
794    if (e->ops->elevator_put_req_fn)
795        e->ops->elevator_put_req_fn(rq);
796}
797
798int elv_may_queue(struct request_queue *q, int rw)
799{
800    struct elevator_queue *e = q->elevator;
801
802    if (e->ops->elevator_may_queue_fn)
803        return e->ops->elevator_may_queue_fn(q, rw);
804
805    return ELV_MQUEUE_MAY;
806}
807
808void elv_abort_queue(struct request_queue *q)
809{
810    struct request *rq;
811
812    while (!list_empty(&q->queue_head)) {
813        rq = list_entry_rq(q->queue_head.next);
814        rq->cmd_flags |= REQ_QUIET;
815        trace_block_rq_abort(q, rq);
816        /*
817         * Mark this request as started so we don't trigger
818         * any debug logic in the end I/O path.
819         */
820        blk_start_request(rq);
821        __blk_end_request_all(rq, -EIO);
822    }
823}
824EXPORT_SYMBOL(elv_abort_queue);
825
826void elv_completed_request(struct request_queue *q, struct request *rq)
827{
828    struct elevator_queue *e = q->elevator;
829
830    /*
831     * request is released from the driver, io must be done
832     */
833    if (blk_account_rq(rq)) {
834        q->in_flight[rq_is_sync(rq)]--;
835        if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn)
836            e->ops->elevator_completed_req_fn(q, rq);
837    }
838
839    /*
840     * Check if the queue is waiting for fs requests to be
841     * drained for flush sequence.
842     */
843    if (unlikely(q->ordseq)) {
844        struct request *next = NULL;
845
846        if (!list_empty(&q->queue_head))
847            next = list_entry_rq(q->queue_head.next);
848
849        if (!queue_in_flight(q) &&
850            blk_ordered_cur_seq(q) == QUEUE_ORDSEQ_DRAIN &&
851            (!next || blk_ordered_req_seq(next) > QUEUE_ORDSEQ_DRAIN)) {
852            blk_ordered_complete_seq(q, QUEUE_ORDSEQ_DRAIN, 0);
853            __blk_run_queue(q);
854        }
855    }
856}
857
858#define to_elv(atr) container_of((atr), struct elv_fs_entry, attr)
859
860static ssize_t
861elv_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
862{
863    struct elv_fs_entry *entry = to_elv(attr);
864    struct elevator_queue *e;
865    ssize_t error;
866
867    if (!entry->show)
868        return -EIO;
869
870    e = container_of(kobj, struct elevator_queue, kobj);
871    mutex_lock(&e->sysfs_lock);
872    error = e->ops ? entry->show(e, page) : -ENOENT;
873    mutex_unlock(&e->sysfs_lock);
874    return error;
875}
876
877static ssize_t
878elv_attr_store(struct kobject *kobj, struct attribute *attr,
879           const char *page, size_t length)
880{
881    struct elv_fs_entry *entry = to_elv(attr);
882    struct elevator_queue *e;
883    ssize_t error;
884
885    if (!entry->store)
886        return -EIO;
887
888    e = container_of(kobj, struct elevator_queue, kobj);
889    mutex_lock(&e->sysfs_lock);
890    error = e->ops ? entry->store(e, page, length) : -ENOENT;
891    mutex_unlock(&e->sysfs_lock);
892    return error;
893}
894
895static const struct sysfs_ops elv_sysfs_ops = {
896    .show = elv_attr_show,
897    .store = elv_attr_store,
898};
899
900static struct kobj_type elv_ktype = {
901    .sysfs_ops = &elv_sysfs_ops,
902    .release = elevator_release,
903};
904
905int elv_register_queue(struct request_queue *q)
906{
907    struct elevator_queue *e = q->elevator;
908    int error;
909
910    error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
911    if (!error) {
912        struct elv_fs_entry *attr = e->elevator_type->elevator_attrs;
913        if (attr) {
914            while (attr->attr.name) {
915                if (sysfs_create_file(&e->kobj, &attr->attr))
916                    break;
917                attr++;
918            }
919        }
920        kobject_uevent(&e->kobj, KOBJ_ADD);
921    }
922    return error;
923}
924
925static void __elv_unregister_queue(struct elevator_queue *e)
926{
927    kobject_uevent(&e->kobj, KOBJ_REMOVE);
928    kobject_del(&e->kobj);
929}
930
931void elv_unregister_queue(struct request_queue *q)
932{
933    if (q)
934        __elv_unregister_queue(q->elevator);
935}
936
937void elv_register(struct elevator_type *e)
938{
939    char *def = "";
940
941    spin_lock(&elv_list_lock);
942    BUG_ON(elevator_find(e->elevator_name));
943    list_add_tail(&e->list, &elv_list);
944    spin_unlock(&elv_list_lock);
945
946    if (!strcmp(e->elevator_name, chosen_elevator) ||
947            (!*chosen_elevator &&
948             !strcmp(e->elevator_name, CONFIG_DEFAULT_IOSCHED)))
949                def = " (default)";
950
951    printk(KERN_INFO "io scheduler %s registered%s\n", e->elevator_name,
952                                def);
953}
954EXPORT_SYMBOL_GPL(elv_register);
955
956void elv_unregister(struct elevator_type *e)
957{
958    struct task_struct *g, *p;
959
960    /*
961     * Iterate every thread in the process to remove the io contexts.
962     */
963    if (e->ops.trim) {
964        read_lock(&tasklist_lock);
965        do_each_thread(g, p) {
966            task_lock(p);
967            if (p->io_context)
968                e->ops.trim(p->io_context);
969            task_unlock(p);
970        } while_each_thread(g, p);
971        read_unlock(&tasklist_lock);
972    }
973
974    spin_lock(&elv_list_lock);
975    list_del_init(&e->list);
976    spin_unlock(&elv_list_lock);
977}
978EXPORT_SYMBOL_GPL(elv_unregister);
979
980/*
981 * switch to new_e io scheduler. be careful not to introduce deadlocks -
982 * we don't free the old io scheduler, before we have allocated what we
983 * need for the new one. this way we have a chance of going back to the old
984 * one, if the new one fails init for some reason.
985 */
986static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
987{
988    struct elevator_queue *old_elevator, *e;
989    void *data;
990
991    /*
992     * Allocate new elevator
993     */
994    e = elevator_alloc(q, new_e);
995    if (!e)
996        return 0;
997
998    data = elevator_init_queue(q, e);
999    if (!data) {
1000        kobject_put(&e->kobj);
1001        return 0;
1002    }
1003
1004    /*
1005     * Turn on BYPASS and drain all requests w/ elevator private data
1006     */
1007    spin_lock_irq(q->queue_lock);
1008    elv_quiesce_start(q);
1009
1010    /*
1011     * Remember old elevator.
1012     */
1013    old_elevator = q->elevator;
1014
1015    /*
1016     * attach and start new elevator
1017     */
1018    elevator_attach(q, e, data);
1019
1020    spin_unlock_irq(q->queue_lock);
1021
1022    __elv_unregister_queue(old_elevator);
1023
1024    if (elv_register_queue(q))
1025        goto fail_register;
1026
1027    /*
1028     * finally exit old elevator and turn off BYPASS.
1029     */
1030    elevator_exit(old_elevator);
1031    spin_lock_irq(q->queue_lock);
1032    elv_quiesce_end(q);
1033    spin_unlock_irq(q->queue_lock);
1034
1035    blk_add_trace_msg(q, "elv switch: %s", e->elevator_type->elevator_name);
1036
1037    return 1;
1038
1039fail_register:
1040    /*
1041     * switch failed, exit the new io scheduler and reattach the old
1042     * one again (along with re-adding the sysfs dir)
1043     */
1044    elevator_exit(e);
1045    q->elevator = old_elevator;
1046    elv_register_queue(q);
1047
1048    spin_lock_irq(q->queue_lock);
1049    queue_flag_clear(QUEUE_FLAG_ELVSWITCH, q);
1050    spin_unlock_irq(q->queue_lock);
1051
1052    return 0;
1053}
1054
1055ssize_t elv_iosched_store(struct request_queue *q, const char *name,
1056              size_t count)
1057{
1058    char elevator_name[ELV_NAME_MAX];
1059    struct elevator_type *e;
1060
1061    if (!q->elevator)
1062        return count;
1063
1064    strlcpy(elevator_name, name, sizeof(elevator_name));
1065    e = elevator_get(strstrip(elevator_name));
1066    if (!e) {
1067        printk(KERN_ERR "elevator: type %s not found\n", elevator_name);
1068        return -EINVAL;
1069    }
1070
1071    if (!strcmp(elevator_name, q->elevator->elevator_type->elevator_name)) {
1072        elevator_put(e);
1073        return count;
1074    }
1075
1076    if (!elevator_switch(q, e))
1077        printk(KERN_ERR "elevator: switch to %s failed\n",
1078                            elevator_name);
1079    return count;
1080}
1081
1082ssize_t elv_iosched_show(struct request_queue *q, char *name)
1083{
1084    struct elevator_queue *e = q->elevator;
1085    struct elevator_type *elv;
1086    struct elevator_type *__e;
1087    int len = 0;
1088
1089    if (!q->elevator)
1090        return sprintf(name, "none\n");
1091
1092    elv = e->elevator_type;
1093
1094    spin_lock(&elv_list_lock);
1095    list_for_each_entry(__e, &elv_list, list) {
1096        if (!strcmp(elv->elevator_name, __e->elevator_name))
1097            len += sprintf(name+len, "[%s] ", elv->elevator_name);
1098        else
1099            len += sprintf(name+len, "%s ", __e->elevator_name);
1100    }
1101    spin_unlock(&elv_list_lock);
1102
1103    len += sprintf(len+name, "\n");
1104    return len;
1105}
1106
1107struct request *elv_rb_former_request(struct request_queue *q,
1108                      struct request *rq)
1109{
1110    struct rb_node *rbprev = rb_prev(&rq->rb_node);
1111
1112    if (rbprev)
1113        return rb_entry_rq(rbprev);
1114
1115    return NULL;
1116}
1117EXPORT_SYMBOL(elv_rb_former_request);
1118
1119struct request *elv_rb_latter_request(struct request_queue *q,
1120                      struct request *rq)
1121{
1122    struct rb_node *rbnext = rb_next(&rq->rb_node);
1123
1124    if (rbnext)
1125        return rb_entry_rq(rbnext);
1126
1127    return NULL;
1128}
1129EXPORT_SYMBOL(elv_rb_latter_request);
1130

Archive Download this file



interactive