Root/
1 | #include "ceph_debug.h" |
2 | |
3 | #include <linux/sort.h> |
4 | #include <linux/slab.h> |
5 | |
6 | #include "super.h" |
7 | #include "decode.h" |
8 | |
9 | /* |
10 | * Snapshots in ceph are driven in large part by cooperation from the |
11 | * client. In contrast to local file systems or file servers that |
12 | * implement snapshots at a single point in the system, ceph's |
13 | * distributed access to storage requires clients to help decide |
14 | * whether a write logically occurs before or after a recently created |
15 | * snapshot. |
16 | * |
17 | * This provides a perfect instantanous client-wide snapshot. Between |
18 | * clients, however, snapshots may appear to be applied at slightly |
19 | * different points in time, depending on delays in delivering the |
20 | * snapshot notification. |
21 | * |
22 | * Snapshots are _not_ file system-wide. Instead, each snapshot |
23 | * applies to the subdirectory nested beneath some directory. This |
24 | * effectively divides the hierarchy into multiple "realms," where all |
25 | * of the files contained by each realm share the same set of |
26 | * snapshots. An individual realm's snap set contains snapshots |
27 | * explicitly created on that realm, as well as any snaps in its |
28 | * parent's snap set _after_ the point at which the parent became it's |
29 | * parent (due to, say, a rename). Similarly, snaps from prior parents |
30 | * during the time intervals during which they were the parent are included. |
31 | * |
32 | * The client is spared most of this detail, fortunately... it must only |
33 | * maintains a hierarchy of realms reflecting the current parent/child |
34 | * realm relationship, and for each realm has an explicit list of snaps |
35 | * inherited from prior parents. |
36 | * |
37 | * A snap_realm struct is maintained for realms containing every inode |
38 | * with an open cap in the system. (The needed snap realm information is |
39 | * provided by the MDS whenever a cap is issued, i.e., on open.) A 'seq' |
40 | * version number is used to ensure that as realm parameters change (new |
41 | * snapshot, new parent, etc.) the client's realm hierarchy is updated. |
42 | * |
43 | * The realm hierarchy drives the generation of a 'snap context' for each |
44 | * realm, which simply lists the resulting set of snaps for the realm. This |
45 | * is attached to any writes sent to OSDs. |
46 | */ |
47 | /* |
48 | * Unfortunately error handling is a bit mixed here. If we get a snap |
49 | * update, but don't have enough memory to update our realm hierarchy, |
50 | * it's not clear what we can do about it (besides complaining to the |
51 | * console). |
52 | */ |
53 | |
54 | |
55 | /* |
56 | * increase ref count for the realm |
57 | * |
58 | * caller must hold snap_rwsem for write. |
59 | */ |
60 | void ceph_get_snap_realm(struct ceph_mds_client *mdsc, |
61 | struct ceph_snap_realm *realm) |
62 | { |
63 | dout("get_realm %p %d -> %d\n", realm, |
64 | atomic_read(&realm->nref), atomic_read(&realm->nref)+1); |
65 | /* |
66 | * since we _only_ increment realm refs or empty the empty |
67 | * list with snap_rwsem held, adjusting the empty list here is |
68 | * safe. we do need to protect against concurrent empty list |
69 | * additions, however. |
70 | */ |
71 | if (atomic_read(&realm->nref) == 0) { |
72 | spin_lock(&mdsc->snap_empty_lock); |
73 | list_del_init(&realm->empty_item); |
74 | spin_unlock(&mdsc->snap_empty_lock); |
75 | } |
76 | |
77 | atomic_inc(&realm->nref); |
78 | } |
79 | |
80 | static void __insert_snap_realm(struct rb_root *root, |
81 | struct ceph_snap_realm *new) |
82 | { |
83 | struct rb_node **p = &root->rb_node; |
84 | struct rb_node *parent = NULL; |
85 | struct ceph_snap_realm *r = NULL; |
86 | |
87 | while (*p) { |
88 | parent = *p; |
89 | r = rb_entry(parent, struct ceph_snap_realm, node); |
90 | if (new->ino < r->ino) |
91 | p = &(*p)->rb_left; |
92 | else if (new->ino > r->ino) |
93 | p = &(*p)->rb_right; |
94 | else |
95 | BUG(); |
96 | } |
97 | |
98 | rb_link_node(&new->node, parent, p); |
99 | rb_insert_color(&new->node, root); |
100 | } |
101 | |
102 | /* |
103 | * create and get the realm rooted at @ino and bump its ref count. |
104 | * |
105 | * caller must hold snap_rwsem for write. |
106 | */ |
107 | static struct ceph_snap_realm *ceph_create_snap_realm( |
108 | struct ceph_mds_client *mdsc, |
109 | u64 ino) |
110 | { |
111 | struct ceph_snap_realm *realm; |
112 | |
113 | realm = kzalloc(sizeof(*realm), GFP_NOFS); |
114 | if (!realm) |
115 | return ERR_PTR(-ENOMEM); |
116 | |
117 | atomic_set(&realm->nref, 0); /* tree does not take a ref */ |
118 | realm->ino = ino; |
119 | INIT_LIST_HEAD(&realm->children); |
120 | INIT_LIST_HEAD(&realm->child_item); |
121 | INIT_LIST_HEAD(&realm->empty_item); |
122 | INIT_LIST_HEAD(&realm->inodes_with_caps); |
123 | spin_lock_init(&realm->inodes_with_caps_lock); |
124 | __insert_snap_realm(&mdsc->snap_realms, realm); |
125 | dout("create_snap_realm %llx %p\n", realm->ino, realm); |
126 | return realm; |
127 | } |
128 | |
129 | /* |
130 | * lookup the realm rooted at @ino. |
131 | * |
132 | * caller must hold snap_rwsem for write. |
133 | */ |
134 | struct ceph_snap_realm *ceph_lookup_snap_realm(struct ceph_mds_client *mdsc, |
135 | u64 ino) |
136 | { |
137 | struct rb_node *n = mdsc->snap_realms.rb_node; |
138 | struct ceph_snap_realm *r; |
139 | |
140 | while (n) { |
141 | r = rb_entry(n, struct ceph_snap_realm, node); |
142 | if (ino < r->ino) |
143 | n = n->rb_left; |
144 | else if (ino > r->ino) |
145 | n = n->rb_right; |
146 | else { |
147 | dout("lookup_snap_realm %llx %p\n", r->ino, r); |
148 | return r; |
149 | } |
150 | } |
151 | return NULL; |
152 | } |
153 | |
154 | static void __put_snap_realm(struct ceph_mds_client *mdsc, |
155 | struct ceph_snap_realm *realm); |
156 | |
157 | /* |
158 | * called with snap_rwsem (write) |
159 | */ |
160 | static void __destroy_snap_realm(struct ceph_mds_client *mdsc, |
161 | struct ceph_snap_realm *realm) |
162 | { |
163 | dout("__destroy_snap_realm %p %llx\n", realm, realm->ino); |
164 | |
165 | rb_erase(&realm->node, &mdsc->snap_realms); |
166 | |
167 | if (realm->parent) { |
168 | list_del_init(&realm->child_item); |
169 | __put_snap_realm(mdsc, realm->parent); |
170 | } |
171 | |
172 | kfree(realm->prior_parent_snaps); |
173 | kfree(realm->snaps); |
174 | ceph_put_snap_context(realm->cached_context); |
175 | kfree(realm); |
176 | } |
177 | |
178 | /* |
179 | * caller holds snap_rwsem (write) |
180 | */ |
181 | static void __put_snap_realm(struct ceph_mds_client *mdsc, |
182 | struct ceph_snap_realm *realm) |
183 | { |
184 | dout("__put_snap_realm %llx %p %d -> %d\n", realm->ino, realm, |
185 | atomic_read(&realm->nref), atomic_read(&realm->nref)-1); |
186 | if (atomic_dec_and_test(&realm->nref)) |
187 | __destroy_snap_realm(mdsc, realm); |
188 | } |
189 | |
190 | /* |
191 | * caller needn't hold any locks |
192 | */ |
193 | void ceph_put_snap_realm(struct ceph_mds_client *mdsc, |
194 | struct ceph_snap_realm *realm) |
195 | { |
196 | dout("put_snap_realm %llx %p %d -> %d\n", realm->ino, realm, |
197 | atomic_read(&realm->nref), atomic_read(&realm->nref)-1); |
198 | if (!atomic_dec_and_test(&realm->nref)) |
199 | return; |
200 | |
201 | if (down_write_trylock(&mdsc->snap_rwsem)) { |
202 | __destroy_snap_realm(mdsc, realm); |
203 | up_write(&mdsc->snap_rwsem); |
204 | } else { |
205 | spin_lock(&mdsc->snap_empty_lock); |
206 | list_add(&mdsc->snap_empty, &realm->empty_item); |
207 | spin_unlock(&mdsc->snap_empty_lock); |
208 | } |
209 | } |
210 | |
211 | /* |
212 | * Clean up any realms whose ref counts have dropped to zero. Note |
213 | * that this does not include realms who were created but not yet |
214 | * used. |
215 | * |
216 | * Called under snap_rwsem (write) |
217 | */ |
218 | static void __cleanup_empty_realms(struct ceph_mds_client *mdsc) |
219 | { |
220 | struct ceph_snap_realm *realm; |
221 | |
222 | spin_lock(&mdsc->snap_empty_lock); |
223 | while (!list_empty(&mdsc->snap_empty)) { |
224 | realm = list_first_entry(&mdsc->snap_empty, |
225 | struct ceph_snap_realm, empty_item); |
226 | list_del(&realm->empty_item); |
227 | spin_unlock(&mdsc->snap_empty_lock); |
228 | __destroy_snap_realm(mdsc, realm); |
229 | spin_lock(&mdsc->snap_empty_lock); |
230 | } |
231 | spin_unlock(&mdsc->snap_empty_lock); |
232 | } |
233 | |
234 | void ceph_cleanup_empty_realms(struct ceph_mds_client *mdsc) |
235 | { |
236 | down_write(&mdsc->snap_rwsem); |
237 | __cleanup_empty_realms(mdsc); |
238 | up_write(&mdsc->snap_rwsem); |
239 | } |
240 | |
241 | /* |
242 | * adjust the parent realm of a given @realm. adjust child list, and parent |
243 | * pointers, and ref counts appropriately. |
244 | * |
245 | * return true if parent was changed, 0 if unchanged, <0 on error. |
246 | * |
247 | * caller must hold snap_rwsem for write. |
248 | */ |
249 | static int adjust_snap_realm_parent(struct ceph_mds_client *mdsc, |
250 | struct ceph_snap_realm *realm, |
251 | u64 parentino) |
252 | { |
253 | struct ceph_snap_realm *parent; |
254 | |
255 | if (realm->parent_ino == parentino) |
256 | return 0; |
257 | |
258 | parent = ceph_lookup_snap_realm(mdsc, parentino); |
259 | if (!parent) { |
260 | parent = ceph_create_snap_realm(mdsc, parentino); |
261 | if (IS_ERR(parent)) |
262 | return PTR_ERR(parent); |
263 | } |
264 | dout("adjust_snap_realm_parent %llx %p: %llx %p -> %llx %p\n", |
265 | realm->ino, realm, realm->parent_ino, realm->parent, |
266 | parentino, parent); |
267 | if (realm->parent) { |
268 | list_del_init(&realm->child_item); |
269 | ceph_put_snap_realm(mdsc, realm->parent); |
270 | } |
271 | realm->parent_ino = parentino; |
272 | realm->parent = parent; |
273 | ceph_get_snap_realm(mdsc, parent); |
274 | list_add(&realm->child_item, &parent->children); |
275 | return 1; |
276 | } |
277 | |
278 | |
279 | static int cmpu64_rev(const void *a, const void *b) |
280 | { |
281 | if (*(u64 *)a < *(u64 *)b) |
282 | return 1; |
283 | if (*(u64 *)a > *(u64 *)b) |
284 | return -1; |
285 | return 0; |
286 | } |
287 | |
288 | /* |
289 | * build the snap context for a given realm. |
290 | */ |
291 | static int build_snap_context(struct ceph_snap_realm *realm) |
292 | { |
293 | struct ceph_snap_realm *parent = realm->parent; |
294 | struct ceph_snap_context *snapc; |
295 | int err = 0; |
296 | int i; |
297 | int num = realm->num_prior_parent_snaps + realm->num_snaps; |
298 | |
299 | /* |
300 | * build parent context, if it hasn't been built. |
301 | * conservatively estimate that all parent snaps might be |
302 | * included by us. |
303 | */ |
304 | if (parent) { |
305 | if (!parent->cached_context) { |
306 | err = build_snap_context(parent); |
307 | if (err) |
308 | goto fail; |
309 | } |
310 | num += parent->cached_context->num_snaps; |
311 | } |
312 | |
313 | /* do i actually need to update? not if my context seq |
314 | matches realm seq, and my parents' does to. (this works |
315 | because we rebuild_snap_realms() works _downward_ in |
316 | hierarchy after each update.) */ |
317 | if (realm->cached_context && |
318 | realm->cached_context->seq == realm->seq && |
319 | (!parent || |
320 | realm->cached_context->seq >= parent->cached_context->seq)) { |
321 | dout("build_snap_context %llx %p: %p seq %lld (%d snaps)" |
322 | " (unchanged)\n", |
323 | realm->ino, realm, realm->cached_context, |
324 | realm->cached_context->seq, |
325 | realm->cached_context->num_snaps); |
326 | return 0; |
327 | } |
328 | |
329 | /* alloc new snap context */ |
330 | err = -ENOMEM; |
331 | if (num > ULONG_MAX / sizeof(u64) - sizeof(*snapc)) |
332 | goto fail; |
333 | snapc = kzalloc(sizeof(*snapc) + num*sizeof(u64), GFP_NOFS); |
334 | if (!snapc) |
335 | goto fail; |
336 | atomic_set(&snapc->nref, 1); |
337 | |
338 | /* build (reverse sorted) snap vector */ |
339 | num = 0; |
340 | snapc->seq = realm->seq; |
341 | if (parent) { |
342 | /* include any of parent's snaps occuring _after_ my |
343 | parent became my parent */ |
344 | for (i = 0; i < parent->cached_context->num_snaps; i++) |
345 | if (parent->cached_context->snaps[i] >= |
346 | realm->parent_since) |
347 | snapc->snaps[num++] = |
348 | parent->cached_context->snaps[i]; |
349 | if (parent->cached_context->seq > snapc->seq) |
350 | snapc->seq = parent->cached_context->seq; |
351 | } |
352 | memcpy(snapc->snaps + num, realm->snaps, |
353 | sizeof(u64)*realm->num_snaps); |
354 | num += realm->num_snaps; |
355 | memcpy(snapc->snaps + num, realm->prior_parent_snaps, |
356 | sizeof(u64)*realm->num_prior_parent_snaps); |
357 | num += realm->num_prior_parent_snaps; |
358 | |
359 | sort(snapc->snaps, num, sizeof(u64), cmpu64_rev, NULL); |
360 | snapc->num_snaps = num; |
361 | dout("build_snap_context %llx %p: %p seq %lld (%d snaps)\n", |
362 | realm->ino, realm, snapc, snapc->seq, snapc->num_snaps); |
363 | |
364 | if (realm->cached_context) |
365 | ceph_put_snap_context(realm->cached_context); |
366 | realm->cached_context = snapc; |
367 | return 0; |
368 | |
369 | fail: |
370 | /* |
371 | * if we fail, clear old (incorrect) cached_context... hopefully |
372 | * we'll have better luck building it later |
373 | */ |
374 | if (realm->cached_context) { |
375 | ceph_put_snap_context(realm->cached_context); |
376 | realm->cached_context = NULL; |
377 | } |
378 | pr_err("build_snap_context %llx %p fail %d\n", realm->ino, |
379 | realm, err); |
380 | return err; |
381 | } |
382 | |
383 | /* |
384 | * rebuild snap context for the given realm and all of its children. |
385 | */ |
386 | static void rebuild_snap_realms(struct ceph_snap_realm *realm) |
387 | { |
388 | struct ceph_snap_realm *child; |
389 | |
390 | dout("rebuild_snap_realms %llx %p\n", realm->ino, realm); |
391 | build_snap_context(realm); |
392 | |
393 | list_for_each_entry(child, &realm->children, child_item) |
394 | rebuild_snap_realms(child); |
395 | } |
396 | |
397 | |
398 | /* |
399 | * helper to allocate and decode an array of snapids. free prior |
400 | * instance, if any. |
401 | */ |
402 | static int dup_array(u64 **dst, __le64 *src, int num) |
403 | { |
404 | int i; |
405 | |
406 | kfree(*dst); |
407 | if (num) { |
408 | *dst = kcalloc(num, sizeof(u64), GFP_NOFS); |
409 | if (!*dst) |
410 | return -ENOMEM; |
411 | for (i = 0; i < num; i++) |
412 | (*dst)[i] = get_unaligned_le64(src + i); |
413 | } else { |
414 | *dst = NULL; |
415 | } |
416 | return 0; |
417 | } |
418 | |
419 | |
420 | /* |
421 | * When a snapshot is applied, the size/mtime inode metadata is queued |
422 | * in a ceph_cap_snap (one for each snapshot) until writeback |
423 | * completes and the metadata can be flushed back to the MDS. |
424 | * |
425 | * However, if a (sync) write is currently in-progress when we apply |
426 | * the snapshot, we have to wait until the write succeeds or fails |
427 | * (and a final size/mtime is known). In this case the |
428 | * cap_snap->writing = 1, and is said to be "pending." When the write |
429 | * finishes, we __ceph_finish_cap_snap(). |
430 | * |
431 | * Caller must hold snap_rwsem for read (i.e., the realm topology won't |
432 | * change). |
433 | */ |
434 | void ceph_queue_cap_snap(struct ceph_inode_info *ci) |
435 | { |
436 | struct inode *inode = &ci->vfs_inode; |
437 | struct ceph_cap_snap *capsnap; |
438 | int used; |
439 | |
440 | capsnap = kzalloc(sizeof(*capsnap), GFP_NOFS); |
441 | if (!capsnap) { |
442 | pr_err("ENOMEM allocating ceph_cap_snap on %p\n", inode); |
443 | return; |
444 | } |
445 | |
446 | spin_lock(&inode->i_lock); |
447 | used = __ceph_caps_used(ci); |
448 | if (__ceph_have_pending_cap_snap(ci)) { |
449 | /* there is no point in queuing multiple "pending" cap_snaps, |
450 | as no new writes are allowed to start when pending, so any |
451 | writes in progress now were started before the previous |
452 | cap_snap. lucky us. */ |
453 | dout("queue_cap_snap %p already pending\n", inode); |
454 | kfree(capsnap); |
455 | } else if (ci->i_wrbuffer_ref_head || (used & CEPH_CAP_FILE_WR)) { |
456 | struct ceph_snap_context *snapc = ci->i_head_snapc; |
457 | |
458 | igrab(inode); |
459 | |
460 | atomic_set(&capsnap->nref, 1); |
461 | capsnap->ci = ci; |
462 | INIT_LIST_HEAD(&capsnap->ci_item); |
463 | INIT_LIST_HEAD(&capsnap->flushing_item); |
464 | |
465 | capsnap->follows = snapc->seq - 1; |
466 | capsnap->issued = __ceph_caps_issued(ci, NULL); |
467 | capsnap->dirty = __ceph_caps_dirty(ci); |
468 | |
469 | capsnap->mode = inode->i_mode; |
470 | capsnap->uid = inode->i_uid; |
471 | capsnap->gid = inode->i_gid; |
472 | |
473 | /* fixme? */ |
474 | capsnap->xattr_blob = NULL; |
475 | capsnap->xattr_len = 0; |
476 | |
477 | /* dirty page count moved from _head to this cap_snap; |
478 | all subsequent writes page dirties occur _after_ this |
479 | snapshot. */ |
480 | capsnap->dirty_pages = ci->i_wrbuffer_ref_head; |
481 | ci->i_wrbuffer_ref_head = 0; |
482 | capsnap->context = snapc; |
483 | ci->i_head_snapc = NULL; |
484 | list_add_tail(&capsnap->ci_item, &ci->i_cap_snaps); |
485 | |
486 | if (used & CEPH_CAP_FILE_WR) { |
487 | dout("queue_cap_snap %p cap_snap %p snapc %p" |
488 | " seq %llu used WR, now pending\n", inode, |
489 | capsnap, snapc, snapc->seq); |
490 | capsnap->writing = 1; |
491 | } else { |
492 | /* note mtime, size NOW. */ |
493 | __ceph_finish_cap_snap(ci, capsnap); |
494 | } |
495 | } else { |
496 | dout("queue_cap_snap %p nothing dirty|writing\n", inode); |
497 | kfree(capsnap); |
498 | } |
499 | |
500 | spin_unlock(&inode->i_lock); |
501 | } |
502 | |
503 | /* |
504 | * Finalize the size, mtime for a cap_snap.. that is, settle on final values |
505 | * to be used for the snapshot, to be flushed back to the mds. |
506 | * |
507 | * If capsnap can now be flushed, add to snap_flush list, and return 1. |
508 | * |
509 | * Caller must hold i_lock. |
510 | */ |
511 | int __ceph_finish_cap_snap(struct ceph_inode_info *ci, |
512 | struct ceph_cap_snap *capsnap) |
513 | { |
514 | struct inode *inode = &ci->vfs_inode; |
515 | struct ceph_mds_client *mdsc = &ceph_client(inode->i_sb)->mdsc; |
516 | |
517 | BUG_ON(capsnap->writing); |
518 | capsnap->size = inode->i_size; |
519 | capsnap->mtime = inode->i_mtime; |
520 | capsnap->atime = inode->i_atime; |
521 | capsnap->ctime = inode->i_ctime; |
522 | capsnap->time_warp_seq = ci->i_time_warp_seq; |
523 | if (capsnap->dirty_pages) { |
524 | dout("finish_cap_snap %p cap_snap %p snapc %p %llu %s s=%llu " |
525 | "still has %d dirty pages\n", inode, capsnap, |
526 | capsnap->context, capsnap->context->seq, |
527 | ceph_cap_string(capsnap->dirty), capsnap->size, |
528 | capsnap->dirty_pages); |
529 | return 0; |
530 | } |
531 | dout("finish_cap_snap %p cap_snap %p snapc %p %llu %s s=%llu\n", |
532 | inode, capsnap, capsnap->context, |
533 | capsnap->context->seq, ceph_cap_string(capsnap->dirty), |
534 | capsnap->size); |
535 | |
536 | spin_lock(&mdsc->snap_flush_lock); |
537 | list_add_tail(&ci->i_snap_flush_item, &mdsc->snap_flush_list); |
538 | spin_unlock(&mdsc->snap_flush_lock); |
539 | return 1; /* caller may want to ceph_flush_snaps */ |
540 | } |
541 | |
542 | |
543 | /* |
544 | * Parse and apply a snapblob "snap trace" from the MDS. This specifies |
545 | * the snap realm parameters from a given realm and all of its ancestors, |
546 | * up to the root. |
547 | * |
548 | * Caller must hold snap_rwsem for write. |
549 | */ |
550 | int ceph_update_snap_trace(struct ceph_mds_client *mdsc, |
551 | void *p, void *e, bool deletion) |
552 | { |
553 | struct ceph_mds_snap_realm *ri; /* encoded */ |
554 | __le64 *snaps; /* encoded */ |
555 | __le64 *prior_parent_snaps; /* encoded */ |
556 | struct ceph_snap_realm *realm; |
557 | int invalidate = 0; |
558 | int err = -ENOMEM; |
559 | |
560 | dout("update_snap_trace deletion=%d\n", deletion); |
561 | more: |
562 | ceph_decode_need(&p, e, sizeof(*ri), bad); |
563 | ri = p; |
564 | p += sizeof(*ri); |
565 | ceph_decode_need(&p, e, sizeof(u64)*(le32_to_cpu(ri->num_snaps) + |
566 | le32_to_cpu(ri->num_prior_parent_snaps)), bad); |
567 | snaps = p; |
568 | p += sizeof(u64) * le32_to_cpu(ri->num_snaps); |
569 | prior_parent_snaps = p; |
570 | p += sizeof(u64) * le32_to_cpu(ri->num_prior_parent_snaps); |
571 | |
572 | realm = ceph_lookup_snap_realm(mdsc, le64_to_cpu(ri->ino)); |
573 | if (!realm) { |
574 | realm = ceph_create_snap_realm(mdsc, le64_to_cpu(ri->ino)); |
575 | if (IS_ERR(realm)) { |
576 | err = PTR_ERR(realm); |
577 | goto fail; |
578 | } |
579 | } |
580 | |
581 | if (le64_to_cpu(ri->seq) > realm->seq) { |
582 | dout("update_snap_trace updating %llx %p %lld -> %lld\n", |
583 | realm->ino, realm, realm->seq, le64_to_cpu(ri->seq)); |
584 | /* |
585 | * if the realm seq has changed, queue a cap_snap for every |
586 | * inode with open caps. we do this _before_ we update |
587 | * the realm info so that we prepare for writeback under the |
588 | * _previous_ snap context. |
589 | * |
590 | * ...unless it's a snap deletion! |
591 | */ |
592 | if (!deletion) { |
593 | struct ceph_inode_info *ci; |
594 | struct inode *lastinode = NULL; |
595 | |
596 | spin_lock(&realm->inodes_with_caps_lock); |
597 | list_for_each_entry(ci, &realm->inodes_with_caps, |
598 | i_snap_realm_item) { |
599 | struct inode *inode = igrab(&ci->vfs_inode); |
600 | if (!inode) |
601 | continue; |
602 | spin_unlock(&realm->inodes_with_caps_lock); |
603 | if (lastinode) |
604 | iput(lastinode); |
605 | lastinode = inode; |
606 | ceph_queue_cap_snap(ci); |
607 | spin_lock(&realm->inodes_with_caps_lock); |
608 | } |
609 | spin_unlock(&realm->inodes_with_caps_lock); |
610 | if (lastinode) |
611 | iput(lastinode); |
612 | dout("update_snap_trace cap_snaps queued\n"); |
613 | } |
614 | |
615 | } else { |
616 | dout("update_snap_trace %llx %p seq %lld unchanged\n", |
617 | realm->ino, realm, realm->seq); |
618 | } |
619 | |
620 | /* ensure the parent is correct */ |
621 | err = adjust_snap_realm_parent(mdsc, realm, le64_to_cpu(ri->parent)); |
622 | if (err < 0) |
623 | goto fail; |
624 | invalidate += err; |
625 | |
626 | if (le64_to_cpu(ri->seq) > realm->seq) { |
627 | /* update realm parameters, snap lists */ |
628 | realm->seq = le64_to_cpu(ri->seq); |
629 | realm->created = le64_to_cpu(ri->created); |
630 | realm->parent_since = le64_to_cpu(ri->parent_since); |
631 | |
632 | realm->num_snaps = le32_to_cpu(ri->num_snaps); |
633 | err = dup_array(&realm->snaps, snaps, realm->num_snaps); |
634 | if (err < 0) |
635 | goto fail; |
636 | |
637 | realm->num_prior_parent_snaps = |
638 | le32_to_cpu(ri->num_prior_parent_snaps); |
639 | err = dup_array(&realm->prior_parent_snaps, prior_parent_snaps, |
640 | realm->num_prior_parent_snaps); |
641 | if (err < 0) |
642 | goto fail; |
643 | |
644 | invalidate = 1; |
645 | } else if (!realm->cached_context) { |
646 | invalidate = 1; |
647 | } |
648 | |
649 | dout("done with %llx %p, invalidated=%d, %p %p\n", realm->ino, |
650 | realm, invalidate, p, e); |
651 | |
652 | if (p < e) |
653 | goto more; |
654 | |
655 | /* invalidate when we reach the _end_ (root) of the trace */ |
656 | if (invalidate) |
657 | rebuild_snap_realms(realm); |
658 | |
659 | __cleanup_empty_realms(mdsc); |
660 | return 0; |
661 | |
662 | bad: |
663 | err = -EINVAL; |
664 | fail: |
665 | pr_err("update_snap_trace error %d\n", err); |
666 | return err; |
667 | } |
668 | |
669 | |
670 | /* |
671 | * Send any cap_snaps that are queued for flush. Try to carry |
672 | * s_mutex across multiple snap flushes to avoid locking overhead. |
673 | * |
674 | * Caller holds no locks. |
675 | */ |
676 | static void flush_snaps(struct ceph_mds_client *mdsc) |
677 | { |
678 | struct ceph_inode_info *ci; |
679 | struct inode *inode; |
680 | struct ceph_mds_session *session = NULL; |
681 | |
682 | dout("flush_snaps\n"); |
683 | spin_lock(&mdsc->snap_flush_lock); |
684 | while (!list_empty(&mdsc->snap_flush_list)) { |
685 | ci = list_first_entry(&mdsc->snap_flush_list, |
686 | struct ceph_inode_info, i_snap_flush_item); |
687 | inode = &ci->vfs_inode; |
688 | igrab(inode); |
689 | spin_unlock(&mdsc->snap_flush_lock); |
690 | spin_lock(&inode->i_lock); |
691 | __ceph_flush_snaps(ci, &session); |
692 | spin_unlock(&inode->i_lock); |
693 | iput(inode); |
694 | spin_lock(&mdsc->snap_flush_lock); |
695 | } |
696 | spin_unlock(&mdsc->snap_flush_lock); |
697 | |
698 | if (session) { |
699 | mutex_unlock(&session->s_mutex); |
700 | ceph_put_mds_session(session); |
701 | } |
702 | dout("flush_snaps done\n"); |
703 | } |
704 | |
705 | |
706 | /* |
707 | * Handle a snap notification from the MDS. |
708 | * |
709 | * This can take two basic forms: the simplest is just a snap creation |
710 | * or deletion notification on an existing realm. This should update the |
711 | * realm and its children. |
712 | * |
713 | * The more difficult case is realm creation, due to snap creation at a |
714 | * new point in the file hierarchy, or due to a rename that moves a file or |
715 | * directory into another realm. |
716 | */ |
717 | void ceph_handle_snap(struct ceph_mds_client *mdsc, |
718 | struct ceph_mds_session *session, |
719 | struct ceph_msg *msg) |
720 | { |
721 | struct super_block *sb = mdsc->client->sb; |
722 | int mds = session->s_mds; |
723 | u64 split; |
724 | int op; |
725 | int trace_len; |
726 | struct ceph_snap_realm *realm = NULL; |
727 | void *p = msg->front.iov_base; |
728 | void *e = p + msg->front.iov_len; |
729 | struct ceph_mds_snap_head *h; |
730 | int num_split_inos, num_split_realms; |
731 | __le64 *split_inos = NULL, *split_realms = NULL; |
732 | int i; |
733 | int locked_rwsem = 0; |
734 | |
735 | /* decode */ |
736 | if (msg->front.iov_len < sizeof(*h)) |
737 | goto bad; |
738 | h = p; |
739 | op = le32_to_cpu(h->op); |
740 | split = le64_to_cpu(h->split); /* non-zero if we are splitting an |
741 | * existing realm */ |
742 | num_split_inos = le32_to_cpu(h->num_split_inos); |
743 | num_split_realms = le32_to_cpu(h->num_split_realms); |
744 | trace_len = le32_to_cpu(h->trace_len); |
745 | p += sizeof(*h); |
746 | |
747 | dout("handle_snap from mds%d op %s split %llx tracelen %d\n", mds, |
748 | ceph_snap_op_name(op), split, trace_len); |
749 | |
750 | mutex_lock(&session->s_mutex); |
751 | session->s_seq++; |
752 | mutex_unlock(&session->s_mutex); |
753 | |
754 | down_write(&mdsc->snap_rwsem); |
755 | locked_rwsem = 1; |
756 | |
757 | if (op == CEPH_SNAP_OP_SPLIT) { |
758 | struct ceph_mds_snap_realm *ri; |
759 | |
760 | /* |
761 | * A "split" breaks part of an existing realm off into |
762 | * a new realm. The MDS provides a list of inodes |
763 | * (with caps) and child realms that belong to the new |
764 | * child. |
765 | */ |
766 | split_inos = p; |
767 | p += sizeof(u64) * num_split_inos; |
768 | split_realms = p; |
769 | p += sizeof(u64) * num_split_realms; |
770 | ceph_decode_need(&p, e, sizeof(*ri), bad); |
771 | /* we will peek at realm info here, but will _not_ |
772 | * advance p, as the realm update will occur below in |
773 | * ceph_update_snap_trace. */ |
774 | ri = p; |
775 | |
776 | realm = ceph_lookup_snap_realm(mdsc, split); |
777 | if (!realm) { |
778 | realm = ceph_create_snap_realm(mdsc, split); |
779 | if (IS_ERR(realm)) |
780 | goto out; |
781 | } |
782 | ceph_get_snap_realm(mdsc, realm); |
783 | |
784 | dout("splitting snap_realm %llx %p\n", realm->ino, realm); |
785 | for (i = 0; i < num_split_inos; i++) { |
786 | struct ceph_vino vino = { |
787 | .ino = le64_to_cpu(split_inos[i]), |
788 | .snap = CEPH_NOSNAP, |
789 | }; |
790 | struct inode *inode = ceph_find_inode(sb, vino); |
791 | struct ceph_inode_info *ci; |
792 | |
793 | if (!inode) |
794 | continue; |
795 | ci = ceph_inode(inode); |
796 | |
797 | spin_lock(&inode->i_lock); |
798 | if (!ci->i_snap_realm) |
799 | goto skip_inode; |
800 | /* |
801 | * If this inode belongs to a realm that was |
802 | * created after our new realm, we experienced |
803 | * a race (due to another split notifications |
804 | * arriving from a different MDS). So skip |
805 | * this inode. |
806 | */ |
807 | if (ci->i_snap_realm->created > |
808 | le64_to_cpu(ri->created)) { |
809 | dout(" leaving %p in newer realm %llx %p\n", |
810 | inode, ci->i_snap_realm->ino, |
811 | ci->i_snap_realm); |
812 | goto skip_inode; |
813 | } |
814 | dout(" will move %p to split realm %llx %p\n", |
815 | inode, realm->ino, realm); |
816 | /* |
817 | * Remove the inode from the realm's inode |
818 | * list, but don't add it to the new realm |
819 | * yet. We don't want the cap_snap to be |
820 | * queued (again) by ceph_update_snap_trace() |
821 | * below. Queue it _now_, under the old context. |
822 | */ |
823 | spin_lock(&realm->inodes_with_caps_lock); |
824 | list_del_init(&ci->i_snap_realm_item); |
825 | spin_unlock(&realm->inodes_with_caps_lock); |
826 | spin_unlock(&inode->i_lock); |
827 | |
828 | ceph_queue_cap_snap(ci); |
829 | |
830 | iput(inode); |
831 | continue; |
832 | |
833 | skip_inode: |
834 | spin_unlock(&inode->i_lock); |
835 | iput(inode); |
836 | } |
837 | |
838 | /* we may have taken some of the old realm's children. */ |
839 | for (i = 0; i < num_split_realms; i++) { |
840 | struct ceph_snap_realm *child = |
841 | ceph_lookup_snap_realm(mdsc, |
842 | le64_to_cpu(split_realms[i])); |
843 | if (!child) |
844 | continue; |
845 | adjust_snap_realm_parent(mdsc, child, realm->ino); |
846 | } |
847 | } |
848 | |
849 | /* |
850 | * update using the provided snap trace. if we are deleting a |
851 | * snap, we can avoid queueing cap_snaps. |
852 | */ |
853 | ceph_update_snap_trace(mdsc, p, e, |
854 | op == CEPH_SNAP_OP_DESTROY); |
855 | |
856 | if (op == CEPH_SNAP_OP_SPLIT) { |
857 | /* |
858 | * ok, _now_ add the inodes into the new realm. |
859 | */ |
860 | for (i = 0; i < num_split_inos; i++) { |
861 | struct ceph_vino vino = { |
862 | .ino = le64_to_cpu(split_inos[i]), |
863 | .snap = CEPH_NOSNAP, |
864 | }; |
865 | struct inode *inode = ceph_find_inode(sb, vino); |
866 | struct ceph_inode_info *ci; |
867 | |
868 | if (!inode) |
869 | continue; |
870 | ci = ceph_inode(inode); |
871 | spin_lock(&inode->i_lock); |
872 | if (!ci->i_snap_realm) |
873 | goto split_skip_inode; |
874 | ceph_put_snap_realm(mdsc, ci->i_snap_realm); |
875 | spin_lock(&realm->inodes_with_caps_lock); |
876 | list_add(&ci->i_snap_realm_item, |
877 | &realm->inodes_with_caps); |
878 | ci->i_snap_realm = realm; |
879 | spin_unlock(&realm->inodes_with_caps_lock); |
880 | ceph_get_snap_realm(mdsc, realm); |
881 | split_skip_inode: |
882 | spin_unlock(&inode->i_lock); |
883 | iput(inode); |
884 | } |
885 | |
886 | /* we took a reference when we created the realm, above */ |
887 | ceph_put_snap_realm(mdsc, realm); |
888 | } |
889 | |
890 | __cleanup_empty_realms(mdsc); |
891 | |
892 | up_write(&mdsc->snap_rwsem); |
893 | |
894 | flush_snaps(mdsc); |
895 | return; |
896 | |
897 | bad: |
898 | pr_err("corrupt snap message from mds%d\n", mds); |
899 | ceph_msg_dump(msg); |
900 | out: |
901 | if (locked_rwsem) |
902 | up_write(&mdsc->snap_rwsem); |
903 | return; |
904 | } |
905 | |
906 | |
907 | |
908 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9