1/* 2 * Copyright (C) 2003 Christophe Saout <christophe@saout.de> 3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 4 * Copyright (C) 2006 Red Hat, Inc. All rights reserved. 5 * 6 * This file is released under the GPL. 7 */ 8 9#include <linux/err.h> 10#include <linux/module.h> 11#include <linux/init.h> 12#include <linux/kernel.h> 13#include <linux/bio.h> 14#include <linux/blkdev.h> 15#include <linux/mempool.h> 16#include <linux/slab.h> 17#include <linux/crypto.h> 18#include <linux/workqueue.h> 19#include <linux/backing-dev.h> 20#include <asm/atomic.h> 21#include <linux/scatterlist.h> 22#include <asm/page.h> 23#include <asm/unaligned.h> 24 25#include "dm.h" 26 27#define DM_MSG_PREFIX "crypt" 28#define MESG_STR(x) x, sizeof(x) 29 30/* 31 * per bio private data 32 */ 33struct crypt_io { 34 struct dm_target *target; 35 struct bio *base_bio; 36 struct work_struct work; 37 atomic_t pending; 38 int error; 39 int post_process; 40}; 41 42/* 43 * context holding the current state of a multi-part conversion 44 */ 45struct convert_context { 46 struct bio *bio_in; 47 struct bio *bio_out; 48 unsigned int offset_in; 49 unsigned int offset_out; 50 unsigned int idx_in; 51 unsigned int idx_out; 52 sector_t sector; 53 int write; 54}; 55 56struct crypt_config; 57 58struct crypt_iv_operations { 59 int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 60 const char *opts); 61 void (*dtr)(struct crypt_config *cc); 62 const char *(*status)(struct crypt_config *cc); 63 int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector); 64}; 65 66/* 67 * Crypt: maps a linear range of a block device 68 * and encrypts / decrypts at the same time. 69 */ 70enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID }; 71struct crypt_config { 72 struct dm_dev *dev; 73 sector_t start; 74 75 /* 76 * pool for per bio private data and 77 * for encryption buffer pages 78 */ 79 mempool_t *io_pool; 80 mempool_t *page_pool; 81 struct bio_set *bs; 82 83 /* 84 * crypto related data 85 */ 86 struct crypt_iv_operations *iv_gen_ops; 87 char *iv_mode; 88 union { 89 struct crypto_cipher *essiv_tfm; 90 int benbi_shift; 91 } iv_gen_private; 92 sector_t iv_offset; 93 unsigned int iv_size; 94 95 char cipher[CRYPTO_MAX_ALG_NAME]; 96 char chainmode[CRYPTO_MAX_ALG_NAME]; 97 struct crypto_blkcipher *tfm; 98 unsigned long flags; 99 unsigned int key_size; 100 u8 key[0]; 101}; 102 103#define MIN_IOS 16 104#define MIN_POOL_PAGES 32 105#define MIN_BIO_PAGES 8 106 107static struct kmem_cache *_crypt_io_pool; 108 109static void clone_init(struct crypt_io *, struct bio *); 110 111/* 112 * Different IV generation algorithms: 113 * 114 * plain: the initial vector is the 32-bit little-endian version of the sector 115 * number, padded with zeros if neccessary. 116 * 117 * essiv: "encrypted sector|salt initial vector", the sector number is 118 * encrypted with the bulk cipher using a salt as key. The salt 119 * should be derived from the bulk cipher's key via hashing. 120 * 121 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 122 * (needed for LRW-32-AES and possible other narrow block modes) 123 * 124 * null: the initial vector is always zero. Provides compatibility with 125 * obsolete loop_fish2 devices. Do not use for new devices. 126 * 127 * plumb: unimplemented, see: 128 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454 129 */ 130 131static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 132{ 133 memset(iv, 0, cc->iv_size); 134 *(u32 *)iv = cpu_to_le32(sector & 0xffffffff); 135 136 return 0; 137} 138 139static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, 140 const char *opts) 141{ 142 struct crypto_cipher *essiv_tfm; 143 struct crypto_hash *hash_tfm; 144 struct hash_desc desc; 145 struct scatterlist sg; 146 unsigned int saltsize; 147 u8 *salt; 148 int err; 149 150 if (opts == NULL) { 151 ti->error = "Digest algorithm missing for ESSIV mode"; 152 return -EINVAL; 153 } 154 155 /* Hash the cipher key with the given hash algorithm */ 156 hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC); 157 if (IS_ERR(hash_tfm)) { 158 ti->error = "Error initializing ESSIV hash"; 159 return PTR_ERR(hash_tfm); 160 } 161 162 saltsize = crypto_hash_digestsize(hash_tfm); 163 salt = kmalloc(saltsize, GFP_KERNEL); 164 if (salt == NULL) { 165 ti->error = "Error kmallocing salt storage in ESSIV"; 166 crypto_free_hash(hash_tfm); 167 return -ENOMEM; 168 } 169 170 sg_set_buf(&sg, cc->key, cc->key_size); 171 desc.tfm = hash_tfm; 172 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; 173 err = crypto_hash_digest(&desc, &sg, cc->key_size, salt); 174 crypto_free_hash(hash_tfm); 175 176 if (err) { 177 ti->error = "Error calculating hash in ESSIV"; 178 return err; 179 } 180 181 /* Setup the essiv_tfm with the given salt */ 182 essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 183 if (IS_ERR(essiv_tfm)) { 184 ti->error = "Error allocating crypto tfm for ESSIV"; 185 kfree(salt); 186 return PTR_ERR(essiv_tfm); 187 } 188 if (crypto_cipher_blocksize(essiv_tfm) != 189 crypto_blkcipher_ivsize(cc->tfm)) { 190 ti->error = "Block size of ESSIV cipher does " 191 "not match IV size of block cipher"; 192 crypto_free_cipher(essiv_tfm); 193 kfree(salt); 194 return -EINVAL; 195 } 196 err = crypto_cipher_setkey(essiv_tfm, salt, saltsize); 197 if (err) { 198 ti->error = "Failed to set key for ESSIV cipher"; 199 crypto_free_cipher(essiv_tfm); 200 kfree(salt); 201 return err; 202 } 203 kfree(salt); 204 205 cc->iv_gen_private.essiv_tfm = essiv_tfm; 206 return 0; 207} 208 209static void crypt_iv_essiv_dtr(struct crypt_config *cc) 210{ 211 crypto_free_cipher(cc->iv_gen_private.essiv_tfm); 212 cc->iv_gen_private.essiv_tfm = NULL; 213} 214 215static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 216{ 217 memset(iv, 0, cc->iv_size); 218 *(u64 *)iv = cpu_to_le64(sector); 219 crypto_cipher_encrypt_one(cc->iv_gen_private.essiv_tfm, iv, iv); 220 return 0; 221} 222 223static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 224 const char *opts) 225{ 226 unsigned int bs = crypto_blkcipher_blocksize(cc->tfm); 227 int log = ilog2(bs); 228 229 /* we need to calculate how far we must shift the sector count 230 * to get the cipher block count, we use this shift in _gen */ 231 232 if (1 << log != bs) { 233 ti->error = "cypher blocksize is not a power of 2"; 234 return -EINVAL; 235 } 236 237 if (log > 9) { 238 ti->error = "cypher blocksize is > 512"; 239 return -EINVAL; 240 } 241 242 cc->iv_gen_private.benbi_shift = 9 - log; 243 244 return 0; 245} 246 247static void crypt_iv_benbi_dtr(struct crypt_config *cc) 248{ 249} 250 251static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 252{ 253 __be64 val; 254 255 memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 256 257 val = cpu_to_be64(((u64)sector << cc->iv_gen_private.benbi_shift) + 1); 258 put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 259 260 return 0; 261} 262 263static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 264{ 265 memset(iv, 0, cc->iv_size); 266 267 return 0; 268} 269 270static struct crypt_iv_operations crypt_iv_plain_ops = { 271 .generator = crypt_iv_plain_gen 272}; 273 274static struct crypt_iv_operations crypt_iv_essiv_ops = { 275 .ctr = crypt_iv_essiv_ctr, 276 .dtr = crypt_iv_essiv_dtr, 277 .generator = crypt_iv_essiv_gen 278}; 279 280static struct crypt_iv_operations crypt_iv_benbi_ops = { 281 .ctr = crypt_iv_benbi_ctr, 282 .dtr = crypt_iv_benbi_dtr, 283 .generator = crypt_iv_benbi_gen 284}; 285 286static struct crypt_iv_operations crypt_iv_null_ops = { 287 .generator = crypt_iv_null_gen 288}; 289 290static int 291crypt_convert_scatterlist(struct crypt_config *cc, struct scatterlist *out, 292 struct scatterlist *in, unsigned int length, 293 int write, sector_t sector) 294{ 295 u8 iv[cc->iv_size] __attribute__ ((aligned(__alignof__(u64)))); 296 struct blkcipher_desc desc = { 297 .tfm = cc->tfm, 298 .info = iv, 299 .flags = CRYPTO_TFM_REQ_MAY_SLEEP, 300 }; 301 int r; 302 303 if (cc->iv_gen_ops) { 304 r = cc->iv_gen_ops->generator(cc, iv, sector); 305 if (r < 0) 306 return r; 307 308 if (write) 309 r = crypto_blkcipher_encrypt_iv(&desc, out, in, length); 310 else 311 r = crypto_blkcipher_decrypt_iv(&desc, out, in, length); 312 } else { 313 if (write) 314 r = crypto_blkcipher_encrypt(&desc, out, in, length); 315 else 316 r = crypto_blkcipher_decrypt(&desc, out, in, length); 317 } 318 319 return r; 320} 321 322static void 323crypt_convert_init(struct crypt_config *cc, struct convert_context *ctx, 324 struct bio *bio_out, struct bio *bio_in, 325 sector_t sector, int write) 326{ 327 ctx->bio_in = bio_in; 328 ctx->bio_out = bio_out; 329 ctx->offset_in = 0; 330 ctx->offset_out = 0; 331 ctx->idx_in = bio_in ? bio_in->bi_idx : 0; 332 ctx->idx_out = bio_out ? bio_out->bi_idx : 0; 333 ctx->sector = sector + cc->iv_offset; 334 ctx->write = write; 335} 336 337/* 338 * Encrypt / decrypt data from one bio to another one (can be the same one) 339 */ 340static int crypt_convert(struct crypt_config *cc, 341 struct convert_context *ctx) 342{ 343 int r = 0; 344 345 while(ctx->idx_in < ctx->bio_in->bi_vcnt && 346 ctx->idx_out < ctx->bio_out->bi_vcnt) { 347 struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in); 348 struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out); 349 struct scatterlist sg_in = { 350 .page = bv_in->bv_page, 351 .offset = bv_in->bv_offset + ctx->offset_in, 352 .length = 1 << SECTOR_SHIFT 353 }; 354 struct scatterlist sg_out = { 355 .page = bv_out->bv_page, 356 .offset = bv_out->bv_offset + ctx->offset_out, 357 .length = 1 << SECTOR_SHIFT 358 }; 359 360 ctx->offset_in += sg_in.length; 361 if (ctx->offset_in >= bv_in->bv_len) { 362 ctx->offset_in = 0; 363 ctx->idx_in++; 364 } 365 366 ctx->offset_out += sg_out.length; 367 if (ctx->offset_out >= bv_out->bv_len) { 368 ctx->offset_out = 0; 369 ctx->idx_out++; 370 } 371 372 r = crypt_convert_scatterlist(cc, &sg_out, &sg_in, sg_in.length, 373 ctx->write, ctx->sector); 374 if (r < 0) 375 break; 376 377 ctx->sector++; 378 } 379 380 return r; 381} 382 383 static void dm_crypt_bio_destructor(struct bio *bio) 384 { 385 struct crypt_io *io = bio->bi_private; 386 struct crypt_config *cc = io->target->private; 387 388 bio_free(bio, cc->bs); 389 } 390 391/* 392 * Generate a new unfragmented bio with the given size 393 * This should never violate the device limitations 394 * May return a smaller bio when running out of pages 395 */ 396static struct bio *crypt_alloc_buffer(struct crypt_io *io, unsigned int size) 397{ 398 struct crypt_config *cc = io->target->private; 399 struct bio *clone; 400 unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 401 gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; 402 unsigned int i; 403 404 clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); 405 if (!clone) 406 return NULL; 407 408 clone_init(io, clone); 409 410 for (i = 0; i < nr_iovecs; i++) { 411 struct bio_vec *bv = bio_iovec_idx(clone, i); 412 413 bv->bv_page = mempool_alloc(cc->page_pool, gfp_mask); 414 if (!bv->bv_page) 415 break; 416 417 /* 418 * if additional pages cannot be allocated without waiting, 419 * return a partially allocated bio, the caller will then try 420 * to allocate additional bios while submitting this partial bio 421 */ 422 if (i == (MIN_BIO_PAGES - 1)) 423 gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT; 424 425 bv->bv_offset = 0; 426 if (size > PAGE_SIZE) 427 bv->bv_len = PAGE_SIZE; 428 else 429 bv->bv_len = size; 430 431 clone->bi_size += bv->bv_len; 432 clone->bi_vcnt++; 433 size -= bv->bv_len; 434 } 435 436 if (!clone->bi_size) { 437 bio_put(clone); 438 return NULL; 439 } 440 441 return clone; 442} 443 444static void crypt_free_buffer_pages(struct crypt_config *cc, 445 struct bio *clone, unsigned int bytes) 446{ 447 unsigned int i, start, end; 448 struct bio_vec *bv; 449 450 /* 451 * This is ugly, but Jens Axboe thinks that using bi_idx in the 452 * endio function is too dangerous at the moment, so I calculate the 453 * correct position using bi_vcnt and bi_size. 454 * The bv_offset and bv_len fields might already be modified but we 455 * know that we always allocated whole pages. 456 * A fix to the bi_idx issue in the kernel is in the works, so 457 * we will hopefully be able to revert to the cleaner solution soon. 458 */ 459 i = clone->bi_vcnt - 1; 460 bv = bio_iovec_idx(clone, i); 461 end = (i << PAGE_SHIFT) + (bv->bv_offset + bv->bv_len) - clone->bi_size; 462 start = end - bytes; 463 464 start >>= PAGE_SHIFT; 465 if (!clone->bi_size) 466 end = clone->bi_vcnt; 467 else 468 end >>= PAGE_SHIFT; 469 470 for (i = start; i < end; i++) { 471 bv = bio_iovec_idx(clone, i); 472 BUG_ON(!bv->bv_page); 473 mempool_free(bv->bv_page, cc->page_pool); 474 bv->bv_page = NULL; 475 } 476} 477 478/* 479 * One of the bios was finished. Check for completion of 480 * the whole request and correctly clean up the buffer. 481 */ 482static void dec_pending(struct crypt_io *io, int error) 483{ 484 struct crypt_config *cc = (struct crypt_config *) io->target->private; 485 486 if (error < 0) 487 io->error = error; 488 489 if (!atomic_dec_and_test(&io->pending)) 490 return; 491 492 bio_endio(io->base_bio, io->base_bio->bi_size, io->error); 493 494 mempool_free(io, cc->io_pool); 495} 496 497/* 498 * kcryptd: 499 * 500 * Needed because it would be very unwise to do decryption in an 501 * interrupt context. 502 */ 503static struct workqueue_struct *_kcryptd_workqueue; 504static void kcryptd_do_work(struct work_struct *work); 505 506static void kcryptd_queue_io(struct crypt_io *io) 507{ 508 INIT_WORK(&io->work, kcryptd_do_work); 509 queue_work(_kcryptd_workqueue, &io->work); 510} 511 512static int crypt_endio(struct bio *clone, unsigned int done, int error) 513{ 514 struct crypt_io *io = clone->bi_private; 515 struct crypt_config *cc = io->target->private; 516 unsigned read_io = bio_data_dir(clone) == READ; 517 518 /* 519 * free the processed pages, even if 520 * it's only a partially completed write 521 */ 522 if (!read_io) 523 crypt_free_buffer_pages(cc, clone, done); 524 525 /* keep going - not finished yet */ 526 if (unlikely(clone->bi_size)) 527 return 1; 528 529 if (!read_io) 530 goto out; 531 532 if (unlikely(!bio_flagged(clone, BIO_UPTODATE))) { 533 error = -EIO; 534 goto out; 535 } 536 537 bio_put(clone); 538 io->post_process = 1; 539 kcryptd_queue_io(io); 540 return 0; 541 542out: 543 bio_put(clone); 544 dec_pending(io, error); 545 return error; 546} 547 548static void clone_init(struct crypt_io *io, struct bio *clone) 549{ 550 struct crypt_config *cc = io->target->private; 551 552 clone->bi_private = io; 553 clone->bi_end_io = crypt_endio; 554 clone->bi_bdev = cc->dev->bdev; 555 clone->bi_rw = io->base_bio->bi_rw; 556 clone->bi_destructor = dm_crypt_bio_destructor; 557} 558 559static void process_read(struct crypt_io *io) 560{ 561 struct crypt_config *cc = io->target->private; 562 struct bio *base_bio = io->base_bio; 563 struct bio *clone; 564 sector_t sector = base_bio->bi_sector - io->target->begin; 565 566 atomic_inc(&io->pending); 567 568 /* 569 * The block layer might modify the bvec array, so always 570 * copy the required bvecs because we need the original 571 * one in order to decrypt the whole bio data *afterwards*. 572 */ 573 clone = bio_alloc_bioset(GFP_NOIO, bio_segments(base_bio), cc->bs); 574 if (unlikely(!clone)) { 575 dec_pending(io, -ENOMEM); 576 return; 577 } 578 579 clone_init(io, clone); 580 clone->bi_idx = 0; 581 clone->bi_vcnt = bio_segments(base_bio); 582 clone->bi_size = base_bio->bi_size; 583 clone->bi_sector = cc->start + sector; 584 memcpy(clone->bi_io_vec, bio_iovec(base_bio), 585 sizeof(struct bio_vec) * clone->bi_vcnt); 586 587 generic_make_request(clone); 588} 589 590static void process_write(struct crypt_io *io) 591{ 592 struct crypt_config *cc = io->target->private; 593 struct bio *base_bio = io->base_bio; 594 struct bio *clone; 595 struct convert_context ctx; 596 unsigned remaining = base_bio->bi_size; 597 sector_t sector = base_bio->bi_sector - io->target->begin; 598 599 atomic_inc(&io->pending); 600 601 crypt_convert_init(cc, &ctx, NULL, base_bio, sector, 1); 602 603 /* 604 * The allocated buffers can be smaller than the whole bio, 605 * so repeat the whole process until all the data can be handled. 606 */ 607 while (remaining) { 608 clone = crypt_alloc_buffer(io, remaining); 609 if (unlikely(!clone)) { 610 dec_pending(io, -ENOMEM); 611 return; 612 } 613 614 ctx.bio_out = clone; 615 ctx.idx_out = 0; 616 617 if (unlikely(crypt_convert(cc, &ctx) < 0)) { 618 crypt_free_buffer_pages(cc, clone, clone->bi_size); 619 bio_put(clone); 620 dec_pending(io, -EIO); 621 return; 622 } 623 624 /* crypt_convert should have filled the clone bio */ 625 BUG_ON(ctx.idx_out < clone->bi_vcnt); 626 627 clone->bi_sector = cc->start + sector; 628 remaining -= clone->bi_size; 629 sector += bio_sectors(clone); 630 631 /* Grab another reference to the io struct 632 * before we kick off the request */ 633 if (remaining) 634 atomic_inc(&io->pending); 635 636 generic_make_request(clone); 637 638 /* Do not reference clone after this - it 639 * may be gone already. */ 640 641 /* out of memory -> run queues */ 642 if (remaining) 643 congestion_wait(WRITE, HZ/100); 644 } 645} 646 647static void process_read_endio(struct crypt_io *io) 648{ 649 struct crypt_config *cc = io->target->private; 650 struct convert_context ctx; 651 652 crypt_convert_init(cc, &ctx, io->base_bio, io->base_bio, 653 io->base_bio->bi_sector - io->target->begin, 0); 654 655 dec_pending(io, crypt_convert(cc, &ctx)); 656} 657 658static void kcryptd_do_work(struct work_struct *work) 659{ 660 struct crypt_io *io = container_of(work, struct crypt_io, work); 661 662 if (io->post_process) 663 process_read_endio(io); 664 else if (bio_data_dir(io->base_bio) == READ) 665 process_read(io); 666 else 667 process_write(io); 668} 669 670/* 671 * Decode key from its hex representation 672 */ 673static int crypt_decode_key(u8 *key, char *hex, unsigned int size) 674{ 675 char buffer[3]; 676 char *endp; 677 unsigned int i; 678 679 buffer[2] = '\0'; 680 681 for (i = 0; i < size; i++) { 682 buffer[0] = *hex++; 683 buffer[1] = *hex++; 684 685 key[i] = (u8)simple_strtoul(buffer, &endp, 16); 686 687 if (endp != &buffer[2]) 688 return -EINVAL; 689 } 690 691 if (*hex != '\0') 692 return -EINVAL; 693 694 return 0; 695} 696 697/* 698 * Encode key into its hex representation 699 */ 700static void crypt_encode_key(char *hex, u8 *key, unsigned int size) 701{ 702 unsigned int i; 703 704 for (i = 0; i < size; i++) { 705 sprintf(hex, "%02x", *key); 706 hex += 2; 707 key++; 708 } 709} 710 711static int crypt_set_key(struct crypt_config *cc, char *key) 712{ 713 unsigned key_size = strlen(key) >> 1; 714 715 if (cc->key_size && cc->key_size != key_size) 716 return -EINVAL; 717 718 cc->key_size = key_size; /* initial settings */ 719 720 if ((!key_size && strcmp(key, "-")) || 721 (key_size && crypt_decode_key(cc->key, key, key_size) < 0)) 722 return -EINVAL; 723 724 set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 725 726 return 0; 727} 728 729static int crypt_wipe_key(struct crypt_config *cc) 730{ 731 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 732 memset(&cc->key, 0, cc->key_size * sizeof(u8)); 733 return 0; 734} 735 736/* 737 * Construct an encryption mapping: 738 * <cipher> <key> <iv_offset> <dev_path> <start> 739 */ 740static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 741{ 742 struct crypt_config *cc; 743 struct crypto_blkcipher *tfm; 744 char *tmp; 745 char *cipher; 746 char *chainmode; 747 char *ivmode; 748 char *ivopts; 749 unsigned int key_size; 750 unsigned long long tmpll; 751 752 if (argc != 5) { 753 ti->error = "Not enough arguments"; 754 return -EINVAL; 755 } 756 757 tmp = argv[0]; 758 cipher = strsep(&tmp, "-"); 759 chainmode = strsep(&tmp, "-"); 760 ivopts = strsep(&tmp, "-"); 761 ivmode = strsep(&ivopts, ":"); 762 763 if (tmp) 764 DMWARN("Unexpected additional cipher options"); 765 766 key_size = strlen(argv[1]) >> 1; 767 768 cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL); 769 if (cc == NULL) { 770 ti->error = 771 "Cannot allocate transparent encryption context"; 772 return -ENOMEM; 773 } 774 775 if (crypt_set_key(cc, argv[1])) { 776 ti->error = "Error decoding key"; 777 goto bad1; 778 } 779 780 /* Compatiblity mode for old dm-crypt cipher strings */ 781 if (!chainmode || (strcmp(chainmode, "plain") == 0 && !ivmode)) { 782 chainmode = "cbc"; 783 ivmode = "plain"; 784 } 785 786 if (strcmp(chainmode, "ecb") && !ivmode) { 787 ti->error = "This chaining mode requires an IV mechanism"; 788 goto bad1; 789 } 790 791 if (snprintf(cc->cipher, CRYPTO_MAX_ALG_NAME, "%s(%s)", chainmode, 792 cipher) >= CRYPTO_MAX_ALG_NAME) { 793 ti->error = "Chain mode + cipher name is too long"; 794 goto bad1; 795 } 796 797 tfm = crypto_alloc_blkcipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 798 if (IS_ERR(tfm)) { 799 ti->error = "Error allocating crypto tfm"; 800 goto bad1; 801 } 802 803 strcpy(cc->cipher, cipher); 804 strcpy(cc->chainmode, chainmode); 805 cc->tfm = tfm; 806 807 /* 808 * Choose ivmode. Valid modes: "plain", "essiv:<esshash>", "benbi". 809 * See comments at iv code 810 */ 811 812 if (ivmode == NULL) 813 cc->iv_gen_ops = NULL; 814 else if (strcmp(ivmode, "plain") == 0) 815 cc->iv_gen_ops = &crypt_iv_plain_ops; 816 else if (strcmp(ivmode, "essiv") == 0) 817 cc->iv_gen_ops = &crypt_iv_essiv_ops; 818 else if (strcmp(ivmode, "benbi") == 0) 819 cc->iv_gen_ops = &crypt_iv_benbi_ops; 820 else if (strcmp(ivmode, "null") == 0) 821 cc->iv_gen_ops = &crypt_iv_null_ops; 822 else { 823 ti->error = "Invalid IV mode"; 824 goto bad2; 825 } 826 827 if (cc->iv_gen_ops && cc->iv_gen_ops->ctr && 828 cc->iv_gen_ops->ctr(cc, ti, ivopts) < 0) 829 goto bad2; 830 831 cc->iv_size = crypto_blkcipher_ivsize(tfm); 832 if (cc->iv_size) 833 /* at least a 64 bit sector number should fit in our buffer */ 834 cc->iv_size = max(cc->iv_size, 835 (unsigned int)(sizeof(u64) / sizeof(u8))); 836 else { 837 if (cc->iv_gen_ops) { 838 DMWARN("Selected cipher does not support IVs"); 839 if (cc->iv_gen_ops->dtr) 840 cc->iv_gen_ops->dtr(cc); 841 cc->iv_gen_ops = NULL; 842 } 843 } 844 845 cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool); 846 if (!cc->io_pool) { 847 ti->error = "Cannot allocate crypt io mempool"; 848 goto bad3; 849 } 850 851 cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); 852 if (!cc->page_pool) { 853 ti->error = "Cannot allocate page mempool"; 854 goto bad4; 855 } 856 857 cc->bs = bioset_create(MIN_IOS, MIN_IOS); 858 if (!cc->bs) { 859 ti->error = "Cannot allocate crypt bioset"; 860 goto bad_bs; 861 } 862 863 if (crypto_blkcipher_setkey(tfm, cc->key, key_size) < 0) { 864 ti->error = "Error setting key"; 865 goto bad5; 866 } 867 868 if (sscanf(argv[2], "%llu", &tmpll) != 1) { 869 ti->error = "Invalid iv_offset sector"; 870 goto bad5; 871 } 872 cc->iv_offset = tmpll; 873 874 if (sscanf(argv[4], "%llu", &tmpll) != 1) { 875 ti->error = "Invalid device sector"; 876 goto bad5; 877 } 878 cc->start = tmpll; 879 880 if (dm_get_device(ti, argv[3], cc->start, ti->len, 881 dm_table_get_mode(ti->table), &cc->dev)) { 882 ti->error = "Device lookup failed"; 883 goto bad5; 884 } 885 886 if (ivmode && cc->iv_gen_ops) { 887 if (ivopts) 888 *(ivopts - 1) = ':'; 889 cc->iv_mode = kmalloc(strlen(ivmode) + 1, GFP_KERNEL); 890 if (!cc->iv_mode) { 891 ti->error = "Error kmallocing iv_mode string"; 892 goto bad5; 893 } 894 strcpy(cc->iv_mode, ivmode); 895 } else 896 cc->iv_mode = NULL; 897 898 ti->private = cc; 899 return 0; 900 901bad5: 902 bioset_free(cc->bs); 903bad_bs: 904 mempool_destroy(cc->page_pool); 905bad4: 906 mempool_destroy(cc->io_pool); 907bad3: 908 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 909 cc->iv_gen_ops->dtr(cc); 910bad2: 911 crypto_free_blkcipher(tfm); 912bad1: 913 /* Must zero key material before freeing */ 914 memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8)); 915 kfree(cc); 916 return -EINVAL; 917} 918 919static void crypt_dtr(struct dm_target *ti) 920{ 921 struct crypt_config *cc = (struct crypt_config *) ti->private; 922 923 bioset_free(cc->bs); 924 mempool_destroy(cc->page_pool); 925 mempool_destroy(cc->io_pool); 926 927 kfree(cc->iv_mode); 928 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 929 cc->iv_gen_ops->dtr(cc); 930 crypto_free_blkcipher(cc->tfm); 931 dm_put_device(ti, cc->dev); 932 933 /* Must zero key material before freeing */ 934 memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8)); 935 kfree(cc); 936} 937 938static int crypt_map(struct dm_target *ti, struct bio *bio, 939 union map_info *map_context) 940{ 941 struct crypt_config *cc = ti->private; 942 struct crypt_io *io; 943 944 if (bio_barrier(bio)) 945 return -EOPNOTSUPP; 946 947 io = mempool_alloc(cc->io_pool, GFP_NOIO); 948 io->target = ti; 949 io->base_bio = bio; 950 io->error = io->post_process = 0; 951 atomic_set(&io->pending, 0); 952 kcryptd_queue_io(io); 953 954 return DM_MAPIO_SUBMITTED; 955} 956 957static int crypt_status(struct dm_target *ti, status_type_t type, 958 char *result, unsigned int maxlen) 959{ 960 struct crypt_config *cc = (struct crypt_config *) ti->private; 961 unsigned int sz = 0; 962 963 switch (type) { 964 case STATUSTYPE_INFO: 965 result[0] = '\0'; 966 break; 967 968 case STATUSTYPE_TABLE: 969 if (cc->iv_mode) 970 DMEMIT("%s-%s-%s ", cc->cipher, cc->chainmode, 971 cc->iv_mode); 972 else 973 DMEMIT("%s-%s ", cc->cipher, cc->chainmode); 974 975 if (cc->key_size > 0) { 976 if ((maxlen - sz) < ((cc->key_size << 1) + 1)) 977 return -ENOMEM; 978 979 crypt_encode_key(result + sz, cc->key, cc->key_size); 980 sz += cc->key_size << 1; 981 } else { 982 if (sz >= maxlen) 983 return -ENOMEM; 984 result[sz++] = '-'; 985 } 986 987 DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 988 cc->dev->name, (unsigned long long)cc->start); 989 break; 990 } 991 return 0; 992} 993 994static void crypt_postsuspend(struct dm_target *ti) 995{ 996 struct crypt_config *cc = ti->private; 997 998 set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 999} 1000 1001static int crypt_preresume(struct dm_target *ti) 1002{ 1003 struct crypt_config *cc = ti->private; 1004 1005 if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 1006 DMERR("aborting resume - crypt key is not set."); 1007 return -EAGAIN; 1008 } 1009 1010 return 0; 1011} 1012 1013static void crypt_resume(struct dm_target *ti) 1014{ 1015 struct crypt_config *cc = ti->private; 1016 1017 clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1018} 1019 1020/* Message interface 1021 * key set <key> 1022 * key wipe 1023 */ 1024static int crypt_message(struct dm_target *ti, unsigned argc, char **argv) 1025{ 1026 struct crypt_config *cc = ti->private; 1027 1028 if (argc < 2) 1029 goto error; 1030 1031 if (!strnicmp(argv[0], MESG_STR("key"))) { 1032 if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 1033 DMWARN("not suspended during key manipulation."); 1034 return -EINVAL; 1035 } 1036 if (argc == 3 && !strnicmp(argv[1], MESG_STR("set"))) 1037 return crypt_set_key(cc, argv[2]); 1038 if (argc == 2 && !strnicmp(argv[1], MESG_STR("wipe"))) 1039 return crypt_wipe_key(cc); 1040 } 1041 1042error: 1043 DMWARN("unrecognised message received."); 1044 return -EINVAL; 1045} 1046 1047static struct target_type crypt_target = { 1048 .name = "crypt", 1049 .version= {1, 5, 0}, 1050 .module = THIS_MODULE, 1051 .ctr = crypt_ctr, 1052 .dtr = crypt_dtr, 1053 .map = crypt_map, 1054 .status = crypt_status, 1055 .postsuspend = crypt_postsuspend, 1056 .preresume = crypt_preresume, 1057 .resume = crypt_resume, 1058 .message = crypt_message, 1059}; 1060 1061static int __init dm_crypt_init(void) 1062{ 1063 int r; 1064 1065 _crypt_io_pool = kmem_cache_create("dm-crypt_io", 1066 sizeof(struct crypt_io), 1067 0, 0, NULL, NULL); 1068 if (!_crypt_io_pool) 1069 return -ENOMEM; 1070 1071 _kcryptd_workqueue = create_workqueue("kcryptd"); 1072 if (!_kcryptd_workqueue) { 1073 r = -ENOMEM; 1074 DMERR("couldn't create kcryptd"); 1075 goto bad1; 1076 } 1077 1078 r = dm_register_target(&crypt_target); 1079 if (r < 0) { 1080 DMERR("register failed %d", r); 1081 goto bad2; 1082 } 1083 1084 return 0; 1085 1086bad2: 1087 destroy_workqueue(_kcryptd_workqueue); 1088bad1: 1089 kmem_cache_destroy(_crypt_io_pool); 1090 return r; 1091} 1092 1093static void __exit dm_crypt_exit(void) 1094{ 1095 int r = dm_unregister_target(&crypt_target); 1096 1097 if (r < 0) 1098 DMERR("unregister failed %d", r); 1099 1100 destroy_workqueue(_kcryptd_workqueue); 1101 kmem_cache_destroy(_crypt_io_pool); 1102} 1103 1104module_init(dm_crypt_init); 1105module_exit(dm_crypt_exit); 1106 1107MODULE_AUTHOR("Christophe Saout <christophe@saout.de>"); 1108MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 1109MODULE_LICENSE("GPL"); 1110