117680Spst/*- 239300Sfenner * SPDX-License-Identifier: BSD-2-Clause 317680Spst * 417680Spst * Copyright (c) 2005-2011 Pawel Jakub Dawidek <pawel@dawidek.net> 517680Spst * All rights reserved. 617680Spst * 717680Spst * Redistribution and use in source and binary forms, with or without 817680Spst * modification, are permitted provided that the following conditions 917680Spst * are met: 1017680Spst * 1. Redistributions of source code must retain the above copyright 1117680Spst * notice, this list of conditions and the following disclaimer. 1217680Spst * 2. Redistributions in binary form must reproduce the above copyright 1317680Spst * notice, this list of conditions and the following disclaimer in the 1417680Spst * documentation and/or other materials provided with the distribution. 1517680Spst * 1617680Spst * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 1717680Spst * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1817680Spst * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1917680Spst * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 2026183Sfenner * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2117680Spst * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2217680Spst * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2356896Sfenner * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2456896Sfenner * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2517680Spst * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2617680Spst * SUCH DAMAGE. 2717680Spst */ 28127675Sbms 29190207Srpaulo#include <sys/param.h> 3017680Spst#include <sys/systm.h> 3117680Spst#include <sys/kernel.h> 3256896Sfenner#include <sys/linker.h> 3356896Sfenner#include <sys/module.h> 3456896Sfenner#include <sys/lock.h> 3556896Sfenner#include <sys/mutex.h> 36127675Sbms#include <sys/bio.h> 3717680Spst#include <sys/sysctl.h> 3817680Spst#include <sys/kthread.h> 3917680Spst#include <sys/proc.h> 4017680Spst#include <sys/sched.h> 4117680Spst#include <sys/smp.h> 4217680Spst#include <sys/vnode.h> 4317680Spst 4417680Spst#include <vm/uma.h> 4517680Spst 4698527Sfenner#include <geom/geom.h> 47146778Ssam#include <geom/geom_dbg.h> 4817680Spst#include <geom/eli/g_eli.h> 49127675Sbms#include <geom/eli/pkcs5v2.h> 50127675Sbms 51127675Sbms/* 52127675Sbms * The data layout description when integrity verification is configured. 53127675Sbms * 54127675Sbms * One of the most important assumption here is that authenticated data and its 55162021Ssam * HMAC has to be stored in the same place (namely in the same sector) to make 56127675Sbms * it work reliable. 57127675Sbms * The problem is that file systems work only with sectors that are multiple of 58127675Sbms * 512 bytes and a power of two number. 59127675Sbms * My idea to implement it is as follows. 60127675Sbms * Let's store HMAC in sector. This is a must. This leaves us 480 bytes for 61127675Sbms * data. We can't use that directly (ie. we can't create provider with 480 bytes 62127675Sbms * sector size). We need another sector from where we take only 32 bytes of data 63127675Sbms * and we store HMAC of this data as well. This takes two sectors from the 64127675Sbms * original provider at the input and leaves us one sector of authenticated data 65147904Ssam * at the output. Not very efficient, but you got the idea. 66127675Sbms * Now, let's assume, we want to create provider with 4096 bytes sector. 67127675Sbms * To output 4096 bytes of authenticated data we need 8x480 plus 1x256, so we 68162021Ssam * need nine 512-bytes sectors at the input to get one 4096-bytes sector at the 6917680Spst * output. That's better. With 4096 bytes sector we can use 89% of size of the 7017680Spst * original provider. I find it as an acceptable cost. 7117680Spst * The reliability comes from the fact, that every HMAC stored inside the sector 7217680Spst * is calculated only for the data in the same sector, so its impossible to 7317680Spst * write new data and leave old HMAC or vice versa. 7417680Spst * 7517680Spst * And here is the picture: 7617680Spst * 7717680Spst * da0: +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+ 7817680Spst * |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |256b | 7917680Spst * |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data | 80162021Ssam * +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+ 81162021Ssam * |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |288 bytes | 82162021Ssam * +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ |224 unused| 83162021Ssam * +----------+ 84162021Ssam * da0.eli: +----+----+----+----+----+----+----+----+----+ 85172686Smlaier * |480b|480b|480b|480b|480b|480b|480b|480b|256b| 86162021Ssam * +----+----+----+----+----+----+----+----+----+ 87162021Ssam * | 4096 bytes | 88162021Ssam * +--------------------------------------------+ 89162021Ssam * 90172686Smlaier * PS. You can use any sector size with geli(8). My example is using 4kB, 91172686Smlaier * because it's most efficient. For 8kB sectors you need 2 extra sectors, 92172686Smlaier * so the cost is the same as for 4kB sectors. 93172686Smlaier */ 94172686Smlaier 95172686Smlaier/* 96172686Smlaier * Code paths: 97172686Smlaier * BIO_READ: 98162021Ssam * g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> g_eli_auth_read_done -> g_io_deliver 99162021Ssam * BIO_WRITE: 100172686Smlaier * g_eli_start -> g_eli_auth_run -> g_eli_auth_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver 101172686Smlaier */ 102162021Ssam 103162021Ssam/* 104162021Ssam * Here we generate key for HMAC. Every sector has its own HMAC key, so it is 105162021Ssam * not possible to copy sectors. 106147904Ssam * We cannot depend on fact, that every sector has its own IV, because different 107147904Ssam * IV doesn't change HMAC, when we use encrypt-then-authenticate method. 108172686Smlaier */ 109172686Smlaierstatic void 110190207Srpaulog_eli_auth_keygen(struct g_eli_softc *sc, off_t offset, u_char *key) 111190207Srpaulo{ 112147904Ssam SHA256_CTX ctx; 113147904Ssam 114147904Ssam /* Copy precalculated SHA256 context. */ 115147904Ssam bcopy(&sc->sc_akeyctx, &ctx, sizeof(ctx)); 116147904Ssam SHA256_Update(&ctx, (uint8_t *)&offset, sizeof(offset)); 117147904Ssam SHA256_Final(key, &ctx); 118147904Ssam} 119147904Ssam 120147904Ssam/* 121147904Ssam * The function is called after we read and decrypt data. 122147904Ssam * 123147904Ssam * g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> G_ELI_AUTH_READ_DONE -> g_io_deliver 124147904Ssam */ 125147904Ssamstatic int 126147904Ssamg_eli_auth_read_done(struct cryptop *crp) 127147904Ssam{ 128147904Ssam struct g_eli_softc *sc; 129147904Ssam struct bio *bp; 130172686Smlaier 131172686Smlaier if (crp->crp_etype == EAGAIN) { 132172686Smlaier if (g_eli_crypto_rerun(crp) == 0) 133172686Smlaier return (0); 134147904Ssam } 135147904Ssam bp = (struct bio *)crp->crp_opaque; 136147904Ssam bp->bio_inbed++; 137147904Ssam sc = bp->bio_to->geom->softc; 138147904Ssam if (crp->crp_etype == 0) { 139147904Ssam bp->bio_completed += crp->crp_payload_length; 140147904Ssam G_ELI_DEBUG(3, "Crypto READ request done (%d/%d) (add=%d completed=%jd).", 141147904Ssam bp->bio_inbed, bp->bio_children, crp->crp_payload_length, (intmax_t)bp->bio_completed); 142147904Ssam } else { 143147904Ssam u_int nsec, decr_secsize, encr_secsize, rel_sec; 144147904Ssam int *errorp; 145147904Ssam 146147904Ssam /* Sectorsize of decrypted provider eg. 4096. */ 147147904Ssam decr_secsize = bp->bio_to->sectorsize; 14817680Spst /* The real sectorsize of encrypted provider, eg. 512. */ 14917680Spst encr_secsize = 15017680Spst LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; 15117680Spst /* Number of sectors from decrypted provider, eg. 2. */ 15217680Spst nsec = bp->bio_length / decr_secsize; 15375118Sfenner /* Number of sectors from encrypted provider, eg. 18. */ 15417680Spst nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; 155162021Ssam /* Which relative sector this request decrypted. */ 15675118Sfenner rel_sec = ((crp->crp_buf.cb_buf + crp->crp_payload_start) - 157147904Ssam (char *)bp->bio_driver2) / encr_secsize; 15817680Spst 15917680Spst errorp = (int *)((char *)bp->bio_driver2 + encr_secsize * nsec + 160172686Smlaier sizeof(int) * rel_sec); 161172686Smlaier *errorp = crp->crp_etype; 16217680Spst G_ELI_DEBUG(1, 16317680Spst "Crypto READ request failed (%d/%d) error=%d.", 16417680Spst bp->bio_inbed, bp->bio_children, crp->crp_etype); 16517680Spst if (bp->bio_error == 0 || bp->bio_error == EINTEGRITY) 16617680Spst bp->bio_error = crp->crp_etype == EBADMSG ? 16717680Spst EINTEGRITY : crp->crp_etype; 168162021Ssam } 169162021Ssam if (crp->crp_cipher_key != NULL) 17017680Spst g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key)); 171147904Ssam crypto_freereq(crp); 172147904Ssam /* 173147904Ssam * Do we have all sectors already? 174147904Ssam */ 175147904Ssam if (bp->bio_inbed < bp->bio_children) 176147904Ssam return (0); 177147904Ssam 178147904Ssam if (bp->bio_error == 0) { 179147904Ssam u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize; 180147904Ssam u_char *srcdata, *dstdata; 181147904Ssam 182147904Ssam /* Sectorsize of decrypted provider eg. 4096. */ 183147904Ssam decr_secsize = bp->bio_to->sectorsize; 184147904Ssam /* The real sectorsize of encrypted provider, eg. 512. */ 185147904Ssam encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; 186147904Ssam /* Number of data bytes in one encrypted sector, eg. 480. */ 187147904Ssam data_secsize = sc->sc_data_per_sector; 188147904Ssam /* Number of sectors from decrypted provider, eg. 2. */ 189147904Ssam nsec = bp->bio_length / decr_secsize; 190147904Ssam /* Number of sectors from encrypted provider, eg. 18. */ 191147904Ssam nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; 192147904Ssam /* Last sector number in every big sector, eg. 9. */ 193127675Sbms lsec = sc->sc_bytes_per_sector / encr_secsize; 19498527Sfenner 195147904Ssam srcdata = bp->bio_driver2; 196147904Ssam dstdata = bp->bio_data; 197147904Ssam 198147904Ssam for (i = 1; i <= nsec; i++) { 199147904Ssam data_secsize = sc->sc_data_per_sector; 200147904Ssam if ((i % lsec) == 0) 201172686Smlaier data_secsize = decr_secsize % data_secsize; 202147904Ssam bcopy(srcdata + sc->sc_alen, dstdata, data_secsize); 20398527Sfenner srcdata += encr_secsize; 20498527Sfenner dstdata += data_secsize; 20598527Sfenner } 20698527Sfenner } else if (bp->bio_error == EINTEGRITY) { 20798527Sfenner u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize; 20898527Sfenner int *errorp; 20998527Sfenner off_t coroff, corsize, dstoff; 21098527Sfenner 21198527Sfenner /* Sectorsize of decrypted provider eg. 4096. */ 21298527Sfenner decr_secsize = bp->bio_to->sectorsize; 21398527Sfenner /* The real sectorsize of encrypted provider, eg. 512. */ 21498527Sfenner encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; 21598527Sfenner /* Number of data bytes in one encrypted sector, eg. 480. */ 216146778Ssam data_secsize = sc->sc_data_per_sector; 217146778Ssam /* Number of sectors from decrypted provider, eg. 2. */ 218162021Ssam nsec = bp->bio_length / decr_secsize; 219146778Ssam /* Number of sectors from encrypted provider, eg. 18. */ 220146778Ssam nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; 221146778Ssam /* Last sector number in every big sector, eg. 9. */ 22217680Spst lsec = sc->sc_bytes_per_sector / encr_secsize; 22375118Sfenner 224172686Smlaier errorp = (int *)((char *)bp->bio_driver2 + encr_secsize * nsec); 225172686Smlaier coroff = -1; 226172686Smlaier corsize = 0; 227147904Ssam dstoff = bp->bio_offset; 228172686Smlaier 229162021Ssam for (i = 1; i <= nsec; i++) { 230162021Ssam data_secsize = sc->sc_data_per_sector; 231172686Smlaier if ((i % lsec) == 0) 232162021Ssam data_secsize = decr_secsize % data_secsize; 233172686Smlaier if (errorp[i - 1] == EBADMSG) { 234172686Smlaier /* 235162021Ssam * Corruption detected, remember the offset if 236147904Ssam * this is the first corrupted sector and 237172686Smlaier * increase size. 238147904Ssam */ 239172686Smlaier if (coroff == -1) 240147904Ssam coroff = dstoff; 241147904Ssam corsize += data_secsize; 242147904Ssam } else { 243147904Ssam /* 244147904Ssam * No corruption, good. 245147904Ssam * Report previous corruption if there was one. 24675118Sfenner */ 24775118Sfenner if (coroff != -1) { 24898527Sfenner G_ELI_DEBUG(0, "%s: Failed to authenticate %jd " 249147904Ssam "bytes of data at offset %jd.", 250147904Ssam sc->sc_name, (intmax_t)corsize, 251146778Ssam (intmax_t)coroff); 252127675Sbms coroff = -1; 253127675Sbms corsize = 0; 254127675Sbms } 255147904Ssam } 256147904Ssam dstoff += data_secsize; 25798527Sfenner } 25898527Sfenner /* Report previous corruption if there was one. */ 25998527Sfenner if (coroff != -1) { 26098527Sfenner G_ELI_DEBUG(0, "%s: Failed to authenticate %jd " 26198527Sfenner "bytes of data at offset %jd.", 26298527Sfenner sc->sc_name, (intmax_t)corsize, (intmax_t)coroff); 26398527Sfenner } 264162021Ssam } 265162021Ssam g_eli_free_data(bp); 266162021Ssam if (bp->bio_error != 0) { 267147904Ssam if (bp->bio_error != EINTEGRITY) { 26898527Sfenner G_ELI_LOGREQ(0, bp, 26998527Sfenner "Crypto READ request failed (error=%d).", 27098527Sfenner bp->bio_error); 27198527Sfenner } 272147904Ssam bp->bio_completed = 0; 273147904Ssam } 27456896Sfenner /* 27556896Sfenner * Read is finished, send it up. 27656896Sfenner */ 27756896Sfenner g_io_deliver(bp, bp->bio_error); 27875118Sfenner atomic_subtract_int(&sc->sc_inflight, 1); 27975118Sfenner return (0); 28075118Sfenner} 28175118Sfenner 28275118Sfenner/* 28356896Sfenner * The function is called after data encryption. 28475118Sfenner * 28575118Sfenner * g_eli_start -> g_eli_auth_run -> G_ELI_AUTH_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver 286147904Ssam */ 28775118Sfennerstatic int 288147904Ssamg_eli_auth_write_done(struct cryptop *crp) 289147904Ssam{ 290147904Ssam struct g_eli_softc *sc; 291147904Ssam struct g_consumer *cp; 29275118Sfenner struct bio *bp, *cbp, *cbp2; 293147904Ssam u_int nsec; 294147904Ssam 295147904Ssam if (crp->crp_etype == EAGAIN) { 29675118Sfenner if (g_eli_crypto_rerun(crp) == 0) 29798527Sfenner return (0); 29856896Sfenner } 29956896Sfenner bp = (struct bio *)crp->crp_opaque; 30098527Sfenner bp->bio_inbed++; 301147904Ssam if (crp->crp_etype == 0) { 302147904Ssam G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).", 303127675Sbms bp->bio_inbed, bp->bio_children); 30417680Spst } else { 30517680Spst G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.", 30617680Spst bp->bio_inbed, bp->bio_children, crp->crp_etype); 307147904Ssam if (bp->bio_error == 0) 308147904Ssam bp->bio_error = crp->crp_etype; 309127675Sbms } 310127675Sbms sc = bp->bio_to->geom->softc; 311127675Sbms if (crp->crp_cipher_key != NULL) 312127675Sbms g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key)); 313127675Sbms crypto_freereq(crp); 314214478Srpaulo /* 315127675Sbms * All sectors are already encrypted? 316127675Sbms */ 31717680Spst if (bp->bio_inbed < bp->bio_children) 31817680Spst return (0); 319147904Ssam if (bp->bio_error != 0) { 320162021Ssam G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).", 321147904Ssam bp->bio_error); 322162021Ssam g_eli_free_data(bp); 323147904Ssam cbp = bp->bio_driver1; 324147904Ssam bp->bio_driver1 = NULL; 325147904Ssam g_destroy_bio(cbp); 326147904Ssam g_io_deliver(bp, bp->bio_error); 327162021Ssam atomic_subtract_int(&sc->sc_inflight, 1); 328147904Ssam return (0); 329147904Ssam } 330147904Ssam cp = LIST_FIRST(&sc->sc_geom->consumer); 331162021Ssam cbp = bp->bio_driver1; 332162021Ssam bp->bio_driver1 = NULL; 333147904Ssam cbp->bio_to = cp->provider; 334147904Ssam cbp->bio_done = g_eli_write_done; 33517680Spst 336162021Ssam /* Number of sectors from decrypted provider, eg. 1. */ 33717680Spst nsec = bp->bio_length / bp->bio_to->sectorsize; 338162021Ssam /* Number of sectors from encrypted provider, eg. 9. */ 339147904Ssam nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize; 34017680Spst 34117680Spst cbp->bio_length = cp->provider->sectorsize * nsec; 342147904Ssam cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector; 343162021Ssam cbp->bio_data = bp->bio_driver2; 344162021Ssam 345172686Smlaier /* 346162021Ssam * We write more than what is requested, so we have to be ready to write 34775118Sfenner * more than maxphys. 34817680Spst */ 34917680Spst cbp2 = NULL; 35017680Spst if (cbp->bio_length > maxphys) { 35117680Spst cbp2 = g_duplicate_bio(bp); 352147904Ssam cbp2->bio_length = cbp->bio_length - maxphys; 35356896Sfenner cbp2->bio_data = cbp->bio_data + maxphys; 35456896Sfenner cbp2->bio_offset = cbp->bio_offset + maxphys; 35556896Sfenner cbp2->bio_to = cp->provider; 35656896Sfenner cbp2->bio_done = g_eli_write_done; 35756896Sfenner cbp->bio_length = maxphys; 35856896Sfenner } 35917680Spst /* 36017680Spst * Send encrypted data to the provider. 36175118Sfenner */ 362162021Ssam G_ELI_LOGREQ(2, cbp, "Sending request."); 363162021Ssam bp->bio_inbed = 0; 36475118Sfenner bp->bio_children = (cbp2 != NULL ? 2 : 1); 365172686Smlaier g_io_request(cbp, cp); 366162021Ssam if (cbp2 != NULL) { 36717680Spst G_ELI_LOGREQ(2, cbp2, "Sending request."); 368162021Ssam g_io_request(cbp2, cp); 36975118Sfenner } 37075118Sfenner return (0); 371172686Smlaier} 372162021Ssam 37317680Spstvoid 37417680Spstg_eli_auth_read(struct g_eli_softc *sc, struct bio *bp) 37517680Spst{ 37617680Spst struct g_consumer *cp; 37717680Spst struct bio *cbp, *cbp2; 37817680Spst size_t size; 37917680Spst off_t nsec; 380127675Sbms 381127675Sbms G_ELI_SETWORKER(bp->bio_pflags, 0); 382214478Srpaulo 383127675Sbms cp = LIST_FIRST(&sc->sc_geom->consumer); 384147904Ssam cbp = bp->bio_driver1; 385147904Ssam bp->bio_driver1 = NULL; 386127675Sbms cbp->bio_to = cp->provider; 387127675Sbms cbp->bio_done = g_eli_read_done; 388147904Ssam 389147904Ssam /* Number of sectors from decrypted provider, eg. 1. */ 390147904Ssam nsec = bp->bio_length / bp->bio_to->sectorsize; 391147904Ssam /* Number of sectors from encrypted provider, eg. 9. */ 392147904Ssam nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize; 393172686Smlaier 394147904Ssam cbp->bio_length = cp->provider->sectorsize * nsec; 395147904Ssam size = cbp->bio_length; 396147904Ssam size += sizeof(int) * nsec; 397147904Ssam size += G_ELI_AUTH_SECKEYLEN * nsec; 398147904Ssam cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector; 399147904Ssam if (!g_eli_alloc_data(bp, size)) { 400147904Ssam G_ELI_LOGREQ(0, bp, "Crypto auth read request failed (ENOMEM)"); 401147904Ssam g_destroy_bio(cbp); 402147904Ssam bp->bio_error = ENOMEM; 403147904Ssam g_io_deliver(bp, bp->bio_error); 404147904Ssam atomic_subtract_int(&sc->sc_inflight, 1); 405147904Ssam return; 406147904Ssam } 407147904Ssam cbp->bio_data = bp->bio_driver2; 408147904Ssam 409147904Ssam /* Clear the error array. */ 410147904Ssam memset((char *)bp->bio_driver2 + cbp->bio_length, 0, 411147904Ssam sizeof(int) * nsec); 412147904Ssam 413127675Sbms /* 414127675Sbms * We read more than what is requested, so we have to be ready to read 415127675Sbms * more than maxphys. 416127675Sbms */ 417127675Sbms cbp2 = NULL; 418127675Sbms if (cbp->bio_length > maxphys) { 419127675Sbms cbp2 = g_duplicate_bio(bp); 420127675Sbms cbp2->bio_length = cbp->bio_length - maxphys; 421127675Sbms cbp2->bio_data = cbp->bio_data + maxphys; 422235530Sdelphij cbp2->bio_offset = cbp->bio_offset + maxphys; 423127675Sbms cbp2->bio_to = cp->provider; 424127675Sbms cbp2->bio_done = g_eli_read_done; 425127675Sbms cbp->bio_length = maxphys; 426127675Sbms } 427127675Sbms /* 428127675Sbms * Read encrypted data from provider. 429127675Sbms */ 430127675Sbms G_ELI_LOGREQ(2, cbp, "Sending request."); 431127675Sbms g_io_request(cbp, cp); 432127675Sbms if (cbp2 != NULL) { 433127675Sbms G_ELI_LOGREQ(2, cbp2, "Sending request."); 434127675Sbms g_io_request(cbp2, cp); 435127675Sbms } 436127675Sbms} 437235530Sdelphij 438127675Sbms/* 439127675Sbms * This is the main function responsible for cryptography (ie. communication 440127675Sbms * with crypto(9) subsystem). 441127675Sbms * 442127675Sbms * BIO_READ: 443127675Sbms * g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> G_ELI_AUTH_RUN -> g_eli_auth_read_done -> g_io_deliver 444190207Srpaulo * BIO_WRITE: 445190207Srpaulo * g_eli_start -> G_ELI_AUTH_RUN -> g_eli_auth_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver 446190207Srpaulo */ 447190207Srpaulovoid 448190207Srpaulog_eli_auth_run(struct g_eli_worker *wr, struct bio *bp) 449190207Srpaulo{ 450190207Srpaulo struct g_eli_softc *sc; 451190207Srpaulo struct cryptopq crpq; 452190207Srpaulo struct cryptop *crp; 453190207Srpaulo u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize; 454190207Srpaulo off_t dstoff; 455190207Srpaulo u_char *p, *data, *authkey, *plaindata; 456190207Srpaulo int error __diagused; 457190207Srpaulo bool batch; 458190207Srpaulo 459190207Srpaulo G_ELI_LOGREQ(3, bp, "%s", __func__); 460190207Srpaulo 461190207Srpaulo G_ELI_SETWORKER(bp->bio_pflags, wr->w_number); 462190207Srpaulo sc = wr->w_softc; 463127675Sbms /* Sectorsize of decrypted provider eg. 4096. */ 464127675Sbms decr_secsize = bp->bio_to->sectorsize; 465127675Sbms /* The real sectorsize of encrypted provider, eg. 512. */ 466127675Sbms encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; 467127675Sbms /* Number of data bytes in one encrypted sector, eg. 480. */ 468127675Sbms data_secsize = sc->sc_data_per_sector; 469127675Sbms /* Number of sectors from decrypted provider, eg. 2. */ 470127675Sbms nsec = bp->bio_length / decr_secsize; 471127675Sbms /* Number of sectors from encrypted provider, eg. 18. */ 472127675Sbms nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; 473127675Sbms /* Last sector number in every big sector, eg. 9. */ 474127675Sbms lsec = sc->sc_bytes_per_sector / encr_secsize; 475127675Sbms /* Destination offset, used for IV generation. */ 476147904Ssam dstoff = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector; 477127675Sbms 478127675Sbms plaindata = bp->bio_data; 479127675Sbms if (bp->bio_cmd == BIO_READ) { 480127675Sbms data = bp->bio_driver2; 481127675Sbms p = data + encr_secsize * nsec; 482127675Sbms p += sizeof(int) * nsec; 483127675Sbms } else { 484235530Sdelphij size_t size; 485127675Sbms 486127675Sbms size = encr_secsize * nsec; 487127675Sbms size += G_ELI_AUTH_SECKEYLEN * nsec; 488127675Sbms size += sizeof(uintptr_t); /* Space for alignment. */ 489127675Sbms if (!g_eli_alloc_data(bp, size)) { 490127675Sbms G_ELI_LOGREQ(0, bp, "Crypto request failed (ENOMEM)"); 491127675Sbms if (bp->bio_driver1 != NULL) { 492127675Sbms g_destroy_bio(bp->bio_driver1); 493127675Sbms bp->bio_driver1 = NULL; 494127675Sbms } 495127675Sbms bp->bio_error = ENOMEM; 496127675Sbms g_io_deliver(bp, bp->bio_error); 497147904Ssam if (sc != NULL) 498127675Sbms atomic_subtract_int(&sc->sc_inflight, 1); 499127675Sbms return; 500127675Sbms } 501127675Sbms data = bp->bio_driver2; 502127675Sbms p = data + encr_secsize * nsec; 503127675Sbms } 504127675Sbms bp->bio_inbed = 0; 505127675Sbms bp->bio_children = nsec; 506127675Sbms 507127675Sbms#if defined(__mips_n64) || defined(__mips_o64) 508127675Sbms p = (char *)roundup((uintptr_t)p, sizeof(uintptr_t)); 509127675Sbms#endif 510127675Sbms 511127675Sbms TAILQ_INIT(&crpq); 512127675Sbms batch = atomic_load_int(&g_eli_batch) != 0; 513127675Sbms 514127675Sbms for (i = 1; i <= nsec; i++, dstoff += encr_secsize) { 515127675Sbms crp = crypto_getreq(wr->w_sid, M_WAITOK); 516127675Sbms authkey = (u_char *)p; p += G_ELI_AUTH_SECKEYLEN; 517127675Sbms 518147904Ssam data_secsize = sc->sc_data_per_sector; 519127675Sbms if ((i % lsec) == 0) { 520127675Sbms data_secsize = decr_secsize % data_secsize; 521127675Sbms /* 522127675Sbms * Last encrypted sector of each decrypted sector is 523127675Sbms * only partially filled. 524127675Sbms */ 525127675Sbms if (bp->bio_cmd == BIO_WRITE) 526127675Sbms memset(data + sc->sc_alen + data_secsize, 0, 527127675Sbms encr_secsize - sc->sc_alen - data_secsize); 528127675Sbms } else if (data_secsize + sc->sc_alen != encr_secsize) { 529127675Sbms /* 530127675Sbms * If the HMAC size is not a multiple of 128 bits, the 531127675Sbms * per-sector data size is rounded down to ensure that 532127675Sbms * encryption can be performed without requiring any 533127675Sbms * padding. In this case, each sector contains unused 534127675Sbms * bytes. 535147904Ssam */ 536147904Ssam if (bp->bio_cmd == BIO_WRITE) 537147904Ssam memset(data + sc->sc_alen + data_secsize, 0, 538147904Ssam encr_secsize - sc->sc_alen - data_secsize); 539127675Sbms } 540146778Ssam 541146778Ssam if (bp->bio_cmd == BIO_WRITE) { 542146778Ssam bcopy(plaindata, data + sc->sc_alen, data_secsize); 543146778Ssam plaindata += data_secsize; 544146778Ssam } 545146778Ssam 546146778Ssam crypto_use_buf(crp, data, sc->sc_alen + data_secsize); 547146778Ssam crp->crp_opaque = (void *)bp; 548 data += encr_secsize; 549 crp->crp_flags = CRYPTO_F_CBIFSYNC; 550 if (bp->bio_cmd == BIO_WRITE) { 551 crp->crp_callback = g_eli_auth_write_done; 552 crp->crp_op = CRYPTO_OP_ENCRYPT | 553 CRYPTO_OP_COMPUTE_DIGEST; 554 } else { 555 crp->crp_callback = g_eli_auth_read_done; 556 crp->crp_op = CRYPTO_OP_DECRYPT | 557 CRYPTO_OP_VERIFY_DIGEST; 558 } 559 560 crp->crp_digest_start = 0; 561 crp->crp_payload_start = sc->sc_alen; 562 crp->crp_payload_length = data_secsize; 563 if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) == 0) { 564 crp->crp_cipher_key = g_eli_key_hold(sc, dstoff, 565 encr_secsize); 566 } 567 if (g_eli_ivlen(sc->sc_ealgo) != 0) { 568 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 569 g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv, 570 sizeof(crp->crp_iv)); 571 } 572 573 g_eli_auth_keygen(sc, dstoff, authkey); 574 crp->crp_auth_key = authkey; 575 576 if (batch) { 577 TAILQ_INSERT_TAIL(&crpq, crp, crp_next); 578 } else { 579 error = crypto_dispatch(crp); 580 KASSERT(error == 0, 581 ("crypto_dispatch() failed (error=%d)", error)); 582 } 583 } 584 585 if (batch) 586 crypto_dispatch_batch(&crpq, 0); 587} 588