module/zfs/vdev_raidz.c

395 vdev_raidz_map_free(raidz_map_t *rm)
397 	for (int i = 0; i < rm->rm_nrows; i++)
398 		vdev_raidz_row_free(rm->rm_row[i]);
400 	if (rm->rm_nphys_cols) {
401 		for (int i = 0; i < rm->rm_nphys_cols; i++) {
402 			if (rm->rm_phys_col[i].rc_abd != NULL)
403 				abd_free(rm->rm_phys_col[i].rc_abd);
406 		kmem_free(rm->rm_phys_col, sizeof (raidz_col_t) *
407 		    rm->rm_nphys_cols);
410 	ASSERT3P(rm->rm_lr, ==, NULL);
411 	kmem_free(rm, offsetof(raidz_map_t, rm_row[rm->rm_nrows]));
417 	raidz_map_t *rm = zio->io_vsd;
419 	vdev_raidz_map_free(rm);
454 vdev_raidz_map_alloc_write(zio_t *zio, raidz_map_t *rm, uint64_t ashift)
459 	raidz_row_t *rr = rm->rm_row[0];
462 	ASSERT3U(rm->rm_nrows, ==, 1);
468 	if (rm->rm_skipstart < rr->rr_firstdatacol) {
469 		ASSERT0(rm->rm_skipstart);
470 		nwrapped = rm->rm_nskip;
471 	} else if (rr->rr_scols < (rm->rm_skipstart + rm->rm_nskip)) {
473 		    (rm->rm_skipstart + rm->rm_nskip) % rr->rr_scols;
524 		if (c >= rm->rm_skipstart && skipped < rm->rm_nskip) {
537 	ASSERT3S(skipped, ==, rm->rm_nskip);
541 vdev_raidz_map_alloc_read(zio_t *zio, raidz_map_t *rm)
544 	raidz_row_t *rr = rm->rm_row[0];
546 	ASSERT3U(rm->rm_nrows, ==, 1);
583 	raidz_map_t *rm =
585 	rm->rm_nrows = 1;
623 	rm->rm_row[0] = rr;
656 	rm->rm_nskip = roundup(tot, nparity + 1) - tot;
657 	rm->rm_skipstart = bc;
689 		if (rm->rm_skipstart == 0)
690 			rm->rm_skipstart = 1;
694 		vdev_raidz_map_alloc_write(zio, rm, ashift);
696 		vdev_raidz_map_alloc_read(zio, rm);
699 	rm->rm_ops = vdev_raidz_math_get_ops();
701 	return (rm);
758 	raidz_map_t *rm =
761 	rm->rm_nrows = rows;
762 	rm->rm_nskip = roundup(tot, nparity + 1) - tot;
763 	rm->rm_skipstart = bc;
769 		rm->rm_row[row] = rr;
939 		rm->rm_nphys_cols = physical_cols;
940 		rm->rm_phys_col =
941 		    kmem_zalloc(sizeof (raidz_col_t) * rm->rm_nphys_cols,
949 		    i < rm->rm_nrows && rm->rm_phys_col != NULL; i++) {
950 			raidz_row_t *rr = rm->rm_row[i];
954 				    &rm->rm_phys_col[rc->rc_devidx];
972 					kmem_free(rm->rm_phys_col,
974 					    rm->rm_nphys_cols);
975 					rm->rm_phys_col = NULL;
976 					rm->rm_nphys_cols = 0;
983 	if (rm->rm_phys_col != NULL) {
987 		for (int i = 0; i < rm->rm_nphys_cols; i++) {
988 			raidz_col_t *prc = &rm->rm_phys_col[i];
996 			    abd_alloc_linear(rm->rm_phys_col[i].rc_size,
1003 		for (int i = 0; i < rm->rm_nrows; i++) {
1004 			raidz_row_t *rr = rm->rm_row[i];
1008 				    &rm->rm_phys_col[rc->rc_devidx];
1020 		for (int i = 0; i < rm->rm_nrows; i++) {
1021 			raidz_row_t *rr = rm->rm_row[i];
1031 	rm->rm_ops = vdev_raidz_math_get_ops();
1033 	return (rm);
1210 vdev_raidz_generate_parity_row(raidz_map_t *rm, raidz_row_t *rr)
1223 	if (vdev_raidz_math_generate(rm, rr) != RAIDZ_ORIGINAL_IMPL)
1242 vdev_raidz_generate_parity(raidz_map_t *rm)
1244 	for (int i = 0; i < rm->rm_nrows; i++) {
1245 		raidz_row_t *rr = rm->rm_row[i];
1246 		vdev_raidz_generate_parity_row(rm, rr);
1373 		zfs_dbgmsg("reconstruct_p(rm=%px x=%u)", rr, x);
1408 		zfs_dbgmsg("reconstruct_q(rm=%px x=%u)", rr, x);
1457 		zfs_dbgmsg("reconstruct_pq(rm=%px x=%u y=%u)", rr, x, y);
1908 		zfs_dbgmsg("reconstruct_general(rm=%px ntgts=%u)", rr, ntgts);
2033 vdev_raidz_reconstruct_row(raidz_map_t *rm, raidz_row_t *rr,
2043 		zfs_dbgmsg("reconstruct(rm=%px nt=%u cols=%u md=%u mp=%u)",
2053 			zfs_dbgmsg("reconstruct(rm=%px col=%u devid=%u "
2082 	ret = vdev_raidz_math_reconstruct(rm, rr, parity_valid, dt, nbaddata);
2290 vdev_raidz_io_verify(zio_t *zio, raidz_map_t *rm, raidz_row_t *rr, int col)
2292 	(void) rm;
2334 	raidz_map_t *rm = zio->io_vsd;
2336 	vdev_raidz_generate_parity_row(rm, rr);
2343 		vdev_raidz_io_verify(zio, rm, rr, c);
2382 	raidz_map_t *rm = zio->io_vsd;
2383 	ASSERT3U(rm->rm_nrows, ==, 1);
2384 	raidz_row_t *rr = rm->rm_row[0];
2446 vdev_raidz_io_start_read_phys_cols(zio_t *zio, raidz_map_t *rm)
2450 	for (int i = 0; i < rm->rm_nphys_cols; i++) {
2451 		raidz_col_t *prc = &rm->rm_phys_col[i];
2476 vdev_raidz_io_start_read(zio_t *zio, raidz_map_t *rm)
2483 	boolean_t forceparity = rm->rm_nrows > 1;
2485 	if (rm->rm_phys_col) {
2486 		vdev_raidz_io_start_read_phys_cols(zio, rm);
2488 		for (int i = 0; i < rm->rm_nrows; i++) {
2489 			raidz_row_t *rr = rm->rm_row[i];
2518 	raidz_map_t *rm;
2571 		rm = vdev_raidz_map_alloc_expanded(zio,
2575 		rm->rm_lr = lr;
2577 		rm = vdev_raidz_map_alloc(zio,
2580 	rm->rm_original_width = vdrz->vd_original_width;
2582 	zio->io_vsd = rm;
2585 		for (int i = 0; i < rm->rm_nrows; i++) {
2586 			vdev_raidz_io_start_write(zio, rm->rm_row[i]);
2594 		vdev_raidz_io_start_read(zio, rm);
2611 		raidz_map_t *rm = zio->io_vsd;
2614 		zbc.zbc_injected = rm->rm_ecksuminjected;
2633 	raidz_map_t *rm = zio->io_vsd;
2637 		rm->rm_ecksuminjected = 1;
2653 	raidz_map_t *rm = zio->io_vsd;
2685 	vdev_raidz_generate_parity_row(rm, rr);
2832 raidz_restore_orig_data(raidz_map_t *rm)
2834 	for (int i = 0; i < rm->rm_nrows; i++) {
2835 		raidz_row_t *rr = rm->rm_row[i];
2892 	raidz_map_t *rm = zio->io_vsd;
2894 	int original_width = (rm->rm_original_width != 0) ?
2895 	    rm->rm_original_width : physical_width;
2904 	for (int r = 0; r < rm->rm_nrows; r++) {
2905 		raidz_row_t *rr = rm->rm_row[r];
2973 			raidz_restore_orig_data(rm);
2977 			vdev_raidz_reconstruct_row(rm, rr, my_tgts, t);
2984 		for (int i = 0; i < rm->rm_nrows; i++) {
2985 			raidz_row_t *rr = rm->rm_row[i];
3024 	raidz_restore_orig_data(rm);
3085 	raidz_map_t *rm = zio->io_vsd;
3087 	int original_width = (rm->rm_original_width != 0) ?
3088 	    rm->rm_original_width : physical_width;
3090 	for (int i = 0; i < rm->rm_nrows; i++) {
3091 		raidz_row_t *rr = rm->rm_row[i];
3187 vdev_raidz_reconstruct(raidz_map_t *rm, const int *t, int nt)
3189 	for (uint64_t row = 0; row < rm->rm_nrows; row++) {
3190 		raidz_row_t *rr = rm->rm_row[row];
3191 		vdev_raidz_reconstruct_row(rm, rr, t, nt);
3248 vdev_raidz_io_done_reconstruct_known_missing(zio_t *zio, raidz_map_t *rm,
3319 		vdev_raidz_reconstruct_row(rm, rr, tgts, n);
3367 	raidz_map_t *rm = zio->io_vsd;
3369 	for (int i = 0; i < rm->rm_nrows; i++) {
3370 		raidz_row_t *rr = rm->rm_row[i];
3381 			zbc.zbc_injected = rm->rm_ecksuminjected;
3396 	raidz_map_t *rm = zio->io_vsd;
3400 		for (int i = 0; i < rm->rm_nrows; i++) {
3401 			vdev_raidz_io_done_write_impl(zio, rm->rm_row[i]);
3404 		if (rm->rm_phys_col) {
3409 			for (int i = 0; i < rm->rm_nrows; i++) {
3410 				raidz_row_t *rr = rm->rm_row[i];
3418 					    &rm->rm_phys_col[rc->rc_devidx];
3440 		for (int i = 0; i < rm->rm_nrows; i++) {
3441 			raidz_row_t *rr = rm->rm_row[i];
3443 			    rm, rr);
3447 			for (int i = 0; i < rm->rm_nrows; i++) {
3448 				raidz_row_t *rr = rm->rm_row[i];
3471 			for (int i = 0; i < rm->rm_nrows; i++) {
3473 				    rm->rm_row[i]);
3541 	if (rm->rm_lr != NULL) {
3542 		zfs_rangelock_exit(rm->rm_lr);
3543 		rm->rm_lr = NULL;