1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright (c) 2011, 2015 by Delphix. All rights reserved. 24 */ 25 26#include <sys/zfs_context.h> 27#include <sys/zfeature.h> 28#include <sys/dmu.h> 29#include <sys/nvpair.h> 30#include <sys/zap.h> 31#include <sys/dmu_tx.h> 32#include "zfeature_common.h" 33#include <sys/spa_impl.h> 34 35/* 36 * ZFS Feature Flags 37 * ----------------- 38 * 39 * ZFS feature flags are used to provide fine-grained versioning to the ZFS 40 * on-disk format. Once enabled on a pool feature flags replace the old 41 * spa_version() number. 42 * 43 * Each new on-disk format change will be given a uniquely identifying string 44 * GUID rather than a version number. This avoids the problem of different 45 * organizations creating new on-disk formats with the same version number. To 46 * keep feature GUIDs unique they should consist of the reverse dns name of the 47 * organization which implemented the feature and a short name for the feature, 48 * separated by a colon (e.g. com.delphix:async_destroy). 49 * 50 * Reference Counts 51 * ---------------- 52 * 53 * Within each pool features can be in one of three states: disabled, enabled, 54 * or active. These states are differentiated by a reference count stored on 55 * disk for each feature: 56 * 57 * 1) If there is no reference count stored on disk the feature is disabled. 58 * 2) If the reference count is 0 a system administrator has enabled the 59 * feature, but the feature has not been used yet, so no on-disk 60 * format changes have been made. 61 * 3) If the reference count is greater than 0 the feature is active. 62 * The format changes required by the feature are currently on disk. 63 * Note that if the feature's format changes are reversed the feature 64 * may choose to set its reference count back to 0. 65 * 66 * Feature flags makes no differentiation between non-zero reference counts 67 * for an active feature (e.g. a reference count of 1 means the same thing as a 68 * reference count of 27834721), but feature implementations may choose to use 69 * the reference count to store meaningful information. For example, a new RAID 70 * implementation might set the reference count to the number of vdevs using 71 * it. If all those disks are removed from the pool the feature goes back to 72 * having a reference count of 0. 73 * 74 * It is the responsibility of the individual features to maintain a non-zero 75 * reference count as long as the feature's format changes are present on disk. 76 * 77 * Dependencies 78 * ------------ 79 * 80 * Each feature may depend on other features. The only effect of this 81 * relationship is that when a feature is enabled all of its dependencies are 82 * automatically enabled as well. Any future work to support disabling of 83 * features would need to ensure that features cannot be disabled if other 84 * enabled features depend on them. 85 * 86 * On-disk Format 87 * -------------- 88 * 89 * When feature flags are enabled spa_version() is set to SPA_VERSION_FEATURES 90 * (5000). In order for this to work the pool is automatically upgraded to 91 * SPA_VERSION_BEFORE_FEATURES (28) first, so all pre-feature flags on disk 92 * format changes will be in use. 93 * 94 * Information about features is stored in 3 ZAP objects in the pool's MOS. 95 * These objects are linked to by the following names in the pool directory 96 * object: 97 * 98 * 1) features_for_read: feature GUID -> reference count 99 * Features needed to open the pool for reading. 100 * 2) features_for_write: feature GUID -> reference count 101 * Features needed to open the pool for writing. 102 * 3) feature_descriptions: feature GUID -> descriptive string 103 * A human readable string. 104 * 105 * All enabled features appear in either features_for_read or 106 * features_for_write, but not both. 107 * 108 * To open a pool in read-only mode only the features listed in 109 * features_for_read need to be supported. 110 * 111 * To open the pool in read-write mode features in both features_for_read and 112 * features_for_write need to be supported. 113 * 114 * Some features may be required to read the ZAP objects containing feature 115 * information. To allow software to check for compatibility with these features 116 * before the pool is opened their names must be stored in the label in a 117 * new "features_for_read" entry (note that features that are only required 118 * to write to a pool never need to be stored in the label since the 119 * features_for_write ZAP object can be read before the pool is written to). 120 * To save space in the label features must be explicitly marked as needing to 121 * be written to the label. Also, reference counts are not stored in the label, 122 * instead any feature whose reference count drops to 0 is removed from the 123 * label. 124 * 125 * Adding New Features 126 * ------------------- 127 * 128 * Features must be registered in zpool_feature_init() function in 129 * zfeature_common.c using the zfeature_register() function. This function 130 * has arguments to specify if the feature should be stored in the 131 * features_for_read or features_for_write ZAP object and if it needs to be 132 * written to the label when active. 133 * 134 * Once a feature is registered it will appear as a "feature@<feature name>" 135 * property which can be set by an administrator. Feature implementors should 136 * use the spa_feature_is_enabled() and spa_feature_is_active() functions to 137 * query the state of a feature and the spa_feature_incr() and 138 * spa_feature_decr() functions to change an enabled feature's reference count. 139 * Reference counts may only be updated in the syncing context. 140 * 141 * Features may not perform enable-time initialization. Instead, any such 142 * initialization should occur when the feature is first used. This design 143 * enforces that on-disk changes be made only when features are used. Code 144 * should only check if a feature is enabled using spa_feature_is_enabled(), 145 * not by relying on any feature specific metadata existing. If a feature is 146 * enabled, but the feature's metadata is not on disk yet then it should be 147 * created as needed. 148 * 149 * As an example, consider the com.delphix:async_destroy feature. This feature 150 * relies on the existence of a bptree in the MOS that store blocks for 151 * asynchronous freeing. This bptree is not created when async_destroy is 152 * enabled. Instead, when a dataset is destroyed spa_feature_is_enabled() is 153 * called to check if async_destroy is enabled. If it is and the bptree object 154 * does not exist yet, the bptree object is created as part of the dataset 155 * destroy and async_destroy's reference count is incremented to indicate it 156 * has made an on-disk format change. Later, after the destroyed dataset's 157 * blocks have all been asynchronously freed there is no longer any use for the 158 * bptree object, so it is destroyed and async_destroy's reference count is 159 * decremented back to 0 to indicate that it has undone its on-disk format 160 * changes. 161 */ 162 163typedef enum { 164 FEATURE_ACTION_INCR, 165 FEATURE_ACTION_DECR, 166} feature_action_t; 167 168/* 169 * Checks that the active features in the pool are supported by 170 * this software. Adds each unsupported feature (name -> description) to 171 * the supplied nvlist. 172 */ 173boolean_t 174spa_features_check(spa_t *spa, boolean_t for_write, 175 nvlist_t *unsup_feat, nvlist_t *enabled_feat) 176{ 177 objset_t *os = spa->spa_meta_objset; 178 boolean_t supported; 179 zap_cursor_t *zc; 180 zap_attribute_t *za; 181 uint64_t obj = for_write ? 182 spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj; 183 char *buf; 184 185 zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP); 186 za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP); 187 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP); 188 189 supported = B_TRUE; 190 for (zap_cursor_init(zc, os, obj); 191 zap_cursor_retrieve(zc, za) == 0; 192 zap_cursor_advance(zc)) { 193 ASSERT(za->za_integer_length == sizeof (uint64_t) && 194 za->za_num_integers == 1); 195 196 if (NULL != enabled_feat) { 197 fnvlist_add_uint64(enabled_feat, za->za_name, 198 za->za_first_integer); 199 } 200 201 if (za->za_first_integer != 0 && 202 !zfeature_is_supported(za->za_name)) { 203 supported = B_FALSE; 204 205 if (NULL != unsup_feat) { 206 const char *desc = ""; 207 208 if (zap_lookup(os, spa->spa_feat_desc_obj, 209 za->za_name, 1, MAXPATHLEN, buf) == 0) 210 desc = buf; 211 212 VERIFY(nvlist_add_string(unsup_feat, 213 za->za_name, desc) == 0); 214 } 215 } 216 } 217 zap_cursor_fini(zc); 218 219 kmem_free(buf, MAXPATHLEN); 220 kmem_free(za, sizeof (zap_attribute_t)); 221 kmem_free(zc, sizeof (zap_cursor_t)); 222 223 return (supported); 224} 225 226/* 227 * Use an in-memory cache of feature refcounts for quick retrieval. 228 * 229 * Note: well-designed features will not need to use this; they should 230 * use spa_feature_is_enabled() and spa_feature_is_active() instead. 231 * However, this is non-static for zdb, zhack, and spa_add_feature_stats(). 232 */ 233int 234feature_get_refcount(spa_t *spa, zfeature_info_t *feature, uint64_t *res) 235{ 236 ASSERT(VALID_FEATURE_FID(feature->fi_feature)); 237 if (spa->spa_feat_refcount_cache[feature->fi_feature] == 238 SPA_FEATURE_DISABLED) { 239 return (SET_ERROR(ENOTSUP)); 240 } 241 *res = spa->spa_feat_refcount_cache[feature->fi_feature]; 242 return (0); 243} 244 245/* 246 * Note: well-designed features will not need to use this; they should 247 * use spa_feature_is_enabled() and spa_feature_is_active() instead. 248 * However, this is non-static for zdb and zhack. 249 */ 250int 251feature_get_refcount_from_disk(spa_t *spa, zfeature_info_t *feature, 252 uint64_t *res) 253{ 254 int err; 255 uint64_t refcount; 256 uint64_t zapobj = (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ? 257 spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj; 258 259 /* 260 * If the pool is currently being created, the feature objects may not 261 * have been allocated yet. Act as though all features are disabled. 262 */ 263 if (zapobj == 0) 264 return (SET_ERROR(ENOTSUP)); 265 266 err = zap_lookup(spa->spa_meta_objset, zapobj, 267 feature->fi_guid, sizeof (uint64_t), 1, &refcount); 268 if (err != 0) { 269 if (err == ENOENT) 270 return (SET_ERROR(ENOTSUP)); 271 else 272 return (err); 273 } 274 *res = refcount; 275 return (0); 276} 277 278 279static int 280feature_get_enabled_txg(spa_t *spa, zfeature_info_t *feature, uint64_t *res) 281{ 282 uint64_t enabled_txg_obj __maybe_unused = spa->spa_feat_enabled_txg_obj; 283 284 ASSERT(zfeature_depends_on(feature->fi_feature, 285 SPA_FEATURE_ENABLED_TXG)); 286 287 if (!spa_feature_is_enabled(spa, feature->fi_feature)) { 288 return (SET_ERROR(ENOTSUP)); 289 } 290 291 ASSERT(enabled_txg_obj != 0); 292 293 VERIFY0(zap_lookup(spa->spa_meta_objset, spa->spa_feat_enabled_txg_obj, 294 feature->fi_guid, sizeof (uint64_t), 1, res)); 295 296 return (0); 297} 298 299/* 300 * This function is non-static for zhack; it should otherwise not be used 301 * outside this file. 302 */ 303void 304feature_sync(spa_t *spa, zfeature_info_t *feature, uint64_t refcount, 305 dmu_tx_t *tx) 306{ 307 ASSERT(VALID_FEATURE_OR_NONE(feature->fi_feature)); 308 uint64_t zapobj = (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ? 309 spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj; 310 VERIFY0(zap_update(spa->spa_meta_objset, zapobj, feature->fi_guid, 311 sizeof (uint64_t), 1, &refcount, tx)); 312 313 /* 314 * feature_sync is called directly from zhack, allowing the 315 * creation of arbitrary features whose fi_feature field may 316 * be greater than SPA_FEATURES. When called from zhack, the 317 * zfeature_info_t object's fi_feature field will be set to 318 * SPA_FEATURE_NONE. 319 */ 320 if (feature->fi_feature != SPA_FEATURE_NONE) { 321 uint64_t *refcount_cache = 322 &spa->spa_feat_refcount_cache[feature->fi_feature]; 323 VERIFY3U(*refcount_cache, ==, 324 atomic_swap_64(refcount_cache, refcount)); 325 } 326 327 if (refcount == 0) 328 spa_deactivate_mos_feature(spa, feature->fi_guid); 329 else if (feature->fi_flags & ZFEATURE_FLAG_MOS) 330 spa_activate_mos_feature(spa, feature->fi_guid, tx); 331} 332 333/* 334 * This function is non-static for zhack; it should otherwise not be used 335 * outside this file. 336 */ 337void 338feature_enable_sync(spa_t *spa, zfeature_info_t *feature, dmu_tx_t *tx) 339{ 340 uint64_t initial_refcount = 341 (feature->fi_flags & ZFEATURE_FLAG_ACTIVATE_ON_ENABLE) ? 1 : 0; 342 uint64_t zapobj = (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ? 343 spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj; 344 345 ASSERT(0 != zapobj); 346 ASSERT(zfeature_is_valid_guid(feature->fi_guid)); 347 ASSERT3U(spa_version(spa), >=, SPA_VERSION_FEATURES); 348 349 /* 350 * If the feature is already enabled, ignore the request. 351 */ 352 if (zap_contains(spa->spa_meta_objset, zapobj, feature->fi_guid) == 0) 353 return; 354 355 for (int i = 0; feature->fi_depends[i] != SPA_FEATURE_NONE; i++) 356 spa_feature_enable(spa, feature->fi_depends[i], tx); 357 358 VERIFY0(zap_update(spa->spa_meta_objset, spa->spa_feat_desc_obj, 359 feature->fi_guid, 1, strlen(feature->fi_desc) + 1, 360 feature->fi_desc, tx)); 361 362 feature_sync(spa, feature, initial_refcount, tx); 363 364 if (spa_feature_is_enabled(spa, SPA_FEATURE_ENABLED_TXG)) { 365 uint64_t enabling_txg = dmu_tx_get_txg(tx); 366 367 if (spa->spa_feat_enabled_txg_obj == 0ULL) { 368 spa->spa_feat_enabled_txg_obj = 369 zap_create_link(spa->spa_meta_objset, 370 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 371 DMU_POOL_FEATURE_ENABLED_TXG, tx); 372 } 373 spa_feature_incr(spa, SPA_FEATURE_ENABLED_TXG, tx); 374 375 VERIFY0(zap_add(spa->spa_meta_objset, 376 spa->spa_feat_enabled_txg_obj, feature->fi_guid, 377 sizeof (uint64_t), 1, &enabling_txg, tx)); 378 } 379 380 /* 381 * Errata #4 is mostly a problem with encrypted datasets, but it 382 * is also a problem where the old encryption feature did not 383 * depend on the bookmark_v2 feature. If the pool does not have 384 * any encrypted datasets we can resolve this issue simply by 385 * enabling this dependency. 386 */ 387 if (spa->spa_errata == ZPOOL_ERRATA_ZOL_8308_ENCRYPTION && 388 spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) && 389 !spa_feature_is_active(spa, SPA_FEATURE_ENCRYPTION) && 390 feature->fi_feature == SPA_FEATURE_BOOKMARK_V2) 391 spa->spa_errata = 0; 392 393 /* 394 * Convert the old on-disk error log to the new format when activating 395 * the head_errlog feature. 396 */ 397 if (feature->fi_feature == SPA_FEATURE_HEAD_ERRLOG) 398 spa_upgrade_errlog(spa, tx); 399} 400 401static void 402feature_do_action(spa_t *spa, spa_feature_t fid, feature_action_t action, 403 dmu_tx_t *tx) 404{ 405 uint64_t refcount = 0; 406 zfeature_info_t *feature = &spa_feature_table[fid]; 407 uint64_t zapobj __maybe_unused = 408 (feature->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ? 409 spa->spa_feat_for_write_obj : spa->spa_feat_for_read_obj; 410 411 ASSERT(VALID_FEATURE_FID(fid)); 412 ASSERT(0 != zapobj); 413 ASSERT(zfeature_is_valid_guid(feature->fi_guid)); 414 415 ASSERT(dmu_tx_is_syncing(tx)); 416 ASSERT3U(spa_version(spa), >=, SPA_VERSION_FEATURES); 417 418 VERIFY3U(feature_get_refcount(spa, feature, &refcount), !=, ENOTSUP); 419 420 switch (action) { 421 case FEATURE_ACTION_INCR: 422 VERIFY3U(refcount, !=, UINT64_MAX); 423 refcount++; 424 break; 425 case FEATURE_ACTION_DECR: 426 VERIFY3U(refcount, !=, 0); 427 refcount--; 428 break; 429 default: 430 ASSERT(0); 431 break; 432 } 433 434 feature_sync(spa, feature, refcount, tx); 435} 436 437void 438spa_feature_create_zap_objects(spa_t *spa, dmu_tx_t *tx) 439{ 440 /* 441 * We create feature flags ZAP objects in two instances: during pool 442 * creation and during pool upgrade. 443 */ 444 ASSERT((!spa->spa_sync_on && tx->tx_txg == TXG_INITIAL) || 445 dsl_pool_sync_context(spa_get_dsl(spa))); 446 447 spa->spa_feat_for_read_obj = zap_create_link(spa->spa_meta_objset, 448 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 449 DMU_POOL_FEATURES_FOR_READ, tx); 450 spa->spa_feat_for_write_obj = zap_create_link(spa->spa_meta_objset, 451 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 452 DMU_POOL_FEATURES_FOR_WRITE, tx); 453 spa->spa_feat_desc_obj = zap_create_link(spa->spa_meta_objset, 454 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 455 DMU_POOL_FEATURE_DESCRIPTIONS, tx); 456} 457 458/* 459 * Enable any required dependencies, then enable the requested feature. 460 */ 461void 462spa_feature_enable(spa_t *spa, spa_feature_t fid, dmu_tx_t *tx) 463{ 464 ASSERT3U(spa_version(spa), >=, SPA_VERSION_FEATURES); 465 ASSERT(VALID_FEATURE_FID(fid)); 466 feature_enable_sync(spa, &spa_feature_table[fid], tx); 467} 468 469void 470spa_feature_incr(spa_t *spa, spa_feature_t fid, dmu_tx_t *tx) 471{ 472 feature_do_action(spa, fid, FEATURE_ACTION_INCR, tx); 473} 474 475void 476spa_feature_decr(spa_t *spa, spa_feature_t fid, dmu_tx_t *tx) 477{ 478 feature_do_action(spa, fid, FEATURE_ACTION_DECR, tx); 479} 480 481boolean_t 482spa_feature_is_enabled(spa_t *spa, spa_feature_t fid) 483{ 484 int err; 485 uint64_t refcount = 0; 486 487 ASSERT(VALID_FEATURE_FID(fid)); 488 if (spa_version(spa) < SPA_VERSION_FEATURES) 489 return (B_FALSE); 490 491 err = feature_get_refcount(spa, &spa_feature_table[fid], &refcount); 492 ASSERT(err == 0 || err == ENOTSUP); 493 return (err == 0); 494} 495 496boolean_t 497spa_feature_is_active(spa_t *spa, spa_feature_t fid) 498{ 499 int err; 500 uint64_t refcount = 0; 501 502 ASSERT(VALID_FEATURE_FID(fid)); 503 if (spa_version(spa) < SPA_VERSION_FEATURES) 504 return (B_FALSE); 505 506 err = feature_get_refcount(spa, &spa_feature_table[fid], &refcount); 507 ASSERT(err == 0 || err == ENOTSUP); 508 return (err == 0 && refcount > 0); 509} 510 511/* 512 * For the feature specified by fid (which must depend on 513 * SPA_FEATURE_ENABLED_TXG), return the TXG at which it was enabled in the 514 * OUT txg argument. 515 * 516 * Returns B_TRUE if the feature is enabled, in which case txg will be filled 517 * with the transaction group in which the specified feature was enabled. 518 * Returns B_FALSE otherwise (i.e. if the feature is not enabled). 519 */ 520boolean_t 521spa_feature_enabled_txg(spa_t *spa, spa_feature_t fid, uint64_t *txg) 522{ 523 int err; 524 525 ASSERT(VALID_FEATURE_FID(fid)); 526 if (spa_version(spa) < SPA_VERSION_FEATURES) 527 return (B_FALSE); 528 529 err = feature_get_enabled_txg(spa, &spa_feature_table[fid], txg); 530 ASSERT(err == 0 || err == ENOTSUP); 531 532 return (err == 0); 533} 534