fs/zfs/trim_map.c

43 #define TRIM_MAP_SINC(tm, size)					\
44 	atomic_add_64(&(tm)->tm_bytes, (size))
46 #define TRIM_MAP_SDEC(tm, size)					\
47 	atomic_add_64(&(tm)->tm_bytes, -(size))
49 #define TRIM_MAP_QINC(tm)					\
50 	atomic_inc_64(&(tm)->tm_pending);			\
52 #define TRIM_MAP_QDEC(tm)					\
53 	atomic_dec_64(&(tm)->tm_pending);
156 	trim_map_t *tm;
163 	tm = kmem_zalloc(sizeof (*tm), KM_SLEEP);
164 	mutex_init(&tm->tm_lock, NULL, MUTEX_DEFAULT, NULL);
165 	list_create(&tm->tm_head, sizeof (trim_seg_t),
167 	list_create(&tm->tm_pending_writes, sizeof (zio_t),
169 	avl_create(&tm->tm_queued_frees, trim_map_seg_compare,
171 	avl_create(&tm->tm_inflight_frees, trim_map_seg_compare,
173 	avl_create(&tm->tm_inflight_writes, trim_map_zio_compare,
175 	vd->vdev_trimmap = tm;
181 	trim_map_t *tm;
189 	tm = vd->vdev_trimmap;
190 	if (tm == NULL)
200 	mutex_enter(&tm->tm_lock);
201 	while ((ts = list_head(&tm->tm_head)) != NULL) {
202 		avl_remove(&tm->tm_queued_frees, ts);
203 		list_remove(&tm->tm_head, ts);
205 		TRIM_MAP_SDEC(tm, ts->ts_end - ts->ts_start);
206 		TRIM_MAP_QDEC(tm);
208 	mutex_exit(&tm->tm_lock);
210 	avl_destroy(&tm->tm_queued_frees);
211 	avl_destroy(&tm->tm_inflight_frees);
212 	avl_destroy(&tm->tm_inflight_writes);
213 	list_destroy(&tm->tm_pending_writes);
214 	list_destroy(&tm->tm_head);
215 	mutex_destroy(&tm->tm_lock);
216 	kmem_free(tm, sizeof (*tm));
221 trim_map_segment_add(trim_map_t *tm, uint64_t start, uint64_t end, uint64_t txg)
228 	ASSERT(MUTEX_HELD(&tm->tm_lock));
235 	ts = avl_find(&tm->tm_queued_frees, &tsearch, &where);
238 			trim_map_segment_add(tm, start, ts->ts_start, txg);
240 			trim_map_segment_add(tm, ts->ts_end, end, txg);
244 	ts_before = avl_nearest(&tm->tm_queued_frees, where, AVL_BEFORE);
245 	ts_after = avl_nearest(&tm->tm_queued_frees, where, AVL_AFTER);
251 		TRIM_MAP_SINC(tm, ts_after->ts_start - ts_before->ts_end);
252 		TRIM_MAP_QDEC(tm);
253 		avl_remove(&tm->tm_queued_frees, ts_before);
254 		list_remove(&tm->tm_head, ts_before);
260 		TRIM_MAP_SINC(tm, end - ts_before->ts_end);
265 		TRIM_MAP_SINC(tm, ts_after->ts_start - start);
270 		TRIM_MAP_SINC(tm, end - start);
271 		TRIM_MAP_QINC(tm);
277 		avl_insert(&tm->tm_queued_frees, ts, where);
278 		list_insert_tail(&tm->tm_head, ts);
283 trim_map_segment_remove(trim_map_t *tm, trim_seg_t *ts, uint64_t start,
289 	ASSERT(MUTEX_HELD(&tm->tm_lock));
294 	TRIM_MAP_SDEC(tm, end - start);
302 		avl_insert_here(&tm->tm_queued_frees, nts, ts, AVL_AFTER);
303 		list_insert_after(&tm->tm_head, ts, nts);
304 		TRIM_MAP_QINC(tm);
310 		avl_remove(&tm->tm_queued_frees, ts);
311 		list_remove(&tm->tm_head, ts);
312 		TRIM_MAP_QDEC(tm);
318 trim_map_free_locked(trim_map_t *tm, uint64_t start, uint64_t end, uint64_t txg)
322 	ASSERT(MUTEX_HELD(&tm->tm_lock));
327 	zs = avl_find(&tm->tm_inflight_writes, &zsearch, NULL);
329 		trim_map_segment_add(tm, start, end, txg);
333 		trim_map_free_locked(tm, start, zs->io_offset, txg);
335 		trim_map_free_locked(tm, zs->io_offset + zs->io_size, end, txg);
341 	trim_map_t *tm = vd->vdev_trimmap;
343 	if (!zfs_trim_enabled || vd->vdev_notrim || tm == NULL)
346 	mutex_enter(&tm->tm_lock);
347 	trim_map_free_locked(tm, offset, TRIM_ZIO_END(vd, offset, size), txg);
348 	mutex_exit(&tm->tm_lock);
355 	trim_map_t *tm = vd->vdev_trimmap;
360 	if (!zfs_trim_enabled || vd->vdev_notrim || tm == NULL)
368 	mutex_enter(&tm->tm_lock);
373 	ts = avl_find(&tm->tm_inflight_frees, &tsearch, NULL);
375 		list_insert_tail(&tm->tm_pending_writes, zio);
376 		mutex_exit(&tm->tm_lock);
380 	ts = avl_find(&tm->tm_queued_frees, &tsearch, NULL);
386 			trim_map_segment_remove(tm, ts, start, end);
387 			ts = avl_find(&tm->tm_queued_frees, &tsearch, NULL);
390 	avl_add(&tm->tm_inflight_writes, zio);
392 	mutex_exit(&tm->tm_lock);
401 	trim_map_t *tm = vd->vdev_trimmap;
407 	if (!zfs_trim_enabled || tm == NULL)
410 	mutex_enter(&tm->tm_lock);
418 	    tm->tm_inflight_writes.avl_root == &zio->io_trim_node)
419 		avl_remove(&tm->tm_inflight_writes, zio);
420 	mutex_exit(&tm->tm_lock);
431 trim_map_first(trim_map_t *tm, uint64_t txg, uint64_t txgsafe, hrtime_t time)
435 	ASSERT(MUTEX_HELD(&tm->tm_lock));
438 	ts = list_head(&tm->tm_head);
441 	    tm->tm_bytes > trim_vdev_max_bytes ||
442 	    tm->tm_pending > trim_vdev_max_pending))
450 	trim_map_t *tm = vd->vdev_trimmap;
457 	if (tm == NULL)
472 	mutex_enter(&tm->tm_lock);
474 	while ((ts = trim_map_first(tm, txgtarget, txgsafe, timelimit))
476 		list_remove(&tm->tm_head, ts);
477 		avl_remove(&tm->tm_queued_frees, ts);
478 		avl_add(&tm->tm_inflight_frees, ts);
481 		TRIM_MAP_SDEC(tm, size);
482 		TRIM_MAP_QDEC(tm);
484 	mutex_exit(&tm->tm_lock);
490 	trim_map_t *tm = vd->vdev_trimmap;
499 	if (tm == NULL)
502 	mutex_enter(&tm->tm_lock);
503 	if (!avl_is_empty(&tm->tm_inflight_frees)) {
505 		while ((ts = avl_destroy_nodes(&tm->tm_inflight_frees,
512 	list_move_tail(&pending_writes, &tm->tm_pending_writes);
513 	mutex_exit(&tm->tm_lock);