1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011-2019 Pawel Jakub Dawidek <pawel@dawidek.net>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD$");
31
32#include <sys/param.h>
33#ifdef _KERNEL
34#include <sys/kernel.h>
35#include <sys/malloc.h>
36#include <sys/sysctl.h>
37#include <sys/systm.h>
38#endif /* _KERNEL */
39#include <sys/queue.h>
40#include <sys/tree.h>
41
42#include <geom/geom.h>
43
44#include <geom/eli/g_eli.h>
45
46#ifdef _KERNEL
47MALLOC_DECLARE(M_ELI);
48
49SYSCTL_DECL(_kern_geom_eli);
50/*
51 * The default limit (8192 keys) will allow to cache all keys for 4TB
52 * provider with 512 bytes sectors and will take around 1MB of memory.
53 */
54static u_int g_eli_key_cache_limit = 8192;
55SYSCTL_UINT(_kern_geom_eli, OID_AUTO, key_cache_limit, CTLFLAG_RDTUN,
56    &g_eli_key_cache_limit, 0, "Maximum number of encryption keys to cache");
57static uint64_t g_eli_key_cache_hits;
58SYSCTL_UQUAD(_kern_geom_eli, OID_AUTO, key_cache_hits, CTLFLAG_RW,
59    &g_eli_key_cache_hits, 0, "Key cache hits");
60static uint64_t g_eli_key_cache_misses;
61SYSCTL_UQUAD(_kern_geom_eli, OID_AUTO, key_cache_misses, CTLFLAG_RW,
62    &g_eli_key_cache_misses, 0, "Key cache misses");
63
64static int
65g_eli_key_cmp(const struct g_eli_key *a, const struct g_eli_key *b)
66{
67
68	if (a->gek_keyno > b->gek_keyno)
69		return (1);
70	else if (a->gek_keyno < b->gek_keyno)
71		return (-1);
72	return (0);
73}
74#endif /* _KERNEL */
75
76void
77g_eli_key_fill(struct g_eli_softc *sc, struct g_eli_key *key, uint64_t keyno)
78{
79	const uint8_t *ekey;
80	struct {
81		char magic[4];
82		uint8_t keyno[8];
83	} __packed hmacdata;
84
85	if ((sc->sc_flags & G_ELI_FLAG_ENC_IVKEY) != 0)
86		ekey = sc->sc_mkey;
87	else
88		ekey = sc->sc_ekey;
89
90	bcopy("ekey", hmacdata.magic, 4);
91	le64enc(hmacdata.keyno, keyno);
92	g_eli_crypto_hmac(ekey, G_ELI_MAXKEYLEN, (uint8_t *)&hmacdata,
93	    sizeof(hmacdata), key->gek_key, 0);
94	key->gek_keyno = keyno;
95	key->gek_count = 0;
96	key->gek_magic = G_ELI_KEY_MAGIC;
97}
98
99#ifdef _KERNEL
100RB_PROTOTYPE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp);
101RB_GENERATE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp);
102
103static struct g_eli_key *
104g_eli_key_allocate(struct g_eli_softc *sc, uint64_t keyno)
105{
106	struct g_eli_key *key, *ekey, keysearch;
107
108	mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
109	mtx_unlock(&sc->sc_ekeys_lock);
110
111	key = malloc(sizeof(*key), M_ELI, M_WAITOK);
112	g_eli_key_fill(sc, key, keyno);
113
114	mtx_lock(&sc->sc_ekeys_lock);
115	/*
116	 * Recheck if the key wasn't added while we weren't holding the lock.
117	 */
118	keysearch.gek_keyno = keyno;
119	ekey = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch);
120	if (ekey != NULL) {
121		zfree(key, M_ELI);
122		key = ekey;
123		TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
124	} else {
125		RB_INSERT(g_eli_key_tree, &sc->sc_ekeys_tree, key);
126		sc->sc_ekeys_allocated++;
127	}
128	TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next);
129
130	return (key);
131}
132
133static struct g_eli_key *
134g_eli_key_find_last(struct g_eli_softc *sc)
135{
136	struct g_eli_key *key;
137
138	mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
139
140	TAILQ_FOREACH(key, &sc->sc_ekeys_queue, gek_next) {
141		if (key->gek_count == 0)
142			break;
143	}
144
145	return (key);
146}
147
148static void
149g_eli_key_replace(struct g_eli_softc *sc, struct g_eli_key *key, uint64_t keyno)
150{
151
152	mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
153	KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid magic."));
154
155	RB_REMOVE(g_eli_key_tree, &sc->sc_ekeys_tree, key);
156	TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
157
158	KASSERT(key->gek_count == 0, ("gek_count=%d", key->gek_count));
159
160	g_eli_key_fill(sc, key, keyno);
161
162	RB_INSERT(g_eli_key_tree, &sc->sc_ekeys_tree, key);
163	TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next);
164}
165
166static void
167g_eli_key_remove(struct g_eli_softc *sc, struct g_eli_key *key)
168{
169
170	mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
171	KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid magic."));
172	KASSERT(key->gek_count == 0, ("gek_count=%d", key->gek_count));
173
174	RB_REMOVE(g_eli_key_tree, &sc->sc_ekeys_tree, key);
175	TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
176	sc->sc_ekeys_allocated--;
177	zfree(key, M_ELI);
178}
179
180void
181g_eli_key_init(struct g_eli_softc *sc)
182{
183	uint8_t *mkey;
184
185	mtx_lock(&sc->sc_ekeys_lock);
186
187	mkey = sc->sc_mkey + sizeof(sc->sc_ivkey);
188	if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0)
189		bcopy(mkey, sc->sc_ekey, G_ELI_DATAKEYLEN);
190	else {
191		/*
192		 * The encryption key is: ekey = HMAC_SHA512(Data-Key, 0x10)
193		 */
194		g_eli_crypto_hmac(mkey, G_ELI_MAXKEYLEN, "\x10", 1,
195		    sc->sc_ekey, 0);
196	}
197
198	if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) {
199		sc->sc_ekeys_total = 1;
200		sc->sc_ekeys_allocated = 0;
201	} else {
202		off_t mediasize;
203		size_t blocksize;
204
205		if ((sc->sc_flags & G_ELI_FLAG_AUTH) != 0) {
206			struct g_provider *pp;
207
208			pp = LIST_FIRST(&sc->sc_geom->consumer)->provider;
209			mediasize = pp->mediasize;
210			blocksize = pp->sectorsize;
211		} else {
212			mediasize = sc->sc_mediasize;
213			blocksize = sc->sc_sectorsize;
214		}
215		sc->sc_ekeys_total =
216		    ((mediasize - 1) >> G_ELI_KEY_SHIFT) / blocksize + 1;
217		sc->sc_ekeys_allocated = 0;
218		TAILQ_INIT(&sc->sc_ekeys_queue);
219		RB_INIT(&sc->sc_ekeys_tree);
220		if (sc->sc_ekeys_total <= g_eli_key_cache_limit) {
221			uint64_t keyno;
222
223			for (keyno = 0; keyno < sc->sc_ekeys_total; keyno++)
224				(void)g_eli_key_allocate(sc, keyno);
225			KASSERT(sc->sc_ekeys_total == sc->sc_ekeys_allocated,
226			    ("sc_ekeys_total=%ju != sc_ekeys_allocated=%ju",
227			    (uintmax_t)sc->sc_ekeys_total,
228			    (uintmax_t)sc->sc_ekeys_allocated));
229		}
230	}
231
232	mtx_unlock(&sc->sc_ekeys_lock);
233}
234
235void
236g_eli_key_destroy(struct g_eli_softc *sc)
237{
238
239	mtx_lock(&sc->sc_ekeys_lock);
240	if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) {
241		explicit_bzero(sc->sc_ekey, sizeof(sc->sc_ekey));
242	} else {
243		struct g_eli_key *key;
244
245		while ((key = TAILQ_FIRST(&sc->sc_ekeys_queue)) != NULL)
246			g_eli_key_remove(sc, key);
247		TAILQ_INIT(&sc->sc_ekeys_queue);
248		RB_INIT(&sc->sc_ekeys_tree);
249	}
250	mtx_unlock(&sc->sc_ekeys_lock);
251}
252
253void
254g_eli_key_resize(struct g_eli_softc *sc)
255{
256	uint64_t new_ekeys_total;
257	off_t mediasize;
258	size_t blocksize;
259
260	if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) {
261		return;
262	}
263
264	mtx_lock(&sc->sc_ekeys_lock);
265
266	if ((sc->sc_flags & G_ELI_FLAG_AUTH) != 0) {
267		struct g_provider *pp;
268
269		pp = LIST_FIRST(&sc->sc_geom->consumer)->provider;
270		mediasize = pp->mediasize;
271		blocksize = pp->sectorsize;
272	} else {
273		mediasize = sc->sc_mediasize;
274		blocksize = sc->sc_sectorsize;
275	}
276	new_ekeys_total = ((mediasize - 1) >> G_ELI_KEY_SHIFT) / blocksize + 1;
277	/* We only allow to grow. */
278	KASSERT(new_ekeys_total >= sc->sc_ekeys_total,
279	    ("new_ekeys_total=%ju < sc_ekeys_total=%ju",
280	    (uintmax_t)new_ekeys_total, (uintmax_t)sc->sc_ekeys_total));
281	if (new_ekeys_total <= g_eli_key_cache_limit) {
282		uint64_t keyno;
283
284		for (keyno = sc->sc_ekeys_total; keyno < new_ekeys_total;
285		    keyno++) {
286			(void)g_eli_key_allocate(sc, keyno);
287		}
288		KASSERT(new_ekeys_total == sc->sc_ekeys_allocated,
289		    ("new_ekeys_total=%ju != sc_ekeys_allocated=%ju",
290		    (uintmax_t)new_ekeys_total,
291		    (uintmax_t)sc->sc_ekeys_allocated));
292	}
293
294	sc->sc_ekeys_total = new_ekeys_total;
295
296	mtx_unlock(&sc->sc_ekeys_lock);
297}
298
299/*
300 * Select encryption key. If G_ELI_FLAG_SINGLE_KEY is present we only have one
301 * key available for all the data. If the flag is not present select the key
302 * based on data offset.
303 */
304uint8_t *
305g_eli_key_hold(struct g_eli_softc *sc, off_t offset, size_t blocksize)
306{
307	struct g_eli_key *key, keysearch;
308	uint64_t keyno;
309
310	if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0)
311		return (sc->sc_ekey);
312
313	/* We switch key every 2^G_ELI_KEY_SHIFT blocks. */
314	keyno = (offset >> G_ELI_KEY_SHIFT) / blocksize;
315
316	KASSERT(keyno < sc->sc_ekeys_total,
317	    ("%s: keyno=%ju >= sc_ekeys_total=%ju",
318	    __func__, (uintmax_t)keyno, (uintmax_t)sc->sc_ekeys_total));
319
320	keysearch.gek_keyno = keyno;
321
322	if (sc->sc_ekeys_total == sc->sc_ekeys_allocated) {
323		/* We have all the keys, so avoid some overhead. */
324		key = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch);
325		KASSERT(key != NULL, ("No key %ju found.", (uintmax_t)keyno));
326		KASSERT(key->gek_magic == G_ELI_KEY_MAGIC,
327		    ("Invalid key magic."));
328		return (key->gek_key);
329	}
330
331	mtx_lock(&sc->sc_ekeys_lock);
332	key = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch);
333	if (key != NULL) {
334		g_eli_key_cache_hits++;
335		TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
336		TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next);
337	} else {
338		/*
339		 * No key in cache, find the least recently unreferenced key
340		 * or allocate one if we haven't reached our limit yet.
341		 */
342		if (sc->sc_ekeys_allocated < g_eli_key_cache_limit) {
343			key = g_eli_key_allocate(sc, keyno);
344		} else {
345			g_eli_key_cache_misses++;
346			key = g_eli_key_find_last(sc);
347			if (key != NULL) {
348				g_eli_key_replace(sc, key, keyno);
349			} else {
350				/* All keys are referenced? Allocate one. */
351				key = g_eli_key_allocate(sc, keyno);
352			}
353		}
354	}
355	key->gek_count++;
356	mtx_unlock(&sc->sc_ekeys_lock);
357
358	KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid key magic."));
359
360	return (key->gek_key);
361}
362
363void
364g_eli_key_drop(struct g_eli_softc *sc, uint8_t *rawkey)
365{
366	struct g_eli_key *key = (struct g_eli_key *)rawkey;
367
368	if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0)
369		return;
370
371	KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid key magic."));
372
373	if (sc->sc_ekeys_total == sc->sc_ekeys_allocated)
374		return;
375
376	mtx_lock(&sc->sc_ekeys_lock);
377	KASSERT(key->gek_count > 0, ("key->gek_count=%d", key->gek_count));
378	key->gek_count--;
379	while (sc->sc_ekeys_allocated > g_eli_key_cache_limit) {
380		key = g_eli_key_find_last(sc);
381		if (key == NULL)
382			break;
383		g_eli_key_remove(sc, key);
384	}
385	mtx_unlock(&sc->sc_ekeys_lock);
386}
387#endif /* _KERNEL */
388