1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org>
5 * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
9 * met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in
14 *    the documentation and/or other materials provided with the
15 *    distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30/*
31 * Subroutines used for writing compressed user process and kernel core dumps.
32 */
33
34#include <sys/cdefs.h>
35#include "opt_gzio.h"
36#include "opt_zstdio.h"
37
38#include <sys/param.h>
39#include <sys/systm.h>
40
41#include <sys/compressor.h>
42#include <sys/endian.h>
43#include <sys/kernel.h>
44#include <sys/linker_set.h>
45#include <sys/malloc.h>
46
47MALLOC_DEFINE(M_COMPRESS, "compressor", "kernel compression subroutines");
48
49struct compressor_methods {
50	int format;
51	void *(* const init)(size_t, int);
52	void (* const reset)(void *);
53	int (* const write)(void *, void *, size_t, compressor_cb_t, void *);
54	void (* const fini)(void *);
55};
56
57struct compressor {
58	const struct compressor_methods *methods;
59	compressor_cb_t cb;
60	void *priv;
61	void *arg;
62};
63
64SET_DECLARE(compressors, struct compressor_methods);
65
66#ifdef GZIO
67
68#include <contrib/zlib/zutil.h>
69
70struct gz_stream {
71	uint8_t		*gz_buffer;	/* output buffer */
72	size_t		gz_bufsz;	/* output buffer size */
73	off_t		gz_off;		/* offset into the output stream */
74	uint32_t	gz_crc;		/* stream CRC32 */
75	z_stream	gz_stream;	/* zlib state */
76};
77
78static void 	*gz_init(size_t maxiosize, int level);
79static void	gz_reset(void *stream);
80static int	gz_write(void *stream, void *data, size_t len, compressor_cb_t,
81		    void *);
82static void	gz_fini(void *stream);
83
84static void *
85gz_alloc(void *arg __unused, u_int n, u_int sz)
86{
87
88	/*
89	 * Memory for zlib state is allocated using M_NODUMP since it may be
90	 * used to compress a kernel dump, and we don't want zlib to attempt to
91	 * compress its own state.
92	 */
93	return (malloc(n * sz, M_COMPRESS, M_WAITOK | M_ZERO | M_NODUMP));
94}
95
96static void
97gz_free(void *arg __unused, void *ptr)
98{
99
100	free(ptr, M_COMPRESS);
101}
102
103static void *
104gz_init(size_t maxiosize, int level)
105{
106	struct gz_stream *s;
107	int error;
108
109	s = gz_alloc(NULL, 1, roundup2(sizeof(*s), PAGE_SIZE));
110	s->gz_buffer = gz_alloc(NULL, 1, maxiosize);
111	s->gz_bufsz = maxiosize;
112
113	s->gz_stream.zalloc = gz_alloc;
114	s->gz_stream.zfree = gz_free;
115	s->gz_stream.opaque = NULL;
116	s->gz_stream.next_in = Z_NULL;
117	s->gz_stream.avail_in = 0;
118
119	if (level != Z_DEFAULT_COMPRESSION) {
120		if (level < Z_BEST_SPEED)
121			level = Z_BEST_SPEED;
122		else if (level > Z_BEST_COMPRESSION)
123			level = Z_BEST_COMPRESSION;
124	}
125
126	error = deflateInit2(&s->gz_stream, level, Z_DEFLATED, -MAX_WBITS,
127	    DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
128	if (error != 0)
129		goto fail;
130
131	gz_reset(s);
132
133	return (s);
134
135fail:
136	gz_free(NULL, s);
137	return (NULL);
138}
139
140static void
141gz_reset(void *stream)
142{
143	struct gz_stream *s;
144	uint8_t *hdr;
145	const size_t hdrlen = 10;
146
147	s = stream;
148	s->gz_off = 0;
149	s->gz_crc = crc32(0L, Z_NULL, 0);
150
151	(void)deflateReset(&s->gz_stream);
152	s->gz_stream.avail_out = s->gz_bufsz;
153	s->gz_stream.next_out = s->gz_buffer;
154
155	/* Write the gzip header to the output buffer. */
156	hdr = s->gz_buffer;
157	memset(hdr, 0, hdrlen);
158	hdr[0] = 0x1f;
159	hdr[1] = 0x8b;
160	hdr[2] = Z_DEFLATED;
161	hdr[9] = OS_CODE;
162	s->gz_stream.next_out += hdrlen;
163	s->gz_stream.avail_out -= hdrlen;
164}
165
166static int
167gz_write(void *stream, void *data, size_t len, compressor_cb_t cb,
168    void *arg)
169{
170	struct gz_stream *s;
171	uint8_t trailer[8];
172	size_t room;
173	int error, zerror, zflag;
174
175	s = stream;
176	zflag = data == NULL ? Z_FINISH : Z_NO_FLUSH;
177
178	if (len > 0) {
179		s->gz_stream.avail_in = len;
180		s->gz_stream.next_in = data;
181		s->gz_crc = crc32(s->gz_crc, data, len);
182	}
183
184	error = 0;
185	do {
186		zerror = deflate(&s->gz_stream, zflag);
187		if (zerror != Z_OK && zerror != Z_STREAM_END) {
188			error = EIO;
189			break;
190		}
191
192		if (s->gz_stream.avail_out == 0 || zerror == Z_STREAM_END) {
193			/*
194			 * Our output buffer is full or there's nothing left
195			 * to produce, so we're flushing the buffer.
196			 */
197			len = s->gz_bufsz - s->gz_stream.avail_out;
198			if (zerror == Z_STREAM_END) {
199				/*
200				 * Try to pack as much of the trailer into the
201				 * output buffer as we can.
202				 */
203				((uint32_t *)trailer)[0] = htole32(s->gz_crc);
204				((uint32_t *)trailer)[1] =
205				    htole32(s->gz_stream.total_in);
206				room = MIN(sizeof(trailer),
207				    s->gz_bufsz - len);
208				memcpy(s->gz_buffer + len, trailer, room);
209				len += room;
210			}
211
212			error = cb(s->gz_buffer, len, s->gz_off, arg);
213			if (error != 0)
214				break;
215
216			s->gz_off += len;
217			s->gz_stream.next_out = s->gz_buffer;
218			s->gz_stream.avail_out = s->gz_bufsz;
219
220			/*
221			 * If we couldn't pack the trailer into the output
222			 * buffer, write it out now.
223			 */
224			if (zerror == Z_STREAM_END && room < sizeof(trailer))
225				error = cb(trailer + room,
226				    sizeof(trailer) - room, s->gz_off, arg);
227		}
228	} while (zerror != Z_STREAM_END &&
229	    (zflag == Z_FINISH || s->gz_stream.avail_in > 0));
230
231	return (error);
232}
233
234static void
235gz_fini(void *stream)
236{
237	struct gz_stream *s;
238
239	s = stream;
240	(void)deflateEnd(&s->gz_stream);
241	gz_free(NULL, s->gz_buffer);
242	gz_free(NULL, s);
243}
244
245struct compressor_methods gzip_methods = {
246	.format = COMPRESS_GZIP,
247	.init = gz_init,
248	.reset = gz_reset,
249	.write = gz_write,
250	.fini = gz_fini,
251};
252DATA_SET(compressors, gzip_methods);
253
254#endif /* GZIO */
255
256#ifdef ZSTDIO
257
258#define	ZSTD_STATIC_LINKING_ONLY
259#include <contrib/zstd/lib/zstd.h>
260
261struct zstdio_stream {
262	ZSTD_CCtx	*zst_stream;
263	ZSTD_inBuffer	zst_inbuffer;
264	ZSTD_outBuffer	zst_outbuffer;
265	uint8_t *	zst_buffer;	/* output buffer */
266	size_t		zst_maxiosz;	/* Max output IO size */
267	off_t		zst_off;	/* offset into the output stream */
268	void *		zst_static_wkspc;
269};
270
271static void 	*zstdio_init(size_t maxiosize, int level);
272static void	zstdio_reset(void *stream);
273static int	zstdio_write(void *stream, void *data, size_t len,
274		    compressor_cb_t, void *);
275static void	zstdio_fini(void *stream);
276
277static void *
278zstdio_init(size_t maxiosize, int level)
279{
280	ZSTD_CCtx *dump_compressor;
281	struct zstdio_stream *s;
282	void *wkspc, *owkspc, *buffer;
283	size_t wkspc_size, buf_size, rc;
284
285	s = NULL;
286	wkspc_size = ZSTD_estimateCStreamSize(level);
287	owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS,
288	    M_WAITOK | M_NODUMP);
289	/* Zstd API requires 8-byte alignment. */
290	if ((uintptr_t)wkspc % 8 != 0)
291		wkspc = (void *)roundup2((uintptr_t)wkspc, 8);
292
293	dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size);
294	if (dump_compressor == NULL) {
295		printf("%s: workspace too small.\n", __func__);
296		goto out;
297	}
298
299	rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_checksumFlag, 1);
300	if (ZSTD_isError(rc)) {
301		printf("%s: error setting checksumFlag: %s\n", __func__,
302		    ZSTD_getErrorName(rc));
303		goto out;
304	}
305	rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_compressionLevel,
306	    level);
307	if (ZSTD_isError(rc)) {
308		printf("%s: error setting compressLevel: %s\n", __func__,
309		    ZSTD_getErrorName(rc));
310		goto out;
311	}
312
313	buf_size = ZSTD_CStreamOutSize() * 2;
314	buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP);
315
316	s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK);
317	s->zst_buffer = buffer;
318	s->zst_outbuffer.dst = buffer;
319	s->zst_outbuffer.size = buf_size;
320	s->zst_maxiosz = maxiosize;
321	s->zst_stream = dump_compressor;
322	s->zst_static_wkspc = owkspc;
323
324	zstdio_reset(s);
325
326out:
327	if (s == NULL)
328		free(owkspc, M_COMPRESS);
329	return (s);
330}
331
332static void
333zstdio_reset(void *stream)
334{
335	struct zstdio_stream *s;
336	size_t res;
337
338	s = stream;
339	res = ZSTD_CCtx_reset(s->zst_stream, ZSTD_reset_session_only);
340	if (ZSTD_isError(res))
341		panic("%s: could not reset stream %p: %s\n", __func__, s,
342		    ZSTD_getErrorName(res));
343	res = ZSTD_CCtx_setPledgedSrcSize(s->zst_stream,
344	    ZSTD_CONTENTSIZE_UNKNOWN);
345	if (ZSTD_isError(res))
346		panic("%s: could not set src size on %p: %s\n", __func__, s,
347		    ZSTD_getErrorName(res));
348
349	s->zst_off = 0;
350	s->zst_inbuffer.src = NULL;
351	s->zst_inbuffer.size = 0;
352	s->zst_inbuffer.pos = 0;
353	s->zst_outbuffer.pos = 0;
354}
355
356static int
357zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
358{
359	size_t bytes_to_dump;
360	int error;
361
362	/* Flush as many full output blocks as possible. */
363	/* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */
364	while (s->zst_outbuffer.pos >= 4096) {
365		bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096);
366
367		if (bytes_to_dump > s->zst_maxiosz)
368			bytes_to_dump = s->zst_maxiosz;
369
370		error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg);
371		if (error != 0)
372			return (error);
373
374		/*
375		 * Shift any non-full blocks up to the front of the output
376		 * buffer.
377		 */
378		s->zst_outbuffer.pos -= bytes_to_dump;
379		memmove(s->zst_outbuffer.dst,
380		    (char *)s->zst_outbuffer.dst + bytes_to_dump,
381		    s->zst_outbuffer.pos);
382		s->zst_off += bytes_to_dump;
383	}
384	return (0);
385}
386
387static int
388zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
389{
390	size_t rc, lastpos;
391	int error;
392
393	/*
394	 * Positive return indicates unflushed data remaining; need to call
395	 * endStream again after clearing out room in output buffer.
396	 */
397	rc = 1;
398	lastpos = s->zst_outbuffer.pos;
399	while (rc > 0) {
400		rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer);
401		if (ZSTD_isError(rc)) {
402			printf("%s: ZSTD_endStream failed (%s)\n", __func__,
403			    ZSTD_getErrorName(rc));
404			return (EIO);
405		}
406		if (lastpos == s->zst_outbuffer.pos) {
407			printf("%s: did not make forward progress endStream %zu\n",
408			    __func__, lastpos);
409			return (EIO);
410		}
411
412		error = zst_flush_intermediate(s, cb, arg);
413		if (error != 0)
414			return (error);
415
416		lastpos = s->zst_outbuffer.pos;
417	}
418
419	/*
420	 * We've already done an intermediate flush, so all full blocks have
421	 * been written.  Only a partial block remains.  Padding happens in a
422	 * higher layer.
423	 */
424	if (s->zst_outbuffer.pos != 0) {
425		error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off,
426		    arg);
427		if (error != 0)
428			return (error);
429	}
430
431	return (0);
432}
433
434static int
435zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb,
436    void *arg)
437{
438	struct zstdio_stream *s;
439	size_t lastpos, rc;
440	int error;
441
442	s = stream;
443	if (data == NULL)
444		return (zstdio_flush(s, cb, arg));
445
446	s->zst_inbuffer.src = data;
447	s->zst_inbuffer.size = len;
448	s->zst_inbuffer.pos = 0;
449	lastpos = 0;
450
451	while (s->zst_inbuffer.pos < s->zst_inbuffer.size) {
452		rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer,
453		    &s->zst_inbuffer);
454		if (ZSTD_isError(rc)) {
455			printf("%s: Compress failed on %p! (%s)\n",
456			    __func__, data, ZSTD_getErrorName(rc));
457			return (EIO);
458		}
459
460		if (lastpos == s->zst_inbuffer.pos) {
461			/*
462			 * XXX: May need flushStream to make forward progress
463			 */
464			printf("ZSTD: did not make forward progress @pos %zu\n",
465			    lastpos);
466			return (EIO);
467		}
468		lastpos = s->zst_inbuffer.pos;
469
470		error = zst_flush_intermediate(s, cb, arg);
471		if (error != 0)
472			return (error);
473	}
474	return (0);
475}
476
477static void
478zstdio_fini(void *stream)
479{
480	struct zstdio_stream *s;
481
482	s = stream;
483	if (s->zst_static_wkspc != NULL)
484		free(s->zst_static_wkspc, M_COMPRESS);
485	else
486		ZSTD_freeCCtx(s->zst_stream);
487	free(s->zst_buffer, M_COMPRESS);
488	free(s, M_COMPRESS);
489}
490
491static struct compressor_methods zstd_methods = {
492	.format = COMPRESS_ZSTD,
493	.init = zstdio_init,
494	.reset = zstdio_reset,
495	.write = zstdio_write,
496	.fini = zstdio_fini,
497};
498DATA_SET(compressors, zstd_methods);
499
500#endif /* ZSTDIO */
501
502bool
503compressor_avail(int format)
504{
505	struct compressor_methods **iter;
506
507	SET_FOREACH(iter, compressors) {
508		if ((*iter)->format == format)
509			return (true);
510	}
511	return (false);
512}
513
514struct compressor *
515compressor_init(compressor_cb_t cb, int format, size_t maxiosize, int level,
516    void *arg)
517{
518	struct compressor_methods **iter;
519	struct compressor *s;
520	void *priv;
521
522	SET_FOREACH(iter, compressors) {
523		if ((*iter)->format == format)
524			break;
525	}
526	if (iter == SET_LIMIT(compressors))
527		return (NULL);
528
529	priv = (*iter)->init(maxiosize, level);
530	if (priv == NULL)
531		return (NULL);
532
533	s = malloc(sizeof(*s), M_COMPRESS, M_WAITOK | M_ZERO);
534	s->methods = (*iter);
535	s->priv = priv;
536	s->cb = cb;
537	s->arg = arg;
538	return (s);
539}
540
541void
542compressor_reset(struct compressor *stream)
543{
544
545	stream->methods->reset(stream->priv);
546}
547
548int
549compressor_write(struct compressor *stream, void *data, size_t len)
550{
551
552	return (stream->methods->write(stream->priv, data, len, stream->cb,
553	    stream->arg));
554}
555
556int
557compressor_flush(struct compressor *stream)
558{
559
560	return (stream->methods->write(stream->priv, NULL, 0, stream->cb,
561	    stream->arg));
562}
563
564void
565compressor_fini(struct compressor *stream)
566{
567
568	stream->methods->fini(stream->priv);
569}
570