1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29#include <sys/param.h>
30#include <sys/cpuset.h>
31#include <sys/errno.h>
32#include <sys/mman.h>
33#include <sys/nv.h>
34#include <sys/socket.h>
35#include <sys/sysctl.h>
36#include <sys/un.h>
37
38#include <stdio.h>
39#include <stdlib.h>
40#include <stdbool.h>
41#include <string.h>
42#include <unistd.h>
43#include <libgen.h>
44#include <libutil.h>
45#include <fcntl.h>
46#include <getopt.h>
47#include <libutil.h>
48
49#include <machine/cpufunc.h>
50#include <machine/vmm.h>
51#include <machine/vmm_dev.h>
52#include <vmmapi.h>
53
54#ifdef BHYVE_SNAPSHOT
55#include "snapshot.h"
56#endif
57
58#include "bhyvectl.h"
59
60#define	MB	(1UL << 20)
61#define	GB	(1UL << 30)
62
63static const char *progname;
64
65static int get_stats, getcap, setcap, capval;
66static int force_reset, force_poweroff;
67static const char *capname;
68static int create, destroy, get_memmap, get_memseg;
69static int get_active_cpus, get_debug_cpus, get_suspended_cpus;
70static uint64_t memsize;
71static int run;
72static int get_cpu_topology;
73#ifdef BHYVE_SNAPSHOT
74static int vm_suspend_opt;
75#endif
76
77static int get_all;
78
79enum {
80	VMNAME = OPT_START,	/* avoid collision with return values from getopt */
81	VCPU,
82	SET_MEM,
83	SET_CAP,
84	CAPNAME,
85#ifdef BHYVE_SNAPSHOT
86	SET_CHECKPOINT_FILE,
87	SET_SUSPEND_FILE,
88#endif
89	OPT_LAST,
90};
91
92_Static_assert(OPT_LAST < OPT_START_MD,
93    "OPT_LAST must be less than OPT_START_MD");
94
95static void
96print_cpus(const char *banner, const cpuset_t *cpus)
97{
98	int i, first;
99
100	first = 1;
101	printf("%s:\t", banner);
102	if (!CPU_EMPTY(cpus)) {
103		for (i = 0; i < CPU_SETSIZE; i++) {
104			if (CPU_ISSET(i, cpus)) {
105				printf("%s%d", first ? " " : ", ", i);
106				first = 0;
107			}
108		}
109	} else
110		printf(" (none)");
111	printf("\n");
112}
113
114static struct option *
115setup_options(void)
116{
117	const struct option common_opts[] = {
118		{ "vm",		REQ_ARG,	0,	VMNAME },
119		{ "cpu",	REQ_ARG,	0,	VCPU },
120		{ "set-mem",	REQ_ARG,	0,	SET_MEM },
121		{ "capname",	REQ_ARG,	0,	CAPNAME },
122		{ "setcap",	REQ_ARG,	0,	SET_CAP },
123		{ "getcap",	NO_ARG,		&getcap,	1 },
124		{ "get-stats",	NO_ARG,		&get_stats,	1 },
125		{ "get-memmap",	NO_ARG,		&get_memmap,	1 },
126		{ "get-memseg", NO_ARG,		&get_memseg,	1 },
127		{ "get-all",		NO_ARG,	&get_all,		1 },
128		{ "run",		NO_ARG,	&run,			1 },
129		{ "create",		NO_ARG,	&create,		1 },
130		{ "destroy",		NO_ARG,	&destroy,		1 },
131		{ "force-reset",	NO_ARG,	&force_reset,		1 },
132		{ "force-poweroff", 	NO_ARG,	&force_poweroff, 	1 },
133		{ "get-active-cpus", 	NO_ARG,	&get_active_cpus, 	1 },
134		{ "get-debug-cpus",	NO_ARG,	&get_debug_cpus,	1 },
135		{ "get-suspended-cpus", NO_ARG,	&get_suspended_cpus, 	1 },
136		{ "get-cpu-topology",	NO_ARG, &get_cpu_topology,	1 },
137#ifdef BHYVE_SNAPSHOT
138		{ "checkpoint", 	REQ_ARG, 0,	SET_CHECKPOINT_FILE},
139		{ "suspend", 		REQ_ARG, 0,	SET_SUSPEND_FILE},
140#endif
141	};
142
143	return (bhyvectl_opts(common_opts, nitems(common_opts)));
144}
145
146void
147usage(const struct option *opts)
148{
149	static const char *set_desc[] = {
150	    [VCPU] = "vcpu_number",
151	    [SET_MEM] = "memory in units of MB",
152	    [SET_CAP] = "0|1",
153	    [CAPNAME] = "capname",
154#ifdef BHYVE_SNAPSHOT
155	    [SET_CHECKPOINT_FILE] = "filename",
156	    [SET_SUSPEND_FILE] = "filename",
157#endif
158	};
159	(void)fprintf(stderr, "Usage: %s --vm=<vmname>\n", progname);
160	for (const struct option *o = opts; o->name; o++) {
161		if (strcmp(o->name, "vm") == 0)
162			continue;
163		if (o->has_arg == REQ_ARG) {
164			(void)fprintf(stderr, "       [--%s=<%s>]\n", o->name,
165			    o->val >= OPT_START_MD ? bhyvectl_opt_desc(o->val) :
166			    set_desc[o->val]);
167		} else {
168			(void)fprintf(stderr, "       [--%s]\n", o->name);
169		}
170	}
171	exit(1);
172}
173
174static int
175show_memmap(struct vmctx *ctx)
176{
177	char name[SPECNAMELEN + 1], numbuf[8];
178	vm_ooffset_t segoff;
179	vm_paddr_t gpa;
180	size_t maplen, seglen;
181	int error, flags, prot, segid, delim;
182
183	printf("Address     Length      Segment     Offset      ");
184	printf("Prot  Flags\n");
185
186	gpa = 0;
187	while (1) {
188		error = vm_mmap_getnext(ctx, &gpa, &segid, &segoff, &maplen,
189		    &prot, &flags);
190		if (error)
191			return (errno == ENOENT ? 0 : error);
192
193		error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
194		if (error)
195			return (error);
196
197		printf("%-12lX", gpa);
198		humanize_number(numbuf, sizeof(numbuf), maplen, "B",
199		    HN_AUTOSCALE, HN_NOSPACE);
200		printf("%-12s", numbuf);
201
202		printf("%-12s", name[0] ? name : "sysmem");
203		printf("%-12lX", segoff);
204		printf("%c%c%c   ", prot & PROT_READ ? 'R' : '-',
205		    prot & PROT_WRITE ? 'W' : '-',
206		    prot & PROT_EXEC ? 'X' : '-');
207
208		delim = '\0';
209		if (flags & VM_MEMMAP_F_WIRED) {
210			printf("%cwired", delim);
211			delim = '/';
212		}
213#ifdef __amd64__
214		if (flags & VM_MEMMAP_F_IOMMU) {
215			printf("%ciommu", delim);
216			delim = '/';
217		}
218#endif
219		printf("\n");
220
221		gpa += maplen;
222	}
223}
224
225static int
226show_memseg(struct vmctx *ctx)
227{
228	char name[SPECNAMELEN + 1], numbuf[8];
229	size_t seglen;
230	int error, segid;
231
232	printf("ID  Length      Name\n");
233
234	segid = 0;
235	while (1) {
236		error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
237		if (error)
238			return (errno == EINVAL ? 0 : error);
239
240		if (seglen) {
241			printf("%-4d", segid);
242			humanize_number(numbuf, sizeof(numbuf), seglen, "B",
243			    HN_AUTOSCALE, HN_NOSPACE);
244			printf("%-12s", numbuf);
245			printf("%s", name[0] ? name : "sysmem");
246			printf("\n");
247		}
248		segid++;
249	}
250}
251
252#ifdef BHYVE_SNAPSHOT
253static int
254send_message(const char *vmname, nvlist_t *nvl)
255{
256	struct sockaddr_un addr;
257	int err = 0, socket_fd;
258
259	socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
260	if (socket_fd < 0) {
261		perror("Error creating bhyvectl socket");
262		err = errno;
263		goto done;
264	}
265
266	memset(&addr, 0, sizeof(struct sockaddr_un));
267	snprintf(addr.sun_path, sizeof(addr.sun_path), "%s%s",
268	    BHYVE_RUN_DIR, vmname);
269	addr.sun_family = AF_UNIX;
270	addr.sun_len = SUN_LEN(&addr);
271
272	if (connect(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) {
273		perror("connect() failed");
274		err = errno;
275		goto done;
276	}
277
278	if (nvlist_send(socket_fd, nvl) < 0) {
279		perror("nvlist_send() failed");
280		err = errno;
281	}
282done:
283	nvlist_destroy(nvl);
284
285	if (socket_fd >= 0)
286		close(socket_fd);
287	return (err);
288}
289
290static int
291open_directory(const char *file)
292{
293	char *path;
294	int fd;
295
296	if ((path = strdup(file)) == NULL)
297		return (-1);
298
299	dirname(path);
300	fd = open(path, O_DIRECTORY);
301	free(path);
302
303	return (fd);
304}
305
306static int
307snapshot_request(const char *vmname, char *file, bool suspend)
308{
309	nvlist_t *nvl;
310	int fd;
311
312	if ((fd = open_directory(file)) < 0)
313		return (errno);
314
315	nvl = nvlist_create(0);
316	nvlist_add_string(nvl, "cmd", "checkpoint");
317	nvlist_add_string(nvl, "filename", basename(file));
318	nvlist_add_bool(nvl, "suspend", suspend);
319	nvlist_move_descriptor(nvl, "fddir", fd);
320
321	return (send_message(vmname, nvl));
322}
323#endif
324
325int
326main(int argc, char *argv[])
327{
328	char *vmname;
329	int error, ch, vcpuid;
330	struct vm_run vmrun;
331	struct vmctx *ctx;
332	struct vcpu *vcpu;
333	cpuset_t cpus;
334	struct option *opts;
335#ifdef BHYVE_SNAPSHOT
336	char *checkpoint_file = NULL;
337#endif
338
339	opts = setup_options();
340
341	vcpuid = 0;
342	vmname = NULL;
343	progname = basename(argv[0]);
344
345	while ((ch = getopt_long(argc, argv, "", opts, NULL)) != -1) {
346		if (ch >= OPT_START_MD) {
347			bhyvectl_handle_opt(opts, ch);
348			continue;
349		}
350
351		switch (ch) {
352		case 0:
353			break;
354		case VMNAME:
355			vmname = optarg;
356			break;
357		case VCPU:
358			vcpuid = atoi(optarg);
359			break;
360		case SET_MEM:
361			memsize = atoi(optarg) * MB;
362			memsize = roundup(memsize, 2 * MB);
363			break;
364		case SET_CAP:
365			capval = strtoul(optarg, NULL, 0);
366			setcap = 1;
367			break;
368		case CAPNAME:
369			capname = optarg;
370			break;
371#ifdef BHYVE_SNAPSHOT
372		case SET_CHECKPOINT_FILE:
373		case SET_SUSPEND_FILE:
374			if (checkpoint_file != NULL)
375				usage(opts);
376
377			checkpoint_file = optarg;
378			vm_suspend_opt = (ch == SET_SUSPEND_FILE);
379			break;
380#endif
381		default:
382			usage(opts);
383		}
384	}
385	argc -= optind;
386	argv += optind;
387
388	if (vmname == NULL)
389		usage(opts);
390
391	error = 0;
392
393	if (!error && create)
394		error = vm_create(vmname);
395
396	if (!error) {
397		ctx = vm_open(vmname);
398		if (ctx == NULL) {
399			fprintf(stderr,
400			    "vm_open: %s could not be opened: %s\n",
401			    vmname, strerror(errno));
402			exit(1);
403		}
404		vcpu = vm_vcpu_open(ctx, vcpuid);
405	}
406
407	if (!error && memsize)
408		error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
409
410	if (!error && (get_memseg || get_all))
411		error = show_memseg(ctx);
412
413	if (!error && (get_memmap || get_all))
414		error = show_memmap(ctx);
415
416	if (!error)
417		bhyvectl_md_main(ctx, vcpu, vcpuid, get_all);
418
419	if (!error && setcap) {
420		int captype;
421
422		captype = vm_capability_name2type(capname);
423		error = vm_set_capability(vcpu, captype, capval);
424		if (error != 0 && errno == ENOENT)
425			printf("Capability \"%s\" is not available\n", capname);
426	}
427
428	if (!error && (getcap || get_all)) {
429		int captype, val, getcaptype;
430
431		if (getcap && capname)
432			getcaptype = vm_capability_name2type(capname);
433		else
434			getcaptype = -1;
435
436		for (captype = 0; captype < VM_CAP_MAX; captype++) {
437			if (getcaptype >= 0 && captype != getcaptype)
438				continue;
439			error = vm_get_capability(vcpu, captype, &val);
440			if (error == 0) {
441				printf("Capability \"%s\" is %s on vcpu %d\n",
442					vm_capability_type2name(captype),
443					val ? "set" : "not set", vcpuid);
444			} else if (errno == ENOENT) {
445				error = 0;
446				printf("Capability \"%s\" is not available\n",
447					vm_capability_type2name(captype));
448			} else {
449				break;
450			}
451		}
452	}
453
454	if (!error && (get_active_cpus || get_all)) {
455		error = vm_active_cpus(ctx, &cpus);
456		if (!error)
457			print_cpus("active cpus", &cpus);
458	}
459
460	if (!error && (get_debug_cpus || get_all)) {
461		error = vm_debug_cpus(ctx, &cpus);
462		if (!error)
463			print_cpus("debug cpus", &cpus);
464	}
465
466	if (!error && (get_suspended_cpus || get_all)) {
467		error = vm_suspended_cpus(ctx, &cpus);
468		if (!error)
469			print_cpus("suspended cpus", &cpus);
470	}
471
472	if (!error && (get_stats || get_all)) {
473		int i, num_stats;
474		uint64_t *stats;
475		struct timeval tv;
476		const char *desc;
477
478		stats = vm_get_stats(vcpu, &tv, &num_stats);
479		if (stats != NULL) {
480			printf("vcpu%d stats:\n", vcpuid);
481			for (i = 0; i < num_stats; i++) {
482				desc = vm_get_stat_desc(ctx, i);
483				printf("%-40s\t%ld\n", desc, stats[i]);
484			}
485		}
486	}
487
488	if (!error && (get_cpu_topology || get_all)) {
489		uint16_t sockets, cores, threads, maxcpus;
490
491		vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus);
492		printf("cpu_topology:\tsockets=%hu, cores=%hu, threads=%hu, "
493		    "maxcpus=%hu\n", sockets, cores, threads, maxcpus);
494	}
495
496	if (!error && run) {
497		struct vm_exit vmexit;
498		cpuset_t cpuset;
499
500		vmrun.vm_exit = &vmexit;
501		vmrun.cpuset = &cpuset;
502		vmrun.cpusetsize = sizeof(cpuset);
503		error = vm_run(vcpu, &vmrun);
504		if (error == 0)
505			bhyvectl_dump_vm_run_exitcode(&vmexit, vcpuid);
506		else
507			printf("vm_run error %d\n", error);
508	}
509
510	if (!error && force_reset)
511		error = vm_suspend(ctx, VM_SUSPEND_RESET);
512
513	if (!error && force_poweroff)
514		error = vm_suspend(ctx, VM_SUSPEND_POWEROFF);
515
516	if (error)
517		printf("errno = %d\n", errno);
518
519	if (!error && destroy)
520		vm_destroy(ctx);
521
522#ifdef BHYVE_SNAPSHOT
523	if (!error && checkpoint_file)
524		error = snapshot_request(vmname, checkpoint_file, vm_suspend_opt);
525#endif
526
527	free(opts);
528	exit(error);
529}
530