1/*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2013 Gleb Smirnoff <glebius@FreeBSD.org>
5 * Copyright (c) 2010 Juniper Networks, Inc.
6 * Copyright (c) 2009 Robert N. M. Watson
7 * Copyright (c) 2009 Bjoern A. Zeeb <bz@FreeBSD.org>
8 * Copyright (c) 2008 Yahoo!, Inc.
9 * All rights reserved.
10 *
11 * Written by: John Baldwin <jhb@FreeBSD.org>
12 *
13 * This software was developed by Robert N. M. Watson under contract
14 * to Juniper Networks, Inc.
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 * 1. Redistributions of source code must retain the above copyright
20 *    notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 *    notice, this list of conditions and the following disclaimer in the
23 *    documentation and/or other materials provided with the distribution.
24 * 3. Neither the name of the author nor the names of any co-contributors
25 *    may be used to endorse or promote products derived from this software
26 *    without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 */
40
41#include <sys/param.h>
42#include <sys/pcpu.h>
43#include <sys/sysctl.h>
44#include <kvm.h>
45#include <limits.h>
46#include <stdlib.h>
47
48#include "kvm_private.h"
49
50#ifdef __amd64__
51#define	__OFFSET_BY_PCPU
52#endif
53
54static struct nlist kvm_pcpu_nl[] = {
55	{ .n_name = "_cpuid_to_pcpu" },
56	{ .n_name = "_mp_maxcpus" },
57	{ .n_name = "_mp_ncpus" },
58#ifdef __OFFSET_BY_PCPU
59	{ .n_name = "___pcpu" },
60#endif
61	{ .n_name = NULL },
62};
63#define	NL_CPUID_TO_PCPU	0
64#define	NL_MP_MAXCPUS		1
65#define	NL_MP_NCPUS		2
66#define	NL___PCPU		3
67
68/*
69 * Kernel per-CPU data state.  We cache this stuff on the first
70 * access.
71 *
72 * XXXRW: Possibly, this (and kvmpcpu_nl) should be per-kvm_t, in case the
73 * consumer has multiple handles in flight to differently configured
74 * kernels/crashdumps.
75 */
76static void **pcpu_data;
77static int maxcpu;
78static int mp_ncpus;
79#ifdef __OFFSET_BY_PCPU
80static unsigned long __pcpu;
81#endif
82
83static int
84_kvm_pcpu_init(kvm_t *kd)
85{
86	size_t len;
87	int max;
88	void *data;
89
90	if (kvm_nlist(kd, kvm_pcpu_nl) < 0)
91		return (-1);
92	if (kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value == 0) {
93		_kvm_err(kd, kd->program, "unable to find cpuid_to_pcpu");
94		return (-1);
95	}
96	if (kvm_pcpu_nl[NL_MP_MAXCPUS].n_value == 0) {
97		_kvm_err(kd, kd->program, "unable to find mp_maxcpus");
98		return (-1);
99	}
100	if (kvm_read(kd, kvm_pcpu_nl[NL_MP_MAXCPUS].n_value, &max,
101	    sizeof(max)) != sizeof(max)) {
102		_kvm_err(kd, kd->program, "cannot read mp_maxcpus");
103		return (-1);
104	}
105	if (kvm_pcpu_nl[NL_MP_NCPUS].n_value == 0) {
106		_kvm_err(kd, kd->program, "unable to find mp_ncpus");
107		return (-1);
108	}
109	if (kvm_read(kd, kvm_pcpu_nl[NL_MP_NCPUS].n_value, &mp_ncpus,
110	    sizeof(mp_ncpus)) != sizeof(mp_ncpus)) {
111		_kvm_err(kd, kd->program, "cannot read mp_ncpus");
112		return (-1);
113	}
114#ifdef __OFFSET_BY_PCPU
115	if (kvm_pcpu_nl[NL___PCPU].n_value == 0) {
116		_kvm_err(kd, kd->program, "unable to find __pcpu");
117		return (-1);
118	}
119	if (kvm_read(kd, kvm_pcpu_nl[NL___PCPU].n_value, &__pcpu,
120	    sizeof(__pcpu)) != sizeof(__pcpu)) {
121		_kvm_err(kd, kd->program, "cannot read __pcpu");
122		return (-1);
123	}
124#endif
125	len = max * sizeof(void *);
126	data = malloc(len);
127	if (data == NULL) {
128		_kvm_err(kd, kd->program, "out of memory");
129		return (-1);
130	}
131	if (kvm_read(kd, kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value, data, len) !=
132	   (ssize_t)len) {
133		_kvm_err(kd, kd->program, "cannot read cpuid_to_pcpu array");
134		free(data);
135		return (-1);
136	}
137	pcpu_data = data;
138	maxcpu = max;
139	return (0);
140}
141
142static void
143_kvm_pcpu_clear(void)
144{
145
146	maxcpu = 0;
147	free(pcpu_data);
148	pcpu_data = NULL;
149}
150
151void *
152kvm_getpcpu(kvm_t *kd, int cpu)
153{
154	char *buf;
155
156	if (kd == NULL) {
157		_kvm_pcpu_clear();
158		return (NULL);
159	}
160
161	if (maxcpu == 0)
162		if (_kvm_pcpu_init(kd) < 0)
163			return ((void *)-1);
164
165	if (cpu >= maxcpu || pcpu_data[cpu] == NULL)
166		return (NULL);
167
168	buf = malloc(sizeof(struct pcpu));
169	if (buf == NULL) {
170		_kvm_err(kd, kd->program, "out of memory");
171		return ((void *)-1);
172	}
173	if (kvm_read(kd, (uintptr_t)pcpu_data[cpu], buf,
174	    sizeof(struct pcpu)) != sizeof(struct pcpu)) {
175		_kvm_err(kd, kd->program, "unable to read per-CPU data");
176		free(buf);
177		return ((void *)-1);
178	}
179	return (buf);
180}
181
182int
183kvm_getmaxcpu(kvm_t *kd)
184{
185
186	if (kd == NULL) {
187		_kvm_pcpu_clear();
188		return (0);
189	}
190
191	if (maxcpu == 0)
192		if (_kvm_pcpu_init(kd) < 0)
193			return (-1);
194	return (maxcpu);
195}
196
197int
198kvm_getncpus(kvm_t *kd)
199{
200
201	if (mp_ncpus == 0)
202		if (_kvm_pcpu_init(kd) < 0)
203			return (-1);
204	return (mp_ncpus);
205}
206
207static int
208_kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu, int report_error)
209{
210
211	if (!kd->dpcpu_initialized) {
212		if (report_error)
213			_kvm_err(kd, kd->program, "%s: not initialized",
214			    __func__);
215		return (-1);
216	}
217	if (cpu >= kd->dpcpu_maxcpus) {
218		if (report_error)
219			_kvm_err(kd, kd->program, "%s: CPU %u too big",
220			    __func__, cpu);
221		return (-1);
222	}
223	if (kd->dpcpu_off[cpu] == 0) {
224		if (report_error)
225			_kvm_err(kd, kd->program, "%s: CPU %u not found",
226			    __func__, cpu);
227		return (-1);
228	}
229	kd->dpcpu_curcpu = cpu;
230	kd->dpcpu_curoff = kd->dpcpu_off[cpu];
231	return (0);
232}
233
234/*
235 * Set up libkvm to handle dynamic per-CPU memory.
236 */
237static int
238_kvm_dpcpu_init(kvm_t *kd)
239{
240	struct kvm_nlist nl[] = {
241#define	NLIST_START_SET_PCPU	0
242		{ .n_name = "___start_" DPCPU_SETNAME },
243#define	NLIST_STOP_SET_PCPU	1
244		{ .n_name = "___stop_" DPCPU_SETNAME },
245#define	NLIST_DPCPU_OFF		2
246		{ .n_name = "_dpcpu_off" },
247#define	NLIST_MP_MAXCPUS	3
248		{ .n_name = "_mp_maxcpus" },
249		{ .n_name = NULL },
250	};
251	uintptr_t *dpcpu_off_buf;
252	size_t len;
253	u_int dpcpu_maxcpus;
254
255	/*
256	 * XXX: This only works for native kernels for now.
257	 */
258	if (!kvm_native(kd))
259		return (-1);
260
261	/*
262	 * Locate and cache locations of important symbols using the internal
263	 * version of _kvm_nlist, turning off initialization to avoid
264	 * recursion in case of unresolveable symbols.
265	 */
266	if (_kvm_nlist(kd, nl, 0) != 0)
267		return (-1);
268	if (kvm_read(kd, nl[NLIST_MP_MAXCPUS].n_value, &dpcpu_maxcpus,
269	    sizeof(dpcpu_maxcpus)) != sizeof(dpcpu_maxcpus))
270		return (-1);
271	len = dpcpu_maxcpus * sizeof(*dpcpu_off_buf);
272	dpcpu_off_buf = malloc(len);
273	if (dpcpu_off_buf == NULL)
274		return (-1);
275	if (kvm_read(kd, nl[NLIST_DPCPU_OFF].n_value, dpcpu_off_buf, len) !=
276	    (ssize_t)len) {
277		free(dpcpu_off_buf);
278		return (-1);
279	}
280	kd->dpcpu_start = nl[NLIST_START_SET_PCPU].n_value;
281	kd->dpcpu_stop = nl[NLIST_STOP_SET_PCPU].n_value;
282	kd->dpcpu_maxcpus = dpcpu_maxcpus;
283	kd->dpcpu_off = dpcpu_off_buf;
284	kd->dpcpu_initialized = 1;
285	(void)_kvm_dpcpu_setcpu(kd, 0, 0);
286	return (0);
287}
288
289/*
290 * Check whether the dpcpu module has been initialized successfully or not,
291 * initialize it if permitted.
292 */
293int
294_kvm_dpcpu_initialized(kvm_t *kd, int intialize)
295{
296
297	if (kd->dpcpu_initialized || !intialize)
298		return (kd->dpcpu_initialized);
299
300	(void)_kvm_dpcpu_init(kd);
301
302	return (kd->dpcpu_initialized);
303}
304
305/*
306 * Check whether the value is within the dpcpu symbol range and only if so
307 * adjust the offset relative to the current offset.
308 */
309kvaddr_t
310_kvm_dpcpu_validaddr(kvm_t *kd, kvaddr_t value)
311{
312
313	if (value == 0)
314		return (value);
315
316	if (!kd->dpcpu_initialized)
317		return (value);
318
319	if (value < kd->dpcpu_start || value >= kd->dpcpu_stop)
320		return (value);
321
322	return (kd->dpcpu_curoff + value);
323}
324
325int
326kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu)
327{
328	int ret;
329
330	if (!kd->dpcpu_initialized) {
331		ret = _kvm_dpcpu_init(kd);
332		if (ret != 0) {
333			_kvm_err(kd, kd->program, "%s: init failed",
334			    __func__);
335			return (ret);
336		}
337	}
338
339	return (_kvm_dpcpu_setcpu(kd, cpu, 1));
340}
341
342/*
343 * Obtain a per-CPU copy for given cpu from UMA_ZONE_PCPU allocation.
344 */
345ssize_t
346kvm_read_zpcpu(kvm_t *kd, u_long base, void *buf, size_t size, int cpu)
347{
348
349	if (!kvm_native(kd))
350		return (-1);
351	if (mp_ncpus == 0)
352		if (_kvm_pcpu_init(kd) < 0)
353			return (0);
354
355#ifdef __OFFSET_BY_PCPU
356	base += __pcpu;
357#endif
358	return (kvm_read(kd, (uintptr_t)(base + sizeof(struct pcpu) * cpu),
359	    buf, size));
360}
361
362/*
363 * Fetch value of a counter(9).
364 */
365uint64_t
366kvm_counter_u64_fetch(kvm_t *kd, u_long base)
367{
368	uint64_t r, c;
369
370	if (mp_ncpus == 0)
371		if (_kvm_pcpu_init(kd) < 0)
372			return (0);
373
374	r = 0;
375	for (int i = 0; i < mp_ncpus; i++) {
376		if (kvm_read_zpcpu(kd, base, &c, sizeof(c), i) != sizeof(c))
377			return (0);
378		r += c;
379	}
380
381	return (r);
382}
383