1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011 The University of Melbourne
5 * All rights reserved.
6 *
7 * This software was developed by Julien Ridoux at the University of Melbourne
8 * under sponsorship from the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD$");
34
35#include "opt_ffclock.h"
36
37#include <sys/param.h>
38#include <sys/bus.h>
39#include <sys/kernel.h>
40#include <sys/lock.h>
41#include <sys/module.h>
42#include <sys/mutex.h>
43#include <sys/priv.h>
44#include <sys/proc.h>
45#include <sys/sbuf.h>
46#include <sys/sysent.h>
47#include <sys/sysproto.h>
48#include <sys/sysctl.h>
49#include <sys/systm.h>
50#include <sys/timeffc.h>
51
52#ifdef FFCLOCK
53
54FEATURE(ffclock, "Feed-forward clock support");
55
56extern struct ffclock_estimate ffclock_estimate;
57extern struct bintime ffclock_boottime;
58extern int8_t ffclock_updated;
59extern struct mtx ffclock_mtx;
60
61/*
62 * Feed-forward clock absolute time. This should be the preferred way to read
63 * the feed-forward clock for "wall-clock" type time. The flags allow to compose
64 * various flavours of absolute time (e.g. with or without leap seconds taken
65 * into account). If valid pointers are provided, the ffcounter value and an
66 * upper bound on clock error associated with the bintime are provided.
67 * NOTE: use ffclock_convert_abs() to differ the conversion of a ffcounter value
68 * read earlier.
69 */
70void
71ffclock_abstime(ffcounter *ffcount, struct bintime *bt,
72    struct bintime *error_bound, uint32_t flags)
73{
74	struct ffclock_estimate cest;
75	ffcounter ffc;
76	ffcounter update_ffcount;
77	ffcounter ffdelta_error;
78
79	/* Get counter and corresponding time. */
80	if ((flags & FFCLOCK_FAST) == FFCLOCK_FAST)
81		ffclock_last_tick(&ffc, bt, flags);
82	else {
83		ffclock_read_counter(&ffc);
84		ffclock_convert_abs(ffc, bt, flags);
85	}
86
87	/* Current ffclock estimate, use update_ffcount as generation number. */
88	do {
89		update_ffcount = ffclock_estimate.update_ffcount;
90		bcopy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate));
91	} while (update_ffcount != ffclock_estimate.update_ffcount);
92
93	/*
94	 * Leap second adjustment. Total as seen by synchronisation algorithm
95	 * since it started. cest.leapsec_next is the ffcounter prediction of
96	 * when the next leapsecond occurs.
97	 */
98	if ((flags & FFCLOCK_LEAPSEC) == FFCLOCK_LEAPSEC) {
99		bt->sec -= cest.leapsec_total;
100		if (ffc > cest.leapsec_next)
101			bt->sec -= cest.leapsec;
102	}
103
104	/* Boot time adjustment, for uptime/monotonic clocks. */
105	if ((flags & FFCLOCK_UPTIME) == FFCLOCK_UPTIME) {
106		bintime_sub(bt, &ffclock_boottime);
107	}
108
109	/* Compute error bound if a valid pointer has been passed. */
110	if (error_bound) {
111		ffdelta_error = ffc - cest.update_ffcount;
112		ffclock_convert_diff(ffdelta_error, error_bound);
113		/* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */
114		bintime_mul(error_bound, cest.errb_rate *
115		    (uint64_t)18446744073709LL);
116		/* 18446744073 = int(2^64 / 1e9), since err_abs in [ns] */
117		bintime_addx(error_bound, cest.errb_abs *
118		    (uint64_t)18446744073LL);
119	}
120
121	if (ffcount)
122		*ffcount = ffc;
123}
124
125/*
126 * Feed-forward difference clock. This should be the preferred way to convert a
127 * time interval in ffcounter values into a time interval in seconds. If a valid
128 * pointer is passed, an upper bound on the error in computing the time interval
129 * in seconds is provided.
130 */
131void
132ffclock_difftime(ffcounter ffdelta, struct bintime *bt,
133    struct bintime *error_bound)
134{
135	ffcounter update_ffcount;
136	uint32_t err_rate;
137
138	ffclock_convert_diff(ffdelta, bt);
139
140	if (error_bound) {
141		do {
142			update_ffcount = ffclock_estimate.update_ffcount;
143			err_rate = ffclock_estimate.errb_rate;
144		} while (update_ffcount != ffclock_estimate.update_ffcount);
145
146		ffclock_convert_diff(ffdelta, error_bound);
147		/* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */
148		bintime_mul(error_bound, err_rate * (uint64_t)18446744073709LL);
149	}
150}
151
152/*
153 * Create a new kern.sysclock sysctl node, which will be home to some generic
154 * sysclock configuration variables. Feed-forward clock specific variables will
155 * live under the ffclock subnode.
156 */
157
158SYSCTL_NODE(_kern, OID_AUTO, sysclock, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
159    "System clock related configuration");
160SYSCTL_NODE(_kern_sysclock, OID_AUTO, ffclock, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
161    "Feed-forward clock configuration");
162
163static char *sysclocks[] = {"feedback", "feed-forward"};
164#define	MAX_SYSCLOCK_NAME_LEN 16
165#define	NUM_SYSCLOCKS nitems(sysclocks)
166
167static int ffclock_version = 2;
168SYSCTL_INT(_kern_sysclock_ffclock, OID_AUTO, version, CTLFLAG_RD,
169    &ffclock_version, 0, "Feed-forward clock kernel version");
170
171/* List available sysclocks. */
172static int
173sysctl_kern_sysclock_available(SYSCTL_HANDLER_ARGS)
174{
175	struct sbuf *s;
176	int clk, error;
177
178	s = sbuf_new_for_sysctl(NULL, NULL,
179	    MAX_SYSCLOCK_NAME_LEN * NUM_SYSCLOCKS, req);
180	if (s == NULL)
181		return (ENOMEM);
182
183	for (clk = 0; clk < NUM_SYSCLOCKS; clk++) {
184		sbuf_cat(s, sysclocks[clk]);
185		if (clk + 1 < NUM_SYSCLOCKS)
186			sbuf_cat(s, " ");
187	}
188	error = sbuf_finish(s);
189	sbuf_delete(s);
190
191	return (error);
192}
193
194SYSCTL_PROC(_kern_sysclock, OID_AUTO, available,
195    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, 0, 0,
196    sysctl_kern_sysclock_available, "A",
197    "List of available system clocks");
198
199/*
200 * Return the name of the active system clock if read, or attempt to change
201 * the active system clock to the user specified one if written to. The active
202 * system clock is read when calling any of the [get]{bin,nano,micro}[up]time()
203 * functions.
204 */
205static int
206sysctl_kern_sysclock_active(SYSCTL_HANDLER_ARGS)
207{
208	char newclock[MAX_SYSCLOCK_NAME_LEN];
209	int error;
210	int clk;
211
212	/* Return the name of the current active sysclock. */
213	strlcpy(newclock, sysclocks[sysclock_active], sizeof(newclock));
214	error = sysctl_handle_string(oidp, newclock, sizeof(newclock), req);
215
216	/* Check for error or no change */
217	if (error != 0 || req->newptr == NULL)
218		goto done;
219
220	/* Change the active sysclock to the user specified one: */
221	error = EINVAL;
222	for (clk = 0; clk < NUM_SYSCLOCKS; clk++) {
223		if (strncmp(newclock, sysclocks[clk],
224		    MAX_SYSCLOCK_NAME_LEN - 1)) {
225			continue;
226		}
227		sysclock_active = clk;
228		error = 0;
229		break;
230	}
231done:
232	return (error);
233}
234
235SYSCTL_PROC(_kern_sysclock, OID_AUTO, active,
236    CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 0, 0,
237    sysctl_kern_sysclock_active, "A",
238    "Name of the active system clock which is currently serving time");
239
240static int sysctl_kern_ffclock_ffcounter_bypass = 0;
241SYSCTL_INT(_kern_sysclock_ffclock, OID_AUTO, ffcounter_bypass, CTLFLAG_RW,
242    &sysctl_kern_ffclock_ffcounter_bypass, 0,
243    "Use reliable hardware timecounter as the feed-forward counter");
244
245/*
246 * High level functions to access the Feed-Forward Clock.
247 */
248void
249ffclock_bintime(struct bintime *bt)
250{
251
252	ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC);
253}
254
255void
256ffclock_nanotime(struct timespec *tsp)
257{
258	struct bintime bt;
259
260	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC);
261	bintime2timespec(&bt, tsp);
262}
263
264void
265ffclock_microtime(struct timeval *tvp)
266{
267	struct bintime bt;
268
269	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC);
270	bintime2timeval(&bt, tvp);
271}
272
273void
274ffclock_getbintime(struct bintime *bt)
275{
276
277	ffclock_abstime(NULL, bt, NULL,
278	    FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST);
279}
280
281void
282ffclock_getnanotime(struct timespec *tsp)
283{
284	struct bintime bt;
285
286	ffclock_abstime(NULL, &bt, NULL,
287	    FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST);
288	bintime2timespec(&bt, tsp);
289}
290
291void
292ffclock_getmicrotime(struct timeval *tvp)
293{
294	struct bintime bt;
295
296	ffclock_abstime(NULL, &bt, NULL,
297	    FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST);
298	bintime2timeval(&bt, tvp);
299}
300
301void
302ffclock_binuptime(struct bintime *bt)
303{
304
305	ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME);
306}
307
308void
309ffclock_nanouptime(struct timespec *tsp)
310{
311	struct bintime bt;
312
313	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME);
314	bintime2timespec(&bt, tsp);
315}
316
317void
318ffclock_microuptime(struct timeval *tvp)
319{
320	struct bintime bt;
321
322	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME);
323	bintime2timeval(&bt, tvp);
324}
325
326void
327ffclock_getbinuptime(struct bintime *bt)
328{
329
330	ffclock_abstime(NULL, bt, NULL,
331	    FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST);
332}
333
334void
335ffclock_getnanouptime(struct timespec *tsp)
336{
337	struct bintime bt;
338
339	ffclock_abstime(NULL, &bt, NULL,
340	    FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST);
341	bintime2timespec(&bt, tsp);
342}
343
344void
345ffclock_getmicrouptime(struct timeval *tvp)
346{
347	struct bintime bt;
348
349	ffclock_abstime(NULL, &bt, NULL,
350	    FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST);
351	bintime2timeval(&bt, tvp);
352}
353
354void
355ffclock_bindifftime(ffcounter ffdelta, struct bintime *bt)
356{
357
358	ffclock_difftime(ffdelta, bt, NULL);
359}
360
361void
362ffclock_nanodifftime(ffcounter ffdelta, struct timespec *tsp)
363{
364	struct bintime bt;
365
366	ffclock_difftime(ffdelta, &bt, NULL);
367	bintime2timespec(&bt, tsp);
368}
369
370void
371ffclock_microdifftime(ffcounter ffdelta, struct timeval *tvp)
372{
373	struct bintime bt;
374
375	ffclock_difftime(ffdelta, &bt, NULL);
376	bintime2timeval(&bt, tvp);
377}
378
379/*
380 * System call allowing userland applications to retrieve the current value of
381 * the Feed-Forward Clock counter.
382 */
383#ifndef _SYS_SYSPROTO_H_
384struct ffclock_getcounter_args {
385	ffcounter *ffcount;
386};
387#endif
388/* ARGSUSED */
389int
390sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap)
391{
392	ffcounter ffcount;
393	int error;
394
395	ffcount = 0;
396	ffclock_read_counter(&ffcount);
397	if (ffcount == 0)
398		return (EAGAIN);
399	error = copyout(&ffcount, uap->ffcount, sizeof(ffcounter));
400
401	return (error);
402}
403
404/*
405 * System call allowing the synchronisation daemon to push new feed-foward clock
406 * estimates to the kernel. Acquire ffclock_mtx to prevent concurrent updates
407 * and ensure data consistency.
408 * NOTE: ffclock_updated signals the fftimehands that new estimates are
409 * available. The updated estimates are picked up by the fftimehands on next
410 * tick, which could take as long as 1/hz seconds (if ticks are not missed).
411 */
412#ifndef _SYS_SYSPROTO_H_
413struct ffclock_setestimate_args {
414	struct ffclock_estimate *cest;
415};
416#endif
417/* ARGSUSED */
418int
419sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap)
420{
421	struct ffclock_estimate cest;
422	int error;
423
424	/* Reuse of PRIV_CLOCK_SETTIME. */
425	if ((error = priv_check(td, PRIV_CLOCK_SETTIME)) != 0)
426		return (error);
427
428	if ((error = copyin(uap->cest, &cest, sizeof(struct ffclock_estimate)))
429	    != 0)
430		return (error);
431
432	mtx_lock(&ffclock_mtx);
433	memcpy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate));
434	ffclock_updated++;
435	mtx_unlock(&ffclock_mtx);
436	return (error);
437}
438
439/*
440 * System call allowing userland applications to retrieve the clock estimates
441 * stored within the kernel. It is useful to kickstart the synchronisation
442 * daemon with the kernel's knowledge of hardware timecounter.
443 */
444#ifndef _SYS_SYSPROTO_H_
445struct ffclock_getestimate_args {
446	struct ffclock_estimate *cest;
447};
448#endif
449/* ARGSUSED */
450int
451sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap)
452{
453	struct ffclock_estimate cest;
454	int error;
455
456	mtx_lock(&ffclock_mtx);
457	memcpy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate));
458	mtx_unlock(&ffclock_mtx);
459	error = copyout(&cest, uap->cest, sizeof(struct ffclock_estimate));
460	return (error);
461}
462
463#else /* !FFCLOCK */
464
465int
466sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap)
467{
468
469	return (ENOSYS);
470}
471
472int
473sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap)
474{
475
476	return (ENOSYS);
477}
478
479int
480sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap)
481{
482
483	return (ENOSYS);
484}
485
486#endif /* FFCLOCK */
487