Bug Summary

File:kern/kern_timeout.c
Warning:line 792, column 2
Value stored to 's' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name kern_timeout.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/kern/kern_timeout.c
1/* $OpenBSD: kern_timeout.c,v 1.85 2021/06/19 02:05:33 cheloha Exp $ */
2/*
3 * Copyright (c) 2001 Thomas Nordin <nordin@openbsd.org>
4 * Copyright (c) 2000-2001 Artur Grabowski <art@openbsd.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
17 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
18 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
19 * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include <sys/param.h>
29#include <sys/systm.h>
30#include <sys/kthread.h>
31#include <sys/proc.h>
32#include <sys/timeout.h>
33#include <sys/mutex.h>
34#include <sys/kernel.h>
35#include <sys/queue.h> /* _Q_INVALIDATE */
36#include <sys/sysctl.h>
37#include <sys/witness.h>
38
39#ifdef DDB1
40#include <machine/db_machdep.h>
41#include <ddb/db_interface.h>
42#include <ddb/db_sym.h>
43#include <ddb/db_output.h>
44#endif
45
46#include "kcov.h"
47#if NKCOV0 > 0
48#include <sys/kcov.h>
49#endif
50
51/*
52 * Locks used to protect global variables in this file:
53 *
54 * I immutable after initialization
55 * T timeout_mutex
56 */
57struct mutex timeout_mutex = MUTEX_INITIALIZER(IPL_HIGH){ ((void *)0), ((((0xd)) > 0x0 && ((0xd)) < 0x9
) ? 0x9 : ((0xd))), 0x0 };
58
59void *softclock_si; /* [I] softclock() interrupt handle */
60struct timeoutstat tostat; /* [T] statistics and totals */
61
62/*
63 * Timeouts are kept in a hierarchical timing wheel. The to_time is the value
64 * of the global variable "ticks" when the timeout should be called. There are
65 * four levels with 256 buckets each.
66 */
67#define WHEELCOUNT4 4
68#define WHEELSIZE256 256
69#define WHEELMASK255 255
70#define WHEELBITS8 8
71#define BUCKETS(4 * 256) (WHEELCOUNT4 * WHEELSIZE256)
72
73struct circq timeout_wheel[BUCKETS(4 * 256)]; /* [T] Tick-based timeouts */
74struct circq timeout_wheel_kc[BUCKETS(4 * 256)]; /* [T] Clock-based timeouts */
75struct circq timeout_new; /* [T] New, unscheduled timeouts */
76struct circq timeout_todo; /* [T] Due or needs rescheduling */
77struct circq timeout_proc; /* [T] Due + needs process context */
78
79time_t timeout_level_width[WHEELCOUNT4]; /* [I] Wheel level width (seconds) */
80struct timespec tick_ts; /* [I] Length of a tick (1/hz secs) */
81
82struct kclock {
83 struct timespec kc_lastscan; /* [T] Clock time at last wheel scan */
84 struct timespec kc_late; /* [T] Late if due prior */
85 struct timespec kc_offset; /* [T] Offset from primary kclock */
86} timeout_kclock[KCLOCK_MAX1];
87
88#define MASKWHEEL(wheel, time)(((time) >> ((wheel)*8)) & 255) (((time) >> ((wheel)*WHEELBITS8)) & WHEELMASK255)
89
90#define BUCKET(rel, abs)(timeout_wheel[ ((rel) <= (1 << (2*8))) ? ((rel) <=
(1 << 8)) ? ((((abs)) >> ((0)*8)) & 255) : (
(((abs)) >> ((1)*8)) & 255) + 256 : ((rel) <= (1
<< (3*8))) ? ((((abs)) >> ((2)*8)) & 255) + 2
*256 : ((((abs)) >> ((3)*8)) & 255) + 3*256]) \
91 (timeout_wheel[ \
92 ((rel) <= (1 << (2*WHEELBITS8))) \
93 ? ((rel) <= (1 << WHEELBITS8)) \
94 ? MASKWHEEL(0, (abs))((((abs)) >> ((0)*8)) & 255) \
95 : MASKWHEEL(1, (abs))((((abs)) >> ((1)*8)) & 255) + WHEELSIZE256 \
96 : ((rel) <= (1 << (3*WHEELBITS8))) \
97 ? MASKWHEEL(2, (abs))((((abs)) >> ((2)*8)) & 255) + 2*WHEELSIZE256 \
98 : MASKWHEEL(3, (abs))((((abs)) >> ((3)*8)) & 255) + 3*WHEELSIZE256])
99
100#define MOVEBUCKET(wheel, time)do { if (!(((&timeout_wheel[((((time)) >> (((wheel)
)*8)) & 255) + (wheel)*256])->next) == (&timeout_wheel
[((((time)) >> (((wheel))*8)) & 255) + (wheel)*256]
))) { (&timeout_todo)->prev->next = (&timeout_wheel
[((((time)) >> (((wheel))*8)) & 255) + (wheel)*256]
)->next; (&timeout_wheel[((((time)) >> (((wheel)
)*8)) & 255) + (wheel)*256])->next->prev = (&timeout_todo
)->prev; (&timeout_wheel[((((time)) >> (((wheel)
)*8)) & 255) + (wheel)*256])->prev->next = (&timeout_todo
); (&timeout_todo)->prev = (&timeout_wheel[((((time
)) >> (((wheel))*8)) & 255) + (wheel)*256])->prev
; do { (&timeout_wheel[((((time)) >> (((wheel))*8))
& 255) + (wheel)*256])->next = (&timeout_wheel[((
((time)) >> (((wheel))*8)) & 255) + (wheel)*256]); (
&timeout_wheel[((((time)) >> (((wheel))*8)) & 255
) + (wheel)*256])->prev = (&timeout_wheel[((((time)) >>
(((wheel))*8)) & 255) + (wheel)*256]); } while (0); } } while
(0) \
101 CIRCQ_CONCAT(&timeout_todo, \do { if (!(((&timeout_wheel[((((time)) >> (((wheel)
)*8)) & 255) + (wheel)*256])->next) == (&timeout_wheel
[((((time)) >> (((wheel))*8)) & 255) + (wheel)*256]
))) { (&timeout_todo)->prev->next = (&timeout_wheel
[((((time)) >> (((wheel))*8)) & 255) + (wheel)*256]
)->next; (&timeout_wheel[((((time)) >> (((wheel)
)*8)) & 255) + (wheel)*256])->next->prev = (&timeout_todo
)->prev; (&timeout_wheel[((((time)) >> (((wheel)
)*8)) & 255) + (wheel)*256])->prev->next = (&timeout_todo
); (&timeout_todo)->prev = (&timeout_wheel[((((time
)) >> (((wheel))*8)) & 255) + (wheel)*256])->prev
; do { (&timeout_wheel[((((time)) >> (((wheel))*8))
& 255) + (wheel)*256])->next = (&timeout_wheel[((
((time)) >> (((wheel))*8)) & 255) + (wheel)*256]); (
&timeout_wheel[((((time)) >> (((wheel))*8)) & 255
) + (wheel)*256])->prev = (&timeout_wheel[((((time)) >>
(((wheel))*8)) & 255) + (wheel)*256]); } while (0); } } while
(0)
102 &timeout_wheel[MASKWHEEL((wheel), (time)) + (wheel)*WHEELSIZE])do { if (!(((&timeout_wheel[((((time)) >> (((wheel)
)*8)) & 255) + (wheel)*256])->next) == (&timeout_wheel
[((((time)) >> (((wheel))*8)) & 255) + (wheel)*256]
))) { (&timeout_todo)->prev->next = (&timeout_wheel
[((((time)) >> (((wheel))*8)) & 255) + (wheel)*256]
)->next; (&timeout_wheel[((((time)) >> (((wheel)
)*8)) & 255) + (wheel)*256])->next->prev = (&timeout_todo
)->prev; (&timeout_wheel[((((time)) >> (((wheel)
)*8)) & 255) + (wheel)*256])->prev->next = (&timeout_todo
); (&timeout_todo)->prev = (&timeout_wheel[((((time
)) >> (((wheel))*8)) & 255) + (wheel)*256])->prev
; do { (&timeout_wheel[((((time)) >> (((wheel))*8))
& 255) + (wheel)*256])->next = (&timeout_wheel[((
((time)) >> (((wheel))*8)) & 255) + (wheel)*256]); (
&timeout_wheel[((((time)) >> (((wheel))*8)) & 255
) + (wheel)*256])->prev = (&timeout_wheel[((((time)) >>
(((wheel))*8)) & 255) + (wheel)*256]); } while (0); } } while
(0)
103
104/*
105 * Circular queue definitions.
106 */
107
108#define CIRCQ_INIT(elem)do { (elem)->next = (elem); (elem)->prev = (elem); } while
(0) do { \
109 (elem)->next = (elem); \
110 (elem)->prev = (elem); \
111} while (0)
112
113#define CIRCQ_INSERT_TAIL(list, elem)do { (elem)->prev = (list)->prev; (elem)->next = (list
); (list)->prev->next = (elem); (list)->prev = (elem
); tostat.tos_pending++; } while (0) do { \
114 (elem)->prev = (list)->prev; \
115 (elem)->next = (list); \
116 (list)->prev->next = (elem); \
117 (list)->prev = (elem); \
118 tostat.tos_pending++; \
119} while (0)
120
121#define CIRCQ_CONCAT(fst, snd)do { if (!(((snd)->next) == (snd))) { (fst)->prev->next
= (snd)->next; (snd)->next->prev = (fst)->prev; (
snd)->prev->next = (fst); (fst)->prev = (snd)->prev
; do { (snd)->next = (snd); (snd)->prev = (snd); } while
(0); } } while (0) do { \
122 if (!CIRCQ_EMPTY(snd)(((snd)->next) == (snd))) { \
123 (fst)->prev->next = (snd)->next;\
124 (snd)->next->prev = (fst)->prev;\
125 (snd)->prev->next = (fst); \
126 (fst)->prev = (snd)->prev; \
127 CIRCQ_INIT(snd)do { (snd)->next = (snd); (snd)->prev = (snd); } while (
0); \
128 } \
129} while (0)
130
131#define CIRCQ_REMOVE(elem)do { (elem)->next->prev = (elem)->prev; (elem)->prev
->next = (elem)->next; ((elem)->prev) = ((void *)-1)
; ((elem)->next) = ((void *)-1); tostat.tos_pending--; } while
(0) do { \
132 (elem)->next->prev = (elem)->prev; \
133 (elem)->prev->next = (elem)->next; \
134 _Q_INVALIDATE((elem)->prev)((elem)->prev) = ((void *)-1); \
135 _Q_INVALIDATE((elem)->next)((elem)->next) = ((void *)-1); \
136 tostat.tos_pending--; \
137} while (0)
138
139#define CIRCQ_FIRST(elem)((elem)->next) ((elem)->next)
140
141#define CIRCQ_EMPTY(elem)(((elem)->next) == (elem)) (CIRCQ_FIRST(elem)((elem)->next) == (elem))
142
143#define CIRCQ_FOREACH(elem, list)for ((elem) = ((list)->next); (elem) != (list); (elem) = (
(elem)->next)) \
144 for ((elem) = CIRCQ_FIRST(list)((list)->next); \
145 (elem) != (list); \
146 (elem) = CIRCQ_FIRST(elem)((elem)->next))
147
148#ifdef WITNESS
149struct lock_object timeout_sleeplock_obj = {
150 .lo_name = "timeout",
151 .lo_flags = LO_WITNESS0x00020000 | LO_INITIALIZED0x00010000 | LO_SLEEPABLE0x00100000 |
152 (LO_CLASS_RWLOCK << LO_CLASSSHIFT24)
153};
154struct lock_object timeout_spinlock_obj = {
155 .lo_name = "timeout",
156 .lo_flags = LO_WITNESS0x00020000 | LO_INITIALIZED0x00010000 |
157 (LO_CLASS_MUTEX << LO_CLASSSHIFT24)
158};
159struct lock_type timeout_sleeplock_type = {
160 .lt_name = "timeout"
161};
162struct lock_type timeout_spinlock_type = {
163 .lt_name = "timeout"
164};
165#define TIMEOUT_LOCK_OBJ(needsproc) \
166 ((needsproc) ? &timeout_sleeplock_obj : &timeout_spinlock_obj)
167#endif
168
169void kclock_nanotime(int, struct timespec *);
170void softclock(void *);
171void softclock_create_thread(void *);
172void softclock_process_kclock_timeout(struct timeout *, int);
173void softclock_process_tick_timeout(struct timeout *, int);
174void softclock_thread(void *);
175void timeout_barrier_timeout(void *);
176uint32_t timeout_bucket(const struct timeout *);
177uint32_t timeout_maskwheel(uint32_t, const struct timespec *);
178void timeout_run(struct timeout *);
179
180/*
181 * The first thing in a struct timeout is its struct circq, so we
182 * can get back from a pointer to the latter to a pointer to the
183 * whole timeout with just a cast.
184 */
185static inline struct timeout *
186timeout_from_circq(struct circq *p)
187{
188 return ((struct timeout *)(p));
189}
190
191static inline void
192timeout_sync_order(int needsproc)
193{
194 WITNESS_CHECKORDER(TIMEOUT_LOCK_OBJ(needsproc), LOP_NEWORDER, NULL)(void)0;
195}
196
197static inline void
198timeout_sync_enter(int needsproc)
199{
200 timeout_sync_order(needsproc);
201 WITNESS_LOCK(TIMEOUT_LOCK_OBJ(needsproc), 0)(void)0;
202}
203
204static inline void
205timeout_sync_leave(int needsproc)
206{
207 WITNESS_UNLOCK(TIMEOUT_LOCK_OBJ(needsproc), 0)(void)0;
208}
209
210/*
211 * Some of the "math" in here is a bit tricky.
212 *
213 * We have to beware of wrapping ints.
214 * We use the fact that any element added to the queue must be added with a
215 * positive time. That means that any element `to' on the queue cannot be
216 * scheduled to timeout further in time than INT_MAX, but to->to_time can
217 * be positive or negative so comparing it with anything is dangerous.
218 * The only way we can use the to->to_time value in any predictable way
219 * is when we calculate how far in the future `to' will timeout -
220 * "to->to_time - ticks". The result will always be positive for future
221 * timeouts and 0 or negative for due timeouts.
222 */
223
224void
225timeout_startup(void)
226{
227 int b, level;
228
229 CIRCQ_INIT(&timeout_new)do { (&timeout_new)->next = (&timeout_new); (&
timeout_new)->prev = (&timeout_new); } while (0);
230 CIRCQ_INIT(&timeout_todo)do { (&timeout_todo)->next = (&timeout_todo); (&
timeout_todo)->prev = (&timeout_todo); } while (0);
231 CIRCQ_INIT(&timeout_proc)do { (&timeout_proc)->next = (&timeout_proc); (&
timeout_proc)->prev = (&timeout_proc); } while (0);
232 for (b = 0; b < nitems(timeout_wheel)(sizeof((timeout_wheel)) / sizeof((timeout_wheel)[0])); b++)
233 CIRCQ_INIT(&timeout_wheel[b])do { (&timeout_wheel[b])->next = (&timeout_wheel[b
]); (&timeout_wheel[b])->prev = (&timeout_wheel[b]
); } while (0);
234 for (b = 0; b < nitems(timeout_wheel_kc)(sizeof((timeout_wheel_kc)) / sizeof((timeout_wheel_kc)[0])); b++)
235 CIRCQ_INIT(&timeout_wheel_kc[b])do { (&timeout_wheel_kc[b])->next = (&timeout_wheel_kc
[b]); (&timeout_wheel_kc[b])->prev = (&timeout_wheel_kc
[b]); } while (0);
236
237 for (level = 0; level < nitems(timeout_level_width)(sizeof((timeout_level_width)) / sizeof((timeout_level_width)
[0])); level++)
238 timeout_level_width[level] = 2 << (level * WHEELBITS8);
239 NSEC_TO_TIMESPEC(tick_nsec, &tick_ts);
240}
241
242void
243timeout_proc_init(void)
244{
245 softclock_si = softintr_establish(IPL_SOFTCLOCK0x4, softclock, NULL((void *)0));
246 if (softclock_si == NULL((void *)0))
247 panic("%s: unable to register softclock interrupt", __func__);
248
249 WITNESS_INIT(&timeout_sleeplock_obj, &timeout_sleeplock_type)(void)0;
250 WITNESS_INIT(&timeout_spinlock_obj, &timeout_spinlock_type)(void)0;
251
252 kthread_create_deferred(softclock_create_thread, NULL((void *)0));
253}
254
255static inline void
256_timeout_set(struct timeout *to, void (*fn)(void *), void *arg, int kclock,
257 int flags)
258{
259 to->to_func = fn;
260 to->to_arg = arg;
261 to->to_kclock = kclock;
262 to->to_flags = flags | TIMEOUT_INITIALIZED0x04;
263}
264
265void
266timeout_set(struct timeout *new, void (*fn)(void *), void *arg)
267{
268 _timeout_set(new, fn, arg, KCLOCK_NONE(-1), 0);
269}
270
271void
272timeout_set_flags(struct timeout *to, void (*fn)(void *), void *arg, int flags)
273{
274 _timeout_set(to, fn, arg, KCLOCK_NONE(-1), flags);
275}
276
277void
278timeout_set_proc(struct timeout *new, void (*fn)(void *), void *arg)
279{
280 _timeout_set(new, fn, arg, KCLOCK_NONE(-1), TIMEOUT_PROC0x01);
281}
282
283void
284timeout_set_kclock(struct timeout *to, void (*fn)(void *), void *arg,
285 int kclock, int flags)
286{
287 _timeout_set(to, fn, arg, kclock, flags | TIMEOUT_KCLOCK0x10);
288}
289
290int
291timeout_add(struct timeout *new, int to_ticks)
292{
293 int old_time;
294 int ret = 1;
295
296 KASSERT(ISSET(new->to_flags, TIMEOUT_INITIALIZED))((((new->to_flags) & (0x04))) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/kern/kern_timeout.c", 296, "ISSET(new->to_flags, TIMEOUT_INITIALIZED)"
));
297 KASSERT(!ISSET(new->to_flags, TIMEOUT_KCLOCK))((!((new->to_flags) & (0x10))) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/kern/kern_timeout.c", 297, "!ISSET(new->to_flags, TIMEOUT_KCLOCK)"
));
298 KASSERT(new->to_kclock == KCLOCK_NONE)((new->to_kclock == (-1)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/kern/kern_timeout.c", 298, "new->to_kclock == KCLOCK_NONE"
));
299 KASSERT(to_ticks >= 0)((to_ticks >= 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/kern/kern_timeout.c"
, 299, "to_ticks >= 0"));
300
301 mtx_enter(&timeout_mutex);
302
303 /* Initialize the time here, it won't change. */
304 old_time = new->to_time;
305 new->to_time = to_ticks + ticks;
306 CLR(new->to_flags, TIMEOUT_TRIGGERED)((new->to_flags) &= ~(0x08));
307
308 /*
309 * If this timeout already is scheduled and now is moved
310 * earlier, reschedule it now. Otherwise leave it in place
311 * and let it be rescheduled later.
312 */
313 if (ISSET(new->to_flags, TIMEOUT_ONQUEUE)((new->to_flags) & (0x02))) {
314 if (new->to_time - ticks < old_time - ticks) {
315 CIRCQ_REMOVE(&new->to_list)do { (&new->to_list)->next->prev = (&new->
to_list)->prev; (&new->to_list)->prev->next =
(&new->to_list)->next; ((&new->to_list)->
prev) = ((void *)-1); ((&new->to_list)->next) = ((void
*)-1); tostat.tos_pending--; } while (0);
316 CIRCQ_INSERT_TAIL(&timeout_new, &new->to_list)do { (&new->to_list)->prev = (&timeout_new)->
prev; (&new->to_list)->next = (&timeout_new); (
&timeout_new)->prev->next = (&new->to_list);
(&timeout_new)->prev = (&new->to_list); tostat
.tos_pending++; } while (0);
317 }
318 tostat.tos_readded++;
319 ret = 0;
320 } else {
321 SET(new->to_flags, TIMEOUT_ONQUEUE)((new->to_flags) |= (0x02));
322 CIRCQ_INSERT_TAIL(&timeout_new, &new->to_list)do { (&new->to_list)->prev = (&timeout_new)->
prev; (&new->to_list)->next = (&timeout_new); (
&timeout_new)->prev->next = (&new->to_list);
(&timeout_new)->prev = (&new->to_list); tostat
.tos_pending++; } while (0);
323 }
324#if NKCOV0 > 0
325 new->to_process = curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p;
326#endif
327 tostat.tos_added++;
328 mtx_leave(&timeout_mutex);
329
330 return ret;
331}
332
333int
334timeout_add_tv(struct timeout *to, const struct timeval *tv)
335{
336 uint64_t to_ticks;
337
338 to_ticks = (uint64_t)hz * tv->tv_sec + tv->tv_usec / tick;
339 if (to_ticks > INT_MAX0x7fffffff)
340 to_ticks = INT_MAX0x7fffffff;
341 if (to_ticks == 0 && tv->tv_usec > 0)
342 to_ticks = 1;
343
344 return timeout_add(to, (int)to_ticks);
345}
346
347int
348timeout_add_sec(struct timeout *to, int secs)
349{
350 uint64_t to_ticks;
351
352 to_ticks = (uint64_t)hz * secs;
353 if (to_ticks > INT_MAX0x7fffffff)
354 to_ticks = INT_MAX0x7fffffff;
355 if (to_ticks == 0)
356 to_ticks = 1;
357
358 return timeout_add(to, (int)to_ticks);
359}
360
361int
362timeout_add_msec(struct timeout *to, int msecs)
363{
364 uint64_t to_ticks;
365
366 to_ticks = (uint64_t)msecs * 1000 / tick;
367 if (to_ticks > INT_MAX0x7fffffff)
368 to_ticks = INT_MAX0x7fffffff;
369 if (to_ticks == 0 && msecs > 0)
370 to_ticks = 1;
371
372 return timeout_add(to, (int)to_ticks);
373}
374
375int
376timeout_add_usec(struct timeout *to, int usecs)
377{
378 int to_ticks = usecs / tick;
379
380 if (to_ticks == 0 && usecs > 0)
381 to_ticks = 1;
382
383 return timeout_add(to, to_ticks);
384}
385
386int
387timeout_add_nsec(struct timeout *to, int nsecs)
388{
389 int to_ticks = nsecs / (tick * 1000);
390
391 if (to_ticks == 0 && nsecs > 0)
392 to_ticks = 1;
393
394 return timeout_add(to, to_ticks);
395}
396
397int
398timeout_at_ts(struct timeout *to, const struct timespec *abstime)
399{
400 struct timespec old_abstime;
401 int ret = 1;
402
403 mtx_enter(&timeout_mutex);
404
405 KASSERT(ISSET(to->to_flags, TIMEOUT_INITIALIZED | TIMEOUT_KCLOCK))((((to->to_flags) & (0x04 | 0x10))) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/kern/kern_timeout.c", 405, "ISSET(to->to_flags, TIMEOUT_INITIALIZED | TIMEOUT_KCLOCK)"
));
406 KASSERT(to->to_kclock != KCLOCK_NONE)((to->to_kclock != (-1)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/kern/kern_timeout.c", 406, "to->to_kclock != KCLOCK_NONE"
));
407
408 old_abstime = to->to_abstime;
409 to->to_abstime = *abstime;
410 CLR(to->to_flags, TIMEOUT_TRIGGERED)((to->to_flags) &= ~(0x08));
411
412 if (ISSET(to->to_flags, TIMEOUT_ONQUEUE)((to->to_flags) & (0x02))) {
413 if (timespeccmp(abstime, &old_abstime, <)(((abstime)->tv_sec == (&old_abstime)->tv_sec) ? ((
abstime)->tv_nsec < (&old_abstime)->tv_nsec) : (
(abstime)->tv_sec < (&old_abstime)->tv_sec))) {
414 CIRCQ_REMOVE(&to->to_list)do { (&to->to_list)->next->prev = (&to->to_list
)->prev; (&to->to_list)->prev->next = (&to
->to_list)->next; ((&to->to_list)->prev) = ((
void *)-1); ((&to->to_list)->next) = ((void *)-1); tostat
.tos_pending--; } while (0);
415 CIRCQ_INSERT_TAIL(&timeout_new, &to->to_list)do { (&to->to_list)->prev = (&timeout_new)->
prev; (&to->to_list)->next = (&timeout_new); (&
timeout_new)->prev->next = (&to->to_list); (&
timeout_new)->prev = (&to->to_list); tostat.tos_pending
++; } while (0);
416 }
417 tostat.tos_readded++;
418 ret = 0;
419 } else {
420 SET(to->to_flags, TIMEOUT_ONQUEUE)((to->to_flags) |= (0x02));
421 CIRCQ_INSERT_TAIL(&timeout_new, &to->to_list)do { (&to->to_list)->prev = (&timeout_new)->
prev; (&to->to_list)->next = (&timeout_new); (&
timeout_new)->prev->next = (&to->to_list); (&
timeout_new)->prev = (&to->to_list); tostat.tos_pending
++; } while (0);
422 }
423#if NKCOV0 > 0
424 to->to_process = curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p;
425#endif
426 tostat.tos_added++;
427
428 mtx_leave(&timeout_mutex);
429
430 return ret;
431}
432
433int
434timeout_in_nsec(struct timeout *to, uint64_t nsecs)
435{
436 struct timespec abstime, interval, now;
437
438 kclock_nanotime(to->to_kclock, &now);
439 NSEC_TO_TIMESPEC(nsecs, &interval);
440 timespecadd(&now, &interval, &abstime)do { (&abstime)->tv_sec = (&now)->tv_sec + (&
interval)->tv_sec; (&abstime)->tv_nsec = (&now)
->tv_nsec + (&interval)->tv_nsec; if ((&abstime
)->tv_nsec >= 1000000000L) { (&abstime)->tv_sec++
; (&abstime)->tv_nsec -= 1000000000L; } } while (0);
441
442 return timeout_at_ts(to, &abstime);
443}
444
445void
446kclock_nanotime(int kclock, struct timespec *now)
447{
448 switch (kclock) {
449 case KCLOCK_UPTIME0:
450 nanouptime(now);
451 break;
452 default:
453 panic("invalid kclock: 0x%x", kclock);
454 }
455}
456
457int
458timeout_del(struct timeout *to)
459{
460 int ret = 0;
461
462 mtx_enter(&timeout_mutex);
463 if (ISSET(to->to_flags, TIMEOUT_ONQUEUE)((to->to_flags) & (0x02))) {
464 CIRCQ_REMOVE(&to->to_list)do { (&to->to_list)->next->prev = (&to->to_list
)->prev; (&to->to_list)->prev->next = (&to
->to_list)->next; ((&to->to_list)->prev) = ((
void *)-1); ((&to->to_list)->next) = ((void *)-1); tostat
.tos_pending--; } while (0);
465 CLR(to->to_flags, TIMEOUT_ONQUEUE)((to->to_flags) &= ~(0x02));
466 tostat.tos_cancelled++;
467 ret = 1;
468 }
469 CLR(to->to_flags, TIMEOUT_TRIGGERED)((to->to_flags) &= ~(0x08));
470 tostat.tos_deleted++;
471 mtx_leave(&timeout_mutex);
472
473 return ret;
474}
475
476int
477timeout_del_barrier(struct timeout *to)
478{
479 int removed;
480
481 timeout_sync_order(ISSET(to->to_flags, TIMEOUT_PROC)((to->to_flags) & (0x01)));
482
483 removed = timeout_del(to);
484 if (!removed)
485 timeout_barrier(to);
486
487 return removed;
488}
489
490void
491timeout_barrier(struct timeout *to)
492{
493 struct timeout barrier;
494 struct cond c;
495 int procflag;
496
497 procflag = (to->to_flags & TIMEOUT_PROC0x01);
498 timeout_sync_order(procflag);
499
500 timeout_set_flags(&barrier, timeout_barrier_timeout, &c, procflag);
501 barrier.to_process = curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p;
502 cond_init(&c);
503
504 mtx_enter(&timeout_mutex);
505
506 barrier.to_time = ticks;
507 SET(barrier.to_flags, TIMEOUT_ONQUEUE)((barrier.to_flags) |= (0x02));
508 if (procflag)
509 CIRCQ_INSERT_TAIL(&timeout_proc, &barrier.to_list)do { (&barrier.to_list)->prev = (&timeout_proc)->
prev; (&barrier.to_list)->next = (&timeout_proc); (
&timeout_proc)->prev->next = (&barrier.to_list)
; (&timeout_proc)->prev = (&barrier.to_list); tostat
.tos_pending++; } while (0);
510 else
511 CIRCQ_INSERT_TAIL(&timeout_todo, &barrier.to_list)do { (&barrier.to_list)->prev = (&timeout_todo)->
prev; (&barrier.to_list)->next = (&timeout_todo); (
&timeout_todo)->prev->next = (&barrier.to_list)
; (&timeout_todo)->prev = (&barrier.to_list); tostat
.tos_pending++; } while (0);
512
513 mtx_leave(&timeout_mutex);
514
515 if (procflag)
516 wakeup_one(&timeout_proc)wakeup_n((&timeout_proc), 1);
517 else
518 softintr_schedule(softclock_si)do { struct x86_soft_intrhand *__sih = (softclock_si); struct
x86_soft_intr *__si = __sih->sih_intrhead; mtx_enter(&
__si->softintr_lock); if (__sih->sih_pending == 0) { do
{ (__sih)->sih_q.tqe_next = ((void *)0); (__sih)->sih_q
.tqe_prev = (&__si->softintr_q)->tqh_last; *(&__si
->softintr_q)->tqh_last = (__sih); (&__si->softintr_q
)->tqh_last = &(__sih)->sih_q.tqe_next; } while (0)
; __sih->sih_pending = 1; softintr(__si->softintr_ssir)
; } mtx_leave(&__si->softintr_lock); } while ( 0);
519
520 cond_wait(&c, "tmobar");
521}
522
523void
524timeout_barrier_timeout(void *arg)
525{
526 struct cond *c = arg;
527
528 cond_signal(c);
529}
530
531uint32_t
532timeout_bucket(const struct timeout *to)
533{
534 struct kclock *kc = &timeout_kclock[to->to_kclock];
535 struct timespec diff, shifted_abstime;
536 uint32_t level;
537
538 KASSERT(ISSET(to->to_flags, TIMEOUT_KCLOCK))((((to->to_flags) & (0x10))) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/kern/kern_timeout.c", 538, "ISSET(to->to_flags, TIMEOUT_KCLOCK)"
));
539 KASSERT(timespeccmp(&kc->kc_lastscan, &to->to_abstime, <))(((((&kc->kc_lastscan)->tv_sec == (&to->to_abstime
)->tv_sec) ? ((&kc->kc_lastscan)->tv_nsec < (
&to->to_abstime)->tv_nsec) : ((&kc->kc_lastscan
)->tv_sec < (&to->to_abstime)->tv_sec))) ? (void
)0 : __assert("diagnostic ", "/usr/src/sys/kern/kern_timeout.c"
, 539, "timespeccmp(&kc->kc_lastscan, &to->to_abstime, <)"
));
540
541 timespecsub(&to->to_abstime, &kc->kc_lastscan, &diff)do { (&diff)->tv_sec = (&to->to_abstime)->tv_sec
- (&kc->kc_lastscan)->tv_sec; (&diff)->tv_nsec
= (&to->to_abstime)->tv_nsec - (&kc->kc_lastscan
)->tv_nsec; if ((&diff)->tv_nsec < 0) { (&diff
)->tv_sec--; (&diff)->tv_nsec += 1000000000L; } } while
(0);
542 for (level = 0; level < nitems(timeout_level_width)(sizeof((timeout_level_width)) / sizeof((timeout_level_width)
[0])) - 1; level++) {
543 if (diff.tv_sec < timeout_level_width[level])
544 break;
545 }
546 timespecadd(&to->to_abstime, &kc->kc_offset, &shifted_abstime)do { (&shifted_abstime)->tv_sec = (&to->to_abstime
)->tv_sec + (&kc->kc_offset)->tv_sec; (&shifted_abstime
)->tv_nsec = (&to->to_abstime)->tv_nsec + (&
kc->kc_offset)->tv_nsec; if ((&shifted_abstime)->
tv_nsec >= 1000000000L) { (&shifted_abstime)->tv_sec
++; (&shifted_abstime)->tv_nsec -= 1000000000L; } } while
(0);
547 return level * WHEELSIZE256 + timeout_maskwheel(level, &shifted_abstime);
548}
549
550/*
551 * Hash the absolute time into a bucket on a given level of the wheel.
552 *
553 * The complete hash is 32 bits. The upper 25 bits are seconds, the
554 * lower 7 bits are nanoseconds. tv_nsec is a positive value less
555 * than one billion so we need to divide it to isolate the desired
556 * bits. We can't just shift it.
557 *
558 * The level is used to isolate an 8-bit portion of the hash. The
559 * resulting number indicates which bucket the absolute time belongs
560 * in on the given level of the wheel.
561 */
562uint32_t
563timeout_maskwheel(uint32_t level, const struct timespec *abstime)
564{
565 uint32_t hi, lo;
566
567 hi = abstime->tv_sec << 7;
568 lo = abstime->tv_nsec / 7812500;
569
570 return ((hi | lo) >> (level * WHEELBITS8)) & WHEELMASK255;
571}
572
573/*
574 * This is called from hardclock() on the primary CPU at the start of
575 * every tick.
576 */
577void
578timeout_hardclock_update(void)
579{
580 struct timespec elapsed, now;
581 struct kclock *kc;
582 struct timespec *lastscan;
583 int b, done, first, i, last, level, need_softclock, off;
584
585 nanouptime(&now);
586 lastscan = &timeout_kclock[KCLOCK_UPTIME0].kc_lastscan;
587 timespecsub(&now, lastscan, &elapsed)do { (&elapsed)->tv_sec = (&now)->tv_sec - (lastscan
)->tv_sec; (&elapsed)->tv_nsec = (&now)->tv_nsec
- (lastscan)->tv_nsec; if ((&elapsed)->tv_nsec <
0) { (&elapsed)->tv_sec--; (&elapsed)->tv_nsec
+= 1000000000L; } } while (0);
588 need_softclock = 1;
589
590 mtx_enter(&timeout_mutex);
591
592 MOVEBUCKET(0, ticks)do { if (!(((&timeout_wheel[((((ticks)) >> (((0))*8
)) & 255) + (0)*256])->next) == (&timeout_wheel[((
((ticks)) >> (((0))*8)) & 255) + (0)*256]))) { (&
timeout_todo)->prev->next = (&timeout_wheel[((((ticks
)) >> (((0))*8)) & 255) + (0)*256])->next; (&
timeout_wheel[((((ticks)) >> (((0))*8)) & 255) + (0
)*256])->next->prev = (&timeout_todo)->prev; (&
timeout_wheel[((((ticks)) >> (((0))*8)) & 255) + (0
)*256])->prev->next = (&timeout_todo); (&timeout_todo
)->prev = (&timeout_wheel[((((ticks)) >> (((0))*
8)) & 255) + (0)*256])->prev; do { (&timeout_wheel
[((((ticks)) >> (((0))*8)) & 255) + (0)*256])->next
= (&timeout_wheel[((((ticks)) >> (((0))*8)) & 255
) + (0)*256]); (&timeout_wheel[((((ticks)) >> (((0)
)*8)) & 255) + (0)*256])->prev = (&timeout_wheel[(
(((ticks)) >> (((0))*8)) & 255) + (0)*256]); } while
(0); } } while (0);
593 if (MASKWHEEL(0, ticks)(((ticks) >> ((0)*8)) & 255) == 0) {
594 MOVEBUCKET(1, ticks)do { if (!(((&timeout_wheel[((((ticks)) >> (((1))*8
)) & 255) + (1)*256])->next) == (&timeout_wheel[((
((ticks)) >> (((1))*8)) & 255) + (1)*256]))) { (&
timeout_todo)->prev->next = (&timeout_wheel[((((ticks
)) >> (((1))*8)) & 255) + (1)*256])->next; (&
timeout_wheel[((((ticks)) >> (((1))*8)) & 255) + (1
)*256])->next->prev = (&timeout_todo)->prev; (&
timeout_wheel[((((ticks)) >> (((1))*8)) & 255) + (1
)*256])->prev->next = (&timeout_todo); (&timeout_todo
)->prev = (&timeout_wheel[((((ticks)) >> (((1))*
8)) & 255) + (1)*256])->prev; do { (&timeout_wheel
[((((ticks)) >> (((1))*8)) & 255) + (1)*256])->next
= (&timeout_wheel[((((ticks)) >> (((1))*8)) & 255
) + (1)*256]); (&timeout_wheel[((((ticks)) >> (((1)
)*8)) & 255) + (1)*256])->prev = (&timeout_wheel[(
(((ticks)) >> (((1))*8)) & 255) + (1)*256]); } while
(0); } } while (0);
595 if (MASKWHEEL(1, ticks)(((ticks) >> ((1)*8)) & 255) == 0) {
596 MOVEBUCKET(2, ticks)do { if (!(((&timeout_wheel[((((ticks)) >> (((2))*8
)) & 255) + (2)*256])->next) == (&timeout_wheel[((
((ticks)) >> (((2))*8)) & 255) + (2)*256]))) { (&
timeout_todo)->prev->next = (&timeout_wheel[((((ticks
)) >> (((2))*8)) & 255) + (2)*256])->next; (&
timeout_wheel[((((ticks)) >> (((2))*8)) & 255) + (2
)*256])->next->prev = (&timeout_todo)->prev; (&
timeout_wheel[((((ticks)) >> (((2))*8)) & 255) + (2
)*256])->prev->next = (&timeout_todo); (&timeout_todo
)->prev = (&timeout_wheel[((((ticks)) >> (((2))*
8)) & 255) + (2)*256])->prev; do { (&timeout_wheel
[((((ticks)) >> (((2))*8)) & 255) + (2)*256])->next
= (&timeout_wheel[((((ticks)) >> (((2))*8)) & 255
) + (2)*256]); (&timeout_wheel[((((ticks)) >> (((2)
)*8)) & 255) + (2)*256])->prev = (&timeout_wheel[(
(((ticks)) >> (((2))*8)) & 255) + (2)*256]); } while
(0); } } while (0);
597 if (MASKWHEEL(2, ticks)(((ticks) >> ((2)*8)) & 255) == 0)
598 MOVEBUCKET(3, ticks)do { if (!(((&timeout_wheel[((((ticks)) >> (((3))*8
)) & 255) + (3)*256])->next) == (&timeout_wheel[((
((ticks)) >> (((3))*8)) & 255) + (3)*256]))) { (&
timeout_todo)->prev->next = (&timeout_wheel[((((ticks
)) >> (((3))*8)) & 255) + (3)*256])->next; (&
timeout_wheel[((((ticks)) >> (((3))*8)) & 255) + (3
)*256])->next->prev = (&timeout_todo)->prev; (&
timeout_wheel[((((ticks)) >> (((3))*8)) & 255) + (3
)*256])->prev->next = (&timeout_todo); (&timeout_todo
)->prev = (&timeout_wheel[((((ticks)) >> (((3))*
8)) & 255) + (3)*256])->prev; do { (&timeout_wheel
[((((ticks)) >> (((3))*8)) & 255) + (3)*256])->next
= (&timeout_wheel[((((ticks)) >> (((3))*8)) & 255
) + (3)*256]); (&timeout_wheel[((((ticks)) >> (((3)
)*8)) & 255) + (3)*256])->prev = (&timeout_wheel[(
(((ticks)) >> (((3))*8)) & 255) + (3)*256]); } while
(0); } } while (0);
599 }
600 }
601
602 /*
603 * Dump the buckets that expired while we were away.
604 *
605 * If the elapsed time has exceeded a level's limit then we need
606 * to dump every bucket in the level. We have necessarily completed
607 * a lap of that level, too, so we need to process buckets in the
608 * next level.
609 *
610 * Otherwise we need to compare indices: if the index of the first
611 * expired bucket is greater than that of the last then we have
612 * completed a lap of the level and need to process buckets in the
613 * next level.
614 */
615 for (level = 0; level < nitems(timeout_level_width)(sizeof((timeout_level_width)) / sizeof((timeout_level_width)
[0])); level++) {
616 first = timeout_maskwheel(level, lastscan);
617 if (elapsed.tv_sec >= timeout_level_width[level]) {
618 last = (first == 0) ? WHEELSIZE256 - 1 : first - 1;
619 done = 0;
620 } else {
621 last = timeout_maskwheel(level, &now);
622 done = first <= last;
623 }
624 off = level * WHEELSIZE256;
625 for (b = first;; b = (b + 1) % WHEELSIZE256) {
626 CIRCQ_CONCAT(&timeout_todo, &timeout_wheel_kc[off + b])do { if (!(((&timeout_wheel_kc[off + b])->next) == (&
timeout_wheel_kc[off + b]))) { (&timeout_todo)->prev->
next = (&timeout_wheel_kc[off + b])->next; (&timeout_wheel_kc
[off + b])->next->prev = (&timeout_todo)->prev; (
&timeout_wheel_kc[off + b])->prev->next = (&timeout_todo
); (&timeout_todo)->prev = (&timeout_wheel_kc[off +
b])->prev; do { (&timeout_wheel_kc[off + b])->next
= (&timeout_wheel_kc[off + b]); (&timeout_wheel_kc[off
+ b])->prev = (&timeout_wheel_kc[off + b]); } while (
0); } } while (0);
627 if (b == last)
628 break;
629 }
630 if (done)
631 break;
632 }
633
634 /*
635 * Update the cached state for each kclock.
636 */
637 for (i = 0; i < nitems(timeout_kclock)(sizeof((timeout_kclock)) / sizeof((timeout_kclock)[0])); i++) {
638 kc = &timeout_kclock[i];
639 timespecadd(&now, &kc->kc_offset, &kc->kc_lastscan)do { (&kc->kc_lastscan)->tv_sec = (&now)->tv_sec
+ (&kc->kc_offset)->tv_sec; (&kc->kc_lastscan
)->tv_nsec = (&now)->tv_nsec + (&kc->kc_offset
)->tv_nsec; if ((&kc->kc_lastscan)->tv_nsec >=
1000000000L) { (&kc->kc_lastscan)->tv_sec++; (&
kc->kc_lastscan)->tv_nsec -= 1000000000L; } } while (0);
640 timespecsub(&kc->kc_lastscan, &tick_ts, &kc->kc_late)do { (&kc->kc_late)->tv_sec = (&kc->kc_lastscan
)->tv_sec - (&tick_ts)->tv_sec; (&kc->kc_late
)->tv_nsec = (&kc->kc_lastscan)->tv_nsec - (&
tick_ts)->tv_nsec; if ((&kc->kc_late)->tv_nsec <
0) { (&kc->kc_late)->tv_sec--; (&kc->kc_late
)->tv_nsec += 1000000000L; } } while (0);
641 }
642
643 if (CIRCQ_EMPTY(&timeout_new)(((&timeout_new)->next) == (&timeout_new)) && CIRCQ_EMPTY(&timeout_todo)(((&timeout_todo)->next) == (&timeout_todo)))
644 need_softclock = 0;
645
646 mtx_leave(&timeout_mutex);
647
648 if (need_softclock)
649 softintr_schedule(softclock_si)do { struct x86_soft_intrhand *__sih = (softclock_si); struct
x86_soft_intr *__si = __sih->sih_intrhead; mtx_enter(&
__si->softintr_lock); if (__sih->sih_pending == 0) { do
{ (__sih)->sih_q.tqe_next = ((void *)0); (__sih)->sih_q
.tqe_prev = (&__si->softintr_q)->tqh_last; *(&__si
->softintr_q)->tqh_last = (__sih); (&__si->softintr_q
)->tqh_last = &(__sih)->sih_q.tqe_next; } while (0)
; __sih->sih_pending = 1; softintr(__si->softintr_ssir)
; } mtx_leave(&__si->softintr_lock); } while ( 0);
650}
651
652void
653timeout_run(struct timeout *to)
654{
655 void (*fn)(void *);
656 void *arg;
657 int needsproc;
658
659 MUTEX_ASSERT_LOCKED(&timeout_mutex)do { if (((&timeout_mutex)->mtx_owner != ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
db_active)) panic("mutex %p not held in %s", (&timeout_mutex
), __func__); } while (0);
660
661 CLR(to->to_flags, TIMEOUT_ONQUEUE)((to->to_flags) &= ~(0x02));
662 SET(to->to_flags, TIMEOUT_TRIGGERED)((to->to_flags) |= (0x08));
663
664 fn = to->to_func;
665 arg = to->to_arg;
666 needsproc = ISSET(to->to_flags, TIMEOUT_PROC)((to->to_flags) & (0x01));
667#if NKCOV0 > 0
668 struct process *kcov_process = to->to_process;
669#endif
670
671 mtx_leave(&timeout_mutex);
672 timeout_sync_enter(needsproc);
673#if NKCOV0 > 0
674 kcov_remote_enter(KCOV_REMOTE_COMMON, kcov_process);
675#endif
676 fn(arg);
677#if NKCOV0 > 0
678 kcov_remote_leave(KCOV_REMOTE_COMMON, kcov_process);
679#endif
680 timeout_sync_leave(needsproc);
681 mtx_enter(&timeout_mutex);
682}
683
684void
685softclock_process_kclock_timeout(struct timeout *to, int new)
686{
687 struct kclock *kc = &timeout_kclock[to->to_kclock];
688
689 if (timespeccmp(&to->to_abstime, &kc->kc_lastscan, >)(((&to->to_abstime)->tv_sec == (&kc->kc_lastscan
)->tv_sec) ? ((&to->to_abstime)->tv_nsec > (&
kc->kc_lastscan)->tv_nsec) : ((&to->to_abstime)->
tv_sec > (&kc->kc_lastscan)->tv_sec))) {
690 tostat.tos_scheduled++;
691 if (!new)
692 tostat.tos_rescheduled++;
693 CIRCQ_INSERT_TAIL(&timeout_wheel_kc[timeout_bucket(to)],do { (&to->to_list)->prev = (&timeout_wheel_kc[
timeout_bucket(to)])->prev; (&to->to_list)->next
= (&timeout_wheel_kc[timeout_bucket(to)]); (&timeout_wheel_kc
[timeout_bucket(to)])->prev->next = (&to->to_list
); (&timeout_wheel_kc[timeout_bucket(to)])->prev = (&
to->to_list); tostat.tos_pending++; } while (0)
694 &to->to_list)do { (&to->to_list)->prev = (&timeout_wheel_kc[
timeout_bucket(to)])->prev; (&to->to_list)->next
= (&timeout_wheel_kc[timeout_bucket(to)]); (&timeout_wheel_kc
[timeout_bucket(to)])->prev->next = (&to->to_list
); (&timeout_wheel_kc[timeout_bucket(to)])->prev = (&
to->to_list); tostat.tos_pending++; } while (0);
695 return;
696 }
697 if (!new && timespeccmp(&to->to_abstime, &kc->kc_late, <=)(((&to->to_abstime)->tv_sec == (&kc->kc_late
)->tv_sec) ? ((&to->to_abstime)->tv_nsec <= (
&kc->kc_late)->tv_nsec) : ((&to->to_abstime)
->tv_sec <= (&kc->kc_late)->tv_sec)))
698 tostat.tos_late++;
699 if (ISSET(to->to_flags, TIMEOUT_PROC)((to->to_flags) & (0x01))) {
700 CIRCQ_INSERT_TAIL(&timeout_proc, &to->to_list)do { (&to->to_list)->prev = (&timeout_proc)->
prev; (&to->to_list)->next = (&timeout_proc); (
&timeout_proc)->prev->next = (&to->to_list);
(&timeout_proc)->prev = (&to->to_list); tostat
.tos_pending++; } while (0);
701 return;
702 }
703 timeout_run(to);
704 tostat.tos_run_softclock++;
705}
706
707void
708softclock_process_tick_timeout(struct timeout *to, int new)
709{
710 int delta = to->to_time - ticks;
711
712 if (delta > 0) {
713 tostat.tos_scheduled++;
714 if (!new)
715 tostat.tos_rescheduled++;
716 CIRCQ_INSERT_TAIL(&BUCKET(delta, to->to_time), &to->to_list)do { (&to->to_list)->prev = (&(timeout_wheel[ (
(delta) <= (1 << (2*8))) ? ((delta) <= (1 <<
8)) ? ((((to->to_time)) >> ((0)*8)) & 255) : ((
((to->to_time)) >> ((1)*8)) & 255) + 256 : ((delta
) <= (1 << (3*8))) ? ((((to->to_time)) >> (
(2)*8)) & 255) + 2*256 : ((((to->to_time)) >> ((
3)*8)) & 255) + 3*256]))->prev; (&to->to_list)->
next = (&(timeout_wheel[ ((delta) <= (1 << (2*8)
)) ? ((delta) <= (1 << 8)) ? ((((to->to_time)) >>
((0)*8)) & 255) : ((((to->to_time)) >> ((1)*8))
& 255) + 256 : ((delta) <= (1 << (3*8))) ? ((((
to->to_time)) >> ((2)*8)) & 255) + 2*256 : ((((to
->to_time)) >> ((3)*8)) & 255) + 3*256])); (&
(timeout_wheel[ ((delta) <= (1 << (2*8))) ? ((delta)
<= (1 << 8)) ? ((((to->to_time)) >> ((0)*8
)) & 255) : ((((to->to_time)) >> ((1)*8)) & 255
) + 256 : ((delta) <= (1 << (3*8))) ? ((((to->to_time
)) >> ((2)*8)) & 255) + 2*256 : ((((to->to_time)
) >> ((3)*8)) & 255) + 3*256]))->prev->next =
(&to->to_list); (&(timeout_wheel[ ((delta) <= (
1 << (2*8))) ? ((delta) <= (1 << 8)) ? ((((to->
to_time)) >> ((0)*8)) & 255) : ((((to->to_time))
>> ((1)*8)) & 255) + 256 : ((delta) <= (1 <<
(3*8))) ? ((((to->to_time)) >> ((2)*8)) & 255) +
2*256 : ((((to->to_time)) >> ((3)*8)) & 255) + 3
*256]))->prev = (&to->to_list); tostat.tos_pending++
; } while (0);
717 return;
718 }
719 if (!new && delta < 0)
720 tostat.tos_late++;
721 if (ISSET(to->to_flags, TIMEOUT_PROC)((to->to_flags) & (0x01))) {
722 CIRCQ_INSERT_TAIL(&timeout_proc, &to->to_list)do { (&to->to_list)->prev = (&timeout_proc)->
prev; (&to->to_list)->next = (&timeout_proc); (
&timeout_proc)->prev->next = (&to->to_list);
(&timeout_proc)->prev = (&to->to_list); tostat
.tos_pending++; } while (0);
723 return;
724 }
725 timeout_run(to);
726 tostat.tos_run_softclock++;
727}
728
729/*
730 * Timeouts are processed here instead of timeout_hardclock_update()
731 * to avoid doing any more work at IPL_CLOCK than absolutely necessary.
732 * Down here at IPL_SOFTCLOCK other interrupts can be serviced promptly
733 * so the system remains responsive even if there is a surge of timeouts.
734 */
735void
736softclock(void *arg)
737{
738 struct timeout *first_new, *to;
739 int needsproc, new;
740
741 first_new = NULL((void *)0);
742 new = 0;
743
744 mtx_enter(&timeout_mutex);
745 if (!CIRCQ_EMPTY(&timeout_new)(((&timeout_new)->next) == (&timeout_new)))
746 first_new = timeout_from_circq(CIRCQ_FIRST(&timeout_new)((&timeout_new)->next));
747 CIRCQ_CONCAT(&timeout_todo, &timeout_new)do { if (!(((&timeout_new)->next) == (&timeout_new
))) { (&timeout_todo)->prev->next = (&timeout_new
)->next; (&timeout_new)->next->prev = (&timeout_todo
)->prev; (&timeout_new)->prev->next = (&timeout_todo
); (&timeout_todo)->prev = (&timeout_new)->prev
; do { (&timeout_new)->next = (&timeout_new); (&
timeout_new)->prev = (&timeout_new); } while (0); } } while
(0);
748 while (!CIRCQ_EMPTY(&timeout_todo)(((&timeout_todo)->next) == (&timeout_todo))) {
749 to = timeout_from_circq(CIRCQ_FIRST(&timeout_todo)((&timeout_todo)->next));
750 CIRCQ_REMOVE(&to->to_list)do { (&to->to_list)->next->prev = (&to->to_list
)->prev; (&to->to_list)->prev->next = (&to
->to_list)->next; ((&to->to_list)->prev) = ((
void *)-1); ((&to->to_list)->next) = ((void *)-1); tostat
.tos_pending--; } while (0);
751 if (to == first_new)
752 new = 1;
753 if (ISSET(to->to_flags, TIMEOUT_KCLOCK)((to->to_flags) & (0x10)))
754 softclock_process_kclock_timeout(to, new);
755 else
756 softclock_process_tick_timeout(to, new);
757 }
758 tostat.tos_softclocks++;
759 needsproc = !CIRCQ_EMPTY(&timeout_proc)(((&timeout_proc)->next) == (&timeout_proc));
760 mtx_leave(&timeout_mutex);
761
762 if (needsproc)
763 wakeup(&timeout_proc);
764}
765
766void
767softclock_create_thread(void *arg)
768{
769 if (kthread_create(softclock_thread, NULL((void *)0), NULL((void *)0), "softclock"))
770 panic("fork softclock");
771}
772
773void
774softclock_thread(void *arg)
775{
776 CPU_INFO_ITERATORint cii;
777 struct cpu_info *ci;
778 struct sleep_state sls;
779 struct timeout *to;
780 int s;
781
782 KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/kern/kern_timeout.c"
, 782, "_kernel_lock_held()"));
783
784 /* Be conservative for the moment */
785 CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
786 if (CPU_IS_PRIMARY(ci)((ci)->ci_flags & 0x0008))
787 break;
788 }
789 KASSERT(ci != NULL)((ci != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/kern/kern_timeout.c"
, 789, "ci != NULL"));
790 sched_peg_curproc(ci);
791
792 s = splsoftclock()splraise(0x4);
Value stored to 's' is never read
793 for (;;) {
794 sleep_setup(&sls, &timeout_proc, PSWP0, "bored", 0);
795 sleep_finish(&sls, CIRCQ_EMPTY(&timeout_proc)(((&timeout_proc)->next) == (&timeout_proc)));
796
797 mtx_enter(&timeout_mutex);
798 while (!CIRCQ_EMPTY(&timeout_proc)(((&timeout_proc)->next) == (&timeout_proc))) {
799 to = timeout_from_circq(CIRCQ_FIRST(&timeout_proc)((&timeout_proc)->next));
800 CIRCQ_REMOVE(&to->to_list)do { (&to->to_list)->next->prev = (&to->to_list
)->prev; (&to->to_list)->prev->next = (&to
->to_list)->next; ((&to->to_list)->prev) = ((
void *)-1); ((&to->to_list)->next) = ((void *)-1); tostat
.tos_pending--; } while (0);
801 timeout_run(to);
802 tostat.tos_run_thread++;
803 }
804 tostat.tos_thread_wakeups++;
805 mtx_leave(&timeout_mutex);
806 }
807 splx(s)spllower(s);
808}
809
810#ifndef SMALL_KERNEL
811void
812timeout_adjust_ticks(int adj)
813{
814 struct timeout *to;
815 struct circq *p;
816 int new_ticks, b;
817
818 /* adjusting the monotonic clock backwards would be a Bad Thing */
819 if (adj <= 0)
820 return;
821
822 mtx_enter(&timeout_mutex);
823 new_ticks = ticks + adj;
824 for (b = 0; b < nitems(timeout_wheel)(sizeof((timeout_wheel)) / sizeof((timeout_wheel)[0])); b++) {
825 p = CIRCQ_FIRST(&timeout_wheel[b])((&timeout_wheel[b])->next);
826 while (p != &timeout_wheel[b]) {
827 to = timeout_from_circq(p);
828 p = CIRCQ_FIRST(p)((p)->next);
829
830 /* when moving a timeout forward need to reinsert it */
831 if (to->to_time - ticks < adj)
832 to->to_time = new_ticks;
833 CIRCQ_REMOVE(&to->to_list)do { (&to->to_list)->next->prev = (&to->to_list
)->prev; (&to->to_list)->prev->next = (&to
->to_list)->next; ((&to->to_list)->prev) = ((
void *)-1); ((&to->to_list)->next) = ((void *)-1); tostat
.tos_pending--; } while (0);
834 CIRCQ_INSERT_TAIL(&timeout_todo, &to->to_list)do { (&to->to_list)->prev = (&timeout_todo)->
prev; (&to->to_list)->next = (&timeout_todo); (
&timeout_todo)->prev->next = (&to->to_list);
(&timeout_todo)->prev = (&to->to_list); tostat
.tos_pending++; } while (0);
835 }
836 }
837 ticks = new_ticks;
838 mtx_leave(&timeout_mutex);
839}
840#endif
841
842int
843timeout_sysctl(void *oldp, size_t *oldlenp, void *newp, size_t newlen)
844{
845 struct timeoutstat status;
846
847 mtx_enter(&timeout_mutex);
848 memcpy(&status, &tostat, sizeof(status))__builtin_memcpy((&status), (&tostat), (sizeof(status
)));
849 mtx_leave(&timeout_mutex);
850
851 return sysctl_rdstruct(oldp, oldlenp, newp, &status, sizeof(status));
852}
853
854#ifdef DDB1
855const char *db_kclock(int);
856void db_show_callout_bucket(struct circq *);
857void db_show_timeout(struct timeout *, struct circq *);
858const char *db_timespec(const struct timespec *);
859
860const char *
861db_kclock(int kclock)
862{
863 switch (kclock) {
864 case KCLOCK_UPTIME0:
865 return "uptime";
866 default:
867 return "invalid";
868 }
869}
870
871const char *
872db_timespec(const struct timespec *ts)
873{
874 static char buf[32];
875 struct timespec tmp, zero;
876
877 if (ts->tv_sec >= 0) {
878 snprintf(buf, sizeof(buf), "%lld.%09ld",
879 ts->tv_sec, ts->tv_nsec);
880 return buf;
881 }
882
883 timespecclear(&zero)(&zero)->tv_sec = (&zero)->tv_nsec = 0;
884 timespecsub(&zero, ts, &tmp)do { (&tmp)->tv_sec = (&zero)->tv_sec - (ts)->
tv_sec; (&tmp)->tv_nsec = (&zero)->tv_nsec - (ts
)->tv_nsec; if ((&tmp)->tv_nsec < 0) { (&tmp
)->tv_sec--; (&tmp)->tv_nsec += 1000000000L; } } while
(0);
885 snprintf(buf, sizeof(buf), "-%lld.%09ld", tmp.tv_sec, tmp.tv_nsec);
886 return buf;
887}
888
889void
890db_show_callout_bucket(struct circq *bucket)
891{
892 struct circq *p;
893
894 CIRCQ_FOREACH(p, bucket)for ((p) = ((bucket)->next); (p) != (bucket); (p) = ((p)->
next))
895 db_show_timeout(timeout_from_circq(p), bucket);
896}
897
898void
899db_show_timeout(struct timeout *to, struct circq *bucket)
900{
901 struct timespec remaining;
902 struct kclock *kc;
903 char buf[8];
904 db_expr_t offset;
905 struct circq *wheel;
906 char *name, *where;
907 int width = sizeof(long) * 2;
908
909 db_find_sym_and_offset((vaddr_t)to->to_func, &name, &offset)db_symbol_values(db_search_symbol((vaddr_t)to->to_func,0,&
offset),&name,((void *)0));
910 name = name ? name : "?";
911 if (bucket == &timeout_new)
912 where = "new";
913 else if (bucket == &timeout_todo)
914 where = "softint";
915 else if (bucket == &timeout_proc)
916 where = "thread";
917 else {
918 if (ISSET(to->to_flags, TIMEOUT_KCLOCK)((to->to_flags) & (0x10)))
919 wheel = timeout_wheel_kc;
920 else
921 wheel = timeout_wheel;
922 snprintf(buf, sizeof(buf), "%3ld/%1ld",
923 (bucket - wheel) % WHEELSIZE256,
924 (bucket - wheel) / WHEELSIZE256);
925 where = buf;
926 }
927 if (ISSET(to->to_flags, TIMEOUT_KCLOCK)((to->to_flags) & (0x10))) {
928 kc = &timeout_kclock[to->to_kclock];
929 timespecsub(&to->to_abstime, &kc->kc_lastscan, &remaining)do { (&remaining)->tv_sec = (&to->to_abstime)->
tv_sec - (&kc->kc_lastscan)->tv_sec; (&remaining
)->tv_nsec = (&to->to_abstime)->tv_nsec - (&
kc->kc_lastscan)->tv_nsec; if ((&remaining)->tv_nsec
< 0) { (&remaining)->tv_sec--; (&remaining)->
tv_nsec += 1000000000L; } } while (0);
930 db_printf("%20s %8s %7s 0x%0*lx %s\n",
931 db_timespec(&remaining), db_kclock(to->to_kclock), where,
932 width, (ulong)to->to_arg, name);
933 } else {
934 db_printf("%20d %8s %7s 0x%0*lx %s\n",
935 to->to_time - ticks, "ticks", where,
936 width, (ulong)to->to_arg, name);
937 }
938}
939
940void
941db_show_callout(db_expr_t addr, int haddr, db_expr_t count, char *modif)
942{
943 struct kclock *kc;
944 int width = sizeof(long) * 2 + 2;
945 int b, i;
946
947 db_printf("%20s %8s\n", "lastscan", "clock");
948 db_printf("%20d %8s\n", ticks, "ticks");
949 for (i = 0; i < nitems(timeout_kclock)(sizeof((timeout_kclock)) / sizeof((timeout_kclock)[0])); i++) {
950 kc = &timeout_kclock[i];
951 db_printf("%20s %8s\n",
952 db_timespec(&kc->kc_lastscan), db_kclock(i));
953 }
954 db_printf("\n");
955 db_printf("%20s %8s %7s %*s %s\n",
956 "remaining", "clock", "wheel", width, "arg", "func");
957 db_show_callout_bucket(&timeout_new);
958 db_show_callout_bucket(&timeout_todo);
959 db_show_callout_bucket(&timeout_proc);
960 for (b = 0; b < nitems(timeout_wheel)(sizeof((timeout_wheel)) / sizeof((timeout_wheel)[0])); b++)
961 db_show_callout_bucket(&timeout_wheel[b]);
962 for (b = 0; b < nitems(timeout_wheel_kc)(sizeof((timeout_wheel_kc)) / sizeof((timeout_wheel_kc)[0])); b++)
963 db_show_callout_bucket(&timeout_wheel_kc[b]);
964}
965#endif