/usr/src/sys/net/if

Bug Summary

File:	net/if_wg.c
Warning:	line 1883, column 41 Access to field 't_done' results in a dereference of a null pointer
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name if_wg.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/net/if_wg.c
1/*	$OpenBSD: if_wg.c,v 1.20 2022/01/02 22:36:04 jsg Exp $ */

3/*
* Copyright (C) 2015-2020 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
* Copyright (C) 2019-2020 Matt Dunwoodie <ncon@noconroy.net>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

20#include "bpfilter.h"
21#include "pf.h"

23#include <sys/types.h>
24#include <sys/systm.h>
25#include <sys/param.h>
26#include <sys/pool.h>

28#include <sys/socket.h>
29#include <sys/socketvar.h>
30#include <sys/percpu.h>
31#include <sys/ioctl.h>
32#include <sys/mbuf.h>
33#include <sys/protosw.h>

35#include <net/if.h>
36#include <net/if_var.h>
37#include <net/if_types.h>
38#include <net/if_wg.h>

40#include <net/wg_noise.h>
41#include <net/wg_cookie.h>

43#include <net/pfvar.h>
44#include <net/route.h>
45#include <net/bpf.h>

47#include <netinet/ip.h>
48#include <netinet/ip6.h>
49#include <netinet/udp.h>
50#include <netinet/in_pcb.h>

52#include <crypto/siphash.h>

54#define DEFAULT_MTU1420		1420

56#define MAX_STAGED_PKT128		128
57#define MAX_QUEUED_PKT1024		1024
58#define MAX_QUEUED_PKT_MASK(1024 - 1)	(MAX_QUEUED_PKT1024 - 1)

60#define MAX_QUEUED_HANDSHAKES4096	4096

62#define HASHTABLE_PEER_SIZE(1 << 11)	(1 << 11)
63#define HASHTABLE_INDEX_SIZE(1 << 13)	(1 << 13)
64#define MAX_PEERS_PER_IFACE(1 << 20)	(1 << 20)

66#define REKEY_TIMEOUT5		5
67#define REKEY_TIMEOUT_JITTER334	334 /* 1/3 sec, round for arc4random_uniform */
68#define KEEPALIVE_TIMEOUT10	10
69#define MAX_TIMER_HANDSHAKES(90 / 5)	(90 / REKEY_TIMEOUT5)
70#define NEW_HANDSHAKE_TIMEOUT(5 + 10)	(REKEY_TIMEOUT5 + KEEPALIVE_TIMEOUT10)
71#define UNDERLOAD_TIMEOUT1	1

73#define DPRINTF(sc, str, ...)do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 str, (sc)->sc_if.if_xname, ...); } while (0) do { if (ISSET((sc)->sc_if.if_flags, IFF_DEBUG)(((sc)->sc_if.if_flags) & (0x4)))\
  printf("%s: " str, (sc)->sc_if.if_xname, ##__VA_ARGS__); } while (0)

76#define CONTAINER_OF(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = (ptr); (
type *)( (char *)__mptr - __builtin_offsetof(type, member) );
}) ({			\
const __typeof( ((type *)0)->member ) *__mptr = (ptr);	\
(type *)( (char *)__mptr - offsetof(type,member)__builtin_offsetof(type, member) );})

80/* First byte indicating packet type on the wire */
81#define WG_PKT_INITIATION((__uint32_t)(1)) htole32(1)((__uint32_t)(1))
82#define WG_PKT_RESPONSE((__uint32_t)(2)) htole32(2)((__uint32_t)(2))
83#define WG_PKT_COOKIE((__uint32_t)(3)) htole32(3)((__uint32_t)(3))
84#define WG_PKT_DATA((__uint32_t)(4)) htole32(4)((__uint32_t)(4))

86#define WG_PKT_WITH_PADDING(n)(((n) + (16-1)) & (~(16-1)))	(((n) + (16-1)) & (~(16-1)))
87#define WG_KEY_SIZE32		WG_KEY_LEN32

89struct wg_pkt_initiation {
uint32_t		t;
uint32_t		s_idx;
uint8_t			ue[NOISE_PUBLIC_KEY_LEN32];
uint8_t			es[NOISE_PUBLIC_KEY_LEN32 + NOISE_AUTHTAG_LEN16];
uint8_t			ets[NOISE_TIMESTAMP_LEN(sizeof(uint64_t) + sizeof(uint32_t)) + NOISE_AUTHTAG_LEN16];
struct cookie_macs	m;
96};

98struct wg_pkt_response {
uint32_t		t;
uint32_t		s_idx;
uint32_t		r_idx;
uint8_t			ue[NOISE_PUBLIC_KEY_LEN32];
uint8_t			en[0 + NOISE_AUTHTAG_LEN16];
struct cookie_macs	m;
105};

107struct wg_pkt_cookie {
uint32_t		t;
uint32_t		r_idx;
uint8_t			nonce[COOKIE_NONCE_SIZE24];
uint8_t			ec[COOKIE_ENCRYPTED_SIZE(16 + 16)];
112};

114struct wg_pkt_data {
uint32_t		t;
uint32_t		r_idx;
uint8_t			nonce[sizeof(uint64_t)];
uint8_t			buf[];
119};

121struct wg_endpoint {
union {
struct sockaddr		r_sa;
struct sockaddr_in	r_sin;
125#ifdef INET61
struct sockaddr_in6	r_sin6;
127#endif
} e_remote;
union {
struct in_addr		l_in;
131#ifdef INET61
struct in6_pktinfo	l_pktinfo6;
133#define l_in6l_pktinfo6.ipi6_addr l_pktinfo6.ipi6_addr
134#endif
} e_local;
136};

138struct wg_tag {
struct wg_endpoint	 t_endpoint;
struct wg_peer		*t_peer;
struct mbuf		*t_mbuf;
int			 t_done;
int			 t_mtu;
144};

146struct wg_index {
LIST_ENTRY(wg_index)struct { struct wg_index *le_next; struct wg_index **le_prev;
 }	 i_entry;
SLIST_ENTRY(wg_index)struct { struct wg_index *sle_next; }	 i_unused_entry;
uint32_t		 i_key;
struct noise_remote	*i_value;
151};

153struct wg_timers {
/* t_lock is for blocking wg_timers_event_* when setting t_disabled. */
struct rwlock		 t_lock;

int			 t_disabled;
int			 t_need_another_keepalive;
uint16_t		 t_persistent_keepalive_interval;
struct timeout		 t_new_handshake;
struct timeout		 t_send_keepalive;
struct timeout		 t_retry_handshake;
struct timeout		 t_zero_key_material;
struct timeout		 t_persistent_keepalive;

struct mutex		 t_handshake_mtx;
struct timespec		 t_handshake_last_sent;	/* nanouptime */
struct timespec		 t_handshake_complete;	/* nanotime */
int			 t_handshake_retries;
170};

172struct wg_aip {
struct art_node		 a_node;
LIST_ENTRY(wg_aip)struct { struct wg_aip *le_next; struct wg_aip **le_prev; }	 a_entry;
struct wg_peer		*a_peer;
struct wg_aip_io	 a_data;
177};

179struct wg_queue {
struct mutex		 q_mtx;
struct mbuf_list	 q_list;
182};

184struct wg_ring {
struct mutex	 r_mtx;
uint32_t	 r_head;
uint32_t	 r_tail;
struct mbuf	*r_buf[MAX_QUEUED_PKT1024];
189};

191struct wg_peer {
LIST_ENTRY(wg_peer)struct { struct wg_peer *le_next; struct wg_peer **le_prev; }	 p_pubkey_entry;
TAILQ_ENTRY(wg_peer)struct { struct wg_peer *tqe_next; struct wg_peer **tqe_prev;
 }	 p_seq_entry;
uint64_t		 p_id;
struct wg_softc		*p_sc;

struct noise_remote	 p_remote;
struct cookie_maker	 p_cookie;
struct wg_timers	 p_timers;

struct mutex		 p_counters_mtx;
uint64_t		 p_counters_tx;
uint64_t		 p_counters_rx;

struct mutex		 p_endpoint_mtx;
struct wg_endpoint	 p_endpoint;

struct task		 p_send_initiation;
struct task		 p_send_keepalive;
struct task		 p_clear_secrets;
struct task		 p_deliver_out;
struct task		 p_deliver_in;

struct mbuf_queue	 p_stage_queue;
struct wg_queue		 p_encap_queue;
struct wg_queue		 p_decap_queue;

SLIST_HEAD(,wg_index)struct { struct wg_index *slh_first; }	 p_unused_index;
struct wg_index		 p_index[3];

LIST_HEAD(,wg_aip)struct { struct wg_aip *lh_first; }	 p_aip;

SLIST_ENTRY(wg_peer)struct { struct wg_peer *sle_next; }	 p_start_list;
int			 p_start_onlist;
225};

227struct wg_softc {
struct ifnet		 sc_if;
SIPHASH_KEY		 sc_secret;

struct rwlock		 sc_lock;
struct noise_local	 sc_local;
struct cookie_checker	 sc_cookie;
in_port_t		 sc_udp_port;
int			 sc_udp_rtable;

struct rwlock		 sc_so_lock;
struct socket		*sc_so4;
239#ifdef INET61
struct socket		*sc_so6;
241#endif

size_t			 sc_aip_num;
struct art_root		*sc_aip4;
245#ifdef INET61
struct art_root		*sc_aip6;
247#endif

struct rwlock		 sc_peer_lock;
size_t			 sc_peer_num;
LIST_HEAD(,wg_peer)struct { struct wg_peer *lh_first; }	*sc_peer;
TAILQ_HEAD(,wg_peer)struct { struct wg_peer *tqh_first; struct wg_peer **tqh_last
; }	 sc_peer_seq;
u_long			 sc_peer_mask;

struct mutex		 sc_index_mtx;
LIST_HEAD(,wg_index)struct { struct wg_index *lh_first; }	*sc_index;
u_long			 sc_index_mask;

struct task		 sc_handshake;
struct mbuf_queue	 sc_handshake_queue;

struct task		 sc_encap;
struct task		 sc_decap;
struct wg_ring		 sc_encap_ring;
struct wg_ring		 sc_decap_ring;
266};

268struct wg_peer *
wg_peer_create(struct wg_softc *, uint8_t[WG_KEY_SIZE32]);
270struct wg_peer *
wg_peer_lookup(struct wg_softc *, const uint8_t[WG_KEY_SIZE32]);
272void	wg_peer_destroy(struct wg_peer *);
273void	wg_peer_set_endpoint_from_tag(struct wg_peer *, struct wg_tag *);
274void	wg_peer_set_sockaddr(struct wg_peer *, struct sockaddr *);
275int	wg_peer_get_sockaddr(struct wg_peer *, struct sockaddr *);
276void	wg_peer_clear_src(struct wg_peer *);
277void	wg_peer_get_endpoint(struct wg_peer *, struct wg_endpoint *);
278void	wg_peer_counters_add(struct wg_peer *, uint64_t, uint64_t);

280int	wg_aip_add(struct wg_softc *, struct wg_peer *, struct wg_aip_io *);
281struct wg_peer *
wg_aip_lookup(struct art_root *, void *);
283int	wg_aip_remove(struct wg_softc *, struct wg_peer *,
   struct wg_aip_io *);

286int	wg_socket_open(struct socket **, int, in_port_t *, int *, void *);
287void	wg_socket_close(struct socket **);
288int	wg_bind(struct wg_softc *, in_port_t *, int *);
289void	wg_unbind(struct wg_softc *);
290int	wg_send(struct wg_softc *, struct wg_endpoint *, struct mbuf *);
291void	wg_send_buf(struct wg_softc *, struct wg_endpoint *, uint8_t *,
   size_t);

294struct wg_tag *
wg_tag_get(struct mbuf *);

297void	wg_timers_init(struct wg_timers *);
298void	wg_timers_enable(struct wg_timers *);
299void	wg_timers_disable(struct wg_timers *);
300void	wg_timers_set_persistent_keepalive(struct wg_timers *, uint16_t);
301int	wg_timers_get_persistent_keepalive(struct wg_timers *, uint16_t *);
302void	wg_timers_get_last_handshake(struct wg_timers *, struct timespec *);
303int	wg_timers_expired_handshake_last_sent(struct wg_timers *);
304int	wg_timers_check_handshake_last_sent(struct wg_timers *);

306void	wg_timers_event_data_sent(struct wg_timers *);
307void	wg_timers_event_data_received(struct wg_timers *);
308void	wg_timers_event_any_authenticated_packet_sent(struct wg_timers *);
309void	wg_timers_event_any_authenticated_packet_received(struct wg_timers *);
310void	wg_timers_event_handshake_initiated(struct wg_timers *);
311void	wg_timers_event_handshake_responded(struct wg_timers *);
312void	wg_timers_event_handshake_complete(struct wg_timers *);
313void	wg_timers_event_session_derived(struct wg_timers *);
314void	wg_timers_event_any_authenticated_packet_traversal(struct wg_timers *);
315void	wg_timers_event_want_initiation(struct wg_timers *);
316void	wg_timers_event_reset_handshake_last_sent(struct wg_timers *);

318void	wg_timers_run_send_initiation(void *, int);
319void	wg_timers_run_retry_handshake(void *);
320void	wg_timers_run_send_keepalive(void *);
321void	wg_timers_run_new_handshake(void *);
322void	wg_timers_run_zero_key_material(void *);
323void	wg_timers_run_persistent_keepalive(void *);

325void	wg_peer_send_buf(struct wg_peer *, uint8_t *, size_t);
326void	wg_send_initiation(void *);
327void	wg_send_response(struct wg_peer *);
328void	wg_send_cookie(struct wg_softc *, struct cookie_macs *, uint32_t,
   struct wg_endpoint *e);
330void	wg_send_keepalive(void *);
331void	wg_peer_clear_secrets(void *);
332void	wg_handshake(struct wg_softc *, struct mbuf *);
333void	wg_handshake_worker(void *);

335void	wg_encap(struct wg_softc *, struct mbuf *);
336void	wg_decap(struct wg_softc *, struct mbuf *);
337void	wg_encap_worker(void *);
338void	wg_decap_worker(void *);
339void	wg_deliver_out(void *);
340void	wg_deliver_in(void *);

342int	wg_queue_in(struct wg_softc *, struct wg_peer *, struct mbuf *);
343void	wg_queue_out(struct wg_softc *, struct wg_peer *);
344struct mbuf *
wg_ring_dequeue(struct wg_ring *);
346struct mbuf *
wg_queue_dequeue(struct wg_queue *, struct wg_tag **);
348size_t	wg_queue_len(struct wg_queue *);

350struct noise_remote *
wg_remote_get(void *, uint8_t[NOISE_PUBLIC_KEY_LEN32]);
352uint32_t
wg_index_set(void *, struct noise_remote *);
354struct noise_remote *
wg_index_get(void *, uint32_t);
356void	wg_index_drop(void *, uint32_t);

358struct mbuf *
wg_input(void *, struct mbuf *, struct ip *, struct ip6_hdr *, void *,
   int);
361int	wg_output(struct ifnet *, struct mbuf *, struct sockaddr *,
   struct rtentry *);
363int	wg_ioctl_set(struct wg_softc *, struct wg_data_io *);
364int	wg_ioctl_get(struct wg_softc *, struct wg_data_io *);
365int	wg_ioctl(struct ifnet *, u_long, caddr_t);
366int	wg_up(struct wg_softc *);
367void	wg_down(struct wg_softc *);

369int	wg_clone_create(struct if_clone *, int);
370int	wg_clone_destroy(struct ifnet *);
371void	wgattach(int);

373uint64_t	peer_counter = 0;
374uint64_t	keypair_counter = 0;
375struct pool	wg_aip_pool;
376struct pool	wg_peer_pool;
377struct pool	wg_ratelimit_pool;
378struct timeval	underload_interval = { UNDERLOAD_TIMEOUT1, 0 };

380size_t		 wg_counter = 0;
381struct taskq	*wg_handshake_taskq;
382struct taskq	*wg_crypt_taskq;

384struct if_clone	wg_cloner =
  IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy){ .ifc_list = { ((void *)0), ((void *)0) }, .ifc_name = "wg",
 .ifc_namelen = sizeof("wg") - 1, .ifc_create = wg_clone_create
, .ifc_destroy = wg_clone_destroy, };

387struct wg_peer *
388wg_peer_create(struct wg_softc *sc, uint8_t public[WG_KEY_SIZE32])
389{
struct wg_peer	*peer;
uint64_t	 idx;

rw_assert_wrlock(&sc->sc_lock);

if (sc->sc_peer_num >= MAX_PEERS_PER_IFACE(1 << 20))
return NULL((void *)0);

if ((peer = pool_get(&wg_peer_pool, PR_NOWAIT0x0002)) == NULL((void *)0))
return NULL((void *)0);

peer->p_id = peer_counter++;
peer->p_sc = sc;

noise_remote_init(&peer->p_remote, public, &sc->sc_local);
cookie_maker_init(&peer->p_cookie, public);
wg_timers_init(&peer->p_timers);

mtx_init(&peer->p_counters_mtx, IPL_NET)do { (void)(((void *)0)); (void)(0); __mtx_init((&peer->
p_counters_mtx), ((((0x7)) > 0x0 && ((0x7)) < 0x9
) ? 0x9 : ((0x7)))); } while (0);
peer->p_counters_tx = 0;
peer->p_counters_rx = 0;

mtx_init(&peer->p_endpoint_mtx, IPL_NET)do { (void)(((void *)0)); (void)(0); __mtx_init((&peer->
p_endpoint_mtx), ((((0x7)) > 0x0 && ((0x7)) < 0x9
) ? 0x9 : ((0x7)))); } while (0);
bzero(&peer->p_endpoint, sizeof(peer->p_endpoint))__builtin_bzero((&peer->p_endpoint), (sizeof(peer->
p_endpoint)));

task_set(&peer->p_send_initiation, wg_send_initiation, peer);
task_set(&peer->p_send_keepalive, wg_send_keepalive, peer);
task_set(&peer->p_clear_secrets, wg_peer_clear_secrets, peer);
task_set(&peer->p_deliver_out, wg_deliver_out, peer);
task_set(&peer->p_deliver_in, wg_deliver_in, peer);

mq_init(&peer->p_stage_queue, MAX_STAGED_PKT128, IPL_NET0x7);
mtx_init(&peer->p_encap_queue.q_mtx, IPL_NET)do { (void)(((void *)0)); (void)(0); __mtx_init((&peer->
p_encap_queue.q_mtx), ((((0x7)) > 0x0 && ((0x7)) <
 0x9) ? 0x9 : ((0x7)))); } while (0);
ml_init(&peer->p_encap_queue.q_list);
mtx_init(&peer->p_decap_queue.q_mtx, IPL_NET)do { (void)(((void *)0)); (void)(0); __mtx_init((&peer->
p_decap_queue.q_mtx), ((((0x7)) > 0x0 && ((0x7)) <
 0x9) ? 0x9 : ((0x7)))); } while (0);
ml_init(&peer->p_decap_queue.q_list);

SLIST_INIT(&peer->p_unused_index){ ((&peer->p_unused_index)->slh_first) = ((void *)0
); };
SLIST_INSERT_HEAD(&peer->p_unused_index, &peer->p_index[0],do { (&peer->p_index[0])->i_unused_entry.sle_next =
 (&peer->p_unused_index)->slh_first; (&peer->
p_unused_index)->slh_first = (&peer->p_index[0]); }
 while (0)
   i_unused_entry)do { (&peer->p_index[0])->i_unused_entry.sle_next =
 (&peer->p_unused_index)->slh_first; (&peer->
p_unused_index)->slh_first = (&peer->p_index[0]); }
 while (0);
SLIST_INSERT_HEAD(&peer->p_unused_index, &peer->p_index[1],do { (&peer->p_index[1])->i_unused_entry.sle_next =
 (&peer->p_unused_index)->slh_first; (&peer->
p_unused_index)->slh_first = (&peer->p_index[1]); }
 while (0)
   i_unused_entry)do { (&peer->p_index[1])->i_unused_entry.sle_next =
 (&peer->p_unused_index)->slh_first; (&peer->
p_unused_index)->slh_first = (&peer->p_index[1]); }
 while (0);
SLIST_INSERT_HEAD(&peer->p_unused_index, &peer->p_index[2],do { (&peer->p_index[2])->i_unused_entry.sle_next =
 (&peer->p_unused_index)->slh_first; (&peer->
p_unused_index)->slh_first = (&peer->p_index[2]); }
 while (0)
   i_unused_entry)do { (&peer->p_index[2])->i_unused_entry.sle_next =
 (&peer->p_unused_index)->slh_first; (&peer->
p_unused_index)->slh_first = (&peer->p_index[2]); }
 while (0);

LIST_INIT(&peer->p_aip)do { ((&peer->p_aip)->lh_first) = ((void *)0); } while
 (0);

peer->p_start_onlist = 0;

idx = SipHash24(&sc->sc_secret, public, WG_KEY_SIZE)SipHash((&sc->sc_secret), 2, 4, (public), (32));
idx &= sc->sc_peer_mask;

rw_enter_write(&sc->sc_peer_lock);
LIST_INSERT_HEAD(&sc->sc_peer[idx], peer, p_pubkey_entry)do { if (((peer)->p_pubkey_entry.le_next = (&sc->sc_peer
[idx])->lh_first) != ((void *)0)) (&sc->sc_peer[idx
])->lh_first->p_pubkey_entry.le_prev = &(peer)->
p_pubkey_entry.le_next; (&sc->sc_peer[idx])->lh_first
 = (peer); (peer)->p_pubkey_entry.le_prev = &(&sc->
sc_peer[idx])->lh_first; } while (0);
TAILQ_INSERT_TAIL(&sc->sc_peer_seq, peer, p_seq_entry)do { (peer)->p_seq_entry.tqe_next = ((void *)0); (peer)->
p_seq_entry.tqe_prev = (&sc->sc_peer_seq)->tqh_last
; *(&sc->sc_peer_seq)->tqh_last = (peer); (&sc->
sc_peer_seq)->tqh_last = &(peer)->p_seq_entry.tqe_next
; } while (0);
sc->sc_peer_num++;
rw_exit_write(&sc->sc_peer_lock);

DPRINTF(sc, "Peer %llu created\n", peer->p_id)do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Peer %llu created\n", (sc)->sc_if.if_xname, peer->p_id
); } while (0);
return peer;
450}

452struct wg_peer *
453wg_peer_lookup(struct wg_softc *sc, const uint8_t public[WG_KEY_SIZE32])
454{
uint8_t		 peer_key[WG_KEY_SIZE32];
struct wg_peer	*peer;
uint64_t	 idx;

idx = SipHash24(&sc->sc_secret, public, WG_KEY_SIZE)SipHash((&sc->sc_secret), 2, 4, (public), (32));
idx &= sc->sc_peer_mask;

rw_enter_read(&sc->sc_peer_lock);
LIST_FOREACH(peer, &sc->sc_peer[idx], p_pubkey_entry)for((peer) = ((&sc->sc_peer[idx])->lh_first); (peer
)!= ((void *)0); (peer) = ((peer)->p_pubkey_entry.le_next)
) {
noise_remote_keys(&peer->p_remote, peer_key, NULL((void *)0));
if (timingsafe_bcmp(peer_key, public, WG_KEY_SIZE32) == 0)
	goto done;
}
peer = NULL((void *)0);
469done:
rw_exit_read(&sc->sc_peer_lock);
return peer;
472}

474void
475wg_peer_destroy(struct wg_peer *peer)
476{
struct wg_softc	*sc = peer->p_sc;
struct wg_aip *aip, *taip;

rw_assert_wrlock(&sc->sc_lock);

/*
* Remove peer from the pubkey hashtable and disable all timeouts.
* After this, and flushing wg_handshake_taskq, then no more handshakes
* can be started.
*/
rw_enter_write(&sc->sc_peer_lock);
LIST_REMOVE(peer, p_pubkey_entry)do { if ((peer)->p_pubkey_entry.le_next != ((void *)0)) (peer
)->p_pubkey_entry.le_next->p_pubkey_entry.le_prev = (peer
)->p_pubkey_entry.le_prev; *(peer)->p_pubkey_entry.le_prev
 = (peer)->p_pubkey_entry.le_next; ((peer)->p_pubkey_entry
.le_prev) = ((void *)-1); ((peer)->p_pubkey_entry.le_next)
 = ((void *)-1); } while (0);
TAILQ_REMOVE(&sc->sc_peer_seq, peer, p_seq_entry)do { if (((peer)->p_seq_entry.tqe_next) != ((void *)0)) (peer
)->p_seq_entry.tqe_next->p_seq_entry.tqe_prev = (peer)->
p_seq_entry.tqe_prev; else (&sc->sc_peer_seq)->tqh_last
 = (peer)->p_seq_entry.tqe_prev; *(peer)->p_seq_entry.tqe_prev
 = (peer)->p_seq_entry.tqe_next; ((peer)->p_seq_entry.tqe_prev
) = ((void *)-1); ((peer)->p_seq_entry.tqe_next) = ((void *
)-1); } while (0);
sc->sc_peer_num--;
rw_exit_write(&sc->sc_peer_lock);

wg_timers_disable(&peer->p_timers);

taskq_barrier(wg_handshake_taskq);

/*
* Now we drop all allowed ips, to drop all outgoing packets to the
* peer. Then drop all the indexes to drop all incoming packets to the
* peer. Then we can flush if_snd, wg_crypt_taskq and then nettq to
* ensure no more references to the peer exist.
*/
LIST_FOREACH_SAFE(aip, &peer->p_aip, a_entry, taip)for ((aip) = ((&peer->p_aip)->lh_first); (aip) &&
 ((taip) = ((aip)->a_entry.le_next), 1); (aip) = (taip))
wg_aip_remove(sc, peer, &aip->a_data);

noise_remote_clear(&peer->p_remote);

NET_LOCK()do { rw_enter_write(&netlock); } while (0);
while (!ifq_empty(&sc->sc_if.if_snd)(((&sc->sc_if.if_snd)->ifq_len) == 0)) {
NET_UNLOCK()do { rw_exit_write(&netlock); } while (0);
tsleep_nsec(sc, PWAIT32, "wg_ifq", 1000);
NET_LOCK()do { rw_enter_write(&netlock); } while (0);
}
NET_UNLOCK()do { rw_exit_write(&netlock); } while (0);

taskq_barrier(wg_crypt_taskq);
taskq_barrier(net_tq(sc->sc_if.if_index));

DPRINTF(sc, "Peer %llu destroyed\n", peer->p_id)do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Peer %llu destroyed\n", (sc)->sc_if.if_xname, peer->p_id
); } while (0);
explicit_bzero(peer, sizeof(*peer));
pool_put(&wg_peer_pool, peer);
522}

524void
525wg_peer_set_endpoint_from_tag(struct wg_peer *peer, struct wg_tag *t)
526{
if (memcmp(&t->t_endpoint, &peer->p_endpoint,__builtin_memcmp((&t->t_endpoint), (&peer->p_endpoint
), (sizeof(t->t_endpoint)))
   sizeof(t->t_endpoint))__builtin_memcmp((&t->t_endpoint), (&peer->p_endpoint
), (sizeof(t->t_endpoint))) == 0)
return;

mtx_enter(&peer->p_endpoint_mtx);
peer->p_endpoint = t->t_endpoint;
mtx_leave(&peer->p_endpoint_mtx);
534}

536void
537wg_peer_set_sockaddr(struct wg_peer *peer, struct sockaddr *remote)
538{
mtx_enter(&peer->p_endpoint_mtx);
memcpy(&peer->p_endpoint.e_remote, remote,__builtin_memcpy((&peer->p_endpoint.e_remote), (remote
), (sizeof(peer->p_endpoint.e_remote)))
      sizeof(peer->p_endpoint.e_remote))__builtin_memcpy((&peer->p_endpoint.e_remote), (remote
), (sizeof(peer->p_endpoint.e_remote)));
bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local))__builtin_bzero((&peer->p_endpoint.e_local), (sizeof(peer
->p_endpoint.e_local)));
mtx_leave(&peer->p_endpoint_mtx);
544}

546int
547wg_peer_get_sockaddr(struct wg_peer *peer, struct sockaddr *remote)
548{
int	ret = 0;

mtx_enter(&peer->p_endpoint_mtx);
if (peer->p_endpoint.e_remote.r_sa.sa_family != AF_UNSPEC0)
memcpy(remote, &peer->p_endpoint.e_remote,__builtin_memcpy((remote), (&peer->p_endpoint.e_remote
), (sizeof(peer->p_endpoint.e_remote)))
       sizeof(peer->p_endpoint.e_remote))__builtin_memcpy((remote), (&peer->p_endpoint.e_remote
), (sizeof(peer->p_endpoint.e_remote)));
else
ret = ENOENT2;
mtx_leave(&peer->p_endpoint_mtx);
return ret;
559}

561void
562wg_peer_clear_src(struct wg_peer *peer)
563{
mtx_enter(&peer->p_endpoint_mtx);
bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local))__builtin_bzero((&peer->p_endpoint.e_local), (sizeof(peer
->p_endpoint.e_local)));
mtx_leave(&peer->p_endpoint_mtx);
567}

569void
570wg_peer_get_endpoint(struct wg_peer *peer, struct wg_endpoint *endpoint)
571{
mtx_enter(&peer->p_endpoint_mtx);
memcpy(endpoint, &peer->p_endpoint, sizeof(*endpoint))__builtin_memcpy((endpoint), (&peer->p_endpoint), (sizeof
(*endpoint)));
mtx_leave(&peer->p_endpoint_mtx);
575}

577void
578wg_peer_counters_add(struct wg_peer *peer, uint64_t tx, uint64_t rx)
579{
mtx_enter(&peer->p_counters_mtx);
peer->p_counters_tx += tx;
peer->p_counters_rx += rx;
mtx_leave(&peer->p_counters_mtx);
584}

586int
587wg_aip_add(struct wg_softc *sc, struct wg_peer *peer, struct wg_aip_io *d)
588{
struct art_root	*root;
struct art_node	*node;
struct wg_aip	*aip;
int		 ret = 0;

switch (d->a_af) {
case AF_INET2:	root = sc->sc_aip4; break;
596#ifdef INET61
case AF_INET624:	root = sc->sc_aip6; break;
598#endif
default: return EAFNOSUPPORT47;
}

if ((aip = pool_get(&wg_aip_pool, PR_NOWAIT0x0002|PR_ZERO0x0008)) == NULL((void *)0))
return ENOBUFS55;

rw_enter_write(&root->ar_lock);
node = art_insert(root, &aip->a_node, &d->a_addr, d->a_cidr);

if (node == &aip->a_node) {
aip->a_peer = peer;
aip->a_data = *d;
LIST_INSERT_HEAD(&peer->p_aip, aip, a_entry)do { if (((aip)->a_entry.le_next = (&peer->p_aip)->
lh_first) != ((void *)0)) (&peer->p_aip)->lh_first->
a_entry.le_prev = &(aip)->a_entry.le_next; (&peer->
p_aip)->lh_first = (aip); (aip)->a_entry.le_prev = &
(&peer->p_aip)->lh_first; } while (0);
sc->sc_aip_num++;
} else {
pool_put(&wg_aip_pool, aip);
aip = (struct wg_aip *) node;
if (aip->a_peer != peer) {
	LIST_REMOVE(aip, a_entry)do { if ((aip)->a_entry.le_next != ((void *)0)) (aip)->
a_entry.le_next->a_entry.le_prev = (aip)->a_entry.le_prev
; *(aip)->a_entry.le_prev = (aip)->a_entry.le_next; ((aip
)->a_entry.le_prev) = ((void *)-1); ((aip)->a_entry.le_next
) = ((void *)-1); } while (0);
	LIST_INSERT_HEAD(&peer->p_aip, aip, a_entry)do { if (((aip)->a_entry.le_next = (&peer->p_aip)->
lh_first) != ((void *)0)) (&peer->p_aip)->lh_first->
a_entry.le_prev = &(aip)->a_entry.le_next; (&peer->
p_aip)->lh_first = (aip); (aip)->a_entry.le_prev = &
(&peer->p_aip)->lh_first; } while (0);
	aip->a_peer = peer;
}
}
rw_exit_write(&root->ar_lock);
return ret;
624}

626struct wg_peer *
627wg_aip_lookup(struct art_root *root, void *addr)
628{
struct srp_ref	 sr;
struct art_node	*node;

node = art_match(root, addr, &sr);
srp_leave(&sr);

return node == NULL((void *)0) ? NULL((void *)0) : ((struct wg_aip *) node)->a_peer;
636}

638int
639wg_aip_remove(struct wg_softc *sc, struct wg_peer *peer, struct wg_aip_io *d)
640{
struct srp_ref	 sr;
struct art_root	*root;
struct art_node	*node;
struct wg_aip	*aip;
int		 ret = 0;

switch (d->a_af) {
case AF_INET2:	root = sc->sc_aip4; break;
649#ifdef INET61
case AF_INET624:	root = sc->sc_aip6; break;
651#endif
default: return EAFNOSUPPORT47;
}

rw_enter_write(&root->ar_lock);
if ((node = art_lookup(root, &d->a_addr, d->a_cidr, &sr)) == NULL((void *)0)) {
ret = ENOENT2;
} else if (((struct wg_aip *) node)->a_peer != peer) {
ret = EXDEV18;
} else {
aip = (struct wg_aip *)node;
if (art_delete(root, node, &d->a_addr, d->a_cidr) == NULL((void *)0))
	panic("art_delete failed to delete node %p", node);

sc->sc_aip_num--;
LIST_REMOVE(aip, a_entry)do { if ((aip)->a_entry.le_next != ((void *)0)) (aip)->
a_entry.le_next->a_entry.le_prev = (aip)->a_entry.le_prev
; *(aip)->a_entry.le_prev = (aip)->a_entry.le_next; ((aip
)->a_entry.le_prev) = ((void *)-1); ((aip)->a_entry.le_next
) = ((void *)-1); } while (0);
pool_put(&wg_aip_pool, aip);
}

srp_leave(&sr);
rw_exit_write(&root->ar_lock);
return ret;
673}

675int
676wg_socket_open(struct socket **so, int af, in_port_t *port,
  int *rtable, void *upcall_arg)
678{
struct mbuf		 mhostnam, mrtable;
680#ifdef INET61
struct sockaddr_in6	*sin6;
682#endif
struct sockaddr_in	*sin;
int			 ret, s;

m_inithdr(&mhostnam);
m_inithdr(&mrtable);

bzero(mtod(&mrtable, u_int *), sizeof(u_int))__builtin_bzero((((u_int *)((&mrtable)->m_hdr.mh_data)
)), (sizeof(u_int)));
*mtod(&mrtable, u_int *)((u_int *)((&mrtable)->m_hdr.mh_data)) = *rtable;
mrtable.m_lenm_hdr.mh_len = sizeof(u_int);

if (af == AF_INET2) {
sin = mtod(&mhostnam, struct sockaddr_in *)((struct sockaddr_in *)((&mhostnam)->m_hdr.mh_data));
bzero(sin, sizeof(*sin))__builtin_bzero((sin), (sizeof(*sin)));
sin->sin_len = sizeof(*sin);
sin->sin_family = AF_INET2;
sin->sin_port = *port;
sin->sin_addr.s_addr = INADDR_ANY((u_int32_t) (__uint32_t)(__builtin_constant_p((u_int32_t)(0x00000000
)) ? (__uint32_t)(((__uint32_t)((u_int32_t)(0x00000000)) &
 0xff) << 24 | ((__uint32_t)((u_int32_t)(0x00000000)) &
 0xff00) << 8 | ((__uint32_t)((u_int32_t)(0x00000000)) &
 0xff0000) >> 8 | ((__uint32_t)((u_int32_t)(0x00000000)
) & 0xff000000) >> 24) : __swap32md((u_int32_t)(0x00000000
))));
mhostnam.m_lenm_hdr.mh_len = sin->sin_len;
701#ifdef INET61
} else if (af == AF_INET624) {
sin6 = mtod(&mhostnam, struct sockaddr_in6 *)((struct sockaddr_in6 *)((&mhostnam)->m_hdr.mh_data));
bzero(sin6, sizeof(*sin6))__builtin_bzero((sin6), (sizeof(*sin6)));
sin6->sin6_len = sizeof(*sin6);
sin6->sin6_family = AF_INET624;
sin6->sin6_port = *port;
sin6->sin6_addr = (struct in6_addr) { .s6_addr__u6_addr.__u6_addr8 = { 0 } };
mhostnam.m_lenm_hdr.mh_len = sin6->sin6_len;
710#endif
} else {
return EAFNOSUPPORT47;
}

if ((ret = socreate(af, so, SOCK_DGRAM2, 0)) != 0)
return ret;

s = solock(*so);
sotoinpcb(*so)((struct inpcb *)(*so)->so_pcb)->inp_upcall = wg_input;
sotoinpcb(*so)((struct inpcb *)(*so)->so_pcb)->inp_upcall_arg = upcall_arg;

if ((ret = sosetopt(*so, SOL_SOCKET0xffff, SO_RTABLE0x1021, &mrtable)) == 0) {
if ((ret = sobind(*so, &mhostnam, curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc)) == 0) {
	*port = sotoinpcb(*so)((struct inpcb *)(*so)->so_pcb)->inp_lport;
	*rtable = sotoinpcb(*so)((struct inpcb *)(*so)->so_pcb)->inp_rtableid;
}
}
sounlock(*so, s);

if (ret != 0)
wg_socket_close(so);

return ret;
734}

736void
737wg_socket_close(struct socket **so)
738{
if (*so != NULL((void *)0) && soclose(*so, 0) != 0)
panic("Unable to close wg socket");
*so = NULL((void *)0);
742}

744int
745wg_bind(struct wg_softc *sc, in_port_t *portp, int *rtablep)
746{
int		 ret = 0, rtable = *rtablep;
in_port_t	 port = *portp;
struct socket	*so4;
750#ifdef INET61
struct socket	*so6;
int		 retries = 0;
753retry:
754#endif
if ((ret = wg_socket_open(&so4, AF_INET2, &port, &rtable, sc)) != 0)
return ret;

758#ifdef INET61
if ((ret = wg_socket_open(&so6, AF_INET624, &port, &rtable, sc)) != 0) {
if (ret == EADDRINUSE48 && *portp == 0 && retries++ < 100)
	goto retry;
wg_socket_close(&so4);
return ret;
}
765#endif

rw_enter_write(&sc->sc_so_lock);
wg_socket_close(&sc->sc_so4);
sc->sc_so4 = so4;
770#ifdef INET61
wg_socket_close(&sc->sc_so6);
sc->sc_so6 = so6;
773#endif
rw_exit_write(&sc->sc_so_lock);

*portp = port;
*rtablep = rtable;
return 0;
779}

781void
782wg_unbind(struct wg_softc *sc)
783{
rw_enter_write(&sc->sc_so_lock);
wg_socket_close(&sc->sc_so4);
786#ifdef INET61
wg_socket_close(&sc->sc_so6);
788#endif
rw_exit_write(&sc->sc_so_lock);
790}

792int
793wg_send(struct wg_softc *sc, struct wg_endpoint *e, struct mbuf *m)
794{
struct mbuf	 peernam, *control = NULL((void *)0);
int		 ret;

/* Get local control address before locking */
if (e->e_remote.r_sa.sa_family == AF_INET2) {
if (e->e_local.l_in.s_addr != INADDR_ANY((u_int32_t) (__uint32_t)(__builtin_constant_p((u_int32_t)(0x00000000
)) ? (__uint32_t)(((__uint32_t)((u_int32_t)(0x00000000)) &
 0xff) << 24 | ((__uint32_t)((u_int32_t)(0x00000000)) &
 0xff00) << 8 | ((__uint32_t)((u_int32_t)(0x00000000)) &
 0xff0000) >> 8 | ((__uint32_t)((u_int32_t)(0x00000000)
) & 0xff000000) >> 24) : __swap32md((u_int32_t)(0x00000000
)))))
	control = sbcreatecontrol(&e->e_local.l_in,
	    sizeof(struct in_addr), IP_SENDSRCADDR7,
	    IPPROTO_IP0);
804#ifdef INET61
} else if (e->e_remote.r_sa.sa_family == AF_INET624) {
if (!IN6_IS_ADDR_UNSPECIFIED(&e->e_local.l_in6)((*(const u_int32_t *)(const void *)(&(&e->e_local
.l_pktinfo6.ipi6_addr)->__u6_addr.__u6_addr8[0]) == 0) &&
 (*(const u_int32_t *)(const void *)(&(&e->e_local
.l_pktinfo6.ipi6_addr)->__u6_addr.__u6_addr8[4]) == 0) &&
 (*(const u_int32_t *)(const void *)(&(&e->e_local
.l_pktinfo6.ipi6_addr)->__u6_addr.__u6_addr8[8]) == 0) &&
 (*(const u_int32_t *)(const void *)(&(&e->e_local
.l_pktinfo6.ipi6_addr)->__u6_addr.__u6_addr8[12]) == 0)))
	control = sbcreatecontrol(&e->e_local.l_pktinfo6,
	    sizeof(struct in6_pktinfo), IPV6_PKTINFO46,
	    IPPROTO_IPV641);
810#endif
} else {
m_freem(m);
return EAFNOSUPPORT47;
}

/* Get remote address */
peernam.m_typem_hdr.mh_type = MT_SONAME3;
peernam.m_nextm_hdr.mh_next = NULL((void *)0);
peernam.m_nextpktm_hdr.mh_nextpkt = NULL((void *)0);
peernam.m_datam_hdr.mh_data = (void *)&e->e_remote.r_sa;
peernam.m_lenm_hdr.mh_len = e->e_remote.r_sa.sa_len;
peernam.m_flagsm_hdr.mh_flags = 0;

rw_enter_read(&sc->sc_so_lock);
if (e->e_remote.r_sa.sa_family == AF_INET2 && sc->sc_so4 != NULL((void *)0))
ret = sosend(sc->sc_so4, &peernam, NULL((void *)0), m, control, 0);
827#ifdef INET61
else if (e->e_remote.r_sa.sa_family == AF_INET624 && sc->sc_so6 != NULL((void *)0))
ret = sosend(sc->sc_so6, &peernam, NULL((void *)0), m, control, 0);
830#endif
else {
ret = ENOTCONN57;
m_freem(control);
m_freem(m);
}
rw_exit_read(&sc->sc_so_lock);

return ret;
839}

841void
842wg_send_buf(struct wg_softc *sc, struct wg_endpoint *e, uint8_t *buf,
  size_t len)
844{
struct mbuf	*m;
int		 ret = 0;

848retry:
m = m_gethdr(M_WAIT0x0001, MT_DATA1);
m->m_lenm_hdr.mh_len = 0;
m_copyback(m, 0, len, buf, M_WAIT0x0001);

/* As we're sending a handshake packet here, we want high priority */
m->m_pkthdrM_dat.MH.MH_pkthdr.pf.prio = IFQ_MAXPRIO8 - 1;

if (ret == 0) {
ret = wg_send(sc, e, m);
/* Retry if we couldn't bind to e->e_local */
if (ret == EADDRNOTAVAIL49) {
	bzero(&e->e_local, sizeof(e->e_local))__builtin_bzero((&e->e_local), (sizeof(e->e_local))
);
	goto retry;
}
} else {
ret = wg_send(sc, e, m);
if (ret != 0)
	DPRINTF(sc, "Unable to send packet\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Unable to send packet\n", (sc)->sc_if.if_xname); } while
 (0);
}
868}

870struct wg_tag *
871wg_tag_get(struct mbuf *m)
872{
struct m_tag	*mtag;

if ((mtag = m_tag_find(m, PACKET_TAG_WIREGUARD0x0040, NULL((void *)0))) == NULL((void *)0)) {
mtag = m_tag_get(PACKET_TAG_WIREGUARD0x0040, sizeof(struct wg_tag),
    M_NOWAIT0x0002);
if (mtag == NULL((void *)0))
	return (NULL((void *)0));
bzero(mtag + 1, sizeof(struct wg_tag))__builtin_bzero((mtag + 1), (sizeof(struct wg_tag)));
m_tag_prepend(m, mtag);
}
return ((struct wg_tag *)(mtag + 1));
884}

886/*
* The following section handles the timeout callbacks for a WireGuard session.
* These functions provide an "event based" model for controlling wg(8) session
* timers. All function calls occur after the specified event below.
*
* wg_timers_event_data_sent:
*	tx: data
* wg_timers_event_data_received:
*	rx: data
* wg_timers_event_any_authenticated_packet_sent:
*	tx: keepalive, data, handshake
* wg_timers_event_any_authenticated_packet_received:
*	rx: keepalive, data, handshake
* wg_timers_event_any_authenticated_packet_traversal:
*	tx, rx: keepalive, data, handshake
* wg_timers_event_handshake_initiated:
*	tx: initiation
* wg_timers_event_handshake_responded:
*	tx: response
* wg_timers_event_handshake_complete:
*	rx: response, confirmation data
* wg_timers_event_session_derived:
*	tx: response, rx: response
* wg_timers_event_want_initiation:
*	tx: data failed, old keys expiring
* wg_timers_event_reset_handshake_last_sent:
* 	anytime we may immediately want a new handshake
*/
914void
915wg_timers_init(struct wg_timers *t)
916{
bzero(t, sizeof(*t))__builtin_bzero((t), (sizeof(*t)));
rw_init(&t->t_lock, "wg_timers")_rw_init_flags(&t->t_lock, "wg_timers", 0, ((void *)0)
);
mtx_init(&t->t_handshake_mtx, IPL_NET)do { (void)(((void *)0)); (void)(0); __mtx_init((&t->t_handshake_mtx
), ((((0x7)) > 0x0 && ((0x7)) < 0x9) ? 0x9 : ((
0x7)))); } while (0);

timeout_set(&t->t_new_handshake, wg_timers_run_new_handshake, t);
timeout_set(&t->t_send_keepalive, wg_timers_run_send_keepalive, t);
timeout_set(&t->t_retry_handshake, wg_timers_run_retry_handshake, t);
timeout_set(&t->t_persistent_keepalive,
   wg_timers_run_persistent_keepalive, t);
timeout_set(&t->t_zero_key_material,
   wg_timers_run_zero_key_material, t);
928}

930void
931wg_timers_enable(struct wg_timers *t)
932{
rw_enter_write(&t->t_lock);
t->t_disabled = 0;
rw_exit_write(&t->t_lock);
wg_timers_run_persistent_keepalive(t);
937}

939void
940wg_timers_disable(struct wg_timers *t)
941{
rw_enter_write(&t->t_lock);
t->t_disabled = 1;
t->t_need_another_keepalive = 0;
rw_exit_write(&t->t_lock);

timeout_del_barrier(&t->t_new_handshake);
timeout_del_barrier(&t->t_send_keepalive);
timeout_del_barrier(&t->t_retry_handshake);
timeout_del_barrier(&t->t_persistent_keepalive);
timeout_del_barrier(&t->t_zero_key_material);
952}

954void
955wg_timers_set_persistent_keepalive(struct wg_timers *t, uint16_t interval)
956{
rw_enter_read(&t->t_lock);
if (!t->t_disabled) {
t->t_persistent_keepalive_interval = interval;
wg_timers_run_persistent_keepalive(t);
}
rw_exit_read(&t->t_lock);
963}

965int
966wg_timers_get_persistent_keepalive(struct wg_timers *t, uint16_t *interval)
967{
*interval = t->t_persistent_keepalive_interval;
return *interval > 0 ? 0 : ENOENT2;
970}

972void
973wg_timers_get_last_handshake(struct wg_timers *t, struct timespec *time)
974{
mtx_enter(&t->t_handshake_mtx);
*time = t->t_handshake_complete;
mtx_leave(&t->t_handshake_mtx);
978}

980int
981wg_timers_expired_handshake_last_sent(struct wg_timers *t)
982{
struct timespec uptime;
struct timespec expire = { .tv_sec = REKEY_TIMEOUT5, .tv_nsec = 0 };

getnanouptime(&uptime);
timespecadd(&t->t_handshake_last_sent, &expire, &expire)do { (&expire)->tv_sec = (&t->t_handshake_last_sent
)->tv_sec + (&expire)->tv_sec; (&expire)->tv_nsec
 = (&t->t_handshake_last_sent)->tv_nsec + (&expire
)->tv_nsec; if ((&expire)->tv_nsec >= 1000000000L
) { (&expire)->tv_sec++; (&expire)->tv_nsec -= 1000000000L
; } } while (0);
return timespeccmp(&uptime, &expire, >)(((&uptime)->tv_sec == (&expire)->tv_sec) ? ((&
uptime)->tv_nsec > (&expire)->tv_nsec) : ((&
uptime)->tv_sec > (&expire)->tv_sec)) ? ETIMEDOUT60 : 0;
989}

991int
992wg_timers_check_handshake_last_sent(struct wg_timers *t)
993{
int ret;
mtx_enter(&t->t_handshake_mtx);
if ((ret = wg_timers_expired_handshake_last_sent(t)) == ETIMEDOUT60)
getnanouptime(&t->t_handshake_last_sent);
mtx_leave(&t->t_handshake_mtx);
return ret;
1000}

1002void
1003wg_timers_event_data_sent(struct wg_timers *t)
1004{
int	msecs = NEW_HANDSHAKE_TIMEOUT(5 + 10) * 1000;
msecs += arc4random_uniform(REKEY_TIMEOUT_JITTER334);

rw_enter_read(&t->t_lock);
if (!t->t_disabled && !timeout_pending(&t->t_new_handshake)((&t->t_new_handshake)->to_flags & 0x02))
timeout_add_msec(&t->t_new_handshake, msecs);
rw_exit_read(&t->t_lock);
1012}

1014void
1015wg_timers_event_data_received(struct wg_timers *t)
1016{
rw_enter_read(&t->t_lock);
if (!t->t_disabled) {
if (!timeout_pending(&t->t_send_keepalive)((&t->t_send_keepalive)->to_flags & 0x02))
	timeout_add_sec(&t->t_send_keepalive,
	    KEEPALIVE_TIMEOUT10);
else
	t->t_need_another_keepalive = 1;
}
rw_exit_read(&t->t_lock);
1026}

1028void
1029wg_timers_event_any_authenticated_packet_sent(struct wg_timers *t)
1030{
timeout_del(&t->t_send_keepalive);
1032}

1034void
1035wg_timers_event_any_authenticated_packet_received(struct wg_timers *t)
1036{
timeout_del(&t->t_new_handshake);
1038}

1040void
1041wg_timers_event_any_authenticated_packet_traversal(struct wg_timers *t)
1042{
rw_enter_read(&t->t_lock);
if (!t->t_disabled && t->t_persistent_keepalive_interval > 0)
timeout_add_sec(&t->t_persistent_keepalive,
    t->t_persistent_keepalive_interval);
rw_exit_read(&t->t_lock);
1048}

1050void
1051wg_timers_event_handshake_initiated(struct wg_timers *t)
1052{
int	msecs = REKEY_TIMEOUT5 * 1000;
msecs += arc4random_uniform(REKEY_TIMEOUT_JITTER334);

rw_enter_read(&t->t_lock);
if (!t->t_disabled)
timeout_add_msec(&t->t_retry_handshake, msecs);
rw_exit_read(&t->t_lock);
1060}

1062void
1063wg_timers_event_handshake_responded(struct wg_timers *t)
1064{
mtx_enter(&t->t_handshake_mtx);
getnanouptime(&t->t_handshake_last_sent);
mtx_leave(&t->t_handshake_mtx);
1068}

1070void
1071wg_timers_event_handshake_complete(struct wg_timers *t)
1072{
rw_enter_read(&t->t_lock);
if (!t->t_disabled) {
mtx_enter(&t->t_handshake_mtx);
timeout_del(&t->t_retry_handshake);
t->t_handshake_retries = 0;
getnanotime(&t->t_handshake_complete);
mtx_leave(&t->t_handshake_mtx);
wg_timers_run_send_keepalive(t);
}
rw_exit_read(&t->t_lock);
1083}

1085void
1086wg_timers_event_session_derived(struct wg_timers *t)
1087{
rw_enter_read(&t->t_lock);
if (!t->t_disabled)
timeout_add_sec(&t->t_zero_key_material, REJECT_AFTER_TIME180 * 3);
rw_exit_read(&t->t_lock);
1092}

1094void
1095wg_timers_event_want_initiation(struct wg_timers *t)
1096{
rw_enter_read(&t->t_lock);
if (!t->t_disabled)
wg_timers_run_send_initiation(t, 0);
rw_exit_read(&t->t_lock);
1101}

1103void
1104wg_timers_event_reset_handshake_last_sent(struct wg_timers *t)
1105{
mtx_enter(&t->t_handshake_mtx);
t->t_handshake_last_sent.tv_sec -= (REKEY_TIMEOUT5 + 1);
mtx_leave(&t->t_handshake_mtx);
1109}

1111void
1112wg_timers_run_send_initiation(void *_t, int is_retry)
1113{
struct wg_timers *t = _t;
struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers)({ const __typeof( ((struct wg_peer *)0)->p_timers ) *__mptr
 = (t); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_timers) );});
if (!is_retry)
t->t_handshake_retries = 0;
if (wg_timers_expired_handshake_last_sent(t) == ETIMEDOUT60)
task_add(wg_handshake_taskq, &peer->p_send_initiation);
1120}

1122void
1123wg_timers_run_retry_handshake(void *_t)
1124{
struct wg_timers *t = _t;
struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers)({ const __typeof( ((struct wg_peer *)0)->p_timers ) *__mptr
 = (t); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_timers) );});

mtx_enter(&t->t_handshake_mtx);
if (t->t_handshake_retries <= MAX_TIMER_HANDSHAKES(90 / 5)) {
t->t_handshake_retries++;
mtx_leave(&t->t_handshake_mtx);

DPRINTF(peer->p_sc, "Handshake for peer %llu did not complete "do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Handshake for peer %llu did not complete " "after %d seconds, retrying (try %d)\n"
, (peer->p_sc)->sc_if.if_xname, peer->p_id, 5, t->
t_handshake_retries + 1); } while (0)
    "after %d seconds, retrying (try %d)\n", peer->p_id,do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Handshake for peer %llu did not complete " "after %d seconds, retrying (try %d)\n"
, (peer->p_sc)->sc_if.if_xname, peer->p_id, 5, t->
t_handshake_retries + 1); } while (0)
    REKEY_TIMEOUT, t->t_handshake_retries + 1)do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Handshake for peer %llu did not complete " "after %d seconds, retrying (try %d)\n"
, (peer->p_sc)->sc_if.if_xname, peer->p_id, 5, t->
t_handshake_retries + 1); } while (0);
wg_peer_clear_src(peer);
wg_timers_run_send_initiation(t, 1);
} else {
mtx_leave(&t->t_handshake_mtx);

DPRINTF(peer->p_sc, "Handshake for peer %llu did not complete "do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Handshake for peer %llu did not complete " "after %d retries, giving up\n"
, (peer->p_sc)->sc_if.if_xname, peer->p_id, (90 / 5)
 + 2); } while (0)
    "after %d retries, giving up\n", peer->p_id,do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Handshake for peer %llu did not complete " "after %d retries, giving up\n"
, (peer->p_sc)->sc_if.if_xname, peer->p_id, (90 / 5)
 + 2); } while (0)
    MAX_TIMER_HANDSHAKES + 2)do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Handshake for peer %llu did not complete " "after %d retries, giving up\n"
, (peer->p_sc)->sc_if.if_xname, peer->p_id, (90 / 5)
 + 2); } while (0);

timeout_del(&t->t_send_keepalive);
mq_purge(&peer->p_stage_queue);
if (!timeout_pending(&t->t_zero_key_material)((&t->t_zero_key_material)->to_flags & 0x02))
	timeout_add_sec(&t->t_zero_key_material,
	    REJECT_AFTER_TIME180 * 3);
}
1151}

1153void
1154wg_timers_run_send_keepalive(void *_t)
1155{
struct wg_timers *t = _t;
struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers)({ const __typeof( ((struct wg_peer *)0)->p_timers ) *__mptr
 = (t); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_timers) );});

task_add(wg_crypt_taskq, &peer->p_send_keepalive);
if (t->t_need_another_keepalive) {
t->t_need_another_keepalive = 0;
timeout_add_sec(&t->t_send_keepalive, KEEPALIVE_TIMEOUT10);
}
1164}

1166void
1167wg_timers_run_new_handshake(void *_t)
1168{
struct wg_timers *t = _t;
struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers)({ const __typeof( ((struct wg_peer *)0)->p_timers ) *__mptr
 = (t); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_timers) );});

DPRINTF(peer->p_sc, "Retrying handshake with peer %llu because we "do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Retrying handshake with peer %llu because we " "stopped hearing back after %d seconds\n"
, (peer->p_sc)->sc_if.if_xname, peer->p_id, (5 + 10)
); } while (0)
   "stopped hearing back after %d seconds\n",do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Retrying handshake with peer %llu because we " "stopped hearing back after %d seconds\n"
, (peer->p_sc)->sc_if.if_xname, peer->p_id, (5 + 10)
); } while (0)
   peer->p_id, NEW_HANDSHAKE_TIMEOUT)do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Retrying handshake with peer %llu because we " "stopped hearing back after %d seconds\n"
, (peer->p_sc)->sc_if.if_xname, peer->p_id, (5 + 10)
); } while (0);
wg_peer_clear_src(peer);

wg_timers_run_send_initiation(t, 0);
1178}

1180void
1181wg_timers_run_zero_key_material(void *_t)
1182{
struct wg_timers *t = _t;
struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers)({ const __typeof( ((struct wg_peer *)0)->p_timers ) *__mptr
 = (t); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_timers) );});

DPRINTF(peer->p_sc, "Zeroing out keys for peer %llu\n", peer->p_id)do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Zeroing out keys for peer %llu\n", (peer->p_sc)->
sc_if.if_xname, peer->p_id); } while (0);
task_add(wg_handshake_taskq, &peer->p_clear_secrets);
1188}

1190void
1191wg_timers_run_persistent_keepalive(void *_t)
1192{
struct wg_timers *t = _t;
struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers)({ const __typeof( ((struct wg_peer *)0)->p_timers ) *__mptr
 = (t); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_timers) );});
if (t->t_persistent_keepalive_interval != 0)
task_add(wg_crypt_taskq, &peer->p_send_keepalive);
1197}

1199/* The following functions handle handshakes */
1200void
1201wg_peer_send_buf(struct wg_peer *peer, uint8_t *buf, size_t len)
1202{
struct wg_endpoint	 endpoint;

wg_peer_counters_add(peer, len, 0);
wg_timers_event_any_authenticated_packet_traversal(&peer->p_timers);
wg_timers_event_any_authenticated_packet_sent(&peer->p_timers);
wg_peer_get_endpoint(peer, &endpoint);
wg_send_buf(peer->p_sc, &endpoint, buf, len);
1210}

1212void
1213wg_send_initiation(void *_peer)
1214{
struct wg_peer			*peer = _peer;
struct wg_pkt_initiation	 pkt;

if (wg_timers_check_handshake_last_sent(&peer->p_timers) != ETIMEDOUT60)
return;

DPRINTF(peer->p_sc, "Sending handshake initiation to peer %llu\n",do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Sending handshake initiation to peer %llu\n", (peer->
p_sc)->sc_if.if_xname, peer->p_id); } while (0)
   peer->p_id)do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Sending handshake initiation to peer %llu\n", (peer->
p_sc)->sc_if.if_xname, peer->p_id); } while (0);

if (noise_create_initiation(&peer->p_remote, &pkt.s_idx, pkt.ue, pkt.es,
		    pkt.ets) != 0)
return;
pkt.t = WG_PKT_INITIATION((__uint32_t)(1));
cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt,
   sizeof(pkt)-sizeof(pkt.m));
wg_peer_send_buf(peer, (uint8_t *)&pkt, sizeof(pkt));
wg_timers_event_handshake_initiated(&peer->p_timers);
1232}

1234void
1235wg_send_response(struct wg_peer *peer)
1236{
struct wg_pkt_response	 pkt;

DPRINTF(peer->p_sc, "Sending handshake response to peer %llu\n",do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Sending handshake response to peer %llu\n", (peer->
p_sc)->sc_if.if_xname, peer->p_id); } while (0)
   peer->p_id)do { if ((((peer->p_sc)->sc_if.if_flags) & (0x4))) printf
("%s: " "Sending handshake response to peer %llu\n", (peer->
p_sc)->sc_if.if_xname, peer->p_id); } while (0);

if (noise_create_response(&peer->p_remote, &pkt.s_idx, &pkt.r_idx,
		  pkt.ue, pkt.en) != 0)
return;
if (noise_remote_begin_session(&peer->p_remote) != 0)
return;
wg_timers_event_session_derived(&peer->p_timers);
pkt.t = WG_PKT_RESPONSE((__uint32_t)(2));
cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt,
   sizeof(pkt)-sizeof(pkt.m));
wg_timers_event_handshake_responded(&peer->p_timers);
wg_peer_send_buf(peer, (uint8_t *)&pkt, sizeof(pkt));
1253}

1255void
1256wg_send_cookie(struct wg_softc *sc, struct cookie_macs *cm, uint32_t idx,
  struct wg_endpoint *e)
1258{
struct wg_pkt_cookie	pkt;

DPRINTF(sc, "Sending cookie response for denied handshake message\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Sending cookie response for denied handshake message\n", (sc
)->sc_if.if_xname); } while (0);

pkt.t = WG_PKT_COOKIE((__uint32_t)(3));
pkt.r_idx = idx;

cookie_checker_create_payload(&sc->sc_cookie, cm, pkt.nonce,
   pkt.ec, &e->e_remote.r_sa);

wg_send_buf(sc, e, (uint8_t *)&pkt, sizeof(pkt));
1270}

1272void
1273wg_send_keepalive(void *_peer)
1274{
struct wg_peer	*peer = _peer;
struct wg_softc	*sc = peer->p_sc;
struct wg_tag	*t;
struct mbuf	*m;

if (!mq_empty(&peer->p_stage_queue)((&(&peer->p_stage_queue)->mq_list)->ml_len ==
 0))
goto send;

if ((m = m_gethdr(M_NOWAIT0x0002, MT_DATA1)) == NULL((void *)0))
return;

if ((t = wg_tag_get(m)) == NULL((void *)0)) {
m_freem(m);
return;
}

m->m_lenm_hdr.mh_len = 0;
m_calchdrlen(m);

t->t_peer = peer;
t->t_mbuf = NULL((void *)0);
t->t_done = 0;
t->t_mtu = 0; /* MTU == 0 OK for keepalive */

mq_push(&peer->p_stage_queue, m);
1300send:
if (noise_remote_ready(&peer->p_remote) == 0) {
wg_queue_out(sc, peer);
task_add(wg_crypt_taskq, &sc->sc_encap);
} else {
wg_timers_event_want_initiation(&peer->p_timers);
}
1307}

1309void
1310wg_peer_clear_secrets(void *_peer)
1311{
struct wg_peer *peer = _peer;
noise_remote_clear(&peer->p_remote);
1314}

1316void
1317wg_handshake(struct wg_softc *sc, struct mbuf *m)
1318{
struct wg_tag			*t;
struct wg_pkt_initiation	*init;
struct wg_pkt_response		*resp;
struct wg_pkt_cookie		*cook;
struct wg_peer			*peer;
struct noise_remote		*remote;
int				 res, underload = 0;
static struct timeval		 wg_last_underload; /* microuptime */

if (mq_len(&sc->sc_handshake_queue)((&(&sc->sc_handshake_queue)->mq_list)->ml_len
) >= MAX_QUEUED_HANDSHAKES4096/8) {
getmicrouptime(&wg_last_underload);
underload = 1;
} else if (wg_last_underload.tv_sec != 0) {
if (!ratecheck(&wg_last_underload, &underload_interval))
	underload = 1;
else
	bzero(&wg_last_underload, sizeof(wg_last_underload))__builtin_bzero((&wg_last_underload), (sizeof(wg_last_underload
)));
}

t = wg_tag_get(m);

switch (*mtod(m, uint32_t *)((uint32_t *)((m)->m_hdr.mh_data))) {
case WG_PKT_INITIATION((__uint32_t)(1)):
init = mtod(m, struct wg_pkt_initiation *)((struct wg_pkt_initiation *)((m)->m_hdr.mh_data));

res = cookie_checker_validate_macs(&sc->sc_cookie, &init->m,
		init, sizeof(*init) - sizeof(init->m),
		underload, &t->t_endpoint.e_remote.r_sa);

if (res == EINVAL22) {
	DPRINTF(sc, "Invalid initiation MAC\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Invalid initiation MAC\n", (sc)->sc_if.if_xname); } while
 (0);
	goto error;
} else if (res == ECONNREFUSED61) {
	DPRINTF(sc, "Handshake ratelimited\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Handshake ratelimited\n", (sc)->sc_if.if_xname); } while
 (0);
	goto error;
} else if (res == EAGAIN35) {
	wg_send_cookie(sc, &init->m, init->s_idx,
	    &t->t_endpoint);
	goto error;
} else if (res != 0) {
	panic("unexpected response: %d", res);
}

if (noise_consume_initiation(&sc->sc_local, &remote,
    init->s_idx, init->ue, init->es, init->ets) != 0) {
	DPRINTF(sc, "Invalid handshake initiation\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Invalid handshake initiation\n", (sc)->sc_if.if_xname); }
 while (0);
	goto error;
}

peer = CONTAINER_OF(remote, struct wg_peer, p_remote)({ const __typeof( ((struct wg_peer *)0)->p_remote ) *__mptr
 = (remote); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_remote) );});

DPRINTF(sc, "Receiving handshake initiation from peer %llu\n",do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Receiving handshake initiation from peer %llu\n", (sc)->
sc_if.if_xname, peer->p_id); } while (0)
    peer->p_id)do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Receiving handshake initiation from peer %llu\n", (sc)->
sc_if.if_xname, peer->p_id); } while (0);

wg_peer_counters_add(peer, 0, sizeof(*init));
wg_peer_set_endpoint_from_tag(peer, t);
wg_send_response(peer);
break;
case WG_PKT_RESPONSE((__uint32_t)(2)):
resp = mtod(m, struct wg_pkt_response *)((struct wg_pkt_response *)((m)->m_hdr.mh_data));

res = cookie_checker_validate_macs(&sc->sc_cookie, &resp->m,
		resp, sizeof(*resp) - sizeof(resp->m),
		underload, &t->t_endpoint.e_remote.r_sa);

if (res == EINVAL22) {
	DPRINTF(sc, "Invalid response MAC\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Invalid response MAC\n", (sc)->sc_if.if_xname); } while (
0);
	goto error;
} else if (res == ECONNREFUSED61) {
	DPRINTF(sc, "Handshake ratelimited\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Handshake ratelimited\n", (sc)->sc_if.if_xname); } while
 (0);
	goto error;
} else if (res == EAGAIN35) {
	wg_send_cookie(sc, &resp->m, resp->s_idx,
	    &t->t_endpoint);
	goto error;
} else if (res != 0) {
	panic("unexpected response: %d", res);
}

if ((remote = wg_index_get(sc, resp->r_idx)) == NULL((void *)0)) {
	DPRINTF(sc, "Unknown handshake response\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Unknown handshake response\n", (sc)->sc_if.if_xname); } while
 (0);
	goto error;
}

peer = CONTAINER_OF(remote, struct wg_peer, p_remote)({ const __typeof( ((struct wg_peer *)0)->p_remote ) *__mptr
 = (remote); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_remote) );});

if (noise_consume_response(remote, resp->s_idx, resp->r_idx,
			   resp->ue, resp->en) != 0) {
	DPRINTF(sc, "Invalid handshake response\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Invalid handshake response\n", (sc)->sc_if.if_xname); } while
 (0);
	goto error;
}

DPRINTF(sc, "Receiving handshake response from peer %llu\n",do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Receiving handshake response from peer %llu\n", (sc)->sc_if
.if_xname, peer->p_id); } while (0)
		peer->p_id)do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Receiving handshake response from peer %llu\n", (sc)->sc_if
.if_xname, peer->p_id); } while (0);

wg_peer_counters_add(peer, 0, sizeof(*resp));
wg_peer_set_endpoint_from_tag(peer, t);
if (noise_remote_begin_session(&peer->p_remote) == 0) {
	wg_timers_event_session_derived(&peer->p_timers);
	wg_timers_event_handshake_complete(&peer->p_timers);
}
break;
case WG_PKT_COOKIE((__uint32_t)(3)):
cook = mtod(m, struct wg_pkt_cookie *)((struct wg_pkt_cookie *)((m)->m_hdr.mh_data));

if ((remote = wg_index_get(sc, cook->r_idx)) == NULL((void *)0)) {
	DPRINTF(sc, "Unknown cookie index\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Unknown cookie index\n", (sc)->sc_if.if_xname); } while (
0);
	goto error;
}

peer = CONTAINER_OF(remote, struct wg_peer, p_remote)({ const __typeof( ((struct wg_peer *)0)->p_remote ) *__mptr
 = (remote); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_remote) );});

if (cookie_maker_consume_payload(&peer->p_cookie,
    cook->nonce, cook->ec) != 0) {
	DPRINTF(sc, "Could not decrypt cookie response\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Could not decrypt cookie response\n", (sc)->sc_if.if_xname
); } while (0);
	goto error;
}

DPRINTF(sc, "Receiving cookie response\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Receiving cookie response\n", (sc)->sc_if.if_xname); } while
 (0);
goto error;
default:
panic("invalid packet in handshake queue");
}

wg_timers_event_any_authenticated_packet_received(&peer->p_timers);
wg_timers_event_any_authenticated_packet_traversal(&peer->p_timers);
1445error:
m_freem(m);
1447}

1449void
1450wg_handshake_worker(void *_sc)
1451{
struct mbuf *m;
struct wg_softc *sc = _sc;
while ((m = mq_dequeue(&sc->sc_handshake_queue)) != NULL((void *)0))
wg_handshake(sc, m);
1456}

1458/*
* The following functions handle encapsulation (encryption) and
* decapsulation (decryption). The wg_{en,de}cap functions will run in the
* sc_crypt_taskq, while wg_deliver_{in,out} must be serialised and will run
* in nettq.
*
* The packets are tracked in two queues, a serial queue and a parallel queue.
*  - The parallel queue is used to distribute the encryption across multiple
*    threads.
*  - The serial queue ensures that packets are not reordered and are
*    delievered in sequence.
* The wg_tag attached to the packet contains two flags to help the two queues
* interact.
*  - t_done: The parallel queue has finished with the packet, now the serial
*            queue can do it's work.
*  - t_mbuf: Used to store the *crypted packet. in the case of encryption,
*            this is a newly allocated packet, and in the case of decryption,
*            it is a pointer to the same packet, that has been decrypted and
*            truncated. If t_mbuf is NULL, then *cryption failed and this
*            packet should not be passed.
* wg_{en,de}cap work on the parallel queue, while wg_deliver_{in,out} work
* on the serial queue.
*/
1481void
1482wg_encap(struct wg_softc *sc, struct mbuf *m)
1483{
int res = 0;
struct wg_pkt_data	*data;
struct wg_peer		*peer;
struct wg_tag		*t;
struct mbuf		*mc;
size_t			 padding_len, plaintext_len, out_len;
uint64_t		 nonce;

t = wg_tag_get(m);
peer = t->t_peer;

plaintext_len = min(WG_PKT_WITH_PADDING(m->m_pkthdr.len)(((m->M_dat.MH.MH_pkthdr.len) + (16-1)) & (~(16-1))), t->t_mtu);
padding_len = plaintext_len - m->m_pkthdrM_dat.MH.MH_pkthdr.len;
out_len = sizeof(struct wg_pkt_data) + plaintext_len + NOISE_AUTHTAG_LEN16;

/*
* For the time being we allocate a new packet with sufficient size to
* hold the encrypted data and headers. It would be difficult to
* overcome as p_encap_queue (mbuf_list) holds a reference to the mbuf.
* If we m_makespace or similar, we risk corrupting that list.
* Additionally, we only pass a buf and buf length to
* noise_remote_encrypt. Technically it would be possible to teach
* noise_remote_encrypt about mbufs, but we would need to sort out the
* p_encap_queue situation first.
*/
if ((mc = m_clget(NULL((void *)0), M_NOWAIT0x0002, out_len)) == NULL((void *)0))
goto error;

data = mtod(mc, struct wg_pkt_data *)((struct wg_pkt_data *)((mc)->m_hdr.mh_data));
m_copydata(m, 0, m->m_pkthdrM_dat.MH.MH_pkthdr.len, data->buf);
bzero(data->buf + m->m_pkthdr.len, padding_len)__builtin_bzero((data->buf + m->M_dat.MH.MH_pkthdr.len)
, (padding_len));
data->t = WG_PKT_DATA((__uint32_t)(4));

/*
* Copy the flow hash from the inner packet to the outer packet, so
* that fq_codel can property separate streams, rather than falling
* back to random buckets.
*/
mc->m_pkthdrM_dat.MH.MH_pkthdr.ph_flowid = m->m_pkthdrM_dat.MH.MH_pkthdr.ph_flowid;

res = noise_remote_encrypt(&peer->p_remote, &data->r_idx, &nonce,
		   data->buf, plaintext_len);
nonce = htole64(nonce)((__uint64_t)(nonce)); /* Wire format is little endian. */
memcpy(data->nonce, &nonce, sizeof(data->nonce))__builtin_memcpy((data->nonce), (&nonce), (sizeof(data
->nonce)));

if (__predict_false(res == EINVAL)__builtin_expect(((res == 22) != 0), 0)) {
m_freem(mc);
goto error;
} else if (__predict_false(res == ESTALE)__builtin_expect(((res == 70) != 0), 0)) {
wg_timers_event_want_initiation(&peer->p_timers);
} else if (__predict_false(res != 0)__builtin_expect(((res != 0) != 0), 0)) {
panic("unexpected result: %d", res);
}

/* A packet with length 0 is a keepalive packet */
if (__predict_false(m->m_pkthdr.len == 0)__builtin_expect(((m->M_dat.MH.MH_pkthdr.len == 0) != 0), 0
))
DPRINTF(sc, "Sending keepalive packet to peer %llu\n",do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Sending keepalive packet to peer %llu\n", (sc)->sc_if.if_xname
, peer->p_id); } while (0)
    peer->p_id)do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Sending keepalive packet to peer %llu\n", (sc)->sc_if.if_xname
, peer->p_id); } while (0);

mc->m_pkthdrM_dat.MH.MH_pkthdr.ph_loopcnt = m->m_pkthdrM_dat.MH.MH_pkthdr.ph_loopcnt;
mc->m_flagsm_hdr.mh_flags &= ~(M_MCAST0x0200 | M_BCAST0x0100);
mc->m_lenm_hdr.mh_len = out_len;
m_calchdrlen(mc);

/*
* We would count ifc_opackets, ifc_obytes of m here, except if_snd
* already does that for us, so no need to worry about it.
counters_pkt(sc->sc_if.if_counters, ifc_opackets, ifc_obytes,
   m->m_pkthdr.len);
*/
wg_peer_counters_add(peer, mc->m_pkthdrM_dat.MH.MH_pkthdr.len, 0);

t->t_mbuf = mc;
1557error:
t->t_done = 1;
task_add(net_tq(sc->sc_if.if_index), &peer->p_deliver_out);
1560}

1562void
1563wg_decap(struct wg_softc *sc, struct mbuf *m)
1564{
int			 res, len;
struct ip		*ip;
struct ip6_hdr		*ip6;
struct wg_pkt_data	*data;
struct wg_peer		*peer, *allowed_peer;
struct wg_tag		*t;
size_t			 payload_len;
uint64_t		 nonce;

t = wg_tag_get(m);
peer = t->t_peer;

/*
* Likewise to wg_encap, we pass a buf and buf length to 
* noise_remote_decrypt. Again, possible to teach it about mbufs
* but need to get over the p_decap_queue situation first. However,
* we do not need to allocate a new mbuf as the decrypted packet is
* strictly smaller than encrypted. We just set t_mbuf to m and
* wg_deliver_in knows how to deal with that.
*/
data = mtod(m, struct wg_pkt_data *)((struct wg_pkt_data *)((m)->m_hdr.mh_data));
payload_len = m->m_pkthdrM_dat.MH.MH_pkthdr.len - sizeof(struct wg_pkt_data);
memcpy(&nonce, data->nonce, sizeof(nonce))__builtin_memcpy((&nonce), (data->nonce), (sizeof(nonce
)));
nonce = le64toh(nonce)((__uint64_t)(nonce)); /* Wire format is little endian. */
res = noise_remote_decrypt(&peer->p_remote, data->r_idx, nonce,
		   data->buf, payload_len);

if (__predict_false(res == EINVAL)__builtin_expect(((res == 22) != 0), 0)) {
goto error;
} else if (__predict_false(res == ECONNRESET)__builtin_expect(((res == 54) != 0), 0)) {
wg_timers_event_handshake_complete(&peer->p_timers);
} else if (__predict_false(res == ESTALE)__builtin_expect(((res == 70) != 0), 0)) {
wg_timers_event_want_initiation(&peer->p_timers);
} else if (__predict_false(res != 0)__builtin_expect(((res != 0) != 0), 0)) {
panic("unexpected response: %d", res);
}

wg_peer_set_endpoint_from_tag(peer, t);

wg_peer_counters_add(peer, 0, m->m_pkthdrM_dat.MH.MH_pkthdr.len);

m_adj(m, sizeof(struct wg_pkt_data));
m_adj(m, -NOISE_AUTHTAG_LEN16);

counters_pkt(sc->sc_if.if_counters, ifc_ipackets, ifc_ibytes,
   m->m_pkthdrM_dat.MH.MH_pkthdr.len);

/* A packet with length 0 is a keepalive packet */
if (__predict_false(m->m_pkthdr.len == 0)__builtin_expect(((m->M_dat.MH.MH_pkthdr.len == 0) != 0), 0
)) {
DPRINTF(sc, "Receiving keepalive packet from peer "do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Receiving keepalive packet from peer " "%llu\n", (sc)->sc_if
.if_xname, peer->p_id); } while (0)
    "%llu\n", peer->p_id)do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Receiving keepalive packet from peer " "%llu\n", (sc)->sc_if
.if_xname, peer->p_id); } while (0);
goto done;
}

/*
* We can let the network stack handle the intricate validation of the
* IP header, we just worry about the sizeof and the version, so we can
* read the source address in wg_aip_lookup.
*
* We also need to trim the packet, as it was likely padded before
* encryption. While we could drop it here, it will be more helpful to
* pass it to bpf_mtap and use the counters that people are expecting
* in ipv4_input and ipv6_input. We can rely on ipv4_input and
* ipv6_input to properly validate the headers.
*/
ip = mtod(m, struct ip *)((struct ip *)((m)->m_hdr.mh_data));
ip6 = mtod(m, struct ip6_hdr *)((struct ip6_hdr *)((m)->m_hdr.mh_data));

if (m->m_pkthdrM_dat.MH.MH_pkthdr.len >= sizeof(struct ip) && ip->ip_v == IPVERSION4) {
m->m_pkthdrM_dat.MH.MH_pkthdr.ph_family = AF_INET2;

len = ntohs(ip->ip_len)(__uint16_t)(__builtin_constant_p(ip->ip_len) ? (__uint16_t
)(((__uint16_t)(ip->ip_len) & 0xffU) << 8 | ((__uint16_t
)(ip->ip_len) & 0xff00U) >> 8) : __swap16md(ip->
ip_len));
if (len >= sizeof(struct ip) && len < m->m_pkthdrM_dat.MH.MH_pkthdr.len)
	m_adj(m, len - m->m_pkthdrM_dat.MH.MH_pkthdr.len);

allowed_peer = wg_aip_lookup(sc->sc_aip4, &ip->ip_src);
1641#ifdef INET61
} else if (m->m_pkthdrM_dat.MH.MH_pkthdr.len >= sizeof(struct ip6_hdr) &&
   (ip6->ip6_vfcip6_ctlun.ip6_un2_vfc & IPV6_VERSION_MASK0xf0) == IPV6_VERSION0x60) {
m->m_pkthdrM_dat.MH.MH_pkthdr.ph_family = AF_INET624;

len = ntohs(ip6->ip6_plen)(__uint16_t)(__builtin_constant_p(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen
) ? (__uint16_t)(((__uint16_t)(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen
) & 0xffU) << 8 | ((__uint16_t)(ip6->ip6_ctlun.ip6_un1
.ip6_un1_plen) & 0xff00U) >> 8) : __swap16md(ip6->
ip6_ctlun.ip6_un1.ip6_un1_plen)) + sizeof(struct ip6_hdr);
if (len < m->m_pkthdrM_dat.MH.MH_pkthdr.len)
	m_adj(m, len - m->m_pkthdrM_dat.MH.MH_pkthdr.len);

allowed_peer = wg_aip_lookup(sc->sc_aip6, &ip6->ip6_src);
1651#endif
} else {
DPRINTF(sc, "Packet is neither ipv4 nor ipv6 from "do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Packet is neither ipv4 nor ipv6 from " "peer %llu\n", (sc)->
sc_if.if_xname, peer->p_id); } while (0)
    "peer %llu\n", peer->p_id)do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Packet is neither ipv4 nor ipv6 from " "peer %llu\n", (sc)->
sc_if.if_xname, peer->p_id); } while (0);
goto error;
}

if (__predict_false(peer != allowed_peer)__builtin_expect(((peer != allowed_peer) != 0), 0)) {
DPRINTF(sc, "Packet has unallowed src IP from peer "do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Packet has unallowed src IP from peer " "%llu\n", (sc)->
sc_if.if_xname, peer->p_id); } while (0)
    "%llu\n", peer->p_id)do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Packet has unallowed src IP from peer " "%llu\n", (sc)->
sc_if.if_xname, peer->p_id); } while (0);
goto error;
}

/* tunneled packet was not offloaded */
m->m_pkthdrM_dat.MH.MH_pkthdr.csum_flags = 0;

m->m_pkthdrM_dat.MH.MH_pkthdr.ph_ifidx = sc->sc_if.if_index;
m->m_pkthdrM_dat.MH.MH_pkthdr.ph_rtableid = sc->sc_if.if_rdomainif_data.ifi_rdomain;
m->m_flagsm_hdr.mh_flags &= ~(M_MCAST0x0200 | M_BCAST0x0100);
1670#if NPF1 > 0
pf_pkt_addr_changed(m);
1672#endif /* NPF > 0 */

1674done:
t->t_mbuf = m;
1676error:
t->t_done = 1;
task_add(net_tq(sc->sc_if.if_index), &peer->p_deliver_in);
1679}

1681void
1682wg_encap_worker(void *_sc)
1683{
struct mbuf *m;
struct wg_softc *sc = _sc;
while ((m = wg_ring_dequeue(&sc->sc_encap_ring)) != NULL((void *)0))
wg_encap(sc, m);
1688}

1690void
1691wg_decap_worker(void *_sc)
1692{
struct mbuf *m;
struct wg_softc *sc = _sc;
while ((m = wg_ring_dequeue(&sc->sc_decap_ring)) != NULL((void *)0))
wg_decap(sc, m);
1697}

1699void
1700wg_deliver_out(void *_peer)
1701{
struct wg_peer		*peer = _peer;
struct wg_softc		*sc = peer->p_sc;
struct wg_endpoint	 endpoint;
struct wg_tag		*t;
struct mbuf		*m;
int			 ret;

wg_peer_get_endpoint(peer, &endpoint);

while ((m = wg_queue_dequeue(&peer->p_encap_queue, &t)) != NULL((void *)0)) {
/* t_mbuf will contain the encrypted packet */
if (t->t_mbuf == NULL((void *)0)){
	counters_inc(sc->sc_if.if_counters, ifc_oerrors);
	m_freem(m);
	continue;
}

ret = wg_send(sc, &endpoint, t->t_mbuf);

if (ret == 0) {
	wg_timers_event_any_authenticated_packet_traversal(
	    &peer->p_timers);
	wg_timers_event_any_authenticated_packet_sent(
	    &peer->p_timers);

	if (m->m_pkthdrM_dat.MH.MH_pkthdr.len != 0)
		wg_timers_event_data_sent(&peer->p_timers);
} else if (ret == EADDRNOTAVAIL49) {
	wg_peer_clear_src(peer);
	wg_peer_get_endpoint(peer, &endpoint);
}

m_freem(m);
}
1736}

1738void
1739wg_deliver_in(void *_peer)
1740{
struct wg_peer	*peer = _peer;
struct wg_softc	*sc = peer->p_sc;
struct wg_tag	*t;
struct mbuf	*m;

while ((m = wg_queue_dequeue(&peer->p_decap_queue, &t)) != NULL((void *)0)) {
1
Calling 'wg_queue_dequeue'→
/* t_mbuf will contain the decrypted packet */
if (t->t_mbuf == NULL((void *)0)) {
	counters_inc(sc->sc_if.if_counters, ifc_ierrors);
	m_freem(m);
	continue;
}

/* From here on m == t->t_mbuf */
KASSERT(m == t->t_mbuf)((m == t->t_mbuf) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/net/if_wg.c"
, 1755, "m == t->t_mbuf"));

wg_timers_event_any_authenticated_packet_received(
    &peer->p_timers);
wg_timers_event_any_authenticated_packet_traversal(
    &peer->p_timers);

if (m->m_pkthdrM_dat.MH.MH_pkthdr.len == 0) {
	m_freem(m);
	continue;
}

1767#if NBPFILTER1 > 0
if (sc->sc_if.if_bpf != NULL((void *)0))
	bpf_mtap_af(sc->sc_if.if_bpf,
	    m->m_pkthdrM_dat.MH.MH_pkthdr.ph_family, m, BPF_DIRECTION_IN(1 << 0));
1771#endif

NET_LOCK()do { rw_enter_write(&netlock); } while (0);
if (m->m_pkthdrM_dat.MH.MH_pkthdr.ph_family == AF_INET2)
	ipv4_input(&sc->sc_if, m);
1776#ifdef INET61
else if (m->m_pkthdrM_dat.MH.MH_pkthdr.ph_family == AF_INET624)
	ipv6_input(&sc->sc_if, m);
1779#endif
else
	panic("invalid ph_family");
NET_UNLOCK()do { rw_exit_write(&netlock); } while (0);

wg_timers_event_data_received(&peer->p_timers);
}
1786}

1788int
1789wg_queue_in(struct wg_softc *sc, struct wg_peer *peer, struct mbuf *m)
1790{
struct wg_ring		*parallel = &sc->sc_decap_ring;
struct wg_queue		*serial = &peer->p_decap_queue;
struct wg_tag		*t;

mtx_enter(&serial->q_mtx);
if (serial->q_list.ml_len < MAX_QUEUED_PKT1024) {
ml_enqueue(&serial->q_list, m);
mtx_leave(&serial->q_mtx);
} else {
mtx_leave(&serial->q_mtx);
m_freem(m);
return ENOBUFS55;
}

mtx_enter(&parallel->r_mtx);
if (parallel->r_tail - parallel->r_head < MAX_QUEUED_PKT1024) {
parallel->r_buf[parallel->r_tail & MAX_QUEUED_PKT_MASK(1024 - 1)] = m;
parallel->r_tail++;
mtx_leave(&parallel->r_mtx);
} else {
mtx_leave(&parallel->r_mtx);
t = wg_tag_get(m);
t->t_done = 1;
return ENOBUFS55;
}

return 0;
1818}

1820void
1821wg_queue_out(struct wg_softc *sc, struct wg_peer *peer)
1822{
struct wg_ring		*parallel = &sc->sc_encap_ring;
struct wg_queue		*serial = &peer->p_encap_queue;
struct mbuf_list 	 ml, ml_free;
struct mbuf		*m;
struct wg_tag		*t;
int			 dropped;

/*
* We delist all staged packets and then add them to the queues. This
* can race with wg_qstart when called from wg_send_keepalive, however
* wg_qstart will not race as it is serialised.
*/
mq_delist(&peer->p_stage_queue, &ml);
ml_init(&ml_free);

while ((m = ml_dequeue(&ml)) != NULL((void *)0)) {
mtx_enter(&serial->q_mtx);
if (serial->q_list.ml_len < MAX_QUEUED_PKT1024) {
	ml_enqueue(&serial->q_list, m);
	mtx_leave(&serial->q_mtx);
} else {
	mtx_leave(&serial->q_mtx);
	ml_enqueue(&ml_free, m);
	continue;
}

mtx_enter(&parallel->r_mtx);
if (parallel->r_tail - parallel->r_head < MAX_QUEUED_PKT1024) {
	parallel->r_buf[parallel->r_tail & MAX_QUEUED_PKT_MASK(1024 - 1)] = m;
	parallel->r_tail++;
	mtx_leave(&parallel->r_mtx);
} else {
	mtx_leave(&parallel->r_mtx);
	t = wg_tag_get(m);
	t->t_done = 1;
}
}

if ((dropped = ml_purge(&ml_free)) > 0)
counters_add(sc->sc_if.if_counters, ifc_oqdrops, dropped);
1863}

1865struct mbuf *
1866wg_ring_dequeue(struct wg_ring *r)
1867{
struct mbuf *m = NULL((void *)0);
mtx_enter(&r->r_mtx);
if (r->r_head != r->r_tail) {
m = r->r_buf[r->r_head & MAX_QUEUED_PKT_MASK(1024 - 1)];
r->r_head++;
}
mtx_leave(&r->r_mtx);
return m;
1876}

1878struct mbuf *
1879wg_queue_dequeue(struct wg_queue *q, struct wg_tag **t)
1880{
struct mbuf *m;
mtx_enter(&q->q_mtx);
if ((m = q->q_list.ml_head) != NULL((void *)0) && (*t = wg_tag_get(m))->t_done)
2
←
Assuming the condition is true→
3
←
Access to field 't_done' results in a dereference of a null pointer
ml_dequeue(&q->q_list);
else
m = NULL((void *)0);
mtx_leave(&q->q_mtx);
return m;
1889}

1891size_t
1892wg_queue_len(struct wg_queue *q)
1893{
size_t len;
mtx_enter(&q->q_mtx);
len = q->q_list.ml_len;
mtx_leave(&q->q_mtx);
return len;
1899}

1901struct noise_remote *
1902wg_remote_get(void *_sc, uint8_t public[NOISE_PUBLIC_KEY_LEN32])
1903{
struct wg_peer	*peer;
struct wg_softc	*sc = _sc;
if ((peer = wg_peer_lookup(sc, public)) == NULL((void *)0))
return NULL((void *)0);
return &peer->p_remote;
1909}

1911uint32_t
1912wg_index_set(void *_sc, struct noise_remote *remote)
1913{
struct wg_peer	*peer;
struct wg_softc	*sc = _sc;
struct wg_index *index, *iter;
uint32_t	 key;

/*
* We can modify this without a lock as wg_index_set, wg_index_drop are
* guaranteed to be serialised (per remote).
*/
peer = CONTAINER_OF(remote, struct wg_peer, p_remote)({ const __typeof( ((struct wg_peer *)0)->p_remote ) *__mptr
 = (remote); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_remote) );});
index = SLIST_FIRST(&peer->p_unused_index)((&peer->p_unused_index)->slh_first);
KASSERT(index != NULL)((index != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/net/if_wg.c"
, 1925, "index != NULL"));
SLIST_REMOVE_HEAD(&peer->p_unused_index, i_unused_entry)do { (&peer->p_unused_index)->slh_first = (&peer
->p_unused_index)->slh_first->i_unused_entry.sle_next
; } while (0);

index->i_value = remote;

mtx_enter(&sc->sc_index_mtx);
1931assign_id:
key = index->i_key = arc4random();
key &= sc->sc_index_mask;
LIST_FOREACH(iter, &sc->sc_index[key], i_entry)for((iter) = ((&sc->sc_index[key])->lh_first); (iter
)!= ((void *)0); (iter) = ((iter)->i_entry.le_next))
if (iter->i_key == index->i_key)
	goto assign_id;

LIST_INSERT_HEAD(&sc->sc_index[key], index, i_entry)do { if (((index)->i_entry.le_next = (&sc->sc_index
[key])->lh_first) != ((void *)0)) (&sc->sc_index[key
])->lh_first->i_entry.le_prev = &(index)->i_entry
.le_next; (&sc->sc_index[key])->lh_first = (index);
 (index)->i_entry.le_prev = &(&sc->sc_index[key
])->lh_first; } while (0);

mtx_leave(&sc->sc_index_mtx);

/* Likewise, no need to lock for index here. */
return index->i_key;
1944}

1946struct noise_remote *
1947wg_index_get(void *_sc, uint32_t key0)
1948{
struct wg_softc		*sc = _sc;
struct wg_index		*iter;
struct noise_remote	*remote = NULL((void *)0);
uint32_t		 key = key0 & sc->sc_index_mask;

mtx_enter(&sc->sc_index_mtx);
LIST_FOREACH(iter, &sc->sc_index[key], i_entry)for((iter) = ((&sc->sc_index[key])->lh_first); (iter
)!= ((void *)0); (iter) = ((iter)->i_entry.le_next))
if (iter->i_key == key0) {
	remote = iter->i_value;
	break;
}
mtx_leave(&sc->sc_index_mtx);
return remote;
1962}

1964void
1965wg_index_drop(void *_sc, uint32_t key0)
1966{
struct wg_softc	*sc = _sc;
struct wg_index	*iter;
struct wg_peer	*peer = NULL((void *)0);
uint32_t	 key = key0 & sc->sc_index_mask;

mtx_enter(&sc->sc_index_mtx);
LIST_FOREACH(iter, &sc->sc_index[key], i_entry)for((iter) = ((&sc->sc_index[key])->lh_first); (iter
)!= ((void *)0); (iter) = ((iter)->i_entry.le_next))
if (iter->i_key == key0) {
	LIST_REMOVE(iter, i_entry)do { if ((iter)->i_entry.le_next != ((void *)0)) (iter)->
i_entry.le_next->i_entry.le_prev = (iter)->i_entry.le_prev
; *(iter)->i_entry.le_prev = (iter)->i_entry.le_next; (
(iter)->i_entry.le_prev) = ((void *)-1); ((iter)->i_entry
.le_next) = ((void *)-1); } while (0);
	break;
}
mtx_leave(&sc->sc_index_mtx);

/* We expect a peer */
peer = CONTAINER_OF(iter->i_value, struct wg_peer, p_remote)({ const __typeof( ((struct wg_peer *)0)->p_remote ) *__mptr
 = (iter->i_value); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_remote) );});
KASSERT(peer != NULL)((peer != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/net/if_wg.c"
, 1982, "peer != NULL"));
SLIST_INSERT_HEAD(&peer->p_unused_index, iter, i_unused_entry)do { (iter)->i_unused_entry.sle_next = (&peer->p_unused_index
)->slh_first; (&peer->p_unused_index)->slh_first
 = (iter); } while (0);
1984}

1986struct mbuf *
1987wg_input(void *_sc, struct mbuf *m, struct ip *ip, struct ip6_hdr *ip6,
  void *_uh, int hlen)
1989{
struct wg_pkt_data	*data;
struct noise_remote	*remote;
struct wg_tag		*t;
struct wg_softc		*sc = _sc;
struct udphdr		*uh = _uh;

NET_ASSERT_LOCKED()do { int _s = rw_status(&netlock); if ((splassert_ctl >
 0) && (_s != 0x0001UL && _s != 0x0002UL)) splassert_fail
(0x0002UL, _s, __func__); } while (0);

if ((t = wg_tag_get(m)) == NULL((void *)0)) {
m_freem(m);
return NULL((void *)0);
}

if (ip != NULL((void *)0)) {
t->t_endpoint.e_remote.r_sa.sa_len = sizeof(struct sockaddr_in);
t->t_endpoint.e_remote.r_sa.sa_family = AF_INET2;
t->t_endpoint.e_remote.r_sin.sin_port = uh->uh_sport;
t->t_endpoint.e_remote.r_sin.sin_addr = ip->ip_src;
t->t_endpoint.e_local.l_in = ip->ip_dst;
2009#ifdef INET61
} else if (ip6 != NULL((void *)0)) {
t->t_endpoint.e_remote.r_sa.sa_len = sizeof(struct sockaddr_in6);
t->t_endpoint.e_remote.r_sa.sa_family = AF_INET624;
t->t_endpoint.e_remote.r_sin6.sin6_port = uh->uh_sport;
t->t_endpoint.e_remote.r_sin6.sin6_addr = ip6->ip6_src;
t->t_endpoint.e_local.l_in6l_pktinfo6.ipi6_addr = ip6->ip6_dst;
2016#endif
} else {
m_freem(m);
return NULL((void *)0);
}

/* m has a IP/IPv6 header of hlen length, we don't need it anymore. */
m_adj(m, hlen);

/*
* Ensure mbuf is contiguous over full length of packet. This is done
* os we can directly read the handshake values in wg_handshake, and so
* we can decrypt a transport packet by passing a single buffer to
* noise_remote_decrypt in wg_decap.
*/
if ((m = m_pullup(m, m->m_pkthdrM_dat.MH.MH_pkthdr.len)) == NULL((void *)0))
return NULL((void *)0);

if ((m->m_pkthdrM_dat.MH.MH_pkthdr.len == sizeof(struct wg_pkt_initiation) &&
*mtod(m, uint32_t *)((uint32_t *)((m)->m_hdr.mh_data)) == WG_PKT_INITIATION((__uint32_t)(1))) ||
   (m->m_pkthdrM_dat.MH.MH_pkthdr.len == sizeof(struct wg_pkt_response) &&
*mtod(m, uint32_t *)((uint32_t *)((m)->m_hdr.mh_data)) == WG_PKT_RESPONSE((__uint32_t)(2))) ||
   (m->m_pkthdrM_dat.MH.MH_pkthdr.len == sizeof(struct wg_pkt_cookie) &&
*mtod(m, uint32_t *)((uint32_t *)((m)->m_hdr.mh_data)) == WG_PKT_COOKIE((__uint32_t)(3)))) {

if (mq_enqueue(&sc->sc_handshake_queue, m) != 0)
	DPRINTF(sc, "Dropping handshake packet\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Dropping handshake packet\n", (sc)->sc_if.if_xname); } while
 (0);
task_add(wg_handshake_taskq, &sc->sc_handshake);

} else if (m->m_pkthdrM_dat.MH.MH_pkthdr.len >= sizeof(struct wg_pkt_data) +
   NOISE_AUTHTAG_LEN16 && *mtod(m, uint32_t *)((uint32_t *)((m)->m_hdr.mh_data)) == WG_PKT_DATA((__uint32_t)(4))) {

data = mtod(m, struct wg_pkt_data *)((struct wg_pkt_data *)((m)->m_hdr.mh_data));

if ((remote = wg_index_get(sc, data->r_idx)) != NULL((void *)0)) {
	t->t_peer = CONTAINER_OF(remote, struct wg_peer,({ const __typeof( ((struct wg_peer *)0)->p_remote ) *__mptr
 = (remote); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_remote) );})
	    p_remote)({ const __typeof( ((struct wg_peer *)0)->p_remote ) *__mptr
 = (remote); (struct wg_peer *)( (char *)__mptr - __builtin_offsetof
(struct wg_peer, p_remote) );});
	t->t_mbuf = NULL((void *)0);
	t->t_done = 0;

	if (wg_queue_in(sc, t->t_peer, m) != 0)
		counters_inc(sc->sc_if.if_counters,
		    ifc_iqdrops);
	task_add(wg_crypt_taskq, &sc->sc_decap);
} else {
	counters_inc(sc->sc_if.if_counters, ifc_ierrors);
	m_freem(m);
}
} else {
counters_inc(sc->sc_if.if_counters, ifc_ierrors);
m_freem(m);
}

return NULL((void *)0);
2070}

2072void
2073wg_qstart(struct ifqueue *ifq)
2074{
struct ifnet		*ifp = ifq->ifq_if;
struct wg_softc		*sc = ifp->if_softc;
struct wg_peer		*peer;
struct wg_tag		*t;
struct mbuf		*m;
SLIST_HEAD(,wg_peer)struct { struct wg_peer *slh_first; }	 start_list;

SLIST_INIT(&start_list){ ((&start_list)->slh_first) = ((void *)0); };

/*
* We should be OK to modify p_start_list, p_start_onlist in this
* function as there should only be one ifp->if_qstart invoked at a
* time.
*/
while ((m = ifq_dequeue(ifq)) != NULL((void *)0)) {
t = wg_tag_get(m);
peer = t->t_peer;
if (mq_push(&peer->p_stage_queue, m) != 0)
	counters_inc(ifp->if_counters, ifc_oqdrops);
if (!peer->p_start_onlist) {
	SLIST_INSERT_HEAD(&start_list, peer, p_start_list)do { (peer)->p_start_list.sle_next = (&start_list)->
slh_first; (&start_list)->slh_first = (peer); } while (
0);
	peer->p_start_onlist = 1;
}
}
SLIST_FOREACH(peer, &start_list, p_start_list)for((peer) = ((&start_list)->slh_first); (peer) != ((void
 *)0); (peer) = ((peer)->p_start_list.sle_next)) {
if (noise_remote_ready(&peer->p_remote) == 0)
	wg_queue_out(sc, peer);
else
	wg_timers_event_want_initiation(&peer->p_timers);
peer->p_start_onlist = 0;
}
task_add(wg_crypt_taskq, &sc->sc_encap);
2107}

2109int
2110wg_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
  struct rtentry *rt)
2112{
struct wg_softc	*sc = ifp->if_softc;
struct wg_peer	*peer;
struct wg_tag	*t;
int		 af, ret = EINVAL22;

NET_ASSERT_LOCKED()do { int _s = rw_status(&netlock); if ((splassert_ctl >
 0) && (_s != 0x0001UL && _s != 0x0002UL)) splassert_fail
(0x0002UL, _s, __func__); } while (0);

if ((t = wg_tag_get(m)) == NULL((void *)0)) {
ret = ENOBUFS55;
goto error;
}

m->m_pkthdrM_dat.MH.MH_pkthdr.ph_family = sa->sa_family;
if (sa->sa_family == AF_INET2) {
peer = wg_aip_lookup(sc->sc_aip4,
    &mtod(m, struct ip *)((struct ip *)((m)->m_hdr.mh_data))->ip_dst);
2129#ifdef INET61
} else if (sa->sa_family == AF_INET624) {
peer = wg_aip_lookup(sc->sc_aip6,
    &mtod(m, struct ip6_hdr *)((struct ip6_hdr *)((m)->m_hdr.mh_data))->ip6_dst);
2133#endif
} else {
ret = EAFNOSUPPORT47;
goto error;
}

2139#if NBPFILTER1 > 0
if (sc->sc_if.if_bpf)
bpf_mtap_af(sc->sc_if.if_bpf, sa->sa_family, m,
    BPF_DIRECTION_OUT(1 << 1));
2143#endif

if (peer == NULL((void *)0)) {
ret = ENETUNREACH51;
goto error;
}

af = peer->p_endpoint.e_remote.r_sa.sa_family;
if (af != AF_INET2 && af != AF_INET624) {
DPRINTF(sc, "No valid endpoint has been configured or "do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "No valid endpoint has been configured or " "discovered for peer %llu\n"
, (sc)->sc_if.if_xname, peer->p_id); } while (0)
		"discovered for peer %llu\n", peer->p_id)do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "No valid endpoint has been configured or " "discovered for peer %llu\n"
, (sc)->sc_if.if_xname, peer->p_id); } while (0);
ret = EDESTADDRREQ39;
goto error;
}

if (m->m_pkthdrM_dat.MH.MH_pkthdr.ph_loopcnt++ > M_MAXLOOP128) {
DPRINTF(sc, "Packet looped")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Packet looped", (sc)->sc_if.if_xname); } while (0);
ret = ELOOP62;
goto error;
}

/*
* As we hold a reference to peer in the mbuf, we can't handle a
* delayed packet without doing some refcnting. If a peer is removed
* while a delayed holds a reference, bad things will happen. For the
* time being, delayed packets are unsupported. This may be fixed with
* another aip_lookup in wg_qstart, or refcnting as mentioned before.
*/
if (m->m_pkthdrM_dat.MH.MH_pkthdr.pf.delay > 0) {
DPRINTF(sc, "PF Delay Unsupported")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "PF Delay Unsupported", (sc)->sc_if.if_xname); } while (0
);
ret = EOPNOTSUPP45;
goto error;
}

t->t_peer = peer;
t->t_mbuf = NULL((void *)0);
t->t_done = 0;
t->t_mtu = ifp->if_mtuif_data.ifi_mtu;

/*
* We still have an issue with ifq that will count a packet that gets
* dropped in wg_qstart, or not encrypted. These get counted as
* ofails or oqdrops, so the packet gets counted twice.
*/
return if_enqueue(ifp, m);
2188error:
counters_inc(ifp->if_counters, ifc_oerrors);
m_freem(m);
return ret;
2192}

2194int
2195wg_ioctl_set(struct wg_softc *sc, struct wg_data_io *data)
2196{
struct wg_interface_io	*iface_p, iface_o;
struct wg_peer_io	*peer_p, peer_o;
struct wg_aip_io	*aip_p, aip_o;

struct wg_peer		*peer, *tpeer;
struct wg_aip		*aip, *taip;

in_port_t		 port;
int			 rtable;

uint8_t			 public[WG_KEY_SIZE32], private[WG_KEY_SIZE32];
size_t			 i, j;
int			 ret, has_identity;

if ((ret = suser(curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc)) != 0)
return ret;

rw_enter_write(&sc->sc_lock);

iface_p = data->wgd_interface;
if ((ret = copyin(iface_p, &iface_o, sizeof(iface_o))) != 0)
goto error;

if (iface_o.i_flags & WG_INTERFACE_REPLACE_PEERS(1 << 4))
TAILQ_FOREACH_SAFE(peer, &sc->sc_peer_seq, p_seq_entry, tpeer)for ((peer) = ((&sc->sc_peer_seq)->tqh_first); (peer
) != ((void *)0) && ((tpeer) = ((peer)->p_seq_entry
.tqe_next), 1); (peer) = (tpeer))
	wg_peer_destroy(peer);

if (iface_o.i_flags & WG_INTERFACE_HAS_PRIVATE(1 << 1) &&
   (noise_local_keys(&sc->sc_local, NULL((void *)0), private) ||
    timingsafe_bcmp(private, iface_o.i_private, WG_KEY_SIZE32))) {
if (curve25519_generate_public(public, iface_o.i_private)) {
	if ((peer = wg_peer_lookup(sc, public)) != NULL((void *)0))
		wg_peer_destroy(peer);
}
noise_local_lock_identity(&sc->sc_local);
has_identity = noise_local_set_private(&sc->sc_local,
				       iface_o.i_private);
TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry)for((peer) = ((&sc->sc_peer_seq)->tqh_first); (peer
) != ((void *)0); (peer) = ((peer)->p_seq_entry.tqe_next)) {
	noise_remote_precompute(&peer->p_remote);
	wg_timers_event_reset_handshake_last_sent(&peer->p_timers);
	noise_remote_expire_current(&peer->p_remote);
}
cookie_checker_update(&sc->sc_cookie,
		      has_identity == 0 ? public : NULL((void *)0));
noise_local_unlock_identity(&sc->sc_local);
}

if (iface_o.i_flags & WG_INTERFACE_HAS_PORT(1 << 2))
port = htons(iface_o.i_port)(__uint16_t)(__builtin_constant_p(iface_o.i_port) ? (__uint16_t
)(((__uint16_t)(iface_o.i_port) & 0xffU) << 8 | ((__uint16_t
)(iface_o.i_port) & 0xff00U) >> 8) : __swap16md(iface_o
.i_port));
else
port = sc->sc_udp_port;

if (iface_o.i_flags & WG_INTERFACE_HAS_RTABLE(1 << 3))
rtable = iface_o.i_rtable;
else
rtable = sc->sc_udp_rtable;

if (port != sc->sc_udp_port || rtable != sc->sc_udp_rtable) {
TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry)for((peer) = ((&sc->sc_peer_seq)->tqh_first); (peer
) != ((void *)0); (peer) = ((peer)->p_seq_entry.tqe_next))
	wg_peer_clear_src(peer);

if (sc->sc_if.if_flags & IFF_RUNNING0x40)
	if ((ret = wg_bind(sc, &port, &rtable)) != 0)
		goto error;

sc->sc_udp_port = port;
sc->sc_udp_rtable = rtable;
}

peer_p = &iface_p->i_peers[0];
for (i = 0; i < iface_o.i_peers_count; i++) {
if ((ret = copyin(peer_p, &peer_o, sizeof(peer_o))) != 0)
	goto error;

/* Peer must have public key */
if (!(peer_o.p_flags & WG_PEER_HAS_PUBLIC(1 << 0)))
	goto next_peer;

/* 0 = latest protocol, 1 = this protocol */
if (peer_o.p_protocol_version != 0) {
	if (peer_o.p_protocol_version > 1) {
		ret = EPFNOSUPPORT46;
		goto error;
	}
}

/* Get local public and check that peer key doesn't match */
if (noise_local_keys(&sc->sc_local, public, NULL((void *)0)) == 0 &&
    bcmp(public, peer_o.p_public, WG_KEY_SIZE32) == 0)
	goto next_peer;

/* Lookup peer, or create if it doesn't exist */
if ((peer = wg_peer_lookup(sc, peer_o.p_public)) == NULL((void *)0)) {
	/* If we want to delete, no need creating a new one.
	 * Also, don't create a new one if we only want to
	 * update. */
	if (peer_o.p_flags & (WG_PEER_REMOVE(1 << 5)|WG_PEER_UPDATE(1 << 6)))
		goto next_peer;

	if ((peer = wg_peer_create(sc,
	    peer_o.p_public)) == NULL((void *)0)) {
		ret = ENOMEM12;
		goto error;
	}
}

/* Remove peer and continue if specified */
if (peer_o.p_flags & WG_PEER_REMOVE(1 << 5)) {
	wg_peer_destroy(peer);
	goto next_peer;
}

if (peer_o.p_flags & WG_PEER_HAS_ENDPOINT(1 << 3))
	wg_peer_set_sockaddr(peer, &peer_o.p_sap_endpoint.sa_sa);

if (peer_o.p_flags & WG_PEER_HAS_PSK(1 << 1))
	noise_remote_set_psk(&peer->p_remote, peer_o.p_psk);

if (peer_o.p_flags & WG_PEER_HAS_PKA(1 << 2))
	wg_timers_set_persistent_keepalive(&peer->p_timers,
	    peer_o.p_pka);

if (peer_o.p_flags & WG_PEER_REPLACE_AIPS(1 << 4)) {
	LIST_FOREACH_SAFE(aip, &peer->p_aip, a_entry, taip)for ((aip) = ((&peer->p_aip)->lh_first); (aip) &&
 ((taip) = ((aip)->a_entry.le_next), 1); (aip) = (taip)) {
		wg_aip_remove(sc, peer, &aip->a_data);
	}
}

aip_p = &peer_p->p_aips[0];
for (j = 0; j < peer_o.p_aips_count; j++) {
	if ((ret = copyin(aip_p, &aip_o, sizeof(aip_o))) != 0)
		goto error;
	ret = wg_aip_add(sc, peer, &aip_o);
	if (ret != 0)
		goto error;
	aip_p++;
}

peer_p = (struct wg_peer_io *)aip_p;
continue;
2337next_peer:
aip_p = &peer_p->p_aips[0];
aip_p += peer_o.p_aips_count;
peer_p = (struct wg_peer_io *)aip_p;
}

2343error:
rw_exit_write(&sc->sc_lock);
explicit_bzero(&iface_o, sizeof(iface_o));
explicit_bzero(&peer_o, sizeof(peer_o));
explicit_bzero(&aip_o, sizeof(aip_o));
explicit_bzero(public, sizeof(public));
explicit_bzero(private, sizeof(private));
return ret;
2351}

2353int
2354wg_ioctl_get(struct wg_softc *sc, struct wg_data_io *data)
2355{
struct wg_interface_io	*iface_p, iface_o;
struct wg_peer_io	*peer_p, peer_o;
struct wg_aip_io	*aip_p;

struct wg_peer		*peer;
struct wg_aip		*aip;

size_t			 size, peer_count, aip_count;
int			 ret = 0, is_suser = suser(curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc) == 0;

size = sizeof(struct wg_interface_io);
if (data->wgd_size < size && !is_suser)
goto ret_size;

iface_p = data->wgd_interface;
bzero(&iface_o, sizeof(iface_o))__builtin_bzero((&iface_o), (sizeof(iface_o)));

rw_enter_read(&sc->sc_lock);

if (sc->sc_udp_port != 0) {
iface_o.i_port = ntohs(sc->sc_udp_port)(__uint16_t)(__builtin_constant_p(sc->sc_udp_port) ? (__uint16_t
)(((__uint16_t)(sc->sc_udp_port) & 0xffU) << 8 |
 ((__uint16_t)(sc->sc_udp_port) & 0xff00U) >> 8)
 : __swap16md(sc->sc_udp_port));
iface_o.i_flags |= WG_INTERFACE_HAS_PORT(1 << 2);
}

if (sc->sc_udp_rtable != 0) {
iface_o.i_rtable = sc->sc_udp_rtable;
iface_o.i_flags |= WG_INTERFACE_HAS_RTABLE(1 << 3);
}

if (!is_suser)
goto copy_out_iface;

if (noise_local_keys(&sc->sc_local, iface_o.i_public,
   iface_o.i_private) == 0) {
iface_o.i_flags |= WG_INTERFACE_HAS_PUBLIC(1 << 0);
iface_o.i_flags |= WG_INTERFACE_HAS_PRIVATE(1 << 1);
}

size += sizeof(struct wg_peer_io) * sc->sc_peer_num;
size += sizeof(struct wg_aip_io) * sc->sc_aip_num;
if (data->wgd_size < size)
goto unlock_and_ret_size;

peer_count = 0;
peer_p = &iface_p->i_peers[0];
TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry)for((peer) = ((&sc->sc_peer_seq)->tqh_first); (peer
) != ((void *)0); (peer) = ((peer)->p_seq_entry.tqe_next)) {
bzero(&peer_o, sizeof(peer_o))__builtin_bzero((&peer_o), (sizeof(peer_o)));
peer_o.p_flags = WG_PEER_HAS_PUBLIC(1 << 0);
peer_o.p_protocol_version = 1;

if (noise_remote_keys(&peer->p_remote, peer_o.p_public,
    peer_o.p_psk) == 0)
	peer_o.p_flags |= WG_PEER_HAS_PSK(1 << 1);

if (wg_timers_get_persistent_keepalive(&peer->p_timers,
    &peer_o.p_pka) == 0)
	peer_o.p_flags |= WG_PEER_HAS_PKA(1 << 2);

if (wg_peer_get_sockaddr(peer, &peer_o.p_sap_endpoint.sa_sa) == 0)
	peer_o.p_flags |= WG_PEER_HAS_ENDPOINT(1 << 3);

mtx_enter(&peer->p_counters_mtx);
peer_o.p_txbytes = peer->p_counters_tx;
peer_o.p_rxbytes = peer->p_counters_rx;
mtx_leave(&peer->p_counters_mtx);

wg_timers_get_last_handshake(&peer->p_timers,
    &peer_o.p_last_handshake);

aip_count = 0;
aip_p = &peer_p->p_aips[0];
LIST_FOREACH(aip, &peer->p_aip, a_entry)for((aip) = ((&peer->p_aip)->lh_first); (aip)!= ((void
 *)0); (aip) = ((aip)->a_entry.le_next)) {
	if ((ret = copyout(&aip->a_data, aip_p, sizeof(*aip_p))) != 0)
		goto unlock_and_ret_size;
	aip_p++;
	aip_count++;
}
peer_o.p_aips_count = aip_count;

if ((ret = copyout(&peer_o, peer_p, sizeof(peer_o))) != 0)
	goto unlock_and_ret_size;

peer_p = (struct wg_peer_io *)aip_p;
peer_count++;
}
iface_o.i_peers_count = peer_count;

2443copy_out_iface:
ret = copyout(&iface_o, iface_p, sizeof(iface_o));
2445unlock_and_ret_size:
rw_exit_read(&sc->sc_lock);
explicit_bzero(&iface_o, sizeof(iface_o));
explicit_bzero(&peer_o, sizeof(peer_o));
2449ret_size:
data->wgd_size = size;
return ret;
2452}

2454int
2455wg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
2456{
struct ifreq	*ifr = (struct ifreq *) data;
struct wg_softc	*sc = ifp->if_softc;
int		 ret = 0;

switch (cmd) {
case SIOCSWG(((unsigned long)0x80000000|(unsigned long)0x40000000) | ((sizeof
(struct wg_data_io) & 0x1fff) << 16) | ((('i')) <<
 8) | ((210))):
NET_UNLOCK()do { rw_exit_write(&netlock); } while (0);
ret = wg_ioctl_set(sc, (struct wg_data_io *) data);
NET_LOCK()do { rw_enter_write(&netlock); } while (0);
break;
case SIOCGWG(((unsigned long)0x80000000|(unsigned long)0x40000000) | ((sizeof
(struct wg_data_io) & 0x1fff) << 16) | ((('i')) <<
 8) | ((211))):
NET_UNLOCK()do { rw_exit_write(&netlock); } while (0);
ret = wg_ioctl_get(sc, (struct wg_data_io *) data);
NET_LOCK()do { rw_enter_write(&netlock); } while (0);
break;
/* Interface IOCTLs */
case SIOCSIFADDR((unsigned long)0x80000000 | ((sizeof(struct ifreq) & 0x1fff
) << 16) | ((('i')) << 8) | ((12))):
SET(ifp->if_flags, IFF_UP)((ifp->if_flags) |= (0x1));
/* FALLTHROUGH */
case SIOCSIFFLAGS((unsigned long)0x80000000 | ((sizeof(struct ifreq) & 0x1fff
) << 16) | ((('i')) << 8) | ((16))):
if (ISSET(ifp->if_flags, IFF_UP)((ifp->if_flags) & (0x1)))
	ret = wg_up(sc);
else
	wg_down(sc);
break;
case SIOCSIFMTU((unsigned long)0x80000000 | ((sizeof(struct ifreq) & 0x1fff
) << 16) | ((('i')) << 8) | ((127))):
/* Arbitrary limits */
if (ifr->ifr_mtuifr_ifru.ifru_metric <= 0 || ifr->ifr_mtuifr_ifru.ifru_metric > 9000)
	ret = EINVAL22;
else
	ifp->if_mtuif_data.ifi_mtu = ifr->ifr_mtuifr_ifru.ifru_metric;
break;
case SIOCADDMULTI((unsigned long)0x80000000 | ((sizeof(struct ifreq) & 0x1fff
) << 16) | ((('i')) << 8) | ((49))):
case SIOCDELMULTI((unsigned long)0x80000000 | ((sizeof(struct ifreq) & 0x1fff
) << 16) | ((('i')) << 8) | ((50))):
break;
default:
ret = ENOTTY25;
}

return ret;
2497}

2499int
2500wg_up(struct wg_softc *sc)
2501{
struct wg_peer	*peer;
int		 ret = 0;

NET_ASSERT_LOCKED()do { int _s = rw_status(&netlock); if ((splassert_ctl >
 0) && (_s != 0x0001UL && _s != 0x0002UL)) splassert_fail
(0x0002UL, _s, __func__); } while (0);
/*
* We use IFF_RUNNING as an exclusive access here. We also may want
* an exclusive sc_lock as wg_bind may write to sc_udp_port. We also
* want to drop NET_LOCK as we want to call socreate, sobind, etc. Once
* solock is no longer === NET_LOCK, we may be able to avoid this.
*/
if (!ISSET(sc->sc_if.if_flags, IFF_RUNNING)((sc->sc_if.if_flags) & (0x40))) {
SET(sc->sc_if.if_flags, IFF_RUNNING)((sc->sc_if.if_flags) |= (0x40));
NET_UNLOCK()do { rw_exit_write(&netlock); } while (0);

rw_enter_write(&sc->sc_lock);
/*
 * If we successfully bind the socket, then enable the timers
 * for the peer. This will send all staged packets and a
 * keepalive if necessary.
 */
ret = wg_bind(sc, &sc->sc_udp_port, &sc->sc_udp_rtable);
if (ret == 0) {
	TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry)for((peer) = ((&sc->sc_peer_seq)->tqh_first); (peer
) != ((void *)0); (peer) = ((peer)->p_seq_entry.tqe_next)) {
		wg_timers_enable(&peer->p_timers);
		wg_queue_out(sc, peer);
	}
}
rw_exit_write(&sc->sc_lock);

NET_LOCK()do { rw_enter_write(&netlock); } while (0);
if (ret != 0)
	CLR(sc->sc_if.if_flags, IFF_RUNNING)((sc->sc_if.if_flags) &= ~(0x40));
}
return ret;
2536}

2538void
2539wg_down(struct wg_softc *sc)
2540{
struct wg_peer	*peer;

NET_ASSERT_LOCKED()do { int _s = rw_status(&netlock); if ((splassert_ctl >
 0) && (_s != 0x0001UL && _s != 0x0002UL)) splassert_fail
(0x0002UL, _s, __func__); } while (0);
if (!ISSET(sc->sc_if.if_flags, IFF_RUNNING)((sc->sc_if.if_flags) & (0x40)))
return;
CLR(sc->sc_if.if_flags, IFF_RUNNING)((sc->sc_if.if_flags) &= ~(0x40));
NET_UNLOCK()do { rw_exit_write(&netlock); } while (0);

/*
* We only need a read lock here, as we aren't writing to anything
* that isn't granularly locked.
*/
rw_enter_read(&sc->sc_lock);
TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry)for((peer) = ((&sc->sc_peer_seq)->tqh_first); (peer
) != ((void *)0); (peer) = ((peer)->p_seq_entry.tqe_next)) {
mq_purge(&peer->p_stage_queue);
wg_timers_disable(&peer->p_timers);
}

taskq_barrier(wg_handshake_taskq);
TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry)for((peer) = ((&sc->sc_peer_seq)->tqh_first); (peer
) != ((void *)0); (peer) = ((peer)->p_seq_entry.tqe_next)) {
noise_remote_clear(&peer->p_remote);
wg_timers_event_reset_handshake_last_sent(&peer->p_timers);
}

wg_unbind(sc);
rw_exit_read(&sc->sc_lock);
NET_LOCK()do { rw_enter_write(&netlock); } while (0);
2568}

2570int
2571wg_clone_create(struct if_clone *ifc, int unit)
2572{
struct ifnet		*ifp;
struct wg_softc		*sc;
struct noise_upcall	 local_upcall;

KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/net/if_wg.c"
, 2577, "_kernel_lock_held()"));

if (wg_counter == 0) {
wg_handshake_taskq = taskq_create("wg_handshake",
    2, IPL_NET0x7, TASKQ_MPSAFE(1 << 0));
wg_crypt_taskq = taskq_create("wg_crypt",
    ncpus, IPL_NET0x7, TASKQ_MPSAFE(1 << 0));

if (wg_handshake_taskq == NULL((void *)0) || wg_crypt_taskq == NULL((void *)0)) {
	if (wg_handshake_taskq != NULL((void *)0))
		taskq_destroy(wg_handshake_taskq);
	if (wg_crypt_taskq != NULL((void *)0))
		taskq_destroy(wg_crypt_taskq);
	wg_handshake_taskq = NULL((void *)0);
	wg_crypt_taskq = NULL((void *)0);
	return ENOTRECOVERABLE93;
}
}
wg_counter++;

if ((sc = malloc(sizeof(*sc), M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008)) == NULL((void *)0))
goto ret_00;

local_upcall.u_arg = sc;
local_upcall.u_remote_get = wg_remote_get;
local_upcall.u_index_set = wg_index_set;
local_upcall.u_index_drop = wg_index_drop;

TAILQ_INIT(&sc->sc_peer_seq)do { (&sc->sc_peer_seq)->tqh_first = ((void *)0); (
&sc->sc_peer_seq)->tqh_last = &(&sc->sc_peer_seq
)->tqh_first; } while (0);

/* sc_if is initialised after everything else */
arc4random_buf(&sc->sc_secret, sizeof(sc->sc_secret));

rw_init(&sc->sc_lock, "wg")_rw_init_flags(&sc->sc_lock, "wg", 0, ((void *)0));
noise_local_init(&sc->sc_local, &local_upcall);
if (cookie_checker_init(&sc->sc_cookie, &wg_ratelimit_pool) != 0)
goto ret_01;
sc->sc_udp_port = 0;
sc->sc_udp_rtable = 0;

rw_init(&sc->sc_so_lock, "wg_so")_rw_init_flags(&sc->sc_so_lock, "wg_so", 0, ((void *)0
));
sc->sc_so4 = NULL((void *)0);
2619#ifdef INET61
sc->sc_so6 = NULL((void *)0);
2621#endif

sc->sc_aip_num = 0;
if ((sc->sc_aip4 = art_alloc(0, 32, 0)) == NULL((void *)0))
goto ret_02;
2626#ifdef INET61
if ((sc->sc_aip6 = art_alloc(0, 128, 0)) == NULL((void *)0))
goto ret_03;
2629#endif

rw_init(&sc->sc_peer_lock, "wg_peer")_rw_init_flags(&sc->sc_peer_lock, "wg_peer", 0, ((void
 *)0));
sc->sc_peer_num = 0;
if ((sc->sc_peer = hashinit(HASHTABLE_PEER_SIZE(1 << 11), M_DEVBUF2,
   M_NOWAIT0x0002, &sc->sc_peer_mask)) == NULL((void *)0))
goto ret_04;

mtx_init(&sc->sc_index_mtx, IPL_NET)do { (void)(((void *)0)); (void)(0); __mtx_init((&sc->
sc_index_mtx), ((((0x7)) > 0x0 && ((0x7)) < 0x9
) ? 0x9 : ((0x7)))); } while (0);
if ((sc->sc_index = hashinit(HASHTABLE_INDEX_SIZE(1 << 13), M_DEVBUF2,
   M_NOWAIT0x0002, &sc->sc_index_mask)) == NULL((void *)0))
goto ret_05;

task_set(&sc->sc_handshake, wg_handshake_worker, sc);
mq_init(&sc->sc_handshake_queue, MAX_QUEUED_HANDSHAKES4096, IPL_NET0x7);

task_set(&sc->sc_encap, wg_encap_worker, sc);
task_set(&sc->sc_decap, wg_decap_worker, sc);

bzero(&sc->sc_encap_ring, sizeof(sc->sc_encap_ring))__builtin_bzero((&sc->sc_encap_ring), (sizeof(sc->sc_encap_ring
)));
mtx_init(&sc->sc_encap_ring.r_mtx, IPL_NET)do { (void)(((void *)0)); (void)(0); __mtx_init((&sc->
sc_encap_ring.r_mtx), ((((0x7)) > 0x0 && ((0x7)) <
 0x9) ? 0x9 : ((0x7)))); } while (0);
bzero(&sc->sc_decap_ring, sizeof(sc->sc_decap_ring))__builtin_bzero((&sc->sc_decap_ring), (sizeof(sc->sc_decap_ring
)));
mtx_init(&sc->sc_decap_ring.r_mtx, IPL_NET)do { (void)(((void *)0)); (void)(0); __mtx_init((&sc->
sc_decap_ring.r_mtx), ((((0x7)) > 0x0 && ((0x7)) <
 0x9) ? 0x9 : ((0x7)))); } while (0);

/* We've setup the softc, now we can setup the ifnet */
ifp = &sc->sc_if;
ifp->if_softc = sc;

snprintf(ifp->if_xname, sizeof(ifp->if_xname), "wg%d", unit);

ifp->if_mtuif_data.ifi_mtu = DEFAULT_MTU1420;
ifp->if_flags = IFF_BROADCAST0x2 | IFF_MULTICAST0x8000 | IFF_NOARP0x80;
ifp->if_xflags = IFXF_CLONED0x2 | IFXF_MPSAFE0x1;
ifp->if_txmit = 64; /* Keep our workers active for longer. */

ifp->if_ioctl = wg_ioctl;
ifp->if_qstart = wg_qstart;
ifp->if_output = wg_output;

ifp->if_typeif_data.ifi_type = IFT_WIREGUARD0xfb;
ifp->if_rtrequest = p2p_rtrequest;

if_attach(ifp);
if_alloc_sadl(ifp);
if_counters_alloc(ifp);

2675#if NBPFILTER1 > 0
bpfattach(&ifp->if_bpf, ifp, DLT_LOOP12, sizeof(uint32_t));
2677#endif

DPRINTF(sc, "Interface created\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Interface created\n", (sc)->sc_if.if_xname); } while (0);

return 0;
2682ret_05:
hashfree(sc->sc_peer, HASHTABLE_PEER_SIZE(1 << 11), M_DEVBUF2);
2684ret_04:
2685#ifdef INET61
free(sc->sc_aip6, M_RTABLE5, sizeof(*sc->sc_aip6));
2687ret_03:
2688#endif
free(sc->sc_aip4, M_RTABLE5, sizeof(*sc->sc_aip4));
2690ret_02:
cookie_checker_deinit(&sc->sc_cookie);
2692ret_01:
free(sc, M_DEVBUF2, sizeof(*sc));
2694ret_00:
return ENOBUFS55;
2696}
2697int
2698wg_clone_destroy(struct ifnet *ifp)
2699{
struct wg_softc	*sc = ifp->if_softc;
struct wg_peer	*peer, *tpeer;

KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/net/if_wg.c"
, 2703, "_kernel_lock_held()"));

rw_enter_write(&sc->sc_lock);
TAILQ_FOREACH_SAFE(peer, &sc->sc_peer_seq, p_seq_entry, tpeer)for ((peer) = ((&sc->sc_peer_seq)->tqh_first); (peer
) != ((void *)0) && ((tpeer) = ((peer)->p_seq_entry
.tqe_next), 1); (peer) = (tpeer))
wg_peer_destroy(peer);
rw_exit_write(&sc->sc_lock);

wg_unbind(sc);
if_detach(ifp);

wg_counter--;
if (wg_counter == 0) {
KASSERT(wg_handshake_taskq != NULL && wg_crypt_taskq != NULL)((wg_handshake_taskq != ((void *)0) && wg_crypt_taskq
 != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/net/if_wg.c"
, 2715, "wg_handshake_taskq != NULL && wg_crypt_taskq != NULL"
));
taskq_destroy(wg_handshake_taskq);
taskq_destroy(wg_crypt_taskq);
wg_handshake_taskq = NULL((void *)0);
wg_crypt_taskq = NULL((void *)0);
}

DPRINTF(sc, "Destroyed interface\n")do { if ((((sc)->sc_if.if_flags) & (0x4))) printf("%s: "
 "Destroyed interface\n", (sc)->sc_if.if_xname); } while (
0);

hashfree(sc->sc_index, HASHTABLE_INDEX_SIZE(1 << 13), M_DEVBUF2);
hashfree(sc->sc_peer, HASHTABLE_PEER_SIZE(1 << 11), M_DEVBUF2);
2726#ifdef INET61
free(sc->sc_aip6, M_RTABLE5, sizeof(*sc->sc_aip6));
2728#endif
free(sc->sc_aip4, M_RTABLE5, sizeof(*sc->sc_aip4));
cookie_checker_deinit(&sc->sc_cookie);
free(sc, M_DEVBUF2, sizeof(*sc));
return 0;
2733}

2735void
2736wgattach(int nwg)
2737{
2738#ifdef WGTEST
cookie_test();
noise_test();
2741#endif
if_clone_attach(&wg_cloner);

pool_init(&wg_aip_pool, sizeof(struct wg_aip), 0,
	IPL_NET0x7, 0, "wgaip", NULL((void *)0));
pool_init(&wg_peer_pool, sizeof(struct wg_peer), 0,
	IPL_NET0x7, 0, "wgpeer", NULL((void *)0));
pool_init(&wg_ratelimit_pool, sizeof(struct ratelimit_entry), 0,
	IPL_NET0x7, 0, "wgratelimit", NULL((void *)0));
2750}