/usr/src/sys/dev/pci/if

Bug Summary

File:	dev/pci/if_ix.c
Warning:	line 1450, column 25 Dereference of null pointer
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name if_ix.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/pci/if_ix.c
1/*	$OpenBSD: if_ix.c,v 1.180 2021/07/27 01:44:55 kevlo Exp $	*/

3/******************************************************************************

Copyright (c) 2001-2013, Intel Corporation
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

 1. Redistributions of source code must retain the above copyright notice,
    this list of conditions and the following disclaimer.

 2. Redistributions in binary form must reproduce the above copyright
    notice, this list of conditions and the following disclaimer in the
    documentation and/or other materials provided with the distribution.

 3. Neither the name of the Intel Corporation nor the names of its
    contributors may be used to endorse or promote products derived from
    this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

34******************************************************************************/
35/* FreeBSD: src/sys/dev/ixgbe/ixgbe.c 251964 Jun 18 21:28:19 2013 UTC */

37#include <dev/pci/if_ix.h>
38#include <dev/pci/ixgbe_type.h>

40/*********************************************************************
*  Driver version
*********************************************************************/
43/* char ixgbe_driver_version[] = "2.5.13"; */

45/*********************************************************************
*  PCI Device ID Table
*
*  Used by probe to select devices to load on
*********************************************************************/

51const struct pci_matchid ixgbe_devices[] = {
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_825980x10b6 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598_BX0x1508 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598AF_DUAL0x10c6 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598AF0x10c7 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598AT0x10c8 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598AT20x150b },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598AT_DUAL0x10d7 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598EB_CX40x10dd },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL0x10ec },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598EB_XF_LR0x10f4 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598EB_SFP0x10db },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM0x10e1 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82598_DA_DUAL0x10f1 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_KX40x10f7 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_KX4_MEZZ0x1514 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_XAUI0x10fc },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_COMBO_BP0x10f8 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_BPLANE_FCOE0x152a },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_CX40x10f9 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_T3_LOM0x151c },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_SFP0x10fb },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_SFP_EM0x1507 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_SFP_SF_QP0x154a },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_SFP_SF20x154d },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_SFP_FCOE0x1529 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599EN_SFP0x1557 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_82599_QSFP_SF_QP0x1558 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X540T0x1528 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X540T10x1560 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550T0x1563 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550T10x15d1 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_X_KX40x15aa },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_X_KR0x15ab },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_X_SFP0x15ac },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_X_10G_T0x15ad },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_X_1G_T0x15ae },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_A_KR0x15c2 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_A_KR_L0x15c3 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_A_SFP_N0x15c4 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_A_SFP0x15ce },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_A_SGMII0x15c6 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_A_SGMII_L0x15c7 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_A_10G_T0x15c8 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_A_1G_T0x15e4 },
{ PCI_VENDOR_INTEL0x8086, PCI_PRODUCT_INTEL_X550EM_A_1G_T_L0x15e5 }
97};

99/*********************************************************************
*  Function prototypes
*********************************************************************/
102int	ixgbe_probe(struct device *, void *, void *);
103void	ixgbe_attach(struct device *, struct device *, void *);
104int	ixgbe_detach(struct device *, int);
105void	ixgbe_start(struct ifqueue *);
106int	ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
107int	ixgbe_rxrinfo(struct ix_softc *, struct if_rxrinfo *);
108int	ixgbe_get_sffpage(struct ix_softc *, struct if_sffpage *);
109void	ixgbe_watchdog(struct ifnet *);
110void	ixgbe_init(void *);
111void	ixgbe_stop(void *);
112void	ixgbe_media_status(struct ifnet *, struct ifmediareq *);
113int	ixgbe_media_change(struct ifnet *);
114void	ixgbe_identify_hardware(struct ix_softc *);
115int	ixgbe_allocate_pci_resources(struct ix_softc *);
116int	ixgbe_allocate_legacy(struct ix_softc *);
117int	ixgbe_allocate_msix(struct ix_softc *);
118void	ixgbe_setup_msix(struct ix_softc *);
119int	ixgbe_allocate_queues(struct ix_softc *);
120void	ixgbe_free_pci_resources(struct ix_softc *);
121void	ixgbe_local_timer(void *);
122void	ixgbe_setup_interface(struct ix_softc *);
123void	ixgbe_config_gpie(struct ix_softc *);
124void	ixgbe_config_delay_values(struct ix_softc *);
125void	ixgbe_add_media_types(struct ix_softc *);
126void	ixgbe_config_link(struct ix_softc *);

128int	ixgbe_allocate_transmit_buffers(struct tx_ring *);
129int	ixgbe_setup_transmit_structures(struct ix_softc *);
130int	ixgbe_setup_transmit_ring(struct tx_ring *);
131void	ixgbe_initialize_transmit_units(struct ix_softc *);
132void	ixgbe_free_transmit_structures(struct ix_softc *);
133void	ixgbe_free_transmit_buffers(struct tx_ring *);

135int	ixgbe_allocate_receive_buffers(struct rx_ring *);
136int	ixgbe_setup_receive_structures(struct ix_softc *);
137int	ixgbe_setup_receive_ring(struct rx_ring *);
138void	ixgbe_initialize_receive_units(struct ix_softc *);
139void	ixgbe_free_receive_structures(struct ix_softc *);
140void	ixgbe_free_receive_buffers(struct rx_ring *);
141void	ixgbe_initialize_rss_mapping(struct ix_softc *);
142int	ixgbe_rxfill(struct rx_ring *);
143void	ixgbe_rxrefill(void *);

145int	ixgbe_intr(struct ix_softc *sc);
146void	ixgbe_enable_intr(struct ix_softc *);
147void	ixgbe_disable_intr(struct ix_softc *);
148int	ixgbe_txeof(struct tx_ring *);
149int	ixgbe_rxeof(struct rx_ring *);
150void	ixgbe_rx_checksum(uint32_t, struct mbuf *, uint32_t);
151void	ixgbe_iff(struct ix_softc *);
152void	ixgbe_map_queue_statistics(struct ix_softc *);
153void	ixgbe_update_link_status(struct ix_softc *);
154int	ixgbe_get_buf(struct rx_ring *, int);
155int	ixgbe_encap(struct tx_ring *, struct mbuf *);
156int	ixgbe_dma_malloc(struct ix_softc *, bus_size_t,
    struct ixgbe_dma_alloc *, int);
158void	ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *);
159int	ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *, uint32_t *,
   uint32_t *);
161int	ixgbe_tso_setup(struct tx_ring *, struct mbuf *, uint32_t *,
   uint32_t *);
163void	ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t);
164void	ixgbe_configure_ivars(struct ix_softc *);
165uint8_t	*ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);

167void	ixgbe_setup_vlan_hw_support(struct ix_softc *);

169/* Support for pluggable optic modules */
170void	ixgbe_handle_mod(struct ix_softc *);
171void	ixgbe_handle_msf(struct ix_softc *);
172void	ixgbe_handle_phy(struct ix_softc *);

174/* Legacy (single vector interrupt handler */
175int	ixgbe_legacy_intr(void *);
176void	ixgbe_enable_queue(struct ix_softc *, uint32_t);
177void	ixgbe_enable_queues(struct ix_softc *);
178void	ixgbe_disable_queue(struct ix_softc *, uint32_t);
179void	ixgbe_rearm_queue(struct ix_softc *, uint32_t);

181/* MSI-X (multiple vectors interrupt handlers)  */
182int	ixgbe_link_intr(void *);
183int	ixgbe_queue_intr(void *);

185#if NKSTAT0 > 0
186static void	ix_kstats(struct ix_softc *);
187static void	ix_rxq_kstats(struct ix_softc *, struct rx_ring *);
188static void	ix_txq_kstats(struct ix_softc *, struct tx_ring *);
189static void	ix_kstats_tick(void *);
190#endif

192/*********************************************************************
*  OpenBSD Device Interface Entry Points
*********************************************************************/

196struct cfdriver ix_cd = {
NULL((void *)0), "ix", DV_IFNET
198};

200struct cfattach ix_ca = {
sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach
202};

204int ixgbe_smart_speed = ixgbe_smart_speed_on;
205int ixgbe_enable_msix = 1;

207/*********************************************************************
*  Device identification routine
*
*  ixgbe_probe determines if the driver should be loaded on
*  adapter based on PCI vendor/device id of the adapter.
*
*  return 0 on success, positive on failure
*********************************************************************/

216int
217ixgbe_probe(struct device *parent, void *match, void *aux)
218{
INIT_DEBUGOUT("ixgbe_probe: begin")if (0) printf("ixgbe_probe: begin" "\n");

return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices,
   nitems(ixgbe_devices)(sizeof((ixgbe_devices)) / sizeof((ixgbe_devices)[0]))));
223}

225/*********************************************************************
*  Device initialization routine
*
*  The attach entry point is called when the driver is being loaded.
*  This routine identifies the type of hardware, allocates all resources
*  and initializes the hardware.
*
*  return 0 on success, positive on failure
*********************************************************************/

235void
236ixgbe_attach(struct device *parent, struct device *self, void *aux)
237{
struct pci_attach_args	*pa = (struct pci_attach_args *)aux;
struct ix_softc		*sc = (struct ix_softc *)self;
int			 error = 0;
uint16_t		 csum;
uint32_t			 ctrl_ext;
struct ixgbe_hw		*hw = &sc->hw;

INIT_DEBUGOUT("ixgbe_attach: begin")if (0) printf("ixgbe_attach: begin" "\n");

sc->osdep.os_sc = sc;
sc->osdep.os_pa = *pa;

rw_init(&sc->sfflock, "ixsff")_rw_init_flags(&sc->sfflock, "ixsff", 0, ((void *)0));

252#if NKSTAT0 > 0
ix_kstats(sc);
254#endif

/* Determine hardware revision */
ixgbe_identify_hardware(sc);

/* Indicate to RX setup to use Jumbo Clusters */
sc->num_tx_desc = DEFAULT_TXD256;
sc->num_rx_desc = DEFAULT_RXD256;

/* Do base PCI setup - map BAR0 */
if (ixgbe_allocate_pci_resources(sc))
goto err_out;

/* Allocate our TX/RX Queues */
if (ixgbe_allocate_queues(sc))
goto err_out;

/* Allocate multicast array memory. */
sc->mta = mallocarray(IXGBE_ETH_LENGTH_OF_ADDRESS6,
   MAX_NUM_MULTICAST_ADDRESSES128, M_DEVBUF2, M_NOWAIT0x0002);
if (sc->mta == NULL((void *)0)) {
printf(": Can not allocate multicast setup array\n");
goto err_late;
}

/* Initialize the shared code */
error = ixgbe_init_shared_code(hw);
if (error) {
printf(": Unable to initialize the shared code\n");
goto err_late;
}

/* Make sure we have a good EEPROM before we read from it */
if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) {
printf(": The EEPROM Checksum Is Not Valid\n");
goto err_late;
}

error = ixgbe_init_hw(hw);
if (error == IXGBE_ERR_EEPROM_VERSION-24) {
printf(": This device is a pre-production adapter/"
    "LOM.  Please be aware there may be issues associated "
    "with your hardware.\nIf you are experiencing problems "
    "please contact your Intel or hardware representative "
    "who provided you with this hardware.\n");
} else if (error && (error != IXGBE_ERR_SFP_NOT_PRESENT-20 &&
   error != IXGBE_ERR_SFP_NOT_SUPPORTED-19)) {
printf(": Hardware Initialization Failure\n");
goto err_late;
}

bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr,
   IXGBE_ETH_LENGTH_OF_ADDRESS6);

if (sc->sc_intrmap)
error = ixgbe_allocate_msix(sc);
else
error = ixgbe_allocate_legacy(sc);
if (error)
goto err_late;

/* Enable the optics for 82599 SFP+ fiber */
if (sc->hw.mac.ops.enable_tx_laser)
sc->hw.mac.ops.enable_tx_laser(&sc->hw);

/* Enable power to the phy */
if (hw->phy.ops.set_phy_power)
hw->phy.ops.set_phy_power(&sc->hw, TRUE1);

/* Setup OS specific network interface */
ixgbe_setup_interface(sc);

/* Get the PCI-E bus info and determine LAN ID */
hw->mac.ops.get_bus_info(hw);

/* Set an initial default flow control value */
sc->fc = ixgbe_fc_full;

/* let hardware know driver is loaded */
ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00018)));
ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD0x10000000;
IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00018), (ctrl_ext)));

printf(", address %s\n", ether_sprintf(sc->hw.mac.addr));

INIT_DEBUGOUT("ixgbe_attach: end")if (0) printf("ixgbe_attach: end" "\n");
return;

342err_late:
ixgbe_free_transmit_structures(sc);
ixgbe_free_receive_structures(sc);
345err_out:
ixgbe_free_pci_resources(sc);
free(sc->mta, M_DEVBUF2, IXGBE_ETH_LENGTH_OF_ADDRESS6 *
   MAX_NUM_MULTICAST_ADDRESSES128);
349}

351/*********************************************************************
*  Device removal routine
*
*  The detach entry point is called when the driver is being removed.
*  This routine stops the adapter and deallocates all the resources
*  that were allocated for driver operation.
*
*  return 0 on success, positive on failure
*********************************************************************/

361int
362ixgbe_detach(struct device *self, int flags)
363{
struct ix_softc *sc = (struct ix_softc *)self;
struct ifnet *ifp = &sc->arpcom.ac_if;
uint32_t	ctrl_ext;

INIT_DEBUGOUT("ixgbe_detach: begin")if (0) printf("ixgbe_detach: begin" "\n");

ixgbe_stop(sc);

/* let hardware know driver is unloading */
ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00018)));
ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD0x10000000;
IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00018), (ctrl_ext)));

ether_ifdetach(ifp);
if_detach(ifp);

ixgbe_free_pci_resources(sc);

ixgbe_free_transmit_structures(sc);
ixgbe_free_receive_structures(sc);
free(sc->mta, M_DEVBUF2, IXGBE_ETH_LENGTH_OF_ADDRESS6 *
   MAX_NUM_MULTICAST_ADDRESSES128);

/* XXX kstat */

return (0);
390}

392/*********************************************************************
*  Transmit entry point
*
*  ixgbe_start is called by the stack to initiate a transmit.
*  The driver will remain in this routine as long as there are
*  packets to transmit and transmit resources are available.
*  In case resources are not available stack is notified and
*  the packet is requeued.
**********************************************************************/

402void
403ixgbe_start(struct ifqueue *ifq)
404{
struct ifnet		*ifp = ifq->ifq_if;
struct ix_softc		*sc = ifp->if_softc;
struct tx_ring		*txr = ifq->ifq_softc_ifq_ptr._ifq_softc;
struct mbuf  		*m_head;
unsigned int		 head, free, used;
int			 post = 0;

if (!sc->link_up)
1
Assuming field 'link_up' is true→
2
←
Taking false branch→
return;

head = txr->next_avail_desc;
free = txr->next_to_clean;
if (free <= head)
3
←
Assuming 'free' is > 'head'→
4
←
Taking false branch→
free += sc->num_tx_desc;
free -= head;

membar_consumer()do { __asm volatile("" ::: "memory"); } while (0);
5
←
Loop condition is false.  Exiting loop→

bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x08))
   0, txr->txdma.dma_map->dm_mapsize,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x08))
   BUS_DMASYNC_POSTWRITE)(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x08));

for (;;) {
6
←
Loop condition is true.  Entering loop body→
/* Check that we have the minimal number of TX descriptors. */
if (free <= IXGBE_TX_OP_THRESHOLD(sc->num_segs + 2)) {
7
←
Assuming the condition is false→
8
←
Taking false branch→
	ifq_set_oactive(ifq);
	break;
}

m_head = ifq_dequeue(ifq);
if (m_head == NULL((void *)0))
9
←
Assuming 'm_head' is not equal to NULL→
10
←
Taking false branch→
	break;

used = ixgbe_encap(txr, m_head);
11
←
Calling 'ixgbe_encap'→
if (used == 0) {
	m_freem(m_head);
	continue;
}

free -= used;

446#if NBPFILTER1 > 0
if (ifp->if_bpf)
	bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT(1 << 1));
449#endif

/* Set timeout in case hardware has problems transmitting */
txr->watchdog_timer = IXGBE_TX_TIMEOUT5;
ifp->if_timer = IXGBE_TX_TIMEOUT5;

post = 1;
}

bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x04))
   0, txr->txdma.dma_map->dm_mapsize,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x04))
   BUS_DMASYNC_PREWRITE)(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x04));

/*
* Advance the Transmit Descriptor Tail (Tdt), this tells the
* hardware that this frame is available to transmit.
*/
if (post)
IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me),((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x06018 + ((txr->me) * 0x40))), (txr->
next_avail_desc)))
    txr->next_avail_desc)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x06018 + ((txr->me) * 0x40))), (txr->
next_avail_desc)));
469}

471/*********************************************************************
*  Ioctl entry point
*
*  ixgbe_ioctl is called when the user wants to configure the
*  interface.
*
*  return 0 on success, positive on failure
**********************************************************************/

480int
481ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
482{
struct ix_softc	*sc = ifp->if_softc;
struct ifreq	*ifr = (struct ifreq *) data;
int		s, error = 0;

s = splnet()splraise(0x7);

switch (command) {
case SIOCSIFADDR((unsigned long)0x80000000 | ((sizeof(struct ifreq) & 0x1fff
) << 16) | ((('i')) << 8) | ((12))):
IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)")if (0) printf("ioctl: SIOCxIFADDR (Get/Set Interface Addr)" "\n"
);
ifp->if_flags |= IFF_UP0x1;
if (!(ifp->if_flags & IFF_RUNNING0x40))
	ixgbe_init(sc);
break;

case SIOCSIFFLAGS((unsigned long)0x80000000 | ((sizeof(struct ifreq) & 0x1fff
) << 16) | ((('i')) << 8) | ((16))):
IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)")if (0) printf("ioctl: SIOCSIFFLAGS (Set Interface Flags)" "\n"
);
if (ifp->if_flags & IFF_UP0x1) {
	if (ifp->if_flags & IFF_RUNNING0x40)
		error = ENETRESET52;
	else
		ixgbe_init(sc);
} else {
	if (ifp->if_flags & IFF_RUNNING0x40)
		ixgbe_stop(sc);
}
break;

case SIOCSIFMEDIA(((unsigned long)0x80000000|(unsigned long)0x40000000) | ((sizeof
(struct ifreq) & 0x1fff) << 16) | ((('i')) <<
 8) | ((55))):
case SIOCGIFMEDIA(((unsigned long)0x80000000|(unsigned long)0x40000000) | ((sizeof
(struct ifmediareq) & 0x1fff) << 16) | ((('i')) <<
 8) | ((56))):
IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)")if (0) printf("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"
 "\n");
error = ifmedia_ioctl(ifp, ifr, &sc->media, command);
break;

case SIOCGIFRXR((unsigned long)0x80000000 | ((sizeof(struct ifreq) & 0x1fff
) << 16) | ((('i')) << 8) | ((170))):
error = ixgbe_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_dataifr_ifru.ifru_data);
break;

case SIOCGIFSFFPAGE(((unsigned long)0x80000000|(unsigned long)0x40000000) | ((sizeof
(struct if_sffpage) & 0x1fff) << 16) | ((('i')) <<
 8) | ((57))):
error = rw_enter(&sc->sfflock, RW_WRITE0x0001UL|RW_INTR0x0010UL);
if (error != 0)
	break;

error = ixgbe_get_sffpage(sc, (struct if_sffpage *)data);
rw_exit(&sc->sfflock);
break;

default:
error = ether_ioctl(ifp, &sc->arpcom, command, data);
}

if (error == ENETRESET52) {
if (ifp->if_flags & IFF_RUNNING0x40) {
	ixgbe_disable_intr(sc);
	ixgbe_iff(sc);
	ixgbe_enable_intr(sc);
	ixgbe_enable_queues(sc);
}
error = 0;
}

splx(s)spllower(s);
return (error);
545}

547int
548ixgbe_get_sffpage(struct ix_softc *sc, struct if_sffpage *sff)
549{
struct ixgbe_hw *hw = &sc->hw;
uint32_t swfw_mask = hw->phy.phy_semaphore_mask;
uint8_t page;
size_t i;
int error = EIO5;

if (hw->phy.type == ixgbe_phy_fw)
return (ENODEV19);

if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
return (EBUSY16); /* XXX */

if (sff->sff_addr == IFSFF_ADDR_EEPROM0xa0) {
if (hw->phy.ops.read_i2c_byte_unlocked(hw, 127,
    IFSFF_ADDR_EEPROM0xa0, &page))
	goto error;
if (page != sff->sff_page &&
    hw->phy.ops.write_i2c_byte_unlocked(hw, 127,
    IFSFF_ADDR_EEPROM0xa0, sff->sff_page))
	goto error;
}

for (i = 0; i < sizeof(sff->sff_data); i++) {
if (hw->phy.ops.read_i2c_byte_unlocked(hw, i,
    sff->sff_addr, &sff->sff_data[i]))
	goto error;
}

if (sff->sff_addr == IFSFF_ADDR_EEPROM0xa0) {
if (page != sff->sff_page &&
    hw->phy.ops.write_i2c_byte_unlocked(hw, 127,
    IFSFF_ADDR_EEPROM0xa0, page))
	goto error;
}

error = 0;
586error:
hw->mac.ops.release_swfw_sync(hw, swfw_mask);
return (error);
589}

591int
592ixgbe_rxrinfo(struct ix_softc *sc, struct if_rxrinfo *ifri)
593{
struct if_rxring_info *ifr, ifr1;
struct rx_ring *rxr;
int error, i;
u_int n = 0;

if (sc->num_queues > 1) {
if ((ifr = mallocarray(sc->num_queues, sizeof(*ifr), M_DEVBUF2,
    M_WAITOK0x0001 | M_ZERO0x0008)) == NULL((void *)0))
	return (ENOMEM12);
} else
ifr = &ifr1;

for (i = 0; i < sc->num_queues; i++) {
rxr = &sc->rx_rings[i];
ifr[n].ifr_size = MCLBYTES(1 << 11);
snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "%d", i);
ifr[n].ifr_info = rxr->rx_ring;
n++;
}

error = if_rxr_info_ioctl(ifri, sc->num_queues, ifr);

if (sc->num_queues > 1)
free(ifr, M_DEVBUF2, sc->num_queues * sizeof(*ifr));
return (error);
619}

621/*********************************************************************
*  Watchdog entry point
*
**********************************************************************/

626void
627ixgbe_watchdog(struct ifnet * ifp)
628{
struct ix_softc *sc = (struct ix_softc *)ifp->if_softc;
struct tx_ring *txr = sc->tx_rings;
struct ixgbe_hw *hw = &sc->hw;
int		tx_hang = FALSE0;
int		i;

/*
* The timer is set to 5 every time ixgbe_start() queues a packet.
* Anytime all descriptors are clean the timer is set to 0.
*/
for (i = 0; i < sc->num_queues; i++, txr++) {
if (txr->watchdog_timer == 0 || --txr->watchdog_timer)
	continue;
else {
	tx_hang = TRUE1;
	break;
}
}
if (tx_hang == FALSE0)
return;

/*
* If we are in this routine because of pause frames, then don't
* reset the hardware.
*/
if (!(IXGBE_READ_REG(hw, IXGBE_TFCS)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x0CE00
))) & IXGBE_TFCS_TXON0x00000001)) {
for (i = 0; i < sc->num_queues; i++, txr++)
	txr->watchdog_timer = IXGBE_TX_TIMEOUT5;
ifp->if_timer = IXGBE_TX_TIMEOUT5;
return;
}


printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname);
for (i = 0; i < sc->num_queues; i++, txr++) {
printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i,
    IXGBE_READ_REG(hw, IXGBE_TDH(i))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x06010
 + ((i) * 0x40))))),
    IXGBE_READ_REG(hw, IXGBE_TDT(i))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x06018
 + ((i) * 0x40))))));
printf("%s: TX(%d) Next TX to Clean = %d\n", ifp->if_xname,
    i, txr->next_to_clean);
}
ifp->if_flags &= ~IFF_RUNNING0x40;

ixgbe_init(sc);
673}

675/*********************************************************************
*  Init entry point
*
*  This routine is used in two ways. It is used by the stack as
*  init entry point in network interface structure. It is also used
*  by the driver as a hw/sw initialization routine to get to a
*  consistent state.
*
*  return 0 on success, positive on failure
**********************************************************************/
685#define IXGBE_MHADD_MFS_SHIFT16 16

687void
688ixgbe_init(void *arg)
689{
struct ix_softc	*sc = (struct ix_softc *)arg;
struct ifnet	*ifp = &sc->arpcom.ac_if;
struct rx_ring	*rxr = sc->rx_rings;
uint32_t	 k, txdctl, rxdctl, rxctrl, mhadd, itr;
int		 i, s, err;

INIT_DEBUGOUT("ixgbe_init: begin")if (0) printf("ixgbe_init: begin" "\n");

s = splnet()splraise(0x7);

ixgbe_stop(sc);

/* reprogram the RAR[0] in case user changed it. */
ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV0x80000000);

/* Get the latest mac address, User can use a LAA */
bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr,
     IXGBE_ETH_LENGTH_OF_ADDRESS6);
ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, 1);
sc->hw.addr_ctrl.rar_used_count = 1;

/* Prepare transmit descriptors and buffers */
if (ixgbe_setup_transmit_structures(sc)) {
printf("%s: Could not setup transmit structures\n",
    ifp->if_xname);
ixgbe_stop(sc);
splx(s)spllower(s);
return;
}

ixgbe_init_hw(&sc->hw);
ixgbe_initialize_transmit_units(sc);

/* Use 2k clusters, even for jumbo frames */
sc->rx_mbuf_sz = MCLBYTES(1 << 11) + ETHER_ALIGN2;

/* Prepare receive descriptors and buffers */
if (ixgbe_setup_receive_structures(sc)) {
printf("%s: Could not setup receive structures\n",
    ifp->if_xname);
ixgbe_stop(sc);
splx(s)spllower(s);
return;
}

/* Configure RX settings */
ixgbe_initialize_receive_units(sc);

/* Enable SDP & MSIX interrupts based on adapter */
ixgbe_config_gpie(sc);

/* Program promiscuous mode and multicast filters. */
ixgbe_iff(sc);

/* Set MRU size */
mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x04268)));
mhadd &= ~IXGBE_MHADD_MFS_MASK0xFFFF0000;
mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT16;
IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x04268), (mhadd)));

/* Now enable all the queues */
for (i = 0; i < sc->num_queues; i++) {
txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i))((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x06028 + ((i) * 0x40)))));
txdctl |= IXGBE_TXDCTL_ENABLE0x02000000;
/* Set WTHRESH to 8, burst writeback */
txdctl |= (8 << 16);
/*
 * When the internal queue falls below PTHRESH (16),
 * start prefetching as long as there are at least
 * HTHRESH (1) buffers ready.
 */
txdctl |= (16 << 0) | (1 << 8);
IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x06028 + ((i) * 0x40))), (txdctl)));
}

for (i = 0; i < sc->num_queues; i++) {
rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i))((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((i) < 64) ? (0x01028 + ((i) * 0x40)) : (
0x0D028 + (((i) - 64) * 0x40))))));
if (sc->hw.mac.type == ixgbe_mac_82598EB) {
	/*
	 * PTHRESH = 21
	 * HTHRESH = 4
	 * WTHRESH = 8
	 */
	rxdctl &= ~0x3FFFFF;
	rxdctl |= 0x080420;
}
rxdctl |= IXGBE_RXDCTL_ENABLE0x02000000;
IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((i) < 64) ? (0x01028 + ((i) * 0x40)) : (
0x0D028 + (((i) - 64) * 0x40)))), (rxdctl)));
for (k = 0; k < 10; k++) {
	if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i))((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((i) < 64) ? (0x01028 + ((i) * 0x40)) : (
0x0D028 + (((i) - 64) * 0x40)))))) &
	    IXGBE_RXDCTL_ENABLE0x02000000)
		break;
	else
		msec_delay(1)(*delay_func)(1000 * (1));
}
IXGBE_WRITE_FLUSH(&sc->hw)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00008)));
IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_desc_filled)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((i) < 64) ? (0x01018 + ((i) * 0x40)) : (
0x0D018 + (((i) - 64) * 0x40)))), (rxr->last_desc_filled))
);
}

/* Set up VLAN support and filter */
ixgbe_setup_vlan_hw_support(sc);

/* Enable Receive engine */
rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x03000)));
if (sc->hw.mac.type == ixgbe_mac_82598EB)
rxctrl |= IXGBE_RXCTRL_DMBYPS0x00000002;
rxctrl |= IXGBE_RXCTRL_RXEN0x00000001;
sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl);

/* Set up MSI/X routing */
if (sc->sc_intrmap) {
ixgbe_configure_ivars(sc);
/* Set up auto-mask */
if (sc->hw.mac.type == ixgbe_mac_82598EB)
	IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00890), (0x0000FFFF)));
else {
	IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x00AD0 + (0) * 4)), (0xFFFFFFFF)));
	IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x00AD0 + (1) * 4)), (0xFFFFFFFF)));
}
} else {  /* Simple settings for Legacy/MSI */
ixgbe_set_ivar(sc, 0, 0, 0);
ixgbe_set_ivar(sc, 0, 0, 1);
IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00890), (0x0000FFFF)));
}

/* Check on any SFP devices that need to be kick-started */
if (sc->hw.phy.type == ixgbe_phy_none) {
err = sc->hw.phy.ops.identify(&sc->hw);
if (err == IXGBE_ERR_SFP_NOT_SUPPORTED-19) {
	printf("Unsupported SFP+ module type was detected.\n");
	splx(s)spllower(s);
	return;
}
}

/* Setup interrupt moderation */
itr = (4000000 / IXGBE_INTS_PER_SEC8000) & 0xff8;
if (sc->hw.mac.type != ixgbe_mac_82598EB)
itr |= IXGBE_EITR_LLI_MOD0x00008000 | IXGBE_EITR_CNT_WDIS0x80000000;
IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((0) <= 23) ? (0x00820 + ((0) * 4)) : (0x012300
 + (((0) - 24) * 4)))), (itr)));

if (sc->sc_intrmap) {
/* Set moderation on the Link interrupt */
IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(sc->linkvec),((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((sc->linkvec) <= 23) ? (0x00820 + ((
sc->linkvec) * 4)) : (0x012300 + (((sc->linkvec) - 24) *
 4)))), (1000)))
    IXGBE_LINK_ITR)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((sc->linkvec) <= 23) ? (0x00820 + ((
sc->linkvec) * 4)) : (0x012300 + (((sc->linkvec) - 24) *
 4)))), (1000)));
}

/* Enable power to the phy */
if (sc->hw.phy.ops.set_phy_power)
sc->hw.phy.ops.set_phy_power(&sc->hw, TRUE1);

/* Config/Enable Link */
ixgbe_config_link(sc);

/* Hardware Packet Buffer & Flow Control setup */
ixgbe_config_delay_values(sc);

/* Initialize the FC settings */
sc->hw.mac.ops.start_hw(&sc->hw);

/* And now turn on interrupts */
ixgbe_enable_intr(sc);
ixgbe_enable_queues(sc);

/* Now inform the stack we're ready */
ifp->if_flags |= IFF_RUNNING0x40;
for (i = 0; i < sc->num_queues; i++)
ifq_clr_oactive(ifp->if_ifqs[i]);

859#if NKSTAT0 > 0
ix_kstats_tick(sc);
861#endif

splx(s)spllower(s);
864}

866void
867ixgbe_config_gpie(struct ix_softc *sc)
868{
struct ixgbe_hw	*hw = &sc->hw;
uint32_t gpie;

gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00898)));

/* Fan Failure Interrupt */
if (hw->device_id == IXGBE_DEV_ID_82598AT0x10C8)
gpie |= IXGBE_SDP1_GPIEN0x00000002;

if (sc->hw.mac.type == ixgbe_mac_82599EB) {
/* Add for Module detection */
gpie |= IXGBE_SDP2_GPIEN0x00000004;

/* Media ready */
if (hw->device_id != IXGBE_DEV_ID_82599_QSFP_SF_QP0x1558)
	gpie |= IXGBE_SDP1_GPIEN0x00000002;

/*
 * Set LL interval to max to reduce the number of low latency
 * interrupts hitting the card when the ring is getting full.
 */
gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT7;
}

if (sc->hw.mac.type == ixgbe_mac_X540 ||
   sc->hw.mac.type == ixgbe_mac_X550EM_x ||
   sc->hw.mac.type == ixgbe_mac_X550EM_a) {
/*
 * Thermal Failure Detection (X540)
 * Link Detection (X552 SFP+, X552/X557-AT)
 */
gpie |= IXGBE_SDP0_GPIEN_X5400x00000002;

/*
 * Set LL interval to max to reduce the number of low latency
 * interrupts hitting the card when the ring is getting full.
 */
gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT7;
}

if (sc->sc_intrmap) {
/* Enable Enhanced MSIX mode */
gpie |= IXGBE_GPIE_MSIX_MODE0x00000010;
gpie |= IXGBE_GPIE_EIAME0x40000000 | IXGBE_GPIE_PBA_SUPPORT0x80000000 |
    IXGBE_GPIE_OCD0x00000020;
}

IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00898), (gpie)));
917}

919/*
* Requires sc->max_frame_size to be set.
*/
922void
923ixgbe_config_delay_values(struct ix_softc *sc)
924{
struct ixgbe_hw *hw = &sc->hw;
uint32_t rxpb, frame, size, tmp;

frame = sc->max_frame_size;

/* Calculate High Water */
switch (hw->mac.type) {
case ixgbe_mac_X540:
case ixgbe_mac_X550:
case ixgbe_mac_X550EM_x:
case ixgbe_mac_X550EM_a:
tmp = IXGBE_DV_X540(frame, frame)((36 * ((frame * 8) + 672 + (2 * 5556) + (2 * (8192 + (2 * 2048
) + 25600)) + 6144) / 25 + 1) + 2 * (frame * 8));
break;
default:
tmp = IXGBE_DV(frame, frame)((36 * ((frame * 8) + 672 + (2 * 5556) + (2 * (4096 + (2 * 1024
) + 12800)) + 6144) / 25 + 1) + 2 * (frame * 8));
break;
}
size = IXGBE_BT2KB(tmp)((tmp + (8 * 1024 - 1)) / (8 * 1024));
rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x03C00
 + ((0) * 4))))) >> 10;
hw->fc.high_water[0] = rxpb - size;

/* Now calculate Low Water */
switch (hw->mac.type) {
case ixgbe_mac_X540:
case ixgbe_mac_X550:
case ixgbe_mac_X550EM_x:
case ixgbe_mac_X550EM_a:
tmp = IXGBE_LOW_DV_X540(frame)(2 * (frame * 8) + (36 * 10000 / 25) + 1);
break;
default:
tmp = IXGBE_LOW_DV(frame)(2 * (2 * (frame * 8) + (36 * 10000 / 25) + 1));
break;
}
hw->fc.low_water[0] = IXGBE_BT2KB(tmp)((tmp + (8 * 1024 - 1)) / (8 * 1024));

hw->fc.requested_mode = sc->fc;
hw->fc.pause_time = IXGBE_FC_PAUSE0xFFFF;
hw->fc.send_xon = TRUE1;
963}

965/*
* MSIX Interrupt Handlers
*/
968void
969ixgbe_enable_queue(struct ix_softc *sc, uint32_t vector)
970{
uint64_t queue = 1ULL << vector;
uint32_t mask;

if (sc->hw.mac.type == ixgbe_mac_82598EB) {
mask = (IXGBE_EIMS_RTX_QUEUE0x0000FFFF & queue);
IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, mask)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00880), (mask)));
} else {
mask = (queue & 0xFFFFFFFF);
if (mask)
	IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(0), mask)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x00AA0 + (0) * 4)), (mask)));
mask = (queue >> 32);
if (mask)
	IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(1), mask)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x00AA0 + (1) * 4)), (mask)));
}
985}

987void
988ixgbe_enable_queues(struct ix_softc *sc)
989{
struct ix_queue *que;
int i;

for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++)
ixgbe_enable_queue(sc, que->msix);
995}

997void
998ixgbe_disable_queue(struct ix_softc *sc, uint32_t vector)
999{
uint64_t queue = 1ULL << vector;
uint32_t mask;

if (sc->hw.mac.type == ixgbe_mac_82598EB) {
mask = (IXGBE_EIMS_RTX_QUEUE0x0000FFFF & queue);
IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, mask)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00888), (mask)));
} else {
mask = (queue & 0xFFFFFFFF);
if (mask)
	IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), mask)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x00AB0 + (0) * 4)), (mask)));
mask = (queue >> 32);
if (mask)
	IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), mask)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x00AB0 + (1) * 4)), (mask)));
}
1014}

1016/*
* MSIX Interrupt Handlers
*/
1019int
1020ixgbe_link_intr(void *vsc)
1021{
struct ix_softc	*sc = (struct ix_softc *)vsc;

return ixgbe_intr(sc);
1025}

1027int
1028ixgbe_queue_intr(void *vque)
1029{
struct ix_queue *que = vque;
struct ix_softc	*sc = que->sc;
struct ifnet	*ifp = &sc->arpcom.ac_if;
struct rx_ring	*rxr = que->rxr;
struct tx_ring	*txr = que->txr;

if (ISSET(ifp->if_flags, IFF_RUNNING)((ifp->if_flags) & (0x40))) {
ixgbe_rxeof(rxr);
ixgbe_txeof(txr);
ixgbe_rxrefill(rxr);
}

ixgbe_enable_queue(sc, que->msix);

return (1);
1045}

1047/*********************************************************************
*
*  Legacy Interrupt Service routine
*
**********************************************************************/

1053int
1054ixgbe_legacy_intr(void *arg)
1055{
struct ix_softc	*sc = (struct ix_softc *)arg;
struct ifnet	*ifp = &sc->arpcom.ac_if;
struct rx_ring	*rxr = sc->rx_rings;
struct tx_ring	*txr = sc->tx_rings;
int rv;

rv = ixgbe_intr(sc);
if (rv == 0) {
return (0);
}

if (ISSET(ifp->if_flags, IFF_RUNNING)((ifp->if_flags) & (0x40))) {
ixgbe_rxeof(rxr);
ixgbe_txeof(txr);
ixgbe_rxrefill(rxr);
}

ixgbe_enable_queues(sc);
return (rv);
1075}

1077int
1078ixgbe_intr(struct ix_softc *sc)
1079{
struct ifnet	*ifp = &sc->arpcom.ac_if;
struct ixgbe_hw	*hw = &sc->hw;
uint32_t	 reg_eicr, mod_mask, msf_mask;

if (sc->sc_intrmap) {
/* Pause other interrupts */
IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00888
), (0x80000000)));
/* First get the cause */
reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00808
)));
/* Be sure the queue bits are not cleared */
reg_eicr &= ~IXGBE_EICR_RTX_QUEUE0x0000FFFF;
/* Clear interrupt with write */
IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00800
), (reg_eicr)));
} else {
reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00800
)));
if (reg_eicr == 0) {
	ixgbe_enable_intr(sc);
	ixgbe_enable_queues(sc);
	return (0);
}
}

/* Link status change */
if (reg_eicr & IXGBE_EICR_LSC0x00100000) {
IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00888
), (0x00100000)));
KERNEL_LOCK()_kernel_lock();
ixgbe_update_link_status(sc);
KERNEL_UNLOCK()_kernel_unlock();
}

if (hw->mac.type != ixgbe_mac_82598EB) {
if (reg_eicr & IXGBE_EICR_ECC0x10000000) {
	printf("%s: CRITICAL: ECC ERROR!! "
	    "Please Reboot!!\n", sc->dev.dv_xname);
	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00800
), (0x10000000)));
}
/* Check for over temp condition */
if (reg_eicr & IXGBE_EICR_TS0x00800000) {
	printf("%s: CRITICAL: OVER TEMP!! "
	    "PHY IS SHUT DOWN!!\n", ifp->if_xname);
	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00800
), (0x00800000)));
}
}

/* Pluggable optics-related interrupt */
if (ixgbe_is_sfp(hw)) {
if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP0x15AC) {
	mod_mask = IXGBE_EICR_GPI_SDP0_X5400x02000000;
	msf_mask = IXGBE_EICR_GPI_SDP1_X5400x04000000;
} else if (hw->mac.type == ixgbe_mac_X540 ||
    hw->mac.type == ixgbe_mac_X550 ||
    hw->mac.type == ixgbe_mac_X550EM_x) {
	mod_mask = IXGBE_EICR_GPI_SDP2_X5400x08000000;
	msf_mask = IXGBE_EICR_GPI_SDP1_X5400x04000000;
} else {
	mod_mask = IXGBE_EICR_GPI_SDP20x04000000;
	msf_mask = IXGBE_EICR_GPI_SDP10x02000000;
}
if (reg_eicr & mod_mask) {
	/* Clear the interrupt */
	IXGBE_WRITE_REG(hw, IXGBE_EICR, mod_mask)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00800
), (mod_mask)));
	KERNEL_LOCK()_kernel_lock();
	ixgbe_handle_mod(sc);
	KERNEL_UNLOCK()_kernel_unlock();
} else if ((hw->phy.media_type != ixgbe_media_type_copper) &&
    (reg_eicr & msf_mask)) {
	/* Clear the interrupt */
	IXGBE_WRITE_REG(hw, IXGBE_EICR, msf_mask)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00800
), (msf_mask)));
	KERNEL_LOCK()_kernel_lock();
	ixgbe_handle_msf(sc);
	KERNEL_UNLOCK()_kernel_unlock();
}
}

/* Check for fan failure */
if ((hw->device_id == IXGBE_DEV_ID_82598AT0x10C8) &&
   (reg_eicr & IXGBE_EICR_GPI_SDP10x02000000)) {
printf("%s: CRITICAL: FAN FAILURE!! "
    "REPLACE IMMEDIATELY!!\n", ifp->if_xname);
IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00800
), (0x02000000)));
}

/* External PHY interrupt */
if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T0x15AD &&
   (reg_eicr & IXGBE_EICR_GPI_SDP0_X5400x02000000)) {
/* Clear the interrupt */
IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00800
), (0x02000000)));
KERNEL_LOCK()_kernel_lock();
ixgbe_handle_phy(sc);
KERNEL_UNLOCK()_kernel_unlock();
}

IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00880
), (0x80000000 | 0x00100000)));

return (1);
1175}

1177/*********************************************************************
*
*  Media Ioctl callback
*
*  This routine is called whenever the user queries the status of
*  the interface using ifconfig.
*
**********************************************************************/
1185void
1186ixgbe_media_status(struct ifnet * ifp, struct ifmediareq *ifmr)
1187{
struct ix_softc *sc = ifp->if_softc;
uint64_t layer;

ifmr->ifm_active = IFM_ETHER0x0000000000000100ULL;
ifmr->ifm_status = IFM_AVALID0x0000000000000001ULL;

INIT_DEBUGOUT("ixgbe_media_status: begin")if (0) printf("ixgbe_media_status: begin" "\n");
ixgbe_update_link_status(sc);

if (!LINK_STATE_IS_UP(ifp->if_link_state)((ifp->if_data.ifi_link_state) >= 4 || (ifp->if_data
.ifi_link_state) == 0))
return;

ifmr->ifm_status |= IFM_ACTIVE0x0000000000000002ULL;
layer = sc->phy_layer;

if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T0x00001 ||
   layer & IXGBE_PHYSICAL_LAYER_1000BASE_T0x00002 ||
   layer & IXGBE_PHYSICAL_LAYER_100BASE_TX0x00004 ||
   layer & IXGBE_PHYSICAL_LAYER_10BASE_T0x08000) {
switch (sc->link_speed) {
case IXGBE_LINK_SPEED_10GB_FULL0x0080:
	ifmr->ifm_active |= IFM_10G_T22 | IFM_FDX0x0000010000000000ULL;
	break;
case IXGBE_LINK_SPEED_1GB_FULL0x0020:
	ifmr->ifm_active |= IFM_1000_T16 | IFM_FDX0x0000010000000000ULL;
	break;
case IXGBE_LINK_SPEED_100_FULL0x0008:
	ifmr->ifm_active |= IFM_100_TX6 | IFM_FDX0x0000010000000000ULL;
	break;
case IXGBE_LINK_SPEED_10_FULL0x0002:
	ifmr->ifm_active |= IFM_10_T3 | IFM_FDX0x0000010000000000ULL;
	break;
}
}
if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU0x00008 ||
   layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA0x02000) {
switch (sc->link_speed) {
case IXGBE_LINK_SPEED_10GB_FULL0x0080:
	ifmr->ifm_active |= IFM_10G_SFP_CU23 | IFM_FDX0x0000010000000000ULL;
	break;
}
}
if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR0x00010) {
switch (sc->link_speed) {
case IXGBE_LINK_SPEED_10GB_FULL0x0080:
	ifmr->ifm_active |= IFM_10G_LR18 | IFM_FDX0x0000010000000000ULL;
	break;
case IXGBE_LINK_SPEED_1GB_FULL0x0020:
	ifmr->ifm_active |= IFM_1000_LX14 | IFM_FDX0x0000010000000000ULL;
	break;
}
}
if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR0x00040 ||
   layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX0x04000) {
switch (sc->link_speed) {
case IXGBE_LINK_SPEED_10GB_FULL0x0080:
	ifmr->ifm_active |= IFM_10G_SR19 | IFM_FDX0x0000010000000000ULL;
	break;
case IXGBE_LINK_SPEED_1GB_FULL0x0020:
	ifmr->ifm_active |= IFM_1000_SX11 | IFM_FDX0x0000010000000000ULL;
	break;
}
}
if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX40x00100) {
switch (sc->link_speed) {
case IXGBE_LINK_SPEED_10GB_FULL0x0080:
	ifmr->ifm_active |= IFM_10G_CX420 | IFM_FDX0x0000010000000000ULL;
	break;
}
}
if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR0x00800) {
switch (sc->link_speed) {
case IXGBE_LINK_SPEED_10GB_FULL0x0080:
	ifmr->ifm_active |= IFM_10G_KR30 | IFM_FDX0x0000010000000000ULL;
	break;
case IXGBE_LINK_SPEED_2_5GB_FULL0x0400:
	ifmr->ifm_active |= IFM_2500_KX33 | IFM_FDX0x0000010000000000ULL;
	break;
case IXGBE_LINK_SPEED_1GB_FULL0x0020:
	ifmr->ifm_active |= IFM_1000_KX28 | IFM_FDX0x0000010000000000ULL;
	break;
}
} else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX40x00080 ||
   layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX0x10000 ||
   layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX0x00200) {
switch (sc->link_speed) {
case IXGBE_LINK_SPEED_10GB_FULL0x0080:
	ifmr->ifm_active |= IFM_10G_KX429 | IFM_FDX0x0000010000000000ULL;
	break;
case IXGBE_LINK_SPEED_2_5GB_FULL0x0400:
	ifmr->ifm_active |= IFM_2500_KX33 | IFM_FDX0x0000010000000000ULL;
	break;
case IXGBE_LINK_SPEED_1GB_FULL0x0020:
	ifmr->ifm_active |= IFM_1000_KX28 | IFM_FDX0x0000010000000000ULL;
	break;
}
}

switch (sc->hw.fc.current_mode) {
case ixgbe_fc_tx_pause:
ifmr->ifm_active |= IFM_FLOW0x0000040000000000ULL | IFM_ETH_TXPAUSE0x0000000000040000ULL;
break;
case ixgbe_fc_rx_pause:
ifmr->ifm_active |= IFM_FLOW0x0000040000000000ULL | IFM_ETH_RXPAUSE0x0000000000020000ULL;
break;
case ixgbe_fc_full:
ifmr->ifm_active |= IFM_FLOW0x0000040000000000ULL | IFM_ETH_RXPAUSE0x0000000000020000ULL |
    IFM_ETH_TXPAUSE0x0000000000040000ULL;
break;
default:
ifmr->ifm_active &= ~(IFM_FLOW0x0000040000000000ULL | IFM_ETH_RXPAUSE0x0000000000020000ULL |
    IFM_ETH_TXPAUSE0x0000000000040000ULL);
break;
}
1302}

1304/*********************************************************************
*
*  Media Ioctl callback
*
*  This routine is called when the user changes speed/duplex using
*  media/mediopt option with ifconfig.
*
**********************************************************************/
1312int
1313ixgbe_media_change(struct ifnet *ifp)
1314{
struct ix_softc	*sc = ifp->if_softc;
struct ixgbe_hw	*hw = &sc->hw;
struct ifmedia	*ifm = &sc->media;
ixgbe_link_speed speed = 0;

if (IFM_TYPE(ifm->ifm_media)((ifm->ifm_media) & 0x000000000000ff00ULL) != IFM_ETHER0x0000000000000100ULL)
return (EINVAL22);

if (hw->phy.media_type == ixgbe_media_type_backplane)
return (ENODEV19);

switch (IFM_SUBTYPE(ifm->ifm_media)((ifm->ifm_media) & 0x00000000000000ffULL)) {
case IFM_AUTO0ULL:
case IFM_10G_T22:
	speed |= IXGBE_LINK_SPEED_100_FULL0x0008;
	speed |= IXGBE_LINK_SPEED_1GB_FULL0x0020;
	speed |= IXGBE_LINK_SPEED_10GB_FULL0x0080;
	break;
case IFM_10G_SR19:
case IFM_10G_KR30:
case IFM_10G_LR18:
case IFM_10G_LRM24:
case IFM_10G_CX420:
case IFM_10G_KX429:
	speed |= IXGBE_LINK_SPEED_1GB_FULL0x0020;
	speed |= IXGBE_LINK_SPEED_10GB_FULL0x0080;
	break;
case IFM_10G_SFP_CU23:
	speed |= IXGBE_LINK_SPEED_10GB_FULL0x0080;
	break;
case IFM_1000_T16:
	speed |= IXGBE_LINK_SPEED_100_FULL0x0008;
	speed |= IXGBE_LINK_SPEED_1GB_FULL0x0020;
	break;
case IFM_1000_LX14:
case IFM_1000_SX11:
case IFM_1000_CX15:
case IFM_1000_KX28:
	speed |= IXGBE_LINK_SPEED_1GB_FULL0x0020;
	break;
case IFM_100_TX6:
	speed |= IXGBE_LINK_SPEED_100_FULL0x0008;
	break;
case IFM_10_T3:
	speed |= IXGBE_LINK_SPEED_10_FULL0x0002;
	break;
default:
	return (EINVAL22);
}

hw->mac.autotry_restart = TRUE1;
hw->mac.ops.setup_link(hw, speed, TRUE1);

return (0);
1369}

1371/*********************************************************************
*
*  This routine maps the mbufs to tx descriptors, allowing the
*  TX engine to transmit the packets.
*  	- return 0 on success, positive on failure
*
**********************************************************************/

1379int
1380ixgbe_encap(struct tx_ring *txr, struct mbuf *m_head)
1381{
struct ix_softc *sc = txr->sc;
uint32_t	olinfo_status = 0, cmd_type_len;
int             i, j, ntxc;
int		first, last = 0;
bus_dmamap_t	map;
struct ixgbe_tx_buf *txbuf;
union ixgbe_adv_tx_desc *txd = NULL((void *)0);
12
←
'txd' initialized to a null pointer value→

/* Basic descriptor defines */
cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA0x00300000 |
   IXGBE_ADVTXD_DCMD_IFCS0x02000000 | IXGBE_ADVTXD_DCMD_DEXT0x20000000);

1394#if NVLAN1 > 0
if (m_head->m_flagsm_hdr.mh_flags & M_VLANTAG0x0020)
13
←
Assuming the condition is false→
14
←
Taking false branch→
cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE0x40000000;
1397#endif

/*
* Important to capture the first descriptor
* used because it will contain the index of
* the one we tell the hardware to report back
*/
first = txr->next_avail_desc;
txbuf = &txr->tx_buffers[first];
map = txbuf->map;

/*
* Map the packet for DMA.
*/
switch (bus_dmamap_load_mbuf(txr->txdma.dma_tag, map,(*(txr->txdma.dma_tag)->_dmamap_load_mbuf)((txr->txdma
.dma_tag), (map), (m_head), (0x0001))
15
←
Control jumps to 'case 0:'  at line 1413→
   m_head, BUS_DMA_NOWAIT)(*(txr->txdma.dma_tag)->_dmamap_load_mbuf)((txr->txdma
.dma_tag), (map), (m_head), (0x0001))) {
case 0:
break;
16
←
 Execution continues on line 1429→
case EFBIG27:
if (m_defrag(m_head, M_NOWAIT0x0002) == 0 &&
    bus_dmamap_load_mbuf(txr->txdma.dma_tag, map,(*(txr->txdma.dma_tag)->_dmamap_load_mbuf)((txr->txdma
.dma_tag), (map), (m_head), (0x0001))
     m_head, BUS_DMA_NOWAIT)(*(txr->txdma.dma_tag)->_dmamap_load_mbuf)((txr->txdma
.dma_tag), (map), (m_head), (0x0001)) == 0)
	break;
/* FALLTHROUGH */
default:
return (0);
}

/*
* Set the appropriate offload context
* this will becomes the first descriptor.
*/
ntxc = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
if (ntxc == -1)
17
←
Taking false branch→
goto xmit_fail;

i = txr->next_avail_desc + ntxc;
if (i >= sc->num_tx_desc)
18
←
Assuming 'i' is < field 'num_tx_desc'→
19
←
Taking false branch→
i -= sc->num_tx_desc;

for (j = 0; j < map->dm_nsegs; j++) {
20
←
Assuming 'j' is >= field 'dm_nsegs'→
21
←
Loop condition is false. Execution continues on line 1450→
txd = &txr->tx_base[i];

txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr)((__uint64_t)(map->dm_segs[j].ds_addr));
txd->read.cmd_type_len = htole32(txr->txd_cmd |((__uint32_t)(txr->txd_cmd | cmd_type_len | map->dm_segs
[j].ds_len))
    cmd_type_len | map->dm_segs[j].ds_len)((__uint32_t)(txr->txd_cmd | cmd_type_len | map->dm_segs
[j].ds_len));
txd->read.olinfo_status = htole32(olinfo_status)((__uint32_t)(olinfo_status));
last = i; /* descriptor that will get completion IRQ */

if (++i == sc->num_tx_desc)
	i = 0;
}

txd->read.cmd_type_len |=
22
←
Dereference of null pointer
   htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS)((__uint32_t)(0x01000000 | 0x08000000));

bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (map), (0), (map->dm_mapsize), (0x04))
   BUS_DMASYNC_PREWRITE)(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (map), (0), (map->dm_mapsize), (0x04));

/* Set the index of the descriptor that will be marked done */
txbuf->m_head = m_head;
txbuf->eop_index = last;

membar_producer()do { __asm volatile("" ::: "memory"); } while (0);

txr->next_avail_desc = i;

return (ntxc + j);

1466xmit_fail:
bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map)(*(txr->txdma.dma_tag)->_dmamap_unload)((txr->txdma.
dma_tag), (txbuf->map));
return (0);
1469}

1471void
1472ixgbe_iff(struct ix_softc *sc)
1473{
struct ifnet *ifp = &sc->arpcom.ac_if;
struct arpcom *ac = &sc->arpcom;
uint32_t	fctrl;
uint8_t	*mta;
uint8_t	*update_ptr;
struct ether_multi *enm;
struct ether_multistep step;
int	mcnt = 0;

IOCTL_DEBUGOUT("ixgbe_iff: begin")if (0) printf("ixgbe_iff: begin" "\n");

mta = sc->mta;
bzero(mta, sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS *__builtin_bzero((mta), (sizeof(uint8_t) * 6 * 128))
   MAX_NUM_MULTICAST_ADDRESSES)__builtin_bzero((mta), (sizeof(uint8_t) * 6 * 128));

fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x05080)));
fctrl &= ~(IXGBE_FCTRL_MPE0x00000100 | IXGBE_FCTRL_UPE0x00000200);
ifp->if_flags &= ~IFF_ALLMULTI0x200;

if (ifp->if_flags & IFF_PROMISC0x100 || ac->ac_multirangecnt > 0 ||
   ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES128) {
ifp->if_flags |= IFF_ALLMULTI0x200;
fctrl |= IXGBE_FCTRL_MPE0x00000100;
if (ifp->if_flags & IFF_PROMISC0x100)
	fctrl |= IXGBE_FCTRL_UPE0x00000200;
} else {
ETHER_FIRST_MULTI(step, &sc->arpcom, enm)do { (step).e_enm = ((&(&sc->arpcom)->ac_multiaddrs
)->lh_first); do { if ((((enm)) = ((step)).e_enm) != ((void
 *)0)) ((step)).e_enm = ((((enm)))->enm_list.le_next); } while
 ( 0); } while ( 0);
while (enm != NULL((void *)0)) {
	bcopy(enm->enm_addrlo,
	    &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS6],
	    IXGBE_ETH_LENGTH_OF_ADDRESS6);
	mcnt++;

	ETHER_NEXT_MULTI(step, enm)do { if (((enm) = (step).e_enm) != ((void *)0)) (step).e_enm =
 (((enm))->enm_list.le_next); } while ( 0);
}

update_ptr = mta;
sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt,
    ixgbe_mc_array_itr, TRUE1);
}

IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x05080), (fctrl)));
1516}

1518/*
* This is an iterator function now needed by the multicast
* shared code. It simply feeds the shared code routine the
* addresses in the array of ixgbe_iff() one by one.
*/
1523uint8_t *
1524ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq)
1525{
uint8_t *addr = *update_ptr;
uint8_t *newptr;
*vmdq = 0;

newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS6;
*update_ptr = newptr;
return addr;
1533}

1535void
1536ixgbe_update_link_status(struct ix_softc *sc)
1537{
struct ifnet	*ifp = &sc->arpcom.ac_if;
int		link_state = LINK_STATE_DOWN2;

ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0);

ifp->if_baudrateif_data.ifi_baudrate = 0;
if (sc->link_up) {
link_state = LINK_STATE_FULL_DUPLEX6;

switch (sc->link_speed) {
case IXGBE_LINK_SPEED_UNKNOWN0:
	ifp->if_baudrateif_data.ifi_baudrate = 0;
	break;
case IXGBE_LINK_SPEED_100_FULL0x0008:
	ifp->if_baudrateif_data.ifi_baudrate = IF_Mbps(100)((((100) * 1000ULL) * 1000ULL));
	break;
case IXGBE_LINK_SPEED_1GB_FULL0x0020:
	ifp->if_baudrateif_data.ifi_baudrate = IF_Gbps(1)((((((1) * 1000ULL) * 1000ULL) * 1000ULL)));
	break;
case IXGBE_LINK_SPEED_10GB_FULL0x0080:
	ifp->if_baudrateif_data.ifi_baudrate = IF_Gbps(10)((((((10) * 1000ULL) * 1000ULL) * 1000ULL)));
	break;
}

/* Update any Flow Control changes */
sc->hw.mac.ops.fc_enable(&sc->hw);
}
if (ifp->if_link_stateif_data.ifi_link_state != link_state) {
ifp->if_link_stateif_data.ifi_link_state = link_state;
if_link_state_change(ifp);
}
1569}


1572/*********************************************************************
*
*  This routine disables all traffic on the adapter by issuing a
*  global reset on the MAC and deallocates TX/RX buffers.
*
**********************************************************************/

1579void
1580ixgbe_stop(void *arg)
1581{
struct ix_softc *sc = arg;
struct ifnet   *ifp = &sc->arpcom.ac_if;
int i;

/* Tell the stack that the interface is no longer active */
ifp->if_flags &= ~IFF_RUNNING0x40;

1589#if NKSTAT0 > 0
timeout_del(&sc->sc_kstat_tmo);
1591#endif

INIT_DEBUGOUT("ixgbe_stop: begin\n")if (0) printf("ixgbe_stop: begin\n" "\n");
ixgbe_disable_intr(sc);

sc->hw.mac.ops.reset_hw(&sc->hw);
sc->hw.adapter_stopped = FALSE0;
sc->hw.mac.ops.stop_adapter(&sc->hw);
if (sc->hw.mac.type == ixgbe_mac_82599EB)
sc->hw.mac.ops.stop_mac_link_on_d3(&sc->hw);
/* Turn off the laser */
if (sc->hw.mac.ops.disable_tx_laser)
sc->hw.mac.ops.disable_tx_laser(&sc->hw);

/* reprogram the RAR[0] in case user changed it. */
ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV0x80000000);

intr_barrier(sc->tag);
for (i = 0; i < sc->num_queues; i++) {
struct ifqueue *ifq = ifp->if_ifqs[i];
ifq_barrier(ifq);
ifq_clr_oactive(ifq);

if (sc->queues[i].tag != NULL((void *)0))
	intr_barrier(sc->queues[i].tag);
timeout_del(&sc->rx_rings[i].rx_refill);
}

KASSERT((ifp->if_flags & IFF_RUNNING) == 0)(((ifp->if_flags & 0x40) == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/pci/if_ix.c", 1619, "(ifp->if_flags & IFF_RUNNING) == 0"
));

/* Should we really clear all structures on stop? */
ixgbe_free_transmit_structures(sc);
ixgbe_free_receive_structures(sc);

ixgbe_update_link_status(sc);
1626}


1629/*********************************************************************
*
*  Determine hardware revision.
*
**********************************************************************/
1634void
1635ixgbe_identify_hardware(struct ix_softc *sc)
1636{
struct ixgbe_osdep	*os = &sc->osdep;
struct pci_attach_args	*pa = &os->os_pa;
uint32_t		 reg;

/* Save off the information about this board */
sc->hw.vendor_id = PCI_VENDOR(pa->pa_id)(((pa->pa_id) >> 0) & 0xffff);
sc->hw.device_id = PCI_PRODUCT(pa->pa_id)(((pa->pa_id) >> 16) & 0xffff);

reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG0x08);
sc->hw.revision_id = PCI_REVISION(reg)(((reg) >> 0) & 0xff);

reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG0x2c);
sc->hw.subsystem_vendor_id = PCI_VENDOR(reg)(((reg) >> 0) & 0xffff);
sc->hw.subsystem_device_id = PCI_PRODUCT(reg)(((reg) >> 16) & 0xffff);

/* We need this here to set the num_segs below */
ixgbe_set_mac_type(&sc->hw);

/* Pick up the 82599 and VF settings */
if (sc->hw.mac.type != ixgbe_mac_82598EB)
sc->hw.phy.smart_speed = ixgbe_smart_speed;
sc->num_segs = IXGBE_82599_SCATTER32;
1659}

1661/*********************************************************************
*
*  Setup the Legacy or MSI Interrupt handler
*
**********************************************************************/
1666int
1667ixgbe_allocate_legacy(struct ix_softc *sc)
1668{
struct ixgbe_osdep	*os = &sc->osdep;
struct pci_attach_args	*pa = &os->os_pa;
const char		*intrstr = NULL((void *)0);
pci_chipset_tag_t	pc = pa->pa_pc;
pci_intr_handle_t	ih;

/* We allocate a single interrupt resource */
if (pci_intr_map_msi(pa, &ih) != 0 &&
   pci_intr_map(pa, &ih) != 0) {
printf(": couldn't map interrupt\n");
return (ENXIO6);
}

1682#if 0
/* XXX */
/* Tasklets for Link, SFP and Multispeed Fiber */
TASK_INIT(&sc->link_task, 0, ixgbe_handle_link, sc);
TASK_INIT(&sc->mod_task, 0, ixgbe_handle_mod, sc);
TASK_INIT(&sc->msf_task, 0, ixgbe_handle_msf, sc);
1688#endif

intrstr = pci_intr_string(pc, ih);
sc->tag = pci_intr_establish(pc, ih, IPL_NET0x7 | IPL_MPSAFE0x100,
   ixgbe_legacy_intr, sc, sc->dev.dv_xname);
if (sc->tag == NULL((void *)0)) {
printf(": couldn't establish interrupt");
if (intrstr != NULL((void *)0))
	printf(" at %s", intrstr);
printf("\n");
return (ENXIO6);
}
printf(": %s", intrstr);

/* For simplicity in the handlers */
sc->que_mask = IXGBE_EIMS_ENABLE_MASK( 0x0000FFFF | 0x00100000 | 0x40000000 | 0x80000000);

return (0);
1706}

1708/*********************************************************************
*
*  Setup the MSI-X Interrupt handlers
*
**********************************************************************/
1713int
1714ixgbe_allocate_msix(struct ix_softc *sc)
1715{
struct ixgbe_osdep	*os = &sc->osdep;
struct pci_attach_args	*pa  = &os->os_pa;
int                      i = 0, error = 0;
struct ix_queue         *que;
pci_intr_handle_t	ih;

for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) {
if (pci_intr_map_msix(pa, i, &ih)) {
	printf("ixgbe_allocate_msix: "
	    "pci_intr_map_msix vec %d failed\n", i);
	error = ENOMEM12;
	goto fail;
}

que->tag = pci_intr_establish_cpu(pa->pa_pc, ih,
    IPL_NET0x7 | IPL_MPSAFE0x100, intrmap_cpu(sc->sc_intrmap, i),
    ixgbe_queue_intr, que, que->name);
if (que->tag == NULL((void *)0)) {
	printf("ixgbe_allocate_msix: "
	    "pci_intr_establish vec %d failed\n", i);
	error = ENOMEM12;
	goto fail;
}

que->msix = i;
}

/* Now the link status/control last MSI-X vector */
if (pci_intr_map_msix(pa, i, &ih)) {
printf("ixgbe_allocate_msix: "
    "pci_intr_map_msix link vector failed\n");
error = ENOMEM12;
goto fail;
}

sc->tag = pci_intr_establish(pa->pa_pc, ih, IPL_NET0x7 | IPL_MPSAFE0x100,
   ixgbe_link_intr, sc, sc->dev.dv_xname);
if (sc->tag == NULL((void *)0)) {
printf("ixgbe_allocate_msix: "
    "pci_intr_establish link vector failed\n");
error = ENOMEM12;
goto fail;
}
sc->linkvec = i;
printf(", %s, %d queue%s", pci_intr_string(pa->pa_pc, ih),
   i, (i > 1) ? "s" : "");

return (0);
1764fail:
for (que = sc->queues; i > 0; i--, que++) {
if (que->tag == NULL((void *)0))
	continue;
pci_intr_disestablish(pa->pa_pc, que->tag);
que->tag = NULL((void *)0);
}

return (error);
1773}

1775void
1776ixgbe_setup_msix(struct ix_softc *sc)
1777{
struct ixgbe_osdep	*os = &sc->osdep;
struct pci_attach_args	*pa = &os->os_pa;
int			 nmsix;
unsigned int		 maxq;

if (!ixgbe_enable_msix)
return;

nmsix = pci_intr_msix_count(pa);
if (nmsix <= 1)
return;

/* give one vector to events */
nmsix--;

/* XXX the number of queues is limited to what we can keep stats on */
maxq = (sc->hw.mac.type == ixgbe_mac_82598EB) ? 8 : 16;

sc->sc_intrmap = intrmap_create(&sc->dev, nmsix, maxq, 0);
sc->num_queues = intrmap_count(sc->sc_intrmap);
1798}

1800int
1801ixgbe_allocate_pci_resources(struct ix_softc *sc)
1802{
struct ixgbe_osdep	*os = &sc->osdep;
struct pci_attach_args	*pa = &os->os_pa;
int			 val;

val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)(0x10 + (0) * 4));
if (PCI_MAPREG_TYPE(val)((val) & 0x00000001) != PCI_MAPREG_TYPE_MEM0x00000000) {
printf(": mmba is not mem space\n");
return (ENXIO6);
}

if (pci_mapreg_map(pa, PCIR_BAR(0)(0x10 + (0) * 4), PCI_MAPREG_MEM_TYPE(val)((val) & 0x00000006), 0,
   &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) {
printf(": cannot find mem space\n");
return (ENXIO6);
}
sc->hw.hw_addr = (uint8_t *)os->os_membase;

/* Legacy defaults */
sc->num_queues = 1;
sc->hw.back = os;

/* Now setup MSI or MSI/X, return us the number of supported vectors. */
ixgbe_setup_msix(sc);

return (0);
1828}

1830void
1831ixgbe_free_pci_resources(struct ix_softc * sc)
1832{
struct ixgbe_osdep	*os = &sc->osdep;
struct pci_attach_args	*pa = &os->os_pa;
struct ix_queue *que = sc->queues;
int i;

/* Release all msix queue resources: */
for (i = 0; i < sc->num_queues; i++, que++) {
if (que->tag)
	pci_intr_disestablish(pa->pa_pc, que->tag);
que->tag = NULL((void *)0);
}

if (sc->tag)
pci_intr_disestablish(pa->pa_pc, sc->tag);
sc->tag = NULL((void *)0);
if (os->os_membase != 0)
bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize);
os->os_membase = 0;
1851}

1853/*********************************************************************
*
*  Setup networking device structure and register an interface.
*
**********************************************************************/
1858void
1859ixgbe_setup_interface(struct ix_softc *sc)
1860{
struct ifnet   *ifp = &sc->arpcom.ac_if;
int i;

strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ16);
ifp->if_softc = sc;
ifp->if_flags = IFF_BROADCAST0x2 | IFF_SIMPLEX0x800 | IFF_MULTICAST0x8000;
ifp->if_xflags = IFXF_MPSAFE0x1;
ifp->if_ioctl = ixgbe_ioctl;
ifp->if_qstart = ixgbe_start;
ifp->if_timer = 0;
ifp->if_watchdog = ixgbe_watchdog;
ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE9216 -
   ETHER_HDR_LEN((6 * 2) + 2) - ETHER_CRC_LEN4;
ifq_set_maxlen(&ifp->if_snd, sc->num_tx_desc - 1)((&ifp->if_snd)->ifq_maxlen = (sc->num_tx_desc -
 1));

ifp->if_capabilitiesif_data.ifi_capabilities = IFCAP_VLAN_MTU0x00000010;

1878#if NVLAN1 > 0
ifp->if_capabilitiesif_data.ifi_capabilities |= IFCAP_VLAN_HWTAGGING0x00000020;
1880#endif

ifp->if_capabilitiesif_data.ifi_capabilities |= IFCAP_CSUM_TCPv40x00000002 | IFCAP_CSUM_UDPv40x00000004;

/*
* Specify the media types supported by this sc and register
* callbacks to update media and link information
*/
ifmedia_init(&sc->media, IFM_IMASK0xff00000000000000ULL, ixgbe_media_change,
   ixgbe_media_status);
ixgbe_add_media_types(sc);
ifmedia_set(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_AUTO0ULL);

if_attach(ifp);
ether_ifattach(ifp);

if_attach_queues(ifp, sc->num_queues);
if_attach_iqueues(ifp, sc->num_queues);
for (i = 0; i < sc->num_queues; i++) {
struct ifqueue *ifq = ifp->if_ifqs[i];
struct ifiqueue *ifiq = ifp->if_iqs[i];
struct tx_ring *txr = &sc->tx_rings[i];
struct rx_ring *rxr = &sc->rx_rings[i];

ifq->ifq_softc_ifq_ptr._ifq_softc = txr;
txr->ifq = ifq;

ifiq->ifiq_softc_ifiq_ptr._ifiq_softc = rxr;
rxr->ifiq = ifiq;

1910#if NKSTAT0 > 0
ix_txq_kstats(sc, txr);
ix_rxq_kstats(sc, rxr);
1913#endif
}

sc->max_frame_size = IXGBE_MAX_FRAME_SIZE9216;
1917}

1919void
1920ixgbe_add_media_types(struct ix_softc *sc)
1921{
struct ixgbe_hw	*hw = &sc->hw;
uint64_t layer;

sc->phy_layer = hw->mac.ops.get_supported_physical_layer(hw);
layer = sc->phy_layer;

if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T0x00001)
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_10G_T22, 0, NULL((void *)0));
if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T0x00002)
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_1000_T16, 0, NULL((void *)0));
if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX0x00004)
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_100_TX6, 0, NULL((void *)0));
if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU0x00008 ||
   layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA0x02000)
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_10G_SFP_CU23, 0, NULL((void *)0));
if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR0x00010) {
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_10G_LR18, 0, NULL((void *)0));
if (hw->phy.multispeed_fiber)
	ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_1000_LX14, 0,
	    NULL((void *)0));
}
if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR0x00040) {
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_10G_SR19, 0, NULL((void *)0));
if (hw->phy.multispeed_fiber)
	ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_1000_SX11, 0,
	    NULL((void *)0));
} else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX0x04000)
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_1000_SX11, 0, NULL((void *)0));
if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX40x00100)
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_10G_CX420, 0, NULL((void *)0));
if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR0x00800)
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_10G_KR30, 0, NULL((void *)0));
if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX40x00080)
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_10G_KX429, 0, NULL((void *)0));
if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX0x00200)
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_1000_KX28, 0, NULL((void *)0));
if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX0x10000)
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_2500_KX33, 0, NULL((void *)0));

if (hw->device_id == IXGBE_DEV_ID_82598AT0x10C8) {
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_1000_T16 | IFM_FDX0x0000010000000000ULL, 0,
    NULL((void *)0));
ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_1000_T16, 0, NULL((void *)0));
}

ifmedia_add(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_AUTO0ULL, 0, NULL((void *)0));
1968}

1970void
1971ixgbe_config_link(struct ix_softc *sc)
1972{
uint32_t	autoneg, err = 0;
bool_Bool		negotiate;

if (ixgbe_is_sfp(&sc->hw)) {
if (sc->hw.phy.multispeed_fiber) {
	sc->hw.mac.ops.setup_sfp(&sc->hw);
	if (sc->hw.mac.ops.enable_tx_laser)
		sc->hw.mac.ops.enable_tx_laser(&sc->hw);
	ixgbe_handle_msf(sc);
} else
	ixgbe_handle_mod(sc);
} else {
if (sc->hw.mac.ops.check_link)
	err = sc->hw.mac.ops.check_link(&sc->hw, &autoneg,
	    &sc->link_up, FALSE0);
if (err)
	return;
autoneg = sc->hw.phy.autoneg_advertised;
if ((!autoneg) && (sc->hw.mac.ops.get_link_capabilities))
	err = sc->hw.mac.ops.get_link_capabilities(&sc->hw,
	    &autoneg, &negotiate);
if (err)
	return;
if (sc->hw.mac.ops.setup_link)
	sc->hw.mac.ops.setup_link(&sc->hw,
	    autoneg, sc->link_up);
}
2000}

2002/********************************************************************
* Manage DMA'able memory.
*******************************************************************/
2005int
2006ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size,
struct ixgbe_dma_alloc *dma, int mapflags)
2008{
struct ifnet		*ifp = &sc->arpcom.ac_if;
struct ixgbe_osdep	*os = &sc->osdep;
int			 r;

dma->dma_tag = os->os_pa.pa_dmat;
r = bus_dmamap_create(dma->dma_tag, size, 1,(*(dma->dma_tag)->_dmamap_create)((dma->dma_tag), (size
), (1), (size), (0), (0x0001), (&dma->dma_map))
   size, 0, BUS_DMA_NOWAIT, &dma->dma_map)(*(dma->dma_tag)->_dmamap_create)((dma->dma_tag), (size
), (1), (size), (0), (0x0001), (&dma->dma_map));
if (r != 0) {
printf("%s: ixgbe_dma_malloc: bus_dmamap_create failed; "
       "error %u\n", ifp->if_xname, r);
goto fail_0;
}

r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg,(*(dma->dma_tag)->_dmamem_alloc)((dma->dma_tag), (size
), ((1 << 12)), (0), (&dma->dma_seg), (1), (&
dma->dma_nseg), (0x0001))
   1, &dma->dma_nseg, BUS_DMA_NOWAIT)(*(dma->dma_tag)->_dmamem_alloc)((dma->dma_tag), (size
), ((1 << 12)), (0), (&dma->dma_seg), (1), (&
dma->dma_nseg), (0x0001));
if (r != 0) {
printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; "
       "error %u\n", ifp->if_xname, r);
goto fail_1;
}

r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size,(*(dma->dma_tag)->_dmamem_map)((dma->dma_tag), (&
dma->dma_seg), (dma->dma_nseg), (size), (&dma->dma_vaddr
), (0x0001))
   &dma->dma_vaddr, BUS_DMA_NOWAIT)(*(dma->dma_tag)->_dmamem_map)((dma->dma_tag), (&
dma->dma_seg), (dma->dma_nseg), (size), (&dma->dma_vaddr
), (0x0001));
if (r != 0) {
printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; "
       "error %u\n", ifp->if_xname, r);
goto fail_2;
}

r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,(*(dma->dma_tag)->_dmamap_load)((dma->dma_tag), (dma
->dma_map), (dma->dma_vaddr), (size), (((void *)0)), (mapflags
 | 0x0001))
   size, NULL, mapflags | BUS_DMA_NOWAIT)(*(dma->dma_tag)->_dmamap_load)((dma->dma_tag), (dma
->dma_map), (dma->dma_vaddr), (size), (((void *)0)), (mapflags
 | 0x0001));
if (r != 0) {
printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; "
       "error %u\n", ifp->if_xname, r);
goto fail_3;
}

dma->dma_size = size;
return (0);
2048fail_3:
bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size)(*(dma->dma_tag)->_dmamem_unmap)((dma->dma_tag), (dma
->dma_vaddr), (size));
2050fail_2:
bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg)(*(dma->dma_tag)->_dmamem_free)((dma->dma_tag), (&
dma->dma_seg), (dma->dma_nseg));
2052fail_1:
bus_dmamap_destroy(dma->dma_tag, dma->dma_map)(*(dma->dma_tag)->_dmamap_destroy)((dma->dma_tag), (
dma->dma_map));
2054fail_0:
dma->dma_map = NULL((void *)0);
dma->dma_tag = NULL((void *)0);
return (r);
2058}

2060void
2061ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma)
2062{
if (dma->dma_tag == NULL((void *)0))
return;

if (dma->dma_map != NULL((void *)0)) {
bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0,(*(dma->dma_tag)->_dmamap_sync)((dma->dma_tag), (dma
->dma_map), (0), (dma->dma_map->dm_mapsize), (0x02 |
 0x08))
    dma->dma_map->dm_mapsize,(*(dma->dma_tag)->_dmamap_sync)((dma->dma_tag), (dma
->dma_map), (0), (dma->dma_map->dm_mapsize), (0x02 |
 0x08))
    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)(*(dma->dma_tag)->_dmamap_sync)((dma->dma_tag), (dma
->dma_map), (0), (dma->dma_map->dm_mapsize), (0x02 |
 0x08));
bus_dmamap_unload(dma->dma_tag, dma->dma_map)(*(dma->dma_tag)->_dmamap_unload)((dma->dma_tag), (dma
->dma_map));
bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size)(*(dma->dma_tag)->_dmamem_unmap)((dma->dma_tag), (dma
->dma_vaddr), (dma->dma_size));
bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg)(*(dma->dma_tag)->_dmamem_free)((dma->dma_tag), (&
dma->dma_seg), (dma->dma_nseg));
bus_dmamap_destroy(dma->dma_tag, dma->dma_map)(*(dma->dma_tag)->_dmamap_destroy)((dma->dma_tag), (
dma->dma_map));
dma->dma_map = NULL((void *)0);
}
2076}


2079/*********************************************************************
*
*  Allocate memory for the transmit and receive rings, and then
*  the descriptors associated with each, called only once at attach.
*
**********************************************************************/
2085int
2086ixgbe_allocate_queues(struct ix_softc *sc)
2087{
struct ifnet	*ifp = &sc->arpcom.ac_if;
struct ix_queue *que;
struct tx_ring *txr;
struct rx_ring *rxr;
int rsize, tsize;
int txconf = 0, rxconf = 0, i;

/* First allocate the top level queue structs */
if (!(sc->queues = mallocarray(sc->num_queues,
   sizeof(struct ix_queue), M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008))) {
printf("%s: Unable to allocate queue memory\n", ifp->if_xname);
goto fail;
}

/* Then allocate the TX ring struct memory */
if (!(sc->tx_rings = mallocarray(sc->num_queues,
   sizeof(struct tx_ring), M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008))) {
printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname);
goto fail;
}

/* Next allocate the RX */
if (!(sc->rx_rings = mallocarray(sc->num_queues,
   sizeof(struct rx_ring), M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008))) {
printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname);
goto rx_fail;
}

/* For the ring itself */
tsize = roundup2(sc->num_tx_desc *(((sc->num_tx_desc * sizeof(union ixgbe_adv_tx_desc)) + (128
) - 1) & ~((128) - 1))
   sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN)(((sc->num_tx_desc * sizeof(union ixgbe_adv_tx_desc)) + (128
) - 1) & ~((128) - 1));

/*
* Now set up the TX queues, txconf is needed to handle the
* possibility that things fail midcourse and we need to
* undo memory gracefully
*/
for (i = 0; i < sc->num_queues; i++, txconf++) {
/* Set up some basics */
txr = &sc->tx_rings[i];
txr->sc = sc;
txr->me = i;

if (ixgbe_dma_malloc(sc, tsize,
    &txr->txdma, BUS_DMA_NOWAIT0x0001)) {
	printf("%s: Unable to allocate TX Descriptor memory\n",
	    ifp->if_xname);
	goto err_tx_desc;
}
txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
bzero((void *)txr->tx_base, tsize)__builtin_bzero(((void *)txr->tx_base), (tsize));
}

/*
* Next the RX queues...
*/
rsize = roundup2(sc->num_rx_desc *(((sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)) + (4096
) - 1) & ~((4096) - 1))
   sizeof(union ixgbe_adv_rx_desc), 4096)(((sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)) + (4096
) - 1) & ~((4096) - 1));
for (i = 0; i < sc->num_queues; i++, rxconf++) {
rxr = &sc->rx_rings[i];
/* Set up some basics */
rxr->sc = sc;
rxr->me = i;
timeout_set(&rxr->rx_refill, ixgbe_rxrefill, rxr);

if (ixgbe_dma_malloc(sc, rsize,
	&rxr->rxdma, BUS_DMA_NOWAIT0x0001)) {
	printf("%s: Unable to allocate RxDescriptor memory\n",
	    ifp->if_xname);
	goto err_rx_desc;
}
rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
bzero((void *)rxr->rx_base, rsize)__builtin_bzero(((void *)rxr->rx_base), (rsize));
}

/*
* Finally set up the queue holding structs
*/
for (i = 0; i < sc->num_queues; i++) {
que = &sc->queues[i];
que->sc = sc;
que->txr = &sc->tx_rings[i];
que->rxr = &sc->rx_rings[i];
snprintf(que->name, sizeof(que->name), "%s:%d",
    sc->dev.dv_xname, i);
}

return (0);

2177err_rx_desc:
for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--)
ixgbe_dma_free(sc, &rxr->rxdma);
2180err_tx_desc:
for (txr = sc->tx_rings; txconf > 0; txr++, txconf--)
ixgbe_dma_free(sc, &txr->txdma);
free(sc->rx_rings, M_DEVBUF2, sc->num_queues * sizeof(struct rx_ring));
sc->rx_rings = NULL((void *)0);
2185rx_fail:
free(sc->tx_rings, M_DEVBUF2, sc->num_queues * sizeof(struct tx_ring));
sc->tx_rings = NULL((void *)0);
2188fail:
return (ENOMEM12);
2190}

2192/*********************************************************************
*
*  Allocate memory for tx_buffer structures. The tx_buffer stores all
*  the information needed to transmit a packet on the wire. This is
*  called only once at attach, setup is done every reset.
*
**********************************************************************/
2199int
2200ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2201{
struct ix_softc 	*sc = txr->sc;
struct ifnet		*ifp = &sc->arpcom.ac_if;
struct ixgbe_tx_buf	*txbuf;
int			 error, i;

if (!(txr->tx_buffers = mallocarray(sc->num_tx_desc,
   sizeof(struct ixgbe_tx_buf), M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008))) {
printf("%s: Unable to allocate tx_buffer memory\n",
    ifp->if_xname);
error = ENOMEM12;
goto fail;
}
txr->txtag = txr->txdma.dma_tag;

/* Create the descriptor buffer dma maps */
for (i = 0; i < sc->num_tx_desc; i++) {
txbuf = &txr->tx_buffers[i];
error = bus_dmamap_create(txr->txdma.dma_tag, IXGBE_TSO_SIZE,(*(txr->txdma.dma_tag)->_dmamap_create)((txr->txdma.
dma_tag), (262140), (sc->num_segs), ((1 << 12)), (0)
, (0x0001), (&txbuf->map))
	    sc->num_segs, PAGE_SIZE, 0,(*(txr->txdma.dma_tag)->_dmamap_create)((txr->txdma.
dma_tag), (262140), (sc->num_segs), ((1 << 12)), (0)
, (0x0001), (&txbuf->map))
	    BUS_DMA_NOWAIT, &txbuf->map)(*(txr->txdma.dma_tag)->_dmamap_create)((txr->txdma.
dma_tag), (262140), (sc->num_segs), ((1 << 12)), (0)
, (0x0001), (&txbuf->map));

if (error != 0) {
	printf("%s: Unable to create TX DMA map\n",
	    ifp->if_xname);
	goto fail;
}
}

return 0;
2231fail:
return (error);
2233}

2235/*********************************************************************
*
*  Initialize a transmit ring.
*
**********************************************************************/
2240int
2241ixgbe_setup_transmit_ring(struct tx_ring *txr)
2242{
struct ix_softc		*sc = txr->sc;
int			 error;

/* Now allocate transmit buffers for the ring */
if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0)
return (error);

/* Clear the old ring contents */
bzero((void *)txr->tx_base,__builtin_bzero(((void *)txr->tx_base), ((sizeof(union ixgbe_adv_tx_desc
)) * sc->num_tx_desc))
     (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc)__builtin_bzero(((void *)txr->tx_base), ((sizeof(union ixgbe_adv_tx_desc
)) * sc->num_tx_desc));

/* Reset indices */
txr->next_avail_desc = 0;
txr->next_to_clean = 0;

bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x01 | 0x04))
   0, txr->txdma.dma_map->dm_mapsize,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x01 | 0x04))
   BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x01 | 0x04));

return (0);
2263}

2265/*********************************************************************
*
*  Initialize all transmit rings.
*
**********************************************************************/
2270int
2271ixgbe_setup_transmit_structures(struct ix_softc *sc)
2272{
struct tx_ring *txr = sc->tx_rings;
int		i, error;

for (i = 0; i < sc->num_queues; i++, txr++) {
if ((error = ixgbe_setup_transmit_ring(txr)) != 0)
	goto fail;
}

return (0);
2282fail:
ixgbe_free_transmit_structures(sc);
return (error);
2285}

2287/*********************************************************************
*
*  Enable transmit unit.
*
**********************************************************************/
2292void
2293ixgbe_initialize_transmit_units(struct ix_softc *sc)
2294{
struct ifnet	*ifp = &sc->arpcom.ac_if;
struct tx_ring	*txr;
struct ixgbe_hw	*hw = &sc->hw;
int		 i;
uint64_t	 tdba;
uint32_t	 txctrl;

/* Setup the Base and Length of the Tx Descriptor Ring */

for (i = 0; i < sc->num_queues; i++) {
txr = &sc->tx_rings[i];

/* Setup descriptor base address */
tdba = txr->txdma.dma_map->dm_segs[0].ds_addr;
IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x06000
 + ((i) * 0x40))), ((tdba & 0x00000000ffffffffULL))))
       (tdba & 0x00000000ffffffffULL))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x06000
 + ((i) * 0x40))), ((tdba & 0x00000000ffffffffULL))));
IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x06004
 + ((i) * 0x40))), ((tdba >> 32))));
IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x06008
 + ((i) * 0x40))), (sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc
))))
    sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x06008
 + ((i) * 0x40))), (sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc
))));

/* Setup the HW Tx Head and Tail descriptor pointers */
IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x06010
 + ((i) * 0x40))), (0)));
IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x06018
 + ((i) * 0x40))), (0)));

/* Setup Transmit Descriptor Cmd Settings */
txr->txd_cmd = IXGBE_TXD_CMD_IFCS0x02000000;
txr->queue_status = IXGBE_QUEUE_IDLE;
txr->watchdog_timer = 0;

/* Disable Head Writeback */
switch (hw->mac.type) {
case ixgbe_mac_82598EB:
	txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x07200
 + ((i) * 4)))));
	break;
case ixgbe_mac_82599EB:
case ixgbe_mac_X540:
default:
	txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x0600C
 + ((i) * 0x40)))));
	break;
}
txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN(1 << 11);
switch (hw->mac.type) {
case ixgbe_mac_82598EB:
	IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x07200
 + ((i) * 4))), (txctrl)));
	break;
case ixgbe_mac_82599EB:
case ixgbe_mac_X540:
default:
	IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x0600C
 + ((i) * 0x40))), (txctrl)));
	break;
}
}
ifp->if_timer = 0;

if (hw->mac.type != ixgbe_mac_82598EB) {
uint32_t dmatxctl, rttdcs;
dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x04A80
)));
dmatxctl |= IXGBE_DMATXCTL_TE0x1;
IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x04A80
), (dmatxctl)));
/* Disable arbiter to set MTQC */
rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x04900
)));
rttdcs |= IXGBE_RTTDCS_ARBDIS0x00000040;
IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x04900
), (rttdcs)));
IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x08120
), (0x0)));
rttdcs &= ~IXGBE_RTTDCS_ARBDIS0x00000040;
IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x04900
), (rttdcs)));
}
2362}

2364/*********************************************************************
*
*  Free all transmit rings.
*
**********************************************************************/
2369void
2370ixgbe_free_transmit_structures(struct ix_softc *sc)
2371{
struct tx_ring *txr = sc->tx_rings;
int		i;

for (i = 0; i < sc->num_queues; i++, txr++)
ixgbe_free_transmit_buffers(txr);
2377}

2379/*********************************************************************
*
*  Free transmit ring related data structures.
*
**********************************************************************/
2384void
2385ixgbe_free_transmit_buffers(struct tx_ring *txr)
2386{
struct ix_softc *sc = txr->sc;
struct ixgbe_tx_buf *tx_buffer;
int             i;

INIT_DEBUGOUT("free_transmit_ring: begin")if (0) printf("free_transmit_ring: begin" "\n");

if (txr->tx_buffers == NULL((void *)0))
return;

tx_buffer = txr->tx_buffers;
for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) {
if (tx_buffer->map != NULL((void *)0) && tx_buffer->map->dm_nsegs > 0) {
	bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (tx_buffer->map), (0), (tx_buffer->map->dm_mapsize
), (0x08))
	    0, tx_buffer->map->dm_mapsize,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (tx_buffer->map), (0), (tx_buffer->map->dm_mapsize
), (0x08))
	    BUS_DMASYNC_POSTWRITE)(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (tx_buffer->map), (0), (tx_buffer->map->dm_mapsize
), (0x08));
	bus_dmamap_unload(txr->txdma.dma_tag,(*(txr->txdma.dma_tag)->_dmamap_unload)((txr->txdma.
dma_tag), (tx_buffer->map))
	    tx_buffer->map)(*(txr->txdma.dma_tag)->_dmamap_unload)((txr->txdma.
dma_tag), (tx_buffer->map));
}
if (tx_buffer->m_head != NULL((void *)0)) {
	m_freem(tx_buffer->m_head);
	tx_buffer->m_head = NULL((void *)0);
}
if (tx_buffer->map != NULL((void *)0)) {
	bus_dmamap_destroy(txr->txdma.dma_tag,(*(txr->txdma.dma_tag)->_dmamap_destroy)((txr->txdma
.dma_tag), (tx_buffer->map))
	    tx_buffer->map)(*(txr->txdma.dma_tag)->_dmamap_destroy)((txr->txdma
.dma_tag), (tx_buffer->map));
	tx_buffer->map = NULL((void *)0);
}
}

if (txr->tx_buffers != NULL((void *)0))
free(txr->tx_buffers, M_DEVBUF2,
    sc->num_tx_desc * sizeof(struct ixgbe_tx_buf));
txr->tx_buffers = NULL((void *)0);
txr->txtag = NULL((void *)0);
2421}

2423/*********************************************************************
*
*  Advanced Context Descriptor setup for VLAN or CSUM
*
**********************************************************************/

2429int
2430ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
  uint32_t *cmd_type_len, uint32_t *olinfo_status)
2432{
struct ixgbe_adv_tx_context_desc *TXD;
struct ixgbe_tx_buf *tx_buffer;
2435#if NVLAN1 > 0
struct ether_vlan_header *eh;
2437#else
struct ether_header *eh;
2439#endif
struct ip *ip;
2441#ifdef notyet
struct ip6_hdr *ip6;
2443#endif
struct mbuf *m;
int	ipoff;
uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0;
int 	ehdrlen, ip_hlen = 0;
uint16_t etype;
uint8_t	ipproto = 0;
int	offload = TRUE1;
int	ctxd = txr->next_avail_desc;
2452#if NVLAN1 > 0
uint16_t vtag = 0;
2454#endif

2456#if notyet
/* First check if TSO is to be used */
if (mp->m_pkthdrM_dat.MH.MH_pkthdr.csum_flags & CSUM_TSO)
return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status));
2460#endif

if ((mp->m_pkthdrM_dat.MH.MH_pkthdr.csum_flags & (M_TCP_CSUM_OUT0x0002 | M_UDP_CSUM_OUT0x0004)) == 0)
offload = FALSE0;

/* Indicate the whole packet as payload when not doing TSO */
*olinfo_status |= mp->m_pkthdrM_dat.MH.MH_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT14;

/* Now ready a context descriptor */
TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
tx_buffer = &txr->tx_buffers[ctxd];

/*
* In advanced descriptors the vlan tag must
* be placed into the descriptor itself. Hence
* we need to make one even if not doing offloads.
*/
2477#if NVLAN1 > 0
if (mp->m_flagsm_hdr.mh_flags & M_VLANTAG0x0020) {
vtag = mp->m_pkthdrM_dat.MH.MH_pkthdr.ether_vtag;
vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT16);
} else
2482#endif
if (offload == FALSE0)
return (0);	/* No need for CTX */

/*
* Determine where frame payload starts.
* Jump over vlan headers if already present,
* helpful for QinQ too.
*/
if (mp->m_lenm_hdr.mh_len < sizeof(struct ether_header))
return (-1);
2493#if NVLAN1 > 0
eh = mtod(mp, struct ether_vlan_header *)((struct ether_vlan_header *)((mp)->m_hdr.mh_data));
if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)(__uint16_t)(__builtin_constant_p(0x8100) ? (__uint16_t)(((__uint16_t
)(0x8100) & 0xffU) << 8 | ((__uint16_t)(0x8100) &
 0xff00U) >> 8) : __swap16md(0x8100))) {
if (mp->m_lenm_hdr.mh_len < sizeof(struct ether_vlan_header))
	return (-1);
etype = ntohs(eh->evl_proto)(__uint16_t)(__builtin_constant_p(eh->evl_proto) ? (__uint16_t
)(((__uint16_t)(eh->evl_proto) & 0xffU) << 8 | (
(__uint16_t)(eh->evl_proto) & 0xff00U) >> 8) : __swap16md
(eh->evl_proto));
ehdrlen = ETHER_HDR_LEN((6 * 2) + 2) + ETHER_VLAN_ENCAP_LEN4;
} else {
etype = ntohs(eh->evl_encap_proto)(__uint16_t)(__builtin_constant_p(eh->evl_encap_proto) ? (
__uint16_t)(((__uint16_t)(eh->evl_encap_proto) & 0xffU
) << 8 | ((__uint16_t)(eh->evl_encap_proto) & 0xff00U
) >> 8) : __swap16md(eh->evl_encap_proto));
ehdrlen = ETHER_HDR_LEN((6 * 2) + 2);
}
2504#else
eh = mtod(mp, struct ether_header *)((struct ether_header *)((mp)->m_hdr.mh_data));
etype = ntohs(eh->ether_type)(__uint16_t)(__builtin_constant_p(eh->ether_type) ? (__uint16_t
)(((__uint16_t)(eh->ether_type) & 0xffU) << 8 | (
(__uint16_t)(eh->ether_type) & 0xff00U) >> 8) : __swap16md
(eh->ether_type));
ehdrlen = ETHER_HDR_LEN((6 * 2) + 2);
2508#endif

/* Set the ether header length */
vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT9;

switch (etype) {
case ETHERTYPE_IP0x0800:
if (mp->m_pkthdrM_dat.MH.MH_pkthdr.len < ehdrlen + sizeof(*ip))
	return (-1);
m = m_getptr(mp, ehdrlen, &ipoff);
KASSERT(m != NULL && m->m_len - ipoff >= sizeof(*ip))((m != ((void *)0) && m->m_hdr.mh_len - ipoff >=
 sizeof(*ip)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/if_ix.c"
, 2518, "m != NULL && m->m_len - ipoff >= sizeof(*ip)"
));
ip = (struct ip *)(m->m_datam_hdr.mh_data + ipoff);
ip_hlen = ip->ip_hl << 2;
ipproto = ip->ip_p;
type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV40x00000400;
break;
2524#ifdef notyet
case ETHERTYPE_IPV60x86DD:
if (mp->m_pkthdrM_dat.MH.MH_pkthdr.len < ehdrlen + sizeof(*ip6))
	return (-1);
m = m_getptr(mp, ehdrlen, &ipoff);
KASSERT(m != NULL && m->m_len - ipoff >= sizeof(*ip6))((m != ((void *)0) && m->m_hdr.mh_len - ipoff >=
 sizeof(*ip6)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/if_ix.c"
, 2529, "m != NULL && m->m_len - ipoff >= sizeof(*ip6)"
));
ip6 = (struct ip6 *)(m->m_datam_hdr.mh_data + ipoff);
ip_hlen = sizeof(*ip6);
/* XXX-BZ this will go badly in case of ext hdrs. */
ipproto = ip6->ip6_nxt;
type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV60x00000000;
break;
2536#endif
default:
offload = FALSE0;
break;
}

vlan_macip_lens |= ip_hlen;
type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT0x20000000 | IXGBE_ADVTXD_DTYP_CTXT0x00200000;

switch (ipproto) {
case IPPROTO_TCP6:
if (mp->m_pkthdrM_dat.MH.MH_pkthdr.csum_flags & M_TCP_CSUM_OUT0x0002)
	type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP0x00000800;
break;
case IPPROTO_UDP17:
if (mp->m_pkthdrM_dat.MH.MH_pkthdr.csum_flags & M_UDP_CSUM_OUT0x0004)
	type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP0x00000000;
break;
default:
offload = FALSE0;
break;
}

if (offload) /* For the TX descriptor setup */
*olinfo_status |= IXGBE_TXD_POPTS_TXSM0x02 << 8;

/* Now copy bits into descriptor */
TXD->vlan_macip_lens = htole32(vlan_macip_lens)((__uint32_t)(vlan_macip_lens));
TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl)((__uint32_t)(type_tucmd_mlhl));
TXD->seqnum_seed = htole32(0)((__uint32_t)(0));
TXD->mss_l4len_idx = htole32(0)((__uint32_t)(0));

tx_buffer->m_head = NULL((void *)0);
tx_buffer->eop_index = -1;

return (1);
2572}

2574/**********************************************************************
*
*  Examine each tx_buffer in the used queue. If the hardware is done
*  processing the packet then free associated resources. The
*  tx_buffer is put back on the free queue.
*
**********************************************************************/
2581int
2582ixgbe_txeof(struct tx_ring *txr)
2583{
struct ix_softc			*sc = txr->sc;
struct ifqueue			*ifq = txr->ifq;
struct ifnet			*ifp = &sc->arpcom.ac_if;
unsigned int			 head, tail, last;
struct ixgbe_tx_buf		*tx_buffer;
struct ixgbe_legacy_tx_desc	*tx_desc;

if (!ISSET(ifp->if_flags, IFF_RUNNING)((ifp->if_flags) & (0x40)))
return FALSE0;

head = txr->next_avail_desc;
tail = txr->next_to_clean;

membar_consumer()do { __asm volatile("" ::: "memory"); } while (0);

if (head == tail)
return (FALSE0);

bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x02))
   0, txr->txdma.dma_map->dm_mapsize,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x02))
   BUS_DMASYNC_POSTREAD)(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x02));

for (;;) {
tx_buffer = &txr->tx_buffers[tail];
last = tx_buffer->eop_index;
tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];

if (!ISSET(tx_desc->upper.fields.status, IXGBE_TXD_STAT_DD)((tx_desc->upper.fields.status) & (0x00000001)))
	break;

bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (tx_buffer->map), (0), (tx_buffer->map->dm_mapsize
), (0x08))
    0, tx_buffer->map->dm_mapsize, BUS_DMASYNC_POSTWRITE)(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (tx_buffer->map), (0), (tx_buffer->map->dm_mapsize
), (0x08));
bus_dmamap_unload(txr->txdma.dma_tag, tx_buffer->map)(*(txr->txdma.dma_tag)->_dmamap_unload)((txr->txdma.
dma_tag), (tx_buffer->map));
m_freem(tx_buffer->m_head);

tx_buffer->m_head = NULL((void *)0);
tx_buffer->eop_index = -1;

tail = last + 1;
if (tail == sc->num_tx_desc)
	tail = 0;
if (head == tail) {
	/* All clean, turn off the timer */
	ifp->if_timer = 0;
	break;
}
}

bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x01))
   0, txr->txdma.dma_map->dm_mapsize,(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x01))
   BUS_DMASYNC_PREREAD)(*(txr->txdma.dma_tag)->_dmamap_sync)((txr->txdma.dma_tag
), (txr->txdma.dma_map), (0), (txr->txdma.dma_map->dm_mapsize
), (0x01));

membar_producer()do { __asm volatile("" ::: "memory"); } while (0);

txr->next_to_clean = tail;

if (ifq_is_oactive(ifq))
ifq_restart(ifq);

return TRUE1;
2644}

2646/*********************************************************************
*
*  Get a buffer from system mbuf buffer pool.
*
**********************************************************************/
2651int
2652ixgbe_get_buf(struct rx_ring *rxr, int i)
2653{
struct ix_softc		*sc = rxr->sc;
struct ixgbe_rx_buf	*rxbuf;
struct mbuf		*mp;
int			error;
union ixgbe_adv_rx_desc	*rxdesc;

rxbuf = &rxr->rx_buffers[i];
rxdesc = &rxr->rx_base[i];
if (rxbuf->buf) {
printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n",
    sc->dev.dv_xname, i);
return (ENOBUFS55);
}

/* needed in any case so prealocate since this one will fail for sure */
mp = MCLGETL(NULL, M_DONTWAIT, sc->rx_mbuf_sz)m_clget((((void *)0)), (0x0002), (sc->rx_mbuf_sz));
if (!mp)
return (ENOBUFS55);

mp->m_datam_hdr.mh_data += (mp->m_extM_dat.MH.MH_dat.MH_ext.ext_size - sc->rx_mbuf_sz);
mp->m_lenm_hdr.mh_len = mp->m_pkthdrM_dat.MH.MH_pkthdr.len = sc->rx_mbuf_sz;

error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map,(*(rxr->rxdma.dma_tag)->_dmamap_load_mbuf)((rxr->rxdma
.dma_tag), (rxbuf->map), (mp), (0x0001))
   mp, BUS_DMA_NOWAIT)(*(rxr->rxdma.dma_tag)->_dmamap_load_mbuf)((rxr->rxdma
.dma_tag), (rxbuf->map), (mp), (0x0001));
if (error) {
m_freem(mp);
return (error);
}

bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxbuf->map), (0), (rxbuf->map->dm_mapsize), (0x01
))
   0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD)(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxbuf->map), (0), (rxbuf->map->dm_mapsize), (0x01
));
rxbuf->buf = mp;

rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr)((__uint64_t)(rxbuf->map->dm_segs[0].ds_addr));

return (0);
2690}

2692/*********************************************************************
*
*  Allocate memory for rx_buffer structures. Since we use one
*  rx_buffer per received packet, the maximum number of rx_buffer's
*  that we'll need is equal to the number of receive descriptors
*  that we've allocated.
*
**********************************************************************/
2700int
2701ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
2702{
struct ix_softc		*sc = rxr->sc;
struct ifnet		*ifp = &sc->arpcom.ac_if;
struct ixgbe_rx_buf 	*rxbuf;
int			i, error;

if (!(rxr->rx_buffers = mallocarray(sc->num_rx_desc,
   sizeof(struct ixgbe_rx_buf), M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008))) {
printf("%s: Unable to allocate rx_buffer memory\n",
    ifp->if_xname);
error = ENOMEM12;
goto fail;
}

rxbuf = rxr->rx_buffers;
for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) {
error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1,(*(rxr->rxdma.dma_tag)->_dmamap_create)((rxr->rxdma.
dma_tag), (16 * 1024), (1), (16 * 1024), (0), (0x0001), (&
rxbuf->map))
    16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map)(*(rxr->rxdma.dma_tag)->_dmamap_create)((rxr->rxdma.
dma_tag), (16 * 1024), (1), (16 * 1024), (0), (0x0001), (&
rxbuf->map));
if (error) {
	printf("%s: Unable to create Pack DMA map\n",
	    ifp->if_xname);
	goto fail;
}
}
bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (0), (rxr->rxdma.dma_map->dm_mapsize
), (0x01 | 0x04))
   rxr->rxdma.dma_map->dm_mapsize,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (0), (rxr->rxdma.dma_map->dm_mapsize
), (0x01 | 0x04))
   BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (0), (rxr->rxdma.dma_map->dm_mapsize
), (0x01 | 0x04));

return (0);

2732fail:
return (error);
2734}

2736/*********************************************************************
*
*  Initialize a receive ring and its buffers.
*
**********************************************************************/
2741int
2742ixgbe_setup_receive_ring(struct rx_ring *rxr)
2743{
struct ix_softc		*sc = rxr->sc;
struct ifnet		*ifp = &sc->arpcom.ac_if;
int			 rsize, error;

rsize = roundup2(sc->num_rx_desc *(((sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)) + (4096
) - 1) & ~((4096) - 1))
   sizeof(union ixgbe_adv_rx_desc), 4096)(((sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)) + (4096
) - 1) & ~((4096) - 1));
/* Clear the ring contents */
bzero((void *)rxr->rx_base, rsize)__builtin_bzero(((void *)rxr->rx_base), (rsize));

if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0)
return (error);

/* Setup our descriptor indices */
rxr->next_to_check = 0;
rxr->last_desc_filled = sc->num_rx_desc - 1;

if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES(1 << 11)) + 1),
   sc->num_rx_desc - 1);

ixgbe_rxfill(rxr);
if (if_rxr_inuse(&rxr->rx_ring)((&rxr->rx_ring)->rxr_alive) == 0) {
printf("%s: unable to fill any rx descriptors\n",
    sc->dev.dv_xname);
return (ENOBUFS55);
}

return (0);
2771}

2773int
2774ixgbe_rxfill(struct rx_ring *rxr)
2775{
struct ix_softc *sc = rxr->sc;
int		 post = 0;
u_int		 slots;
int		 i;

bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (0), (rxr->rxdma.dma_map->dm_mapsize
), (0x08))
   0, rxr->rxdma.dma_map->dm_mapsize,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (0), (rxr->rxdma.dma_map->dm_mapsize
), (0x08))
   BUS_DMASYNC_POSTWRITE)(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (0), (rxr->rxdma.dma_map->dm_mapsize
), (0x08));

i = rxr->last_desc_filled;
for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc);
   slots > 0; slots--) {
if (++i == sc->num_rx_desc)
	i = 0;

if (ixgbe_get_buf(rxr, i) != 0)
	break;

rxr->last_desc_filled = i;
post = 1;
}

bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (0), (rxr->rxdma.dma_map->dm_mapsize
), (0x04))
   0, rxr->rxdma.dma_map->dm_mapsize,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (0), (rxr->rxdma.dma_map->dm_mapsize
), (0x04))
   BUS_DMASYNC_PREWRITE)(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (0), (rxr->rxdma.dma_map->dm_mapsize
), (0x04));

if_rxr_put(&rxr->rx_ring, slots)do { (&rxr->rx_ring)->rxr_alive -= (slots); } while
 (0);

return (post);
2805}

2807void
2808ixgbe_rxrefill(void *xrxr)
2809{
struct rx_ring *rxr = xrxr;
struct ix_softc *sc = rxr->sc;

if (ixgbe_rxfill(rxr)) {
/* Advance the Rx Queue "Tail Pointer" */
IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me),((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((rxr->me) < 64) ? (0x01018 + ((rxr->
me) * 0x40)) : (0x0D018 + (((rxr->me) - 64) * 0x40)))), (rxr
->last_desc_filled)))
    rxr->last_desc_filled)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((rxr->me) < 64) ? (0x01018 + ((rxr->
me) * 0x40)) : (0x0D018 + (((rxr->me) - 64) * 0x40)))), (rxr
->last_desc_filled)));
} else if (if_rxr_inuse(&rxr->rx_ring)((&rxr->rx_ring)->rxr_alive) == 0)
timeout_add(&rxr->rx_refill, 1);

2820}

2822/*********************************************************************
*
*  Initialize all receive rings.
*
**********************************************************************/
2827int
2828ixgbe_setup_receive_structures(struct ix_softc *sc)
2829{
struct rx_ring *rxr = sc->rx_rings;
int i;

for (i = 0; i < sc->num_queues; i++, rxr++)
if (ixgbe_setup_receive_ring(rxr))
	goto fail;

return (0);
2838fail:
ixgbe_free_receive_structures(sc);
return (ENOBUFS55);
2841}

2843/*********************************************************************
*
*  Setup receive registers and features.
*
**********************************************************************/
2848#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT2 2

2850void
2851ixgbe_initialize_receive_units(struct ix_softc *sc)
2852{
struct rx_ring	*rxr = sc->rx_rings;
struct ixgbe_hw	*hw = &sc->hw;
uint32_t	bufsz, fctrl, srrctl, rxcsum;
uint32_t	hlreg;
int		i;

/*
* Make sure receives are disabled while
* setting up the descriptor ring
*/
ixgbe_disable_rx(hw);

/* Enable broadcasts */
fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x05080
)));
fctrl |= IXGBE_FCTRL_BAM0x00000400;
if (sc->hw.mac.type == ixgbe_mac_82598EB) {
fctrl |= IXGBE_FCTRL_DPF0x00002000;
fctrl |= IXGBE_FCTRL_PMCF0x00001000;
}
IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x05080
), (fctrl)));

/* Always enable jumbo frame reception */
hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x04240
)));
hlreg |= IXGBE_HLREG0_JUMBOEN0x00000004;
IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x04240
), (hlreg)));

bufsz = (sc->rx_mbuf_sz - ETHER_ALIGN2) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT10;

for (i = 0; i < sc->num_queues; i++, rxr++) {
uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr;

/* Setup the Base and Length of the Rx Descriptor Ring */
IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((((i) <
 64) ? (0x01000 + ((i) * 0x40)) : (0x0D000 + (((i) - 64) * 0x40
)))), ((rdba & 0x00000000ffffffffULL))))
	       (rdba & 0x00000000ffffffffULL))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((((i) <
 64) ? (0x01000 + ((i) * 0x40)) : (0x0D000 + (((i) - 64) * 0x40
)))), ((rdba & 0x00000000ffffffffULL))));
IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((((i) <
 64) ? (0x01004 + ((i) * 0x40)) : (0x0D004 + (((i) - 64) * 0x40
)))), ((rdba >> 32))));
IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((((i) <
 64) ? (0x01008 + ((i) * 0x40)) : (0x0D008 + (((i) - 64) * 0x40
)))), (sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)))
)
    sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((((i) <
 64) ? (0x01008 + ((i) * 0x40)) : (0x0D008 + (((i) - 64) * 0x40
)))), (sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)))
);

/* Set up the SRRCTL register */
srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF0x02000000;
IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((((i) <=
 15) ? (0x02100 + ((i) * 4)) : (((i) < 64) ? (0x01014 + ((
i) * 0x40)) : (0x0D014 + (((i) - 64) * 0x40))))), (srrctl)));

/* Setup the HW Rx Head and Tail Descriptor Pointers */
IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((((i) <
 64) ? (0x01010 + ((i) * 0x40)) : (0x0D010 + (((i) - 64) * 0x40
)))), (0)));
IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((((i) <
 64) ? (0x01018 + ((i) * 0x40)) : (0x0D018 + (((i) - 64) * 0x40
)))), (0)));
}

if (sc->hw.mac.type != ixgbe_mac_82598EB) {
uint32_t psrtype = IXGBE_PSRTYPE_TCPHDR0x00000010 |
	      IXGBE_PSRTYPE_UDPHDR0x00000020 |
	      IXGBE_PSRTYPE_IPV4HDR0x00000100 |
	      IXGBE_PSRTYPE_IPV6HDR0x00000200;
IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((((0) <=
 15) ? (0x05480 + ((0) * 4)) : (0x0EA00 + ((0) * 4)))), (psrtype
)));
}

rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x05000
)));
rxcsum &= ~IXGBE_RXCSUM_PCSD0x00002000;

ixgbe_initialize_rss_mapping(sc);

/* Setup RSS */
if (sc->num_queues > 1) {
/* RSS and RX IPP Checksum are mutually exclusive */
rxcsum |= IXGBE_RXCSUM_PCSD0x00002000;
}

/* Map QPRC/QPRDC/QPTC on a per queue basis */
ixgbe_map_queue_statistics(sc);

/* This is useful for calculating UDP/IP fragment checksums */
if (!(rxcsum & IXGBE_RXCSUM_PCSD0x00002000))
rxcsum |= IXGBE_RXCSUM_IPPCSE0x00001000;

IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x05000
), (rxcsum)));
2927}

2929void
2930ixgbe_initialize_rss_mapping(struct ix_softc *sc)
2931{
struct ixgbe_hw	*hw = &sc->hw;
uint32_t reta = 0, mrqc, rss_key[10];
int i, j, queue_id, table_size, index_mult;

/* set up random bits */
stoeplitz_to_key(&rss_key, sizeof(rss_key));

/* Set multiplier for RETA setup and table size based on MAC */
index_mult = 0x1;
table_size = 128;
switch (sc->hw.mac.type) {
case ixgbe_mac_82598EB:
index_mult = 0x11;
break;
case ixgbe_mac_X550:
case ixgbe_mac_X550EM_x:
case ixgbe_mac_X550EM_a:
table_size = 512;
break;
default:
break;
}

/* Set up the redirection table */
for (i = 0, j = 0; i < table_size; i++, j++) {
if (j == sc->num_queues) j = 0;
queue_id = (j * index_mult);
/*
 * The low 8 bits are for hash value (n+0);
 * The next 8 bits are for hash value (n+1), etc.
 */
reta = reta >> 8;
reta = reta | ( ((uint32_t) queue_id) << 24);
if ((i & 3) == 3) {
	if (i < 128)
		IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x05C00
 + ((i >> 2) * 4))), (reta)));
	else
		IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x0EE80
 + (((i >> 2) - 32) * 4))), (reta)))
		    reta)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x0EE80
 + (((i >> 2) - 32) * 4))), (reta)));
	reta = 0;
}
}

/* Now fill our hash function seeds */
for (i = 0; i < 10; i++)
IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i])((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x05C80
 + ((i) * 4))), (rss_key[i])));

/*
* Disable UDP - IP fragments aren't currently being handled
* and so we end up with a mix of 2-tuple and 4-tuple
* traffic.
*/
mrqc = IXGBE_MRQC_RSSEN0x00000001
    | IXGBE_MRQC_RSS_FIELD_IPV40x00020000
    | IXGBE_MRQC_RSS_FIELD_IPV4_TCP0x00010000
    | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP0x00040000
    | IXGBE_MRQC_RSS_FIELD_IPV6_EX0x00080000
    | IXGBE_MRQC_RSS_FIELD_IPV60x00100000
    | IXGBE_MRQC_RSS_FIELD_IPV6_TCP0x00200000
;
IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x05818
), (mrqc)));
2993}

2995/*********************************************************************
*
*  Free all receive rings.
*
**********************************************************************/
3000void
3001ixgbe_free_receive_structures(struct ix_softc *sc)
3002{
struct rx_ring *rxr;
int		i;

for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++)
if_rxr_init(&rxr->rx_ring, 0, 0);

for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++)
ixgbe_free_receive_buffers(rxr);
3011}

3013/*********************************************************************
*
*  Free receive ring data structures
*
**********************************************************************/
3018void
3019ixgbe_free_receive_buffers(struct rx_ring *rxr)
3020{
struct ix_softc		*sc;
struct ixgbe_rx_buf	*rxbuf;
int			 i;

sc = rxr->sc;
if (rxr->rx_buffers != NULL((void *)0)) {
for (i = 0; i < sc->num_rx_desc; i++) {
	rxbuf = &rxr->rx_buffers[i];
	if (rxbuf->buf != NULL((void *)0)) {
		bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxbuf->map), (0), (rxbuf->map->dm_mapsize), (0x02
))
		    0, rxbuf->map->dm_mapsize,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxbuf->map), (0), (rxbuf->map->dm_mapsize), (0x02
))
		    BUS_DMASYNC_POSTREAD)(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxbuf->map), (0), (rxbuf->map->dm_mapsize), (0x02
));
		bus_dmamap_unload(rxr->rxdma.dma_tag,(*(rxr->rxdma.dma_tag)->_dmamap_unload)((rxr->rxdma.
dma_tag), (rxbuf->map))
		    rxbuf->map)(*(rxr->rxdma.dma_tag)->_dmamap_unload)((rxr->rxdma.
dma_tag), (rxbuf->map));
		m_freem(rxbuf->buf);
		rxbuf->buf = NULL((void *)0);
	}
	bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map)(*(rxr->rxdma.dma_tag)->_dmamap_destroy)((rxr->rxdma
.dma_tag), (rxbuf->map));
	rxbuf->map = NULL((void *)0);
}
free(rxr->rx_buffers, M_DEVBUF2,
    sc->num_rx_desc * sizeof(struct ixgbe_rx_buf));
rxr->rx_buffers = NULL((void *)0);
}
3045}

3047/*********************************************************************
*
*  This routine executes in interrupt context. It replenishes
*  the mbufs in the descriptor and sends data which has been
*  dma'ed into host memory to upper layer.
*
*********************************************************************/
3054int
3055ixgbe_rxeof(struct rx_ring *rxr)
3056{
struct ix_softc 	*sc = rxr->sc;
struct ifnet   		*ifp = &sc->arpcom.ac_if;
struct mbuf_list	 ml = MBUF_LIST_INITIALIZER(){ ((void *)0), ((void *)0), 0 };
struct mbuf    		*mp, *sendmp;
uint8_t		    	 eop = 0;
uint16_t		 len, vtag;
uint32_t		 staterr = 0, ptype;
struct ixgbe_rx_buf	*rxbuf, *nxbuf;
union ixgbe_adv_rx_desc	*rxdesc;
size_t			 dsize = sizeof(union ixgbe_adv_rx_desc);
int			 i, nextp;

if (!ISSET(ifp->if_flags, IFF_RUNNING)((ifp->if_flags) & (0x40)))
return FALSE0;

i = rxr->next_to_check;
while (if_rxr_inuse(&rxr->rx_ring)((&rxr->rx_ring)->rxr_alive) > 0) {
uint32_t hash;
uint16_t hashtype;

bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (dsize * i), (dsize), (0x02))
    dsize * i, dsize, BUS_DMASYNC_POSTREAD)(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (dsize * i), (dsize), (0x02));

rxdesc = &rxr->rx_base[i];
staterr = letoh32(rxdesc->wb.upper.status_error)((__uint32_t)(rxdesc->wb.upper.status_error));
if (!ISSET(staterr, IXGBE_RXD_STAT_DD)((staterr) & (0x01))) {
	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (dsize * i), (dsize), (0x01))
	    dsize * i, dsize,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (dsize * i), (dsize), (0x01))
	    BUS_DMASYNC_PREREAD)(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (dsize * i), (dsize), (0x01));
	break;
}

/* Zero out the receive descriptors status  */
rxdesc->wb.upper.status_error = 0;
rxbuf = &rxr->rx_buffers[i];

/* pull the mbuf off the ring */
bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxbuf->map), (0), (rxbuf->map->dm_mapsize), (0x02
))
    rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD)(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxbuf->map), (0), (rxbuf->map->dm_mapsize), (0x02
));
bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map)(*(rxr->rxdma.dma_tag)->_dmamap_unload)((rxr->rxdma.
dma_tag), (rxbuf->map));

mp = rxbuf->buf;
len = letoh16(rxdesc->wb.upper.length)((__uint16_t)(rxdesc->wb.upper.length));
ptype = letoh32(rxdesc->wb.lower.lo_dword.data)((__uint32_t)(rxdesc->wb.lower.lo_dword.data)) &
    IXGBE_RXDADV_PKTTYPE_MASK0x0000FFF0;
vtag = letoh16(rxdesc->wb.upper.vlan)((__uint16_t)(rxdesc->wb.upper.vlan));
eop = ((staterr & IXGBE_RXD_STAT_EOP0x02) != 0);
hash = lemtoh32(&rxdesc->wb.lower.hi_dword.rss)((__uint32_t)(*(__uint32_t *)(&rxdesc->wb.lower.hi_dword
.rss)));
hashtype =
    lemtoh16(&rxdesc->wb.lower.lo_dword.hs_rss.pkt_info)((__uint16_t)(*(__uint16_t *)(&rxdesc->wb.lower.lo_dword
.hs_rss.pkt_info))) &
    IXGBE_RXDADV_RSSTYPE_MASK0x0000000F;

if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK( 0x01000000 | 0x02000000 | 0x08000000 | 0x10000000 | 0x20000000
)) {
	if (rxbuf->fmp) {
		m_freem(rxbuf->fmp);
		rxbuf->fmp = NULL((void *)0);
	}

	m_freem(mp);
	rxbuf->buf = NULL((void *)0);
	goto next_desc;
}

if (mp == NULL((void *)0)) {
	panic("%s: ixgbe_rxeof: NULL mbuf in slot %d "
	    "(nrx %d, filled %d)", sc->dev.dv_xname,
	    i, if_rxr_inuse(&rxr->rx_ring)((&rxr->rx_ring)->rxr_alive),
	    rxr->last_desc_filled);
}

/* Currently no HW RSC support of 82599 */
if (!eop) {
	/*
	 * Figure out the next descriptor of this frame.
	 */
	nextp = i + 1;
	if (nextp == sc->num_rx_desc)
		nextp = 0;
	nxbuf = &rxr->rx_buffers[nextp];
	/* prefetch(nxbuf); */
}

/*
 * Rather than using the fmp/lmp global pointers
 * we now keep the head of a packet chain in the
 * buffer struct and pass this along from one
 * descriptor to the next, until we get EOP.
 */
mp->m_lenm_hdr.mh_len = len;
/*
 * See if there is a stored head
 * that determines what we are
 */
sendmp = rxbuf->fmp;
rxbuf->buf = rxbuf->fmp = NULL((void *)0);

if (sendmp != NULL((void *)0)) /* secondary frag */
	sendmp->m_pkthdrM_dat.MH.MH_pkthdr.len += mp->m_lenm_hdr.mh_len;
else {
	/* first desc of a non-ps chain */
	sendmp = mp;
	sendmp->m_pkthdrM_dat.MH.MH_pkthdr.len = mp->m_lenm_hdr.mh_len;
3159#if NVLAN1 > 0
	if (staterr & IXGBE_RXD_STAT_VP0x08) {
		sendmp->m_pkthdrM_dat.MH.MH_pkthdr.ether_vtag = vtag;
		sendmp->m_flagsm_hdr.mh_flags |= M_VLANTAG0x0020;
	}
3164#endif
}

/* Pass the head pointer on */
if (eop == 0) {
	nxbuf->fmp = sendmp;
	sendmp = NULL((void *)0);
	mp->m_nextm_hdr.mh_next = nxbuf->buf;
} else { /* Sending this frame? */
	ixgbe_rx_checksum(staterr, sendmp, ptype);

	if (hashtype != IXGBE_RXDADV_RSSTYPE_NONE0x00000000) {
		sendmp->m_pkthdrM_dat.MH.MH_pkthdr.ph_flowid = hash;
		SET(sendmp->m_pkthdr.csum_flags, M_FLOWID)((sendmp->M_dat.MH.MH_pkthdr.csum_flags) |= (0x4000));
	}

	ml_enqueue(&ml, sendmp);
}
3182next_desc:
if_rxr_put(&rxr->rx_ring, 1)do { (&rxr->rx_ring)->rxr_alive -= (1); } while (0);
bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (dsize * i), (dsize), (0x01))
    dsize * i, dsize,(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (dsize * i), (dsize), (0x01))
    BUS_DMASYNC_PREREAD)(*(rxr->rxdma.dma_tag)->_dmamap_sync)((rxr->rxdma.dma_tag
), (rxr->rxdma.dma_map), (dsize * i), (dsize), (0x01));

/* Advance our pointers to the next descriptor. */
if (++i == sc->num_rx_desc)
	i = 0;
}
rxr->next_to_check = i;

if (ifiq_input(rxr->ifiq, &ml))
if_rxr_livelocked(&rxr->rx_ring);

if (!(staterr & IXGBE_RXD_STAT_DD0x01))
return FALSE0;

return TRUE1;
3201}

3203/*********************************************************************
*
*  Verify that the hardware indicated that the checksum is valid.
*  Inform the stack about the status of checksum so that stack
*  doesn't spend time verifying the checksum.
*
*********************************************************************/
3210void
3211ixgbe_rx_checksum(uint32_t staterr, struct mbuf * mp, uint32_t ptype)
3212{
uint16_t status = (uint16_t) staterr;
uint8_t  errors = (uint8_t) (staterr >> 24);

if (status & IXGBE_RXD_STAT_IPCS0x40) {
if (!(errors & IXGBE_RXD_ERR_IPE0x80)) {
	/* IP Checksum Good */
	mp->m_pkthdrM_dat.MH.MH_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK0x0008;
} else
	mp->m_pkthdrM_dat.MH.MH_pkthdr.csum_flags = 0;
}
if (status & IXGBE_RXD_STAT_L4CS0x20) {
if (!(errors & IXGBE_RXD_ERR_TCPE0x40))
	mp->m_pkthdrM_dat.MH.MH_pkthdr.csum_flags |=
		M_TCP_CSUM_IN_OK0x0020 | M_UDP_CSUM_IN_OK0x0080;
}
3228}

3230void
3231ixgbe_setup_vlan_hw_support(struct ix_softc *sc)
3232{
uint32_t	ctrl;
int		i;

/*
* A soft reset zero's out the VFTA, so
* we need to repopulate it now.
*/
for (i = 0; i < IXGBE_VFTA_SIZE128; i++) {
if (sc->shadow_vfta[i] != 0)
	IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTA(i),((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x0A000 + ((i) * 4))), (sc->shadow_vfta[
i])))
	    sc->shadow_vfta[i])((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x0A000 + ((i) * 4))), (sc->shadow_vfta[
i])));
}

ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x05088)));
3247#if 0
/* Enable the Filter Table if enabled */
if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
ctrl &= ~IXGBE_VLNCTRL_CFIEN0x20000000;
ctrl |= IXGBE_VLNCTRL_VFE0x40000000;
}
3253#endif
if (sc->hw.mac.type == ixgbe_mac_82598EB)
ctrl |= IXGBE_VLNCTRL_VME0x80000000;
IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x05088), (ctrl)));

/* On 82599 the VLAN enable is per/queue in RXDCTL */
if (sc->hw.mac.type != ixgbe_mac_82598EB) {
for (i = 0; i < sc->num_queues; i++) {
	ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i))((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((i) < 64) ? (0x01028 + ((i) * 0x40)) : (
0x0D028 + (((i) - 64) * 0x40))))));
	ctrl |= IXGBE_RXDCTL_VME0x40000000;
	IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), ctrl)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((i) < 64) ? (0x01028 + ((i) * 0x40)) : (
0x0D028 + (((i) - 64) * 0x40)))), (ctrl)));
}
}
3266}

3268void
3269ixgbe_enable_intr(struct ix_softc *sc)
3270{
struct ixgbe_hw *hw = &sc->hw;
uint32_t	mask, fwsm;

mask = (IXGBE_EIMS_ENABLE_MASK( 0x0000FFFF | 0x00100000 | 0x40000000 | 0x80000000) & ~IXGBE_EIMS_RTX_QUEUE0x0000FFFF);
/* Enable Fan Failure detection */
if (hw->device_id == IXGBE_DEV_ID_82598AT0x10C8)
    mask |= IXGBE_EIMS_GPI_SDP10x02000000;

switch (sc->hw.mac.type) {
case ixgbe_mac_82599EB:
mask |= IXGBE_EIMS_ECC0x10000000;
/* Temperature sensor on some adapters */
mask |= IXGBE_EIMS_GPI_SDP00x01000000;
/* SFP+ (RX_LOS_N & MOD_ABS_N) */
mask |= IXGBE_EIMS_GPI_SDP10x02000000;
mask |= IXGBE_EIMS_GPI_SDP20x04000000;
break;
case ixgbe_mac_X540:
mask |= IXGBE_EIMS_ECC0x10000000;
/* Detect if Thermal Sensor is enabled */
fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x10148
)));
if (fwsm & IXGBE_FWSM_TS_ENABLED0x1)
	mask |= IXGBE_EIMS_TS0x00800000;
break;
case ixgbe_mac_X550:
case ixgbe_mac_X550EM_x:
case ixgbe_mac_X550EM_a:
mask |= IXGBE_EIMS_ECC0x10000000;
/* MAC thermal sensor is automatically enabled */
mask |= IXGBE_EIMS_TS0x00800000;
/* Some devices use SDP0 for important information */
if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP0x15AC ||
    hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T0x15AD)
	mask |= IXGBE_EIMS_GPI_SDP0_X5400x02000000;
default:
break;
}

IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00880
), (mask)));

/* With MSI-X we use auto clear */
if (sc->sc_intrmap) {
mask = IXGBE_EIMS_ENABLE_MASK( 0x0000FFFF | 0x00100000 | 0x40000000 | 0x80000000);
/* Don't autoclear Link */
mask &= ~IXGBE_EIMS_OTHER0x80000000;
mask &= ~IXGBE_EIMS_LSC0x00100000;
IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00810
), (mask)));
}

IXGBE_WRITE_FLUSH(hw)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00008
)));
3321}

3323void
3324ixgbe_disable_intr(struct ix_softc *sc)
3325{
if (sc->sc_intrmap)
IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00810), (0)));
if (sc->hw.mac.type == ixgbe_mac_82598EB) {
IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00888), (~0)));
} else {
IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00888), (0xFFFF0000)));
IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x00AB0 + (0) * 4)), (~0)));
IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x00AB0 + (1) * 4)), (~0)));
}
IXGBE_WRITE_FLUSH(&sc->hw)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->read_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), (0x00008)));
3336}

3338uint16_t
3339ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg)
3340{
struct pci_attach_args	*pa;
uint32_t value;
int high = 0;

if (reg & 0x2) {
high = 1;
reg &= ~0x2;
}
pa = &((struct ixgbe_osdep *)hw->back)->os_pa;
value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg);

if (high)
value >>= 16;

return (value & 0xffff);
3356}

3358void
3359ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value)
3360{
struct pci_attach_args	*pa;
uint32_t rv;
int high = 0;

/* Need to do read/mask/write... because 16 vs 32 bit!!! */
if (reg & 0x2) {
high = 1;
reg &= ~0x2;
}
pa = &((struct ixgbe_osdep *)hw->back)->os_pa;
rv = pci_conf_read(pa->pa_pc, pa->pa_tag, reg);
if (!high)
rv = (rv & 0xffff0000) | value;
else
rv = (rv & 0xffff) | ((uint32_t)value << 16);
pci_conf_write(pa->pa_pc, pa->pa_tag, reg, rv);
3377}

3379/*
* Setup the correct IVAR register for a particular MSIX interrupt
*   (yes this is all very magic and confusing :)
*  - entry is the register array entry
*  - vector is the MSIX vector for this queue
*  - type is RX/TX/MISC
*/
3386void
3387ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type)
3388{
struct ixgbe_hw *hw = &sc->hw;
uint32_t ivar, index;

vector |= IXGBE_IVAR_ALLOC_VAL0x80;

switch (hw->mac.type) {

case ixgbe_mac_82598EB:
if (type == -1)
	entry = IXGBE_IVAR_OTHER_CAUSES_INDEX97;
else
	entry += (type * 64);
index = (entry >> 2) & 0x1F;
ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x00900
 + ((index) * 4)))));
ivar &= ~(0xFF << (8 * (entry & 0x3)));
ivar |= (vector << (8 * (entry & 0x3)));
IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x00900 + ((index) * 4))), (ivar)));
break;

case ixgbe_mac_82599EB:
case ixgbe_mac_X540:
case ixgbe_mac_X550:
case ixgbe_mac_X550EM_x:
case ixgbe_mac_X550EM_a:
if (type == -1) { /* MISC IVAR */
	index = (entry & 1) * 8;
	ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00A00
)));
	ivar &= ~(0xFF << index);
	ivar |= (vector << index);
	IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x00A00
), (ivar)));
} else {	/* RX/TX IVARS */
	index = (16 * (entry & 1)) + (8 * type);
	ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x00900
 + ((entry >> 1) * 4)))));
	ivar &= ~(0xFF << index);
	ivar |= (vector << index);
	IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->write_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x00900
 + ((entry >> 1) * 4))), (ivar)));
}

default:
break;
}
3430}

3432void
3433ixgbe_configure_ivars(struct ix_softc *sc)
3434{
struct ix_queue *que = sc->queues;
uint32_t newitr;
int i;

newitr = (4000000 / IXGBE_INTS_PER_SEC8000) & 0x0FF8;

for (i = 0; i < sc->num_queues; i++, que++) {
/* First the RX queue entry */
ixgbe_set_ivar(sc, i, que->msix, 0);
/* ... and the TX */
ixgbe_set_ivar(sc, i, que->msix, 1);
/* Set an Initial EITR value */
IXGBE_WRITE_REG(&sc->hw,((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((que->msix) <= 23) ? (0x00820 + ((que
->msix) * 4)) : (0x012300 + (((que->msix) - 24) * 4))))
, (newitr)))
    IXGBE_EITR(que->msix), newitr)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((((que->msix) <= 23) ? (0x00820 + ((que
->msix) * 4)) : (0x012300 + (((que->msix) - 24) * 4))))
, (newitr)));
}

/* For the Link interrupt */
ixgbe_set_ivar(sc, 1, sc->linkvec, -1);
3453}

3455/*
* SFP module interrupts handler
*/
3458void
3459ixgbe_handle_mod(struct ix_softc *sc)
3460{
struct ixgbe_hw *hw = &sc->hw;
uint32_t err;

err = hw->phy.ops.identify_sfp(hw);
if (err == IXGBE_ERR_SFP_NOT_SUPPORTED-19) {
printf("%s: Unsupported SFP+ module type was detected!\n",
    sc->dev.dv_xname);
return;
}
err = hw->mac.ops.setup_sfp(hw);
if (err == IXGBE_ERR_SFP_NOT_SUPPORTED-19) {
printf("%s: Setup failure - unsupported SFP+ module type!\n",
    sc->dev.dv_xname);
return;
}

ixgbe_handle_msf(sc);
3478}


3481/*
* MSF (multispeed fiber) interrupts handler
*/
3484void
3485ixgbe_handle_msf(struct ix_softc *sc)
3486{
struct ixgbe_hw *hw = &sc->hw;
uint32_t autoneg;
bool_Bool negotiate;

autoneg = hw->phy.autoneg_advertised;
if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) {
if (hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate))
	return;
}
if (hw->mac.ops.setup_link)
hw->mac.ops.setup_link(hw, autoneg, TRUE1);

ifmedia_delete_instance(&sc->media, IFM_INST_ANY((uint64_t) -1));
ixgbe_add_media_types(sc);
ifmedia_set(&sc->media, IFM_ETHER0x0000000000000100ULL | IFM_AUTO0ULL);
3502}

3504/*
* External PHY interrupts handler
*/
3507void
3508ixgbe_handle_phy(struct ix_softc *sc)
3509{
struct ixgbe_hw *hw = &sc->hw;
int error;

error = hw->phy.ops.handle_lasi(hw);
if (error == IXGBE_ERR_OVERTEMP-26)
printf("%s: CRITICAL: EXTERNAL PHY OVER TEMP!! "
    " PHY will downshift to lower power state!\n",
    sc->dev.dv_xname);
else if (error)
printf("%s: Error handling LASI interrupt: %d\n",
    sc->dev.dv_xname, error);

3522}

3524#if NKSTAT0 > 0
3525enum ix_counter_idx {
ix_counter_crcerrs,
ix_counter_lxontxc,
ix_counter_lxonrxc,
ix_counter_lxofftxc,
ix_counter_lxoffrxc,
ix_counter_prc64,
ix_counter_prc127,
ix_counter_prc255,
ix_counter_prc511,
ix_counter_prc1023,
ix_counter_prc1522,
ix_counter_gptc,
ix_counter_gorc,
ix_counter_gotc,
ix_counter_ruc,
ix_counter_rfc,
ix_counter_roc,
ix_counter_rjc,
ix_counter_tor,
ix_counter_tpr,
ix_counter_tpt,
ix_counter_gprc,
ix_counter_bprc,
ix_counter_mprc,
ix_counter_ptc64,
ix_counter_ptc127,
ix_counter_ptc255,
ix_counter_ptc511,
ix_counter_ptc1023,
ix_counter_ptc1522,
ix_counter_mptc,
ix_counter_bptc,

ix_counter_num,
3560};

3562CTASSERT(KSTAT_KV_U_PACKETS <= 0xff)extern char _ctassert[(KSTAT_KV_U_PACKETS <= 0xff) ? 1 : -
] __attribute__((__unused__));
3563CTASSERT(KSTAT_KV_U_BYTES <= 0xff)extern char _ctassert[(KSTAT_KV_U_BYTES <= 0xff) ? 1 : -1 ]
 __attribute__((__unused__));

3565struct ix_counter {
char			 name[KSTAT_KV_NAMELEN16];
uint32_t		 reg;
uint8_t			 width;
uint8_t			 unit;
3570};

3572static const struct ix_counter ix_counters[ix_counter_num] = {
[ix_counter_crcerrs] = {	"crc errs",	IXGBE_CRCERRS0x04000,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_lxontxc] = {	"tx link xon",	IXGBE_LXONTXC0x03F60,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_lxonrxc] = {	"rx link xon",	0,		32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_lxofftxc] = {	"tx link xoff",	IXGBE_LXOFFTXC0x03F68,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_lxoffrxc] = {	"rx link xoff",	0,		32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_prc64] = {		"rx 64B",	IXGBE_PRC640x0405C,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_prc127] = {		"rx 65-127B",	IXGBE_PRC1270x04060,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_prc255] = {		"rx 128-255B",	IXGBE_PRC2550x04064,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_prc511] = {		"rx 256-511B",	IXGBE_PRC5110x04068,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_prc1023] = {	"rx 512-1023B",	IXGBE_PRC10230x0406C,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_prc1522] = {	"rx 1024-maxB",	IXGBE_PRC15220x04070,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_gptc] = {		"tx good",	IXGBE_GPTC0x04080,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_gorc] = {		"rx good",	IXGBE_GORCL0x04088,	36,
			    KSTAT_KV_U_BYTES },
[ix_counter_gotc] = {		"tx good",	IXGBE_GOTCL0x04090,	36,
			    KSTAT_KV_U_BYTES },
[ix_counter_ruc] = {		"rx undersize",	IXGBE_RUC0x040A4,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_rfc] = {		"rx fragment",	IXGBE_RFC0x040A8,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_roc] = {		"rx oversize",	IXGBE_ROC0x040AC,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_rjc] = {		"rx jabber",	IXGBE_RJC0x040B0,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_tor] = {		"rx total",	IXGBE_TORL0x040C0,	36,
			    KSTAT_KV_U_BYTES },
[ix_counter_tpr] = {		"rx total",	IXGBE_TPR0x040D0,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_tpt] = {		"tx total",	IXGBE_TPT0x040D4,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_gprc] = {		"rx good",	IXGBE_GPRC0x04074,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_bprc] = {		"rx bcast",	IXGBE_BPRC0x04078,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_mprc] = {		"rx mcast",	IXGBE_MPRC0x0407C,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_ptc64] = {		"tx 64B",	IXGBE_PTC640x040D8,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_ptc127] = {		"tx 65-127B",	IXGBE_PTC1270x040DC,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_ptc255] = {		"tx 128-255B",	IXGBE_PTC2550x040E0,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_ptc511] = {		"tx 256-511B",	IXGBE_PTC5110x040E4,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_ptc1023] = {	"tx 512-1023B",	IXGBE_PTC10230x040E8,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_ptc1522] = {	"tx 1024-maxB",	IXGBE_PTC15220x040EC,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_mptc] = {		"tx mcast",	IXGBE_MPTC0x040F0,	32,
			    KSTAT_KV_U_PACKETS },
[ix_counter_bptc] = {		"tx bcast",	IXGBE_BPTC0x040F4,	32,
			    KSTAT_KV_U_PACKETS },
3637};

3639struct ix_rxq_kstats {
struct kstat_kv	qprc;
struct kstat_kv	qbrc;
struct kstat_kv	qprdc;
3643};

3645static const struct ix_rxq_kstats ix_rxq_kstats_tpl = {
KSTAT_KV_UNIT_INITIALIZER("packets",{ .kv_key = ("packets"), .kv_type = (KSTAT_KV_T_COUNTER64), .
kv_unit = (KSTAT_KV_U_PACKETS), }
   KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS){ .kv_key = ("packets"), .kv_type = (KSTAT_KV_T_COUNTER64), .
kv_unit = (KSTAT_KV_U_PACKETS), },
KSTAT_KV_UNIT_INITIALIZER("bytes",{ .kv_key = ("bytes"), .kv_type = (KSTAT_KV_T_COUNTER64), .kv_unit
 = (KSTAT_KV_U_BYTES), }
   KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES){ .kv_key = ("bytes"), .kv_type = (KSTAT_KV_T_COUNTER64), .kv_unit
 = (KSTAT_KV_U_BYTES), },
KSTAT_KV_UNIT_INITIALIZER("qdrops",{ .kv_key = ("qdrops"), .kv_type = (KSTAT_KV_T_COUNTER64), .kv_unit
 = (KSTAT_KV_U_PACKETS), }
   KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS){ .kv_key = ("qdrops"), .kv_type = (KSTAT_KV_T_COUNTER64), .kv_unit
 = (KSTAT_KV_U_PACKETS), },
3652};

3654struct ix_txq_kstats {
struct kstat_kv	qptc;
struct kstat_kv	qbtc;
3657};

3659static const struct ix_txq_kstats ix_txq_kstats_tpl = {
KSTAT_KV_UNIT_INITIALIZER("packets",{ .kv_key = ("packets"), .kv_type = (KSTAT_KV_T_COUNTER64), .
kv_unit = (KSTAT_KV_U_PACKETS), }
   KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS){ .kv_key = ("packets"), .kv_type = (KSTAT_KV_T_COUNTER64), .
kv_unit = (KSTAT_KV_U_PACKETS), },
KSTAT_KV_UNIT_INITIALIZER("bytes",{ .kv_key = ("bytes"), .kv_type = (KSTAT_KV_T_COUNTER64), .kv_unit
 = (KSTAT_KV_U_BYTES), }
   KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES){ .kv_key = ("bytes"), .kv_type = (KSTAT_KV_T_COUNTER64), .kv_unit
 = (KSTAT_KV_U_BYTES), },
3664};

3666static int	ix_kstats_read(struct kstat *ks);
3667static int	ix_rxq_kstats_read(struct kstat *ks);
3668static int	ix_txq_kstats_read(struct kstat *ks);

3670static void
3671ix_kstats(struct ix_softc *sc)
3672{
struct kstat *ks;
struct kstat_kv *kvs;
unsigned int i;

mtx_init(&sc->sc_kstat_mtx, IPL_SOFTCLOCK)do { (void)(((void *)0)); (void)(0); __mtx_init((&sc->
sc_kstat_mtx), ((((0x4)) > 0x0 && ((0x4)) < 0x9
) ? 0x9 : ((0x4)))); } while (0);
timeout_set(&sc->sc_kstat_tmo, ix_kstats_tick, sc);

ks = kstat_create(sc->dev.dv_xname, 0, "ix-stats", 0,
   KSTAT_T_KV1, 0);
if (ks == NULL((void *)0))
return;

kvs = mallocarray(nitems(ix_counters)(sizeof((ix_counters)) / sizeof((ix_counters)[0])), sizeof(*kvs),
   M_DEVBUF2, M_WAITOK0x0001|M_ZERO0x0008);

for (i = 0; i < nitems(ix_counters)(sizeof((ix_counters)) / sizeof((ix_counters)[0])); i++) {
const struct ix_counter *ixc = &ix_counters[i];

kstat_kv_unit_init(&kvs[i], ixc->name,
    KSTAT_KV_T_COUNTER64, ixc->unit);
}

kstat_set_mutex(ks, &sc->sc_kstat_mtx);
ks->ks_softc = sc;
ks->ks_data = kvs;
ks->ks_datalen = nitems(ix_counters)(sizeof((ix_counters)) / sizeof((ix_counters)[0])) * sizeof(*kvs);
ks->ks_read = ix_kstats_read;

sc->sc_kstat = ks;
kstat_install(ks);
3703}

3705static void
3706ix_rxq_kstats(struct ix_softc *sc, struct rx_ring *rxr)
3707{
struct ix_rxq_kstats *stats;
struct kstat *ks;

ks = kstat_create(sc->dev.dv_xname, 0, "ix-rxq", rxr->me,
   KSTAT_T_KV1, 0);
if (ks == NULL((void *)0))
return;

stats = malloc(sizeof(*stats), M_DEVBUF2, M_WAITOK0x0001|M_ZERO0x0008);
*stats = ix_rxq_kstats_tpl;

kstat_set_mutex(ks, &sc->sc_kstat_mtx);
ks->ks_softc = rxr;
ks->ks_data = stats;
ks->ks_datalen = sizeof(*stats);
ks->ks_read = ix_rxq_kstats_read;

rxr->kstat = ks;
kstat_install(ks);
3727}

3729static void
3730ix_txq_kstats(struct ix_softc *sc, struct tx_ring *txr)
3731{
struct ix_txq_kstats *stats;
struct kstat *ks;

ks = kstat_create(sc->dev.dv_xname, 0, "ix-txq", txr->me,
   KSTAT_T_KV1, 0);
if (ks == NULL((void *)0))
return;

stats = malloc(sizeof(*stats), M_DEVBUF2, M_WAITOK0x0001|M_ZERO0x0008);
*stats = ix_txq_kstats_tpl;

kstat_set_mutex(ks, &sc->sc_kstat_mtx);
ks->ks_softc = txr;
ks->ks_data = stats;
ks->ks_datalen = sizeof(*stats);
ks->ks_read = ix_txq_kstats_read;

txr->kstat = ks;
kstat_install(ks);
3751}

3753/**********************************************************************
*
*  Update the board statistics counters.
*
**********************************************************************/

3759static void
3760ix_kstats_tick(void *arg)
3761{
struct ix_softc *sc = arg;
int i;

timeout_add_sec(&sc->sc_kstat_tmo, 1);

mtx_enter(&sc->sc_kstat_mtx);
ix_kstats_read(sc->sc_kstat);
for (i = 0; i < sc->num_queues; i++) {
ix_rxq_kstats_read(sc->rx_rings[i].kstat);
ix_txq_kstats_read(sc->tx_rings[i].kstat);
}
mtx_leave(&sc->sc_kstat_mtx);
3774}

3776static uint64_t
3777ix_read36(struct ixgbe_hw *hw, bus_size_t loreg, bus_size_t hireg)
3778{
uint64_t lo, hi;

lo = IXGBE_READ_REG(hw, loreg)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (loreg)
));
hi = IXGBE_READ_REG(hw, hireg)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (hireg)
));

return (((hi & 0xf) << 32) | lo);
3785}

3787static int
3788ix_kstats_read(struct kstat *ks)
3789{
struct ix_softc *sc = ks->ks_softc;
struct kstat_kv *kvs = ks->ks_data;
struct ixgbe_hw	*hw = &sc->hw;
unsigned int i;

for (i = 0; i < nitems(ix_counters)(sizeof((ix_counters)) / sizeof((ix_counters)[0])); i++) {
const struct ix_counter *ixc = &ix_counters[i];
uint32_t reg = ixc->reg;
uint64_t v;

if (reg == 0)
	continue;

if (ixc->width > 32) {
	if (sc->hw.mac.type == ixgbe_mac_82598EB)
		v = IXGBE_READ_REG(hw, reg + 4)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (reg + 4
)));
	else
		v = ix_read36(hw, reg, reg + 4);
} else
	v = IXGBE_READ_REG(hw, reg)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (reg)));

kstat_kv_u64(&kvs[i])(&kvs[i])->kv_v.v_u64 += v;
}

/* handle the exceptions */
if (sc->hw.mac.type == ixgbe_mac_82598EB) {
kstat_kv_u64(&kvs[ix_counter_lxonrxc])(&kvs[ix_counter_lxonrxc])->kv_v.v_u64 += 
    IXGBE_READ_REG(hw, IXGBE_LXONRXC)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x0CF60
)));
kstat_kv_u64(&kvs[ix_counter_lxoffrxc])(&kvs[ix_counter_lxoffrxc])->kv_v.v_u64 +=
    IXGBE_READ_REG(hw, IXGBE_LXOFFRXC)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x0CF68
)));
} else {
kstat_kv_u64(&kvs[ix_counter_lxonrxc])(&kvs[ix_counter_lxonrxc])->kv_v.v_u64 += 
    IXGBE_READ_REG(hw, IXGBE_LXONRXCNT)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x041A4
)));
kstat_kv_u64(&kvs[ix_counter_lxoffrxc])(&kvs[ix_counter_lxoffrxc])->kv_v.v_u64 +=
    IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT)((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), (0x041A8
)));
}

getnanouptime(&ks->ks_updated);

return (0);
3830}

3832int
3833ix_rxq_kstats_read(struct kstat *ks)
3834{
struct ix_rxq_kstats *stats = ks->ks_data;
struct rx_ring *rxr = ks->ks_softc;
struct ix_softc *sc = rxr->sc;
struct ixgbe_hw	*hw = &sc->hw;
uint32_t i = rxr->me;

kstat_kv_u64(&stats->qprc)(&stats->qprc)->kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_QPRC(i))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x01030
 + ((i) * 0x40)))));
if (sc->hw.mac.type == ixgbe_mac_82598EB) {
kstat_kv_u64(&stats->qprdc)(&stats->qprdc)->kv_v.v_u64 +=
    IXGBE_READ_REG(hw, IXGBE_RNBC(i))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x03FC0
 + ((i) * 4)))));
kstat_kv_u64(&stats->qbrc)(&stats->qbrc)->kv_v.v_u64 +=
    IXGBE_READ_REG(hw, IXGBE_QBRC(i))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x01034
 + ((i) * 0x40)))));
} else {
kstat_kv_u64(&stats->qprdc)(&stats->qprdc)->kv_v.v_u64 +=
    IXGBE_READ_REG(hw, IXGBE_QPRDC(i))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x01430
 + ((i) * 0x40)))));
kstat_kv_u64(&stats->qbrc)(&stats->qbrc)->kv_v.v_u64 +=
    ix_read36(hw, IXGBE_QBRC_L(i)(0x01034 + ((i) * 0x40)), IXGBE_QBRC_H(i)(0x01038 + ((i) * 0x40)));
}

getnanouptime(&ks->ks_updated);

return (0);
3857}

3859int
3860ix_txq_kstats_read(struct kstat *ks)
3861{
struct ix_txq_kstats *stats = ks->ks_data;
struct tx_ring *txr = ks->ks_softc;
struct ix_softc *sc = txr->sc;
struct ixgbe_hw	*hw = &sc->hw;
uint32_t i = txr->me;

kstat_kv_u64(&stats->qptc)(&stats->qptc)->kv_v.v_u64 += IXGBE_READ_REG(hw, IXGBE_QPTC(i))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x06030
 + ((i) * 0x40)))));
if (sc->hw.mac.type == ixgbe_mac_82598EB) {
kstat_kv_u64(&stats->qbtc)(&stats->qbtc)->kv_v.v_u64 +=
    IXGBE_READ_REG(hw, IXGBE_QBTC(i))((((struct ixgbe_osdep *)(hw)->back)->os_memt)->read_4
((((struct ixgbe_osdep *)(hw)->back)->os_memh), ((0x06034
 + ((i) * 0x40)))));
} else {
kstat_kv_u64(&stats->qbtc)(&stats->qbtc)->kv_v.v_u64 +=
    ix_read36(hw, IXGBE_QBTC_L(i)(0x08700 + ((i) * 0x8)), IXGBE_QBTC_H(i)(0x08704 + ((i) * 0x8)));
}

getnanouptime(&ks->ks_updated);

return (0);
3880}
3881#endif /* NKVSTAT > 0 */

3883void
3884ixgbe_map_queue_statistics(struct ix_softc *sc)
3885{
int i;
uint32_t r;

for (i = 0; i < 32; i++) {
/*
 * Queues 0-15 are mapped 1:1
 * Queue 0 -> Counter 0
 * Queue 1 -> Counter 1
 * Queue 2 -> Counter 2....
 * Queues 16-127 are mapped to Counter 0
 */
if (i < 4) {
	r = (i * 4 + 0);
	r |= (i * 4 + 1) << 8;
	r |= (i * 4 + 2) << 16;
	r |= (i * 4 + 3) << 24;
} else
	r = 0;

IXGBE_WRITE_REG(&sc->hw, IXGBE_RQSMR(i), r)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x02300 + ((i) * 4))), (r)));
IXGBE_WRITE_REG(&sc->hw, IXGBE_TQSM(i), r)((((struct ixgbe_osdep *)(&sc->hw)->back)->os_memt
)->write_4((((struct ixgbe_osdep *)(&sc->hw)->back
)->os_memh), ((0x08600 + ((i) * 4))), (r)));
}
3908}