1171095Ssam/*- 2171095Ssam * Copyright (c) 2002-2007 Neterion, Inc. 3171095Ssam * All rights reserved. 4171095Ssam * 5171095Ssam * Redistribution and use in source and binary forms, with or without 6171095Ssam * modification, are permitted provided that the following conditions 7171095Ssam * are met: 8171095Ssam * 1. Redistributions of source code must retain the above copyright 9171095Ssam * notice, this list of conditions and the following disclaimer. 10171095Ssam * 2. Redistributions in binary form must reproduce the above copyright 11171095Ssam * notice, this list of conditions and the following disclaimer in the 12171095Ssam * documentation and/or other materials provided with the distribution. 13171095Ssam * 14171095Ssam * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15171095Ssam * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16171095Ssam * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17171095Ssam * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18171095Ssam * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19171095Ssam * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20171095Ssam * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21171095Ssam * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22171095Ssam * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23171095Ssam * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24171095Ssam * SUCH DAMAGE. 25171095Ssam * 26171095Ssam * $FreeBSD$ 27171095Ssam */ 28171095Ssam 29171095Ssam#include <dev/nxge/if_nxge.h> 30171095Ssam#include <dev/nxge/xge-osdep.h> 31171095Ssam#include <net/if_arp.h> 32171095Ssam#include <sys/types.h> 33171095Ssam#include <net/if.h> 34171095Ssam#include <net/if_vlan_var.h> 35171095Ssam 36171095Ssamint copyright_print = 0; 37171095Ssamint hal_driver_init_count = 0; 38171095Ssamsize_t size = sizeof(int); 39171095Ssam 40173139Srwatsonstatic void inline xge_flush_txds(xge_hal_channel_h); 41173139Srwatson 42173139Srwatson/** 43171095Ssam * xge_probe 44173139Srwatson * Probes for Xframe devices 45173139Srwatson * 46173139Srwatson * @dev Device handle 47173139Srwatson * 48173139Srwatson * Returns 49173139Srwatson * BUS_PROBE_DEFAULT if device is supported 50173139Srwatson * ENXIO if device is not supported 51173139Srwatson */ 52171095Ssamint 53171095Ssamxge_probe(device_t dev) 54171095Ssam{ 55171095Ssam int devid = pci_get_device(dev); 56171095Ssam int vendorid = pci_get_vendor(dev); 57171095Ssam int retValue = ENXIO; 58171095Ssam 59171095Ssam if(vendorid == XGE_PCI_VENDOR_ID) { 60171095Ssam if((devid == XGE_PCI_DEVICE_ID_XENA_2) || 61171095Ssam (devid == XGE_PCI_DEVICE_ID_HERC_2)) { 62171095Ssam if(!copyright_print) { 63173139Srwatson xge_os_printf(XGE_COPYRIGHT); 64171095Ssam copyright_print = 1; 65171095Ssam } 66171095Ssam device_set_desc_copy(dev, 67171095Ssam "Neterion Xframe 10 Gigabit Ethernet Adapter"); 68171095Ssam retValue = BUS_PROBE_DEFAULT; 69171095Ssam } 70171095Ssam } 71171095Ssam 72171095Ssam return retValue; 73171095Ssam} 74171095Ssam 75173139Srwatson/** 76171095Ssam * xge_init_params 77173139Srwatson * Sets HAL parameter values (from kenv). 78173139Srwatson * 79173139Srwatson * @dconfig Device Configuration 80173139Srwatson * @dev Device Handle 81173139Srwatson */ 82171095Ssamvoid 83171095Ssamxge_init_params(xge_hal_device_config_t *dconfig, device_t dev) 84171095Ssam{ 85173139Srwatson int qindex, tindex, revision; 86171095Ssam device_t checkdev; 87173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev); 88171095Ssam 89171095Ssam dconfig->mtu = XGE_DEFAULT_INITIAL_MTU; 90171095Ssam dconfig->pci_freq_mherz = XGE_DEFAULT_USER_HARDCODED; 91171095Ssam dconfig->device_poll_millis = XGE_HAL_DEFAULT_DEVICE_POLL_MILLIS; 92171095Ssam dconfig->link_stability_period = XGE_HAL_DEFAULT_LINK_STABILITY_PERIOD; 93171095Ssam dconfig->mac.rmac_bcast_en = XGE_DEFAULT_MAC_RMAC_BCAST_EN; 94171095Ssam dconfig->fifo.alignment_size = XGE_DEFAULT_FIFO_ALIGNMENT_SIZE; 95171095Ssam 96173139Srwatson XGE_GET_PARAM("hw.xge.enable_tso", (*lldev), enabled_tso, 97173139Srwatson XGE_DEFAULT_ENABLED_TSO); 98173139Srwatson XGE_GET_PARAM("hw.xge.enable_lro", (*lldev), enabled_lro, 99173139Srwatson XGE_DEFAULT_ENABLED_LRO); 100173139Srwatson XGE_GET_PARAM("hw.xge.enable_msi", (*lldev), enabled_msi, 101173139Srwatson XGE_DEFAULT_ENABLED_MSI); 102173139Srwatson 103173139Srwatson XGE_GET_PARAM("hw.xge.latency_timer", (*dconfig), latency_timer, 104171095Ssam XGE_DEFAULT_LATENCY_TIMER); 105173139Srwatson XGE_GET_PARAM("hw.xge.max_splits_trans", (*dconfig), max_splits_trans, 106171095Ssam XGE_DEFAULT_MAX_SPLITS_TRANS); 107173139Srwatson XGE_GET_PARAM("hw.xge.mmrb_count", (*dconfig), mmrb_count, 108171095Ssam XGE_DEFAULT_MMRB_COUNT); 109173139Srwatson XGE_GET_PARAM("hw.xge.shared_splits", (*dconfig), shared_splits, 110171095Ssam XGE_DEFAULT_SHARED_SPLITS); 111173139Srwatson XGE_GET_PARAM("hw.xge.isr_polling_cnt", (*dconfig), isr_polling_cnt, 112171095Ssam XGE_DEFAULT_ISR_POLLING_CNT); 113173139Srwatson XGE_GET_PARAM("hw.xge.stats_refresh_time_sec", (*dconfig), 114171095Ssam stats_refresh_time_sec, XGE_DEFAULT_STATS_REFRESH_TIME_SEC); 115171095Ssam 116173139Srwatson XGE_GET_PARAM_MAC("hw.xge.mac_tmac_util_period", tmac_util_period, 117171095Ssam XGE_DEFAULT_MAC_TMAC_UTIL_PERIOD); 118173139Srwatson XGE_GET_PARAM_MAC("hw.xge.mac_rmac_util_period", rmac_util_period, 119171095Ssam XGE_DEFAULT_MAC_RMAC_UTIL_PERIOD); 120173139Srwatson XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_gen_en", rmac_pause_gen_en, 121171095Ssam XGE_DEFAULT_MAC_RMAC_PAUSE_GEN_EN); 122173139Srwatson XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_rcv_en", rmac_pause_rcv_en, 123171095Ssam XGE_DEFAULT_MAC_RMAC_PAUSE_RCV_EN); 124173139Srwatson XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_time", rmac_pause_time, 125171095Ssam XGE_DEFAULT_MAC_RMAC_PAUSE_TIME); 126173139Srwatson XGE_GET_PARAM_MAC("hw.xge.mac_mc_pause_threshold_q0q3", 127171095Ssam mc_pause_threshold_q0q3, XGE_DEFAULT_MAC_MC_PAUSE_THRESHOLD_Q0Q3); 128173139Srwatson XGE_GET_PARAM_MAC("hw.xge.mac_mc_pause_threshold_q4q7", 129171095Ssam mc_pause_threshold_q4q7, XGE_DEFAULT_MAC_MC_PAUSE_THRESHOLD_Q4Q7); 130171095Ssam 131173139Srwatson XGE_GET_PARAM_FIFO("hw.xge.fifo_memblock_size", memblock_size, 132171095Ssam XGE_DEFAULT_FIFO_MEMBLOCK_SIZE); 133173139Srwatson XGE_GET_PARAM_FIFO("hw.xge.fifo_reserve_threshold", reserve_threshold, 134171095Ssam XGE_DEFAULT_FIFO_RESERVE_THRESHOLD); 135173139Srwatson XGE_GET_PARAM_FIFO("hw.xge.fifo_max_frags", max_frags, 136171095Ssam XGE_DEFAULT_FIFO_MAX_FRAGS); 137171095Ssam 138173139Srwatson for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) { 139173139Srwatson XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_intr", intr, qindex, 140173139Srwatson XGE_DEFAULT_FIFO_QUEUE_INTR); 141173139Srwatson XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_max", max, qindex, 142173139Srwatson XGE_DEFAULT_FIFO_QUEUE_MAX); 143173139Srwatson XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_initial", initial, 144173139Srwatson qindex, XGE_DEFAULT_FIFO_QUEUE_INITIAL); 145171095Ssam 146173139Srwatson for (tindex = 0; tindex < XGE_HAL_MAX_FIFO_TTI_NUM; tindex++) { 147173139Srwatson dconfig->fifo.queue[qindex].tti[tindex].enabled = 1; 148173139Srwatson dconfig->fifo.queue[qindex].configured = 1; 149171095Ssam 150173139Srwatson XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_a", 151173139Srwatson urange_a, qindex, tindex, 152173139Srwatson XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_A); 153173139Srwatson XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_b", 154173139Srwatson urange_b, qindex, tindex, 155173139Srwatson XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_B); 156173139Srwatson XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_c", 157173139Srwatson urange_c, qindex, tindex, 158173139Srwatson XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_C); 159173139Srwatson XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_a", 160173139Srwatson ufc_a, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_A); 161173139Srwatson XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_b", 162173139Srwatson ufc_b, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_B); 163173139Srwatson XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_c", 164173139Srwatson ufc_c, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_C); 165173139Srwatson XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_d", 166173139Srwatson ufc_d, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_D); 167173139Srwatson XGE_GET_PARAM_FIFO_QUEUE_TTI( 168173139Srwatson "hw.xge.fifo_queue_tti_timer_ci_en", timer_ci_en, qindex, 169173139Srwatson tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_CI_EN); 170173139Srwatson XGE_GET_PARAM_FIFO_QUEUE_TTI( 171173139Srwatson "hw.xge.fifo_queue_tti_timer_ac_en", timer_ac_en, qindex, 172173139Srwatson tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_AC_EN); 173173139Srwatson XGE_GET_PARAM_FIFO_QUEUE_TTI( 174173139Srwatson "hw.xge.fifo_queue_tti_timer_val_us", timer_val_us, qindex, 175173139Srwatson tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_VAL_US); 176173139Srwatson } 177171095Ssam } 178171095Ssam 179173139Srwatson XGE_GET_PARAM_RING("hw.xge.ring_memblock_size", memblock_size, 180171095Ssam XGE_DEFAULT_RING_MEMBLOCK_SIZE); 181173139Srwatson 182173139Srwatson XGE_GET_PARAM_RING("hw.xge.ring_strip_vlan_tag", strip_vlan_tag, 183171095Ssam XGE_DEFAULT_RING_STRIP_VLAN_TAG); 184171095Ssam 185173139Srwatson XGE_GET_PARAM("hw.xge.buffer_mode", (*lldev), buffer_mode, 186173139Srwatson XGE_DEFAULT_BUFFER_MODE); 187173139Srwatson if((lldev->buffer_mode < XGE_HAL_RING_QUEUE_BUFFER_MODE_1) || 188173139Srwatson (lldev->buffer_mode > XGE_HAL_RING_QUEUE_BUFFER_MODE_2)) { 189173139Srwatson xge_trace(XGE_ERR, "Supported buffer modes are 1 and 2"); 190173139Srwatson lldev->buffer_mode = XGE_HAL_RING_QUEUE_BUFFER_MODE_1; 191173139Srwatson } 192173139Srwatson 193173139Srwatson for (qindex = 0; qindex < XGE_RING_COUNT; qindex++) { 194173139Srwatson dconfig->ring.queue[qindex].max_frm_len = XGE_HAL_RING_USE_MTU; 195173139Srwatson dconfig->ring.queue[qindex].priority = 0; 196173139Srwatson dconfig->ring.queue[qindex].configured = 1; 197173139Srwatson dconfig->ring.queue[qindex].buffer_mode = 198173139Srwatson (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) ? 199173139Srwatson XGE_HAL_RING_QUEUE_BUFFER_MODE_3 : lldev->buffer_mode; 200173139Srwatson 201173139Srwatson XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_max", max, qindex, 202171095Ssam XGE_DEFAULT_RING_QUEUE_MAX); 203173139Srwatson XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_initial", initial, 204173139Srwatson qindex, XGE_DEFAULT_RING_QUEUE_INITIAL); 205173139Srwatson XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_dram_size_mb", 206173139Srwatson dram_size_mb, qindex, XGE_DEFAULT_RING_QUEUE_DRAM_SIZE_MB); 207173139Srwatson XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_indicate_max_pkts", 208173139Srwatson indicate_max_pkts, qindex, 209171095Ssam XGE_DEFAULT_RING_QUEUE_INDICATE_MAX_PKTS); 210173139Srwatson XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_backoff_interval_us", 211173139Srwatson backoff_interval_us, qindex, 212171095Ssam XGE_DEFAULT_RING_QUEUE_BACKOFF_INTERVAL_US); 213171095Ssam 214173139Srwatson XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_a", ufc_a, 215173139Srwatson qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_A); 216173139Srwatson XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_b", ufc_b, 217173139Srwatson qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_B); 218173139Srwatson XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_c", ufc_c, 219173139Srwatson qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_C); 220173139Srwatson XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_d", ufc_d, 221173139Srwatson qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_D); 222173139Srwatson XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_timer_ac_en", 223173139Srwatson timer_ac_en, qindex, XGE_DEFAULT_RING_QUEUE_RTI_TIMER_AC_EN); 224173139Srwatson XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_timer_val_us", 225173139Srwatson timer_val_us, qindex, XGE_DEFAULT_RING_QUEUE_RTI_TIMER_VAL_US); 226173139Srwatson XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_a", 227173139Srwatson urange_a, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_A); 228173139Srwatson XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_b", 229173139Srwatson urange_b, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_B); 230173139Srwatson XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_c", 231173139Srwatson urange_c, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_C); 232171095Ssam } 233171095Ssam 234171095Ssam if(dconfig->fifo.max_frags > (PAGE_SIZE/32)) { 235173139Srwatson xge_os_printf("fifo_max_frags = %d", dconfig->fifo.max_frags) 236171095Ssam xge_os_printf("fifo_max_frags should be <= (PAGE_SIZE / 32) = %d", 237173139Srwatson (int)(PAGE_SIZE / 32)) 238173139Srwatson xge_os_printf("Using fifo_max_frags = %d", (int)(PAGE_SIZE / 32)) 239171095Ssam dconfig->fifo.max_frags = (PAGE_SIZE / 32); 240171095Ssam } 241171095Ssam 242171095Ssam checkdev = pci_find_device(VENDOR_ID_AMD, DEVICE_ID_8131_PCI_BRIDGE); 243171095Ssam if(checkdev != NULL) { 244171095Ssam /* Check Revision for 0x12 */ 245173139Srwatson revision = pci_read_config(checkdev, 246171095Ssam xge_offsetof(xge_hal_pci_config_t, revision), 1); 247171095Ssam if(revision <= 0x12) { 248171095Ssam /* Set mmrb_count to 1k and max splits = 2 */ 249171095Ssam dconfig->mmrb_count = 1; 250171095Ssam dconfig->max_splits_trans = XGE_HAL_THREE_SPLIT_TRANSACTION; 251171095Ssam } 252171095Ssam } 253173139Srwatson} 254171095Ssam 255173139Srwatson/** 256173139Srwatson * xge_buffer_sizes_set 257173139Srwatson * Set buffer sizes based on Rx buffer mode 258173139Srwatson * 259173139Srwatson * @lldev Per-adapter Data 260173139Srwatson * @buffer_mode Rx Buffer Mode 261173139Srwatson */ 262173139Srwatsonvoid 263173139Srwatsonxge_rx_buffer_sizes_set(xge_lldev_t *lldev, int buffer_mode, int mtu) 264173139Srwatson{ 265173139Srwatson int index = 0; 266173139Srwatson int frame_header = XGE_HAL_MAC_HEADER_MAX_SIZE; 267173139Srwatson int buffer_size = mtu + frame_header; 268171095Ssam 269173139Srwatson xge_os_memzero(lldev->rxd_mbuf_len, sizeof(lldev->rxd_mbuf_len)); 270173139Srwatson 271173139Srwatson if(buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_5) 272173139Srwatson lldev->rxd_mbuf_len[buffer_mode - 1] = mtu; 273173139Srwatson 274173139Srwatson lldev->rxd_mbuf_len[0] = (buffer_mode == 1) ? buffer_size:frame_header; 275173139Srwatson 276173139Srwatson if(buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) 277173139Srwatson lldev->rxd_mbuf_len[1] = XGE_HAL_TCPIP_HEADER_MAX_SIZE; 278173139Srwatson 279173139Srwatson if(buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) { 280173139Srwatson index = 2; 281173139Srwatson buffer_size -= XGE_HAL_TCPIP_HEADER_MAX_SIZE; 282173139Srwatson while(buffer_size > MJUMPAGESIZE) { 283173139Srwatson lldev->rxd_mbuf_len[index++] = MJUMPAGESIZE; 284173139Srwatson buffer_size -= MJUMPAGESIZE; 285173139Srwatson } 286173139Srwatson XGE_ALIGN_TO(buffer_size, 128); 287173139Srwatson lldev->rxd_mbuf_len[index] = buffer_size; 288173139Srwatson lldev->rxd_mbuf_cnt = index + 1; 289173139Srwatson } 290173139Srwatson 291173139Srwatson for(index = 0; index < buffer_mode; index++) 292173139Srwatson xge_trace(XGE_TRACE, "Buffer[%d] %d\n", index, 293173139Srwatson lldev->rxd_mbuf_len[index]); 294171095Ssam} 295171095Ssam 296173139Srwatson/** 297173139Srwatson * xge_buffer_mode_init 298173139Srwatson * Init Rx buffer mode 299173139Srwatson * 300173139Srwatson * @lldev Per-adapter Data 301173139Srwatson * @mtu Interface MTU 302173139Srwatson */ 303173139Srwatsonvoid 304173139Srwatsonxge_buffer_mode_init(xge_lldev_t *lldev, int mtu) 305173139Srwatson{ 306173139Srwatson int index = 0, buffer_size = 0; 307173139Srwatson xge_hal_ring_config_t *ring_config = &((lldev->devh)->config.ring); 308173139Srwatson 309173139Srwatson buffer_size = mtu + XGE_HAL_MAC_HEADER_MAX_SIZE; 310173139Srwatson 311173139Srwatson if(lldev->enabled_lro) 312173139Srwatson (lldev->ifnetp)->if_capenable |= IFCAP_LRO; 313173139Srwatson else 314173139Srwatson (lldev->ifnetp)->if_capenable &= ~IFCAP_LRO; 315173139Srwatson 316173139Srwatson lldev->rxd_mbuf_cnt = lldev->buffer_mode; 317173139Srwatson if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) { 318173139Srwatson XGE_SET_BUFFER_MODE_IN_RINGS(XGE_HAL_RING_QUEUE_BUFFER_MODE_3); 319173139Srwatson ring_config->scatter_mode = XGE_HAL_RING_QUEUE_SCATTER_MODE_B; 320173139Srwatson } 321173139Srwatson else { 322173139Srwatson XGE_SET_BUFFER_MODE_IN_RINGS(lldev->buffer_mode); 323173139Srwatson ring_config->scatter_mode = XGE_HAL_RING_QUEUE_SCATTER_MODE_A; 324173139Srwatson } 325173139Srwatson xge_rx_buffer_sizes_set(lldev, lldev->buffer_mode, mtu); 326173139Srwatson 327173139Srwatson xge_os_printf("%s: TSO %s", device_get_nameunit(lldev->device), 328173139Srwatson ((lldev->enabled_tso) ? "Enabled":"Disabled")); 329173139Srwatson xge_os_printf("%s: LRO %s", device_get_nameunit(lldev->device), 330173139Srwatson ((lldev->ifnetp)->if_capenable & IFCAP_LRO) ? "Enabled":"Disabled"); 331173139Srwatson xge_os_printf("%s: Rx %d Buffer Mode Enabled", 332173139Srwatson device_get_nameunit(lldev->device), lldev->buffer_mode); 333173139Srwatson} 334173139Srwatson 335173139Srwatson/** 336171095Ssam * xge_driver_initialize 337173139Srwatson * Initializes HAL driver (common for all devices) 338173139Srwatson * 339173139Srwatson * Returns 340173139Srwatson * XGE_HAL_OK if success 341173139Srwatson * XGE_HAL_ERR_BAD_DRIVER_CONFIG if driver configuration parameters are invalid 342173139Srwatson */ 343171095Ssamint 344171095Ssamxge_driver_initialize(void) 345171095Ssam{ 346171095Ssam xge_hal_uld_cbs_t uld_callbacks; 347171095Ssam xge_hal_driver_config_t driver_config; 348171095Ssam xge_hal_status_e status = XGE_HAL_OK; 349171095Ssam 350171095Ssam /* Initialize HAL driver */ 351171095Ssam if(!hal_driver_init_count) { 352171095Ssam xge_os_memzero(&uld_callbacks, sizeof(xge_hal_uld_cbs_t)); 353173139Srwatson xge_os_memzero(&driver_config, sizeof(xge_hal_driver_config_t)); 354171095Ssam 355171095Ssam /* 356171095Ssam * Initial and maximum size of the queue used to store the events 357171095Ssam * like Link up/down (xge_hal_event_e) 358171095Ssam */ 359173139Srwatson driver_config.queue_size_initial = XGE_HAL_MIN_QUEUE_SIZE_INITIAL; 360173139Srwatson driver_config.queue_size_max = XGE_HAL_MAX_QUEUE_SIZE_MAX; 361171095Ssam 362173139Srwatson uld_callbacks.link_up = xge_callback_link_up; 363173139Srwatson uld_callbacks.link_down = xge_callback_link_down; 364173139Srwatson uld_callbacks.crit_err = xge_callback_crit_err; 365173139Srwatson uld_callbacks.event = xge_callback_event; 366171095Ssam 367171095Ssam status = xge_hal_driver_initialize(&driver_config, &uld_callbacks); 368171095Ssam if(status != XGE_HAL_OK) { 369173139Srwatson XGE_EXIT_ON_ERR("xgeX: Initialization of HAL driver failed", 370173139Srwatson xdi_out, status); 371171095Ssam } 372171095Ssam } 373171095Ssam hal_driver_init_count = hal_driver_init_count + 1; 374171095Ssam 375171095Ssam xge_hal_driver_debug_module_mask_set(0xffffffff); 376171095Ssam xge_hal_driver_debug_level_set(XGE_TRACE); 377171095Ssam 378171095Ssamxdi_out: 379171095Ssam return status; 380171095Ssam} 381171095Ssam 382173139Srwatson/** 383173139Srwatson * xge_media_init 384173139Srwatson * Initializes, adds and sets media 385173139Srwatson * 386173139Srwatson * @devc Device Handle 387173139Srwatson */ 388171095Ssamvoid 389171095Ssamxge_media_init(device_t devc) 390171095Ssam{ 391173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(devc); 392171095Ssam 393171095Ssam /* Initialize Media */ 394173139Srwatson ifmedia_init(&lldev->media, IFM_IMASK, xge_ifmedia_change, 395171095Ssam xge_ifmedia_status); 396171095Ssam 397171095Ssam /* Add supported media */ 398173139Srwatson ifmedia_add(&lldev->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 0, NULL); 399173139Srwatson ifmedia_add(&lldev->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 400173139Srwatson ifmedia_add(&lldev->media, IFM_ETHER | IFM_AUTO, 0, NULL); 401173139Srwatson ifmedia_add(&lldev->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 402173139Srwatson ifmedia_add(&lldev->media, IFM_ETHER | IFM_10G_LR, 0, NULL); 403171095Ssam 404171095Ssam /* Set media */ 405173139Srwatson ifmedia_set(&lldev->media, IFM_ETHER | IFM_AUTO); 406171095Ssam} 407171095Ssam 408173139Srwatson/** 409171095Ssam * xge_pci_space_save 410171095Ssam * Save PCI configuration space 411173139Srwatson * 412173139Srwatson * @dev Device Handle 413171095Ssam */ 414171095Ssamvoid 415171095Ssamxge_pci_space_save(device_t dev) 416171095Ssam{ 417171095Ssam struct pci_devinfo *dinfo = NULL; 418171095Ssam 419171095Ssam dinfo = device_get_ivars(dev); 420171095Ssam xge_trace(XGE_TRACE, "Saving PCI configuration space"); 421171095Ssam pci_cfg_save(dev, dinfo, 0); 422171095Ssam} 423171095Ssam 424173139Srwatson/** 425171095Ssam * xge_pci_space_restore 426171095Ssam * Restore saved PCI configuration space 427173139Srwatson * 428173139Srwatson * @dev Device Handle 429171095Ssam */ 430171095Ssamvoid 431171095Ssamxge_pci_space_restore(device_t dev) 432171095Ssam{ 433171095Ssam struct pci_devinfo *dinfo = NULL; 434171095Ssam 435171095Ssam dinfo = device_get_ivars(dev); 436171095Ssam xge_trace(XGE_TRACE, "Restoring PCI configuration space"); 437171095Ssam pci_cfg_restore(dev, dinfo); 438173139Srwatson} 439171095Ssam 440173139Srwatson/** 441173139Srwatson * xge_msi_info_save 442173139Srwatson * Save MSI info 443173139Srwatson * 444173139Srwatson * @lldev Per-adapter Data 445173139Srwatson */ 446173139Srwatsonvoid 447173139Srwatsonxge_msi_info_save(xge_lldev_t * lldev) 448173139Srwatson{ 449173139Srwatson xge_os_pci_read16(lldev->pdev, NULL, 450173139Srwatson xge_offsetof(xge_hal_pci_config_le_t, msi_control), 451173139Srwatson &lldev->msi_info.msi_control); 452173139Srwatson xge_os_pci_read32(lldev->pdev, NULL, 453173139Srwatson xge_offsetof(xge_hal_pci_config_le_t, msi_lower_address), 454173139Srwatson &lldev->msi_info.msi_lower_address); 455173139Srwatson xge_os_pci_read32(lldev->pdev, NULL, 456173139Srwatson xge_offsetof(xge_hal_pci_config_le_t, msi_higher_address), 457173139Srwatson &lldev->msi_info.msi_higher_address); 458173139Srwatson xge_os_pci_read16(lldev->pdev, NULL, 459173139Srwatson xge_offsetof(xge_hal_pci_config_le_t, msi_data), 460173139Srwatson &lldev->msi_info.msi_data); 461171095Ssam} 462171095Ssam 463173139Srwatson/** 464173139Srwatson * xge_msi_info_restore 465173139Srwatson * Restore saved MSI info 466173139Srwatson * 467173139Srwatson * @dev Device Handle 468173139Srwatson */ 469173139Srwatsonvoid 470173139Srwatsonxge_msi_info_restore(xge_lldev_t *lldev) 471173139Srwatson{ 472173139Srwatson /* 473173139Srwatson * If interface is made down and up, traffic fails. It was observed that 474173139Srwatson * MSI information were getting reset on down. Restoring them. 475173139Srwatson */ 476173139Srwatson xge_os_pci_write16(lldev->pdev, NULL, 477173139Srwatson xge_offsetof(xge_hal_pci_config_le_t, msi_control), 478173139Srwatson lldev->msi_info.msi_control); 479173139Srwatson 480173139Srwatson xge_os_pci_write32(lldev->pdev, NULL, 481173139Srwatson xge_offsetof(xge_hal_pci_config_le_t, msi_lower_address), 482173139Srwatson lldev->msi_info.msi_lower_address); 483173139Srwatson 484173139Srwatson xge_os_pci_write32(lldev->pdev, NULL, 485173139Srwatson xge_offsetof(xge_hal_pci_config_le_t, msi_higher_address), 486173139Srwatson lldev->msi_info.msi_higher_address); 487173139Srwatson 488173139Srwatson xge_os_pci_write16(lldev->pdev, NULL, 489173139Srwatson xge_offsetof(xge_hal_pci_config_le_t, msi_data), 490173139Srwatson lldev->msi_info.msi_data); 491173139Srwatson} 492173139Srwatson 493173139Srwatson/** 494173139Srwatson * xge_init_mutex 495173139Srwatson * Initializes mutexes used in driver 496173139Srwatson * 497173139Srwatson * @lldev Per-adapter Data 498173139Srwatson */ 499173139Srwatsonvoid 500173139Srwatsonxge_mutex_init(xge_lldev_t *lldev) 501173139Srwatson{ 502173139Srwatson int qindex; 503173139Srwatson 504173139Srwatson sprintf(lldev->mtx_name_drv, "%s_drv", 505173139Srwatson device_get_nameunit(lldev->device)); 506173139Srwatson mtx_init(&lldev->mtx_drv, lldev->mtx_name_drv, MTX_NETWORK_LOCK, 507173139Srwatson MTX_DEF); 508173139Srwatson 509173139Srwatson for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) { 510173139Srwatson sprintf(lldev->mtx_name_tx[qindex], "%s_tx_%d", 511173139Srwatson device_get_nameunit(lldev->device), qindex); 512173139Srwatson mtx_init(&lldev->mtx_tx[qindex], lldev->mtx_name_tx[qindex], NULL, 513173139Srwatson MTX_DEF); 514173139Srwatson } 515173139Srwatson} 516173139Srwatson 517173139Srwatson/** 518173139Srwatson * xge_mutex_destroy 519173139Srwatson * Destroys mutexes used in driver 520173139Srwatson * 521173139Srwatson * @lldev Per-adapter Data 522173139Srwatson */ 523173139Srwatsonvoid 524173139Srwatsonxge_mutex_destroy(xge_lldev_t *lldev) 525173139Srwatson{ 526173139Srwatson int qindex; 527173139Srwatson 528173139Srwatson for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) 529173139Srwatson mtx_destroy(&lldev->mtx_tx[qindex]); 530173139Srwatson mtx_destroy(&lldev->mtx_drv); 531173139Srwatson} 532173139Srwatson 533173139Srwatson/** 534173139Srwatson * xge_print_info 535173139Srwatson * Print device and driver information 536173139Srwatson * 537173139Srwatson * @lldev Per-adapter Data 538173139Srwatson */ 539173139Srwatsonvoid 540173139Srwatsonxge_print_info(xge_lldev_t *lldev) 541173139Srwatson{ 542173139Srwatson device_t dev = lldev->device; 543173139Srwatson xge_hal_device_t *hldev = lldev->devh; 544173139Srwatson xge_hal_status_e status = XGE_HAL_OK; 545173139Srwatson u64 val64 = 0; 546173139Srwatson const char *xge_pci_bus_speeds[17] = { 547173139Srwatson "PCI 33MHz Bus", 548173139Srwatson "PCI 66MHz Bus", 549173139Srwatson "PCIX(M1) 66MHz Bus", 550173139Srwatson "PCIX(M1) 100MHz Bus", 551173139Srwatson "PCIX(M1) 133MHz Bus", 552173139Srwatson "PCIX(M2) 133MHz Bus", 553173139Srwatson "PCIX(M2) 200MHz Bus", 554173139Srwatson "PCIX(M2) 266MHz Bus", 555173139Srwatson "PCIX(M1) Reserved", 556173139Srwatson "PCIX(M1) 66MHz Bus (Not Supported)", 557173139Srwatson "PCIX(M1) 100MHz Bus (Not Supported)", 558173139Srwatson "PCIX(M1) 133MHz Bus (Not Supported)", 559173139Srwatson "PCIX(M2) Reserved", 560173139Srwatson "PCIX 533 Reserved", 561173139Srwatson "PCI Basic Mode", 562173139Srwatson "PCIX Basic Mode", 563173139Srwatson "PCI Invalid Mode" 564173139Srwatson }; 565173139Srwatson 566173139Srwatson xge_os_printf("%s: Xframe%s %s Revision %d Driver v%s", 567173139Srwatson device_get_nameunit(dev), 568173139Srwatson ((hldev->device_id == XGE_PCI_DEVICE_ID_XENA_2) ? "I" : "II"), 569173139Srwatson hldev->vpd_data.product_name, hldev->revision, XGE_DRIVER_VERSION); 570173139Srwatson xge_os_printf("%s: Serial Number %s", 571173139Srwatson device_get_nameunit(dev), hldev->vpd_data.serial_num); 572173139Srwatson 573173139Srwatson if(pci_get_device(dev) == XGE_PCI_DEVICE_ID_HERC_2) { 574173139Srwatson status = xge_hal_mgmt_reg_read(hldev, 0, 575173139Srwatson xge_offsetof(xge_hal_pci_bar0_t, pci_info), &val64); 576173139Srwatson if(status != XGE_HAL_OK) 577173139Srwatson xge_trace(XGE_ERR, "Error for getting bus speed"); 578173139Srwatson 579173139Srwatson xge_os_printf("%s: Adapter is on %s bit %s", 580173139Srwatson device_get_nameunit(dev), ((val64 & BIT(8)) ? "32":"64"), 581173139Srwatson (xge_pci_bus_speeds[((val64 & XGE_HAL_PCI_INFO) >> 60)])); 582173139Srwatson } 583173139Srwatson 584173139Srwatson xge_os_printf("%s: Using %s Interrupts", 585173139Srwatson device_get_nameunit(dev), 586173139Srwatson (lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) ? "MSI":"Line"); 587173139Srwatson} 588173139Srwatson 589173139Srwatson/** 590173139Srwatson * xge_create_dma_tags 591173139Srwatson * Creates DMA tags for both Tx and Rx 592173139Srwatson * 593173139Srwatson * @dev Device Handle 594173139Srwatson * 595173139Srwatson * Returns XGE_HAL_OK or XGE_HAL_FAIL (if errors) 596173139Srwatson */ 597173139Srwatsonxge_hal_status_e 598173139Srwatsonxge_create_dma_tags(device_t dev) 599173139Srwatson{ 600173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev); 601173139Srwatson xge_hal_status_e status = XGE_HAL_FAIL; 602173139Srwatson int mtu = (lldev->ifnetp)->if_mtu, maxsize; 603173139Srwatson 604173139Srwatson /* DMA tag for Tx */ 605173139Srwatson status = bus_dma_tag_create( 606173139Srwatson bus_get_dma_tag(dev), /* Parent */ 607173139Srwatson PAGE_SIZE, /* Alignment */ 608173139Srwatson 0, /* Bounds */ 609173139Srwatson BUS_SPACE_MAXADDR, /* Low Address */ 610173139Srwatson BUS_SPACE_MAXADDR, /* High Address */ 611173139Srwatson NULL, /* Filter Function */ 612173139Srwatson NULL, /* Filter Function Arguments */ 613173139Srwatson MCLBYTES * XGE_MAX_SEGS, /* Maximum Size */ 614173139Srwatson XGE_MAX_SEGS, /* Number of Segments */ 615173139Srwatson MCLBYTES, /* Maximum Segment Size */ 616173139Srwatson BUS_DMA_ALLOCNOW, /* Flags */ 617173139Srwatson NULL, /* Lock Function */ 618173139Srwatson NULL, /* Lock Function Arguments */ 619173139Srwatson (&lldev->dma_tag_tx)); /* DMA Tag */ 620173139Srwatson if(status != 0) 621173139Srwatson goto _exit; 622173139Srwatson 623173139Srwatson maxsize = mtu + XGE_HAL_MAC_HEADER_MAX_SIZE; 624173139Srwatson if(maxsize <= MCLBYTES) { 625173139Srwatson maxsize = MCLBYTES; 626173139Srwatson } 627173139Srwatson else { 628173139Srwatson if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) 629173139Srwatson maxsize = MJUMPAGESIZE; 630173139Srwatson else 631173139Srwatson maxsize = (maxsize <= MJUMPAGESIZE) ? MJUMPAGESIZE : MJUM9BYTES; 632173139Srwatson } 633173139Srwatson 634173139Srwatson /* DMA tag for Rx */ 635173139Srwatson status = bus_dma_tag_create( 636173139Srwatson bus_get_dma_tag(dev), /* Parent */ 637173139Srwatson PAGE_SIZE, /* Alignment */ 638173139Srwatson 0, /* Bounds */ 639173139Srwatson BUS_SPACE_MAXADDR, /* Low Address */ 640173139Srwatson BUS_SPACE_MAXADDR, /* High Address */ 641173139Srwatson NULL, /* Filter Function */ 642173139Srwatson NULL, /* Filter Function Arguments */ 643173139Srwatson maxsize, /* Maximum Size */ 644173139Srwatson 1, /* Number of Segments */ 645173139Srwatson maxsize, /* Maximum Segment Size */ 646173139Srwatson BUS_DMA_ALLOCNOW, /* Flags */ 647173139Srwatson NULL, /* Lock Function */ 648173139Srwatson NULL, /* Lock Function Arguments */ 649173139Srwatson (&lldev->dma_tag_rx)); /* DMA Tag */ 650173139Srwatson if(status != 0) 651173139Srwatson goto _exit1; 652173139Srwatson 653173139Srwatson status = bus_dmamap_create(lldev->dma_tag_rx, BUS_DMA_NOWAIT, 654173139Srwatson &lldev->extra_dma_map); 655173139Srwatson if(status != 0) 656173139Srwatson goto _exit2; 657173139Srwatson 658173139Srwatson status = XGE_HAL_OK; 659173139Srwatson goto _exit; 660173139Srwatson 661173139Srwatson_exit2: 662173139Srwatson status = bus_dma_tag_destroy(lldev->dma_tag_rx); 663173139Srwatson if(status != 0) 664173139Srwatson xge_trace(XGE_ERR, "Rx DMA tag destroy failed"); 665173139Srwatson_exit1: 666173139Srwatson status = bus_dma_tag_destroy(lldev->dma_tag_tx); 667173139Srwatson if(status != 0) 668173139Srwatson xge_trace(XGE_ERR, "Tx DMA tag destroy failed"); 669173139Srwatson status = XGE_HAL_FAIL; 670173139Srwatson_exit: 671173139Srwatson return status; 672173139Srwatson} 673173139Srwatson 674173139Srwatson/** 675173139Srwatson * xge_confirm_changes 676173139Srwatson * Disables and Enables interface to apply requested change 677173139Srwatson * 678173139Srwatson * @lldev Per-adapter Data 679173139Srwatson * @mtu_set Is it called for changing MTU? (Yes: 1, No: 0) 680173139Srwatson * 681173139Srwatson * Returns 0 or Error Number 682173139Srwatson */ 683173139Srwatsonvoid 684173139Srwatsonxge_confirm_changes(xge_lldev_t *lldev, xge_option_e option) 685173139Srwatson{ 686173139Srwatson if(lldev->initialized == 0) goto _exit1; 687173139Srwatson 688173139Srwatson mtx_lock(&lldev->mtx_drv); 689173139Srwatson if_down(lldev->ifnetp); 690173139Srwatson xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL); 691173139Srwatson 692173139Srwatson if(option == XGE_SET_MTU) 693173139Srwatson (lldev->ifnetp)->if_mtu = lldev->mtu; 694173139Srwatson else 695173139Srwatson xge_buffer_mode_init(lldev, lldev->mtu); 696173139Srwatson 697173139Srwatson xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL); 698173139Srwatson if_up(lldev->ifnetp); 699173139Srwatson mtx_unlock(&lldev->mtx_drv); 700173139Srwatson goto _exit; 701173139Srwatson 702173139Srwatson_exit1: 703173139Srwatson /* Request was to change MTU and device not initialized */ 704173139Srwatson if(option == XGE_SET_MTU) { 705173139Srwatson (lldev->ifnetp)->if_mtu = lldev->mtu; 706173139Srwatson xge_buffer_mode_init(lldev, lldev->mtu); 707173139Srwatson } 708173139Srwatson_exit: 709173139Srwatson return; 710173139Srwatson} 711173139Srwatson 712173139Srwatson/** 713173139Srwatson * xge_change_lro_status 714173139Srwatson * Enable/Disable LRO feature 715173139Srwatson * 716173139Srwatson * @SYSCTL_HANDLER_ARGS sysctl_oid structure with arguments 717173139Srwatson * 718173139Srwatson * Returns 0 or error number. 719173139Srwatson */ 720173139Srwatsonstatic int 721173139Srwatsonxge_change_lro_status(SYSCTL_HANDLER_ARGS) 722173139Srwatson{ 723173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)arg1; 724173139Srwatson int request = lldev->enabled_lro, status = XGE_HAL_OK; 725173139Srwatson 726173139Srwatson status = sysctl_handle_int(oidp, &request, arg2, req); 727173139Srwatson if((status != XGE_HAL_OK) || (!req->newptr)) 728173139Srwatson goto _exit; 729173139Srwatson 730173139Srwatson if((request < 0) || (request > 1)) { 731173139Srwatson status = EINVAL; 732173139Srwatson goto _exit; 733173139Srwatson } 734173139Srwatson 735173139Srwatson /* Return if current and requested states are same */ 736173139Srwatson if(request == lldev->enabled_lro){ 737173139Srwatson xge_trace(XGE_ERR, "LRO is already %s", 738173139Srwatson ((request) ? "enabled" : "disabled")); 739173139Srwatson goto _exit; 740173139Srwatson } 741173139Srwatson 742173139Srwatson lldev->enabled_lro = request; 743173139Srwatson xge_confirm_changes(lldev, XGE_CHANGE_LRO); 744173139Srwatson arg2 = lldev->enabled_lro; 745173139Srwatson 746173139Srwatson_exit: 747173139Srwatson return status; 748173139Srwatson} 749173139Srwatson 750173139Srwatson/** 751173139Srwatson * xge_add_sysctl_handlers 752173139Srwatson * Registers sysctl parameter value update handlers 753173139Srwatson * 754173139Srwatson * @lldev Per-adapter data 755173139Srwatson */ 756173139Srwatsonvoid 757173139Srwatsonxge_add_sysctl_handlers(xge_lldev_t *lldev) 758173139Srwatson{ 759173139Srwatson struct sysctl_ctx_list *context_list = 760173139Srwatson device_get_sysctl_ctx(lldev->device); 761173139Srwatson struct sysctl_oid *oid = device_get_sysctl_tree(lldev->device); 762173139Srwatson 763173139Srwatson SYSCTL_ADD_PROC(context_list, SYSCTL_CHILDREN(oid), OID_AUTO, 764173139Srwatson "enable_lro", CTLTYPE_INT | CTLFLAG_RW, lldev, 0, 765173139Srwatson xge_change_lro_status, "I", "Enable or disable LRO feature"); 766173139Srwatson} 767173139Srwatson 768173139Srwatson/** 769171095Ssam * xge_attach 770173139Srwatson * Connects driver to the system if probe was success 771173139Srwatson * 772173139Srwatson * @dev Device Handle 773173139Srwatson */ 774171095Ssamint 775171095Ssamxge_attach(device_t dev) 776171095Ssam{ 777171095Ssam xge_hal_device_config_t *device_config; 778171095Ssam xge_hal_device_attr_t attr; 779173139Srwatson xge_lldev_t *lldev; 780171095Ssam xge_hal_device_t *hldev; 781173139Srwatson xge_pci_info_t *pci_info; 782171095Ssam struct ifnet *ifnetp; 783173139Srwatson int rid, rid0, rid1, error; 784173139Srwatson int msi_count = 0, status = XGE_HAL_OK; 785173139Srwatson int enable_msi = XGE_HAL_INTR_MODE_IRQLINE; 786171095Ssam 787173139Srwatson device_config = xge_os_malloc(NULL, sizeof(xge_hal_device_config_t)); 788171095Ssam if(!device_config) { 789173139Srwatson XGE_EXIT_ON_ERR("Memory allocation for device configuration failed", 790173139Srwatson attach_out_config, ENOMEM); 791171095Ssam } 792171095Ssam 793173139Srwatson lldev = (xge_lldev_t *) device_get_softc(dev); 794171095Ssam if(!lldev) { 795173139Srwatson XGE_EXIT_ON_ERR("Adapter softc is NULL", attach_out, ENOMEM); 796171095Ssam } 797171095Ssam lldev->device = dev; 798171095Ssam 799173139Srwatson xge_mutex_init(lldev); 800171095Ssam 801171095Ssam error = xge_driver_initialize(); 802171095Ssam if(error != XGE_HAL_OK) { 803173139Srwatson xge_resources_free(dev, xge_free_mutex); 804173139Srwatson XGE_EXIT_ON_ERR("Initializing driver failed", attach_out, ENXIO); 805171095Ssam } 806171095Ssam 807171095Ssam /* HAL device */ 808173139Srwatson hldev = 809173139Srwatson (xge_hal_device_t *)xge_os_malloc(NULL, sizeof(xge_hal_device_t)); 810171095Ssam if(!hldev) { 811173139Srwatson xge_resources_free(dev, xge_free_terminate_hal_driver); 812173139Srwatson XGE_EXIT_ON_ERR("Memory allocation for HAL device failed", 813173139Srwatson attach_out, ENOMEM); 814171095Ssam } 815171095Ssam lldev->devh = hldev; 816171095Ssam 817171095Ssam /* Our private structure */ 818173139Srwatson pci_info = 819173139Srwatson (xge_pci_info_t*) xge_os_malloc(NULL, sizeof(xge_pci_info_t)); 820171095Ssam if(!pci_info) { 821173139Srwatson xge_resources_free(dev, xge_free_hal_device); 822173139Srwatson XGE_EXIT_ON_ERR("Memory allocation for PCI info. failed", 823173139Srwatson attach_out, ENOMEM); 824171095Ssam } 825171095Ssam lldev->pdev = pci_info; 826171095Ssam pci_info->device = dev; 827171095Ssam 828171095Ssam /* Set bus master */ 829171095Ssam pci_enable_busmaster(dev); 830171095Ssam 831171095Ssam /* Get virtual address for BAR0 */ 832171095Ssam rid0 = PCIR_BAR(0); 833171095Ssam pci_info->regmap0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid0, 834171095Ssam RF_ACTIVE); 835171095Ssam if(pci_info->regmap0 == NULL) { 836173139Srwatson xge_resources_free(dev, xge_free_pci_info); 837173139Srwatson XGE_EXIT_ON_ERR("Bus resource allocation for BAR0 failed", 838173139Srwatson attach_out, ENOMEM); 839171095Ssam } 840171095Ssam attr.bar0 = (char *)pci_info->regmap0; 841171095Ssam 842173139Srwatson pci_info->bar0resource = (xge_bus_resource_t*) 843173139Srwatson xge_os_malloc(NULL, sizeof(xge_bus_resource_t)); 844171095Ssam if(pci_info->bar0resource == NULL) { 845173139Srwatson xge_resources_free(dev, xge_free_bar0); 846173139Srwatson XGE_EXIT_ON_ERR("Memory allocation for BAR0 Resources failed", 847173139Srwatson attach_out, ENOMEM); 848171095Ssam } 849173139Srwatson ((xge_bus_resource_t *)(pci_info->bar0resource))->bus_tag = 850171095Ssam rman_get_bustag(pci_info->regmap0); 851173139Srwatson ((xge_bus_resource_t *)(pci_info->bar0resource))->bus_handle = 852171095Ssam rman_get_bushandle(pci_info->regmap0); 853173139Srwatson ((xge_bus_resource_t *)(pci_info->bar0resource))->bar_start_addr = 854171095Ssam pci_info->regmap0; 855171095Ssam 856171095Ssam /* Get virtual address for BAR1 */ 857171095Ssam rid1 = PCIR_BAR(2); 858171095Ssam pci_info->regmap1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid1, 859171095Ssam RF_ACTIVE); 860171095Ssam if(pci_info->regmap1 == NULL) { 861173139Srwatson xge_resources_free(dev, xge_free_bar0_resource); 862173139Srwatson XGE_EXIT_ON_ERR("Bus resource allocation for BAR1 failed", 863173139Srwatson attach_out, ENOMEM); 864171095Ssam } 865171095Ssam attr.bar1 = (char *)pci_info->regmap1; 866171095Ssam 867173139Srwatson pci_info->bar1resource = (xge_bus_resource_t*) 868173139Srwatson xge_os_malloc(NULL, sizeof(xge_bus_resource_t)); 869171095Ssam if(pci_info->bar1resource == NULL) { 870173139Srwatson xge_resources_free(dev, xge_free_bar1); 871173139Srwatson XGE_EXIT_ON_ERR("Memory allocation for BAR1 Resources failed", 872173139Srwatson attach_out, ENOMEM); 873171095Ssam } 874173139Srwatson ((xge_bus_resource_t *)(pci_info->bar1resource))->bus_tag = 875171095Ssam rman_get_bustag(pci_info->regmap1); 876173139Srwatson ((xge_bus_resource_t *)(pci_info->bar1resource))->bus_handle = 877171095Ssam rman_get_bushandle(pci_info->regmap1); 878173139Srwatson ((xge_bus_resource_t *)(pci_info->bar1resource))->bar_start_addr = 879171095Ssam pci_info->regmap1; 880171095Ssam 881171095Ssam /* Save PCI config space */ 882171095Ssam xge_pci_space_save(dev); 883171095Ssam 884173139Srwatson attr.regh0 = (xge_bus_resource_t *) pci_info->bar0resource; 885173139Srwatson attr.regh1 = (xge_bus_resource_t *) pci_info->bar1resource; 886171095Ssam attr.irqh = lldev->irqhandle; 887171095Ssam attr.cfgh = pci_info; 888171095Ssam attr.pdev = pci_info; 889171095Ssam 890171095Ssam /* Initialize device configuration parameters */ 891171095Ssam xge_init_params(device_config, dev); 892171095Ssam 893173139Srwatson rid = 0; 894173139Srwatson if(lldev->enabled_msi) { 895173139Srwatson /* Number of MSI messages supported by device */ 896173139Srwatson msi_count = pci_msi_count(dev); 897173139Srwatson if(msi_count > 1) { 898173139Srwatson /* Device supports MSI */ 899173139Srwatson if(bootverbose) { 900173139Srwatson xge_trace(XGE_ERR, "MSI count: %d", msi_count); 901173139Srwatson xge_trace(XGE_ERR, "Now, driver supporting 1 message"); 902173139Srwatson } 903173139Srwatson msi_count = 1; 904173139Srwatson error = pci_alloc_msi(dev, &msi_count); 905173139Srwatson if(error == 0) { 906173139Srwatson if(bootverbose) 907173139Srwatson xge_trace(XGE_ERR, "Allocated messages: %d", msi_count); 908173139Srwatson enable_msi = XGE_HAL_INTR_MODE_MSI; 909173139Srwatson rid = 1; 910173139Srwatson } 911173139Srwatson else { 912173139Srwatson if(bootverbose) 913173139Srwatson xge_trace(XGE_ERR, "pci_alloc_msi failed, %d", error); 914173139Srwatson } 915171095Ssam } 916171095Ssam } 917173139Srwatson lldev->enabled_msi = enable_msi; 918171095Ssam 919173139Srwatson /* Allocate resource for irq */ 920173139Srwatson lldev->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 921173139Srwatson (RF_SHAREABLE | RF_ACTIVE)); 922173139Srwatson if(lldev->irq == NULL) { 923173139Srwatson xge_trace(XGE_ERR, "Allocating irq resource for %s failed", 924173139Srwatson ((rid == 0) ? "line interrupt" : "MSI")); 925173139Srwatson if(rid == 1) { 926173139Srwatson error = pci_release_msi(dev); 927173139Srwatson if(error != 0) { 928173139Srwatson xge_trace(XGE_ERR, "Releasing MSI resources failed %d", 929173139Srwatson error); 930173139Srwatson xge_trace(XGE_ERR, "Requires reboot to use MSI again"); 931173139Srwatson } 932173139Srwatson xge_trace(XGE_ERR, "Trying line interrupts"); 933173139Srwatson rid = 0; 934173139Srwatson lldev->enabled_msi = XGE_HAL_INTR_MODE_IRQLINE; 935173139Srwatson lldev->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 936173139Srwatson (RF_SHAREABLE | RF_ACTIVE)); 937171095Ssam } 938173139Srwatson if(lldev->irq == NULL) { 939173139Srwatson xge_trace(XGE_ERR, "Allocating irq resource failed"); 940173139Srwatson xge_resources_free(dev, xge_free_bar1_resource); 941173139Srwatson status = ENOMEM; 942171095Ssam goto attach_out; 943171095Ssam } 944173139Srwatson } 945171095Ssam 946173139Srwatson device_config->intr_mode = lldev->enabled_msi; 947173139Srwatson if(bootverbose) { 948173139Srwatson xge_trace(XGE_TRACE, "rid: %d, Mode: %d, MSI count: %d", rid, 949173139Srwatson lldev->enabled_msi, msi_count); 950173139Srwatson } 951171095Ssam 952173139Srwatson /* Initialize HAL device */ 953173139Srwatson error = xge_hal_device_initialize(hldev, &attr, device_config); 954173139Srwatson if(error != XGE_HAL_OK) { 955173139Srwatson xge_resources_free(dev, xge_free_irq_resource); 956173139Srwatson XGE_EXIT_ON_ERR("Initializing HAL device failed", attach_out, 957173139Srwatson ENXIO); 958171095Ssam } 959171095Ssam 960171095Ssam xge_hal_device_private_set(hldev, lldev); 961171095Ssam 962171095Ssam error = xge_interface_setup(dev); 963171095Ssam if(error != 0) { 964173139Srwatson status = error; 965171095Ssam goto attach_out; 966171095Ssam } 967171095Ssam 968171095Ssam ifnetp = lldev->ifnetp; 969171095Ssam ifnetp->if_mtu = device_config->mtu; 970171095Ssam 971171095Ssam xge_media_init(dev); 972171095Ssam 973171095Ssam /* Associate interrupt handler with the device */ 974173139Srwatson if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) { 975173139Srwatson error = bus_setup_intr(dev, lldev->irq, 976173139Srwatson (INTR_TYPE_NET | INTR_MPSAFE), 977171095Ssam#if __FreeBSD_version > 700030 978173139Srwatson NULL, 979171095Ssam#endif 980173139Srwatson xge_isr_msi, lldev, &lldev->irqhandle); 981173139Srwatson xge_msi_info_save(lldev); 982171095Ssam } 983173139Srwatson else { 984173139Srwatson error = bus_setup_intr(dev, lldev->irq, 985173139Srwatson (INTR_TYPE_NET | INTR_MPSAFE), 986173139Srwatson#if __FreeBSD_version > 700030 987173139Srwatson xge_isr_filter, 988173139Srwatson#endif 989173139Srwatson xge_isr_line, lldev, &lldev->irqhandle); 990171095Ssam } 991171095Ssam if(error != 0) { 992173139Srwatson xge_resources_free(dev, xge_free_media_interface); 993173139Srwatson XGE_EXIT_ON_ERR("Associating interrupt handler with device failed", 994173139Srwatson attach_out, ENXIO); 995171095Ssam } 996171095Ssam 997173139Srwatson xge_print_info(lldev); 998171095Ssam 999173139Srwatson xge_add_sysctl_handlers(lldev); 1000171095Ssam 1001173139Srwatson xge_buffer_mode_init(lldev, device_config->mtu); 1002171095Ssam 1003171095Ssamattach_out: 1004173139Srwatson xge_os_free(NULL, device_config, sizeof(xge_hal_device_config_t)); 1005171095Ssamattach_out_config: 1006173139Srwatson return status; 1007171095Ssam} 1008171095Ssam 1009173139Srwatson/** 1010173139Srwatson * xge_resources_free 1011173139Srwatson * Undo what-all we did during load/attach 1012173139Srwatson * 1013173139Srwatson * @dev Device Handle 1014173139Srwatson * @error Identifies what-all to undo 1015173139Srwatson */ 1016171095Ssamvoid 1017173139Srwatsonxge_resources_free(device_t dev, xge_lables_e error) 1018171095Ssam{ 1019173139Srwatson xge_lldev_t *lldev; 1020173139Srwatson xge_pci_info_t *pci_info; 1021171095Ssam xge_hal_device_t *hldev; 1022171095Ssam int rid, status; 1023171095Ssam 1024171095Ssam /* LL Device */ 1025173139Srwatson lldev = (xge_lldev_t *) device_get_softc(dev); 1026171095Ssam pci_info = lldev->pdev; 1027171095Ssam 1028171095Ssam /* HAL Device */ 1029171095Ssam hldev = lldev->devh; 1030171095Ssam 1031171095Ssam switch(error) { 1032173139Srwatson case xge_free_all: 1033171095Ssam /* Teardown interrupt handler - device association */ 1034171095Ssam bus_teardown_intr(dev, lldev->irq, lldev->irqhandle); 1035171095Ssam 1036173139Srwatson case xge_free_media_interface: 1037171095Ssam /* Media */ 1038173139Srwatson ifmedia_removeall(&lldev->media); 1039171095Ssam 1040171095Ssam /* Detach Ether */ 1041171095Ssam ether_ifdetach(lldev->ifnetp); 1042171095Ssam if_free(lldev->ifnetp); 1043171095Ssam 1044171095Ssam xge_hal_device_private_set(hldev, NULL); 1045171095Ssam xge_hal_device_disable(hldev); 1046171095Ssam 1047173139Srwatson case xge_free_terminate_hal_device: 1048171095Ssam /* HAL Device */ 1049171095Ssam xge_hal_device_terminate(hldev); 1050171095Ssam 1051173139Srwatson case xge_free_irq_resource: 1052173139Srwatson /* Release IRQ resource */ 1053173139Srwatson bus_release_resource(dev, SYS_RES_IRQ, 1054173139Srwatson ((lldev->enabled_msi == XGE_HAL_INTR_MODE_IRQLINE) ? 0:1), 1055173139Srwatson lldev->irq); 1056173139Srwatson 1057173139Srwatson if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) { 1058173139Srwatson status = pci_release_msi(dev); 1059173139Srwatson if(status != 0) { 1060173139Srwatson if(bootverbose) { 1061173139Srwatson xge_trace(XGE_ERR, 1062173139Srwatson "pci_release_msi returned %d", status); 1063173139Srwatson } 1064173139Srwatson } 1065173139Srwatson } 1066173139Srwatson 1067173139Srwatson case xge_free_bar1_resource: 1068171095Ssam /* Restore PCI configuration space */ 1069171095Ssam xge_pci_space_restore(dev); 1070171095Ssam 1071171095Ssam /* Free bar1resource */ 1072173139Srwatson xge_os_free(NULL, pci_info->bar1resource, 1073173139Srwatson sizeof(xge_bus_resource_t)); 1074171095Ssam 1075173139Srwatson case xge_free_bar1: 1076171095Ssam /* Release BAR1 */ 1077171095Ssam rid = PCIR_BAR(2); 1078171095Ssam bus_release_resource(dev, SYS_RES_MEMORY, rid, 1079171095Ssam pci_info->regmap1); 1080171095Ssam 1081173139Srwatson case xge_free_bar0_resource: 1082171095Ssam /* Free bar0resource */ 1083173139Srwatson xge_os_free(NULL, pci_info->bar0resource, 1084173139Srwatson sizeof(xge_bus_resource_t)); 1085171095Ssam 1086173139Srwatson case xge_free_bar0: 1087171095Ssam /* Release BAR0 */ 1088171095Ssam rid = PCIR_BAR(0); 1089171095Ssam bus_release_resource(dev, SYS_RES_MEMORY, rid, 1090171095Ssam pci_info->regmap0); 1091171095Ssam 1092173139Srwatson case xge_free_pci_info: 1093171095Ssam /* Disable Bus Master */ 1094171095Ssam pci_disable_busmaster(dev); 1095171095Ssam 1096171095Ssam /* Free pci_info_t */ 1097171095Ssam lldev->pdev = NULL; 1098173139Srwatson xge_os_free(NULL, pci_info, sizeof(xge_pci_info_t)); 1099171095Ssam 1100173139Srwatson case xge_free_hal_device: 1101171095Ssam /* Free device configuration struct and HAL device */ 1102173139Srwatson xge_os_free(NULL, hldev, sizeof(xge_hal_device_t)); 1103171095Ssam 1104173139Srwatson case xge_free_terminate_hal_driver: 1105171095Ssam /* Terminate HAL driver */ 1106171095Ssam hal_driver_init_count = hal_driver_init_count - 1; 1107171095Ssam if(!hal_driver_init_count) { 1108171095Ssam xge_hal_driver_terminate(); 1109171095Ssam } 1110171095Ssam 1111173139Srwatson case xge_free_mutex: 1112173139Srwatson xge_mutex_destroy(lldev); 1113171095Ssam } 1114171095Ssam} 1115171095Ssam 1116173139Srwatson/** 1117171095Ssam * xge_detach 1118173139Srwatson * Detaches driver from the Kernel subsystem 1119173139Srwatson * 1120173139Srwatson * @dev Device Handle 1121173139Srwatson */ 1122171095Ssamint 1123171095Ssamxge_detach(device_t dev) 1124171095Ssam{ 1125173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev); 1126171095Ssam 1127173139Srwatson if(lldev->in_detach == 0) { 1128173139Srwatson lldev->in_detach = 1; 1129173139Srwatson xge_stop(lldev); 1130173139Srwatson xge_resources_free(dev, xge_free_all); 1131173139Srwatson } 1132171095Ssam 1133171095Ssam return 0; 1134171095Ssam} 1135171095Ssam 1136173139Srwatson/** 1137171095Ssam * xge_shutdown 1138173139Srwatson * To shutdown device before system shutdown 1139173139Srwatson * 1140173139Srwatson * @dev Device Handle 1141173139Srwatson */ 1142171095Ssamint 1143171095Ssamxge_shutdown(device_t dev) 1144171095Ssam{ 1145173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *) device_get_softc(dev); 1146173139Srwatson xge_stop(lldev); 1147171095Ssam 1148171095Ssam return 0; 1149171095Ssam} 1150171095Ssam 1151173139Srwatson/** 1152173139Srwatson * xge_interface_setup 1153173139Srwatson * Setup interface 1154173139Srwatson * 1155173139Srwatson * @dev Device Handle 1156173139Srwatson * 1157173139Srwatson * Returns 0 on success, ENXIO/ENOMEM on failure 1158173139Srwatson */ 1159171095Ssamint 1160171095Ssamxge_interface_setup(device_t dev) 1161171095Ssam{ 1162171095Ssam u8 mcaddr[ETHER_ADDR_LEN]; 1163173139Srwatson xge_hal_status_e status; 1164173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev); 1165171095Ssam struct ifnet *ifnetp; 1166171095Ssam xge_hal_device_t *hldev = lldev->devh; 1167171095Ssam 1168171095Ssam /* Get the MAC address of the device */ 1169173139Srwatson status = xge_hal_device_macaddr_get(hldev, 0, &mcaddr); 1170173139Srwatson if(status != XGE_HAL_OK) { 1171173139Srwatson xge_resources_free(dev, xge_free_terminate_hal_device); 1172173139Srwatson XGE_EXIT_ON_ERR("Getting MAC address failed", ifsetup_out, ENXIO); 1173171095Ssam } 1174171095Ssam 1175171095Ssam /* Get interface ifnet structure for this Ether device */ 1176171095Ssam ifnetp = lldev->ifnetp = if_alloc(IFT_ETHER); 1177171095Ssam if(ifnetp == NULL) { 1178173139Srwatson xge_resources_free(dev, xge_free_terminate_hal_device); 1179173139Srwatson XGE_EXIT_ON_ERR("Allocation ifnet failed", ifsetup_out, ENOMEM); 1180171095Ssam } 1181171095Ssam 1182171095Ssam /* Initialize interface ifnet structure */ 1183171095Ssam if_initname(ifnetp, device_get_name(dev), device_get_unit(dev)); 1184173139Srwatson ifnetp->if_mtu = XGE_HAL_DEFAULT_MTU; 1185173139Srwatson ifnetp->if_baudrate = XGE_BAUDRATE; 1186171095Ssam ifnetp->if_init = xge_init; 1187171095Ssam ifnetp->if_softc = lldev; 1188171095Ssam ifnetp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1189171095Ssam ifnetp->if_ioctl = xge_ioctl; 1190171095Ssam ifnetp->if_start = xge_send; 1191171095Ssam 1192171095Ssam /* TODO: Check and assign optimal value */ 1193207554Ssobomax ifnetp->if_snd.ifq_maxlen = ifqmaxlen; 1194171095Ssam 1195173139Srwatson ifnetp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | 1196171095Ssam IFCAP_HWCSUM; 1197173139Srwatson if(lldev->enabled_tso) 1198173139Srwatson ifnetp->if_capabilities |= IFCAP_TSO4; 1199173139Srwatson if(lldev->enabled_lro) 1200173139Srwatson ifnetp->if_capabilities |= IFCAP_LRO; 1201171095Ssam 1202171095Ssam ifnetp->if_capenable = ifnetp->if_capabilities; 1203171095Ssam 1204171095Ssam /* Attach the interface */ 1205171095Ssam ether_ifattach(ifnetp, mcaddr); 1206171095Ssam 1207171095Ssamifsetup_out: 1208173139Srwatson return status; 1209171095Ssam} 1210171095Ssam 1211173139Srwatson/** 1212173139Srwatson * xge_callback_link_up 1213173139Srwatson * Callback for Link-up indication from HAL 1214173139Srwatson * 1215173139Srwatson * @userdata Per-adapter data 1216173139Srwatson */ 1217171095Ssamvoid 1218173139Srwatsonxge_callback_link_up(void *userdata) 1219171095Ssam{ 1220173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)userdata; 1221171095Ssam struct ifnet *ifnetp = lldev->ifnetp; 1222171095Ssam 1223171095Ssam ifnetp->if_flags &= ~IFF_DRV_OACTIVE; 1224171095Ssam if_link_state_change(ifnetp, LINK_STATE_UP); 1225171095Ssam} 1226171095Ssam 1227173139Srwatson/** 1228173139Srwatson * xge_callback_link_down 1229173139Srwatson * Callback for Link-down indication from HAL 1230173139Srwatson * 1231173139Srwatson * @userdata Per-adapter data 1232173139Srwatson */ 1233171095Ssamvoid 1234173139Srwatsonxge_callback_link_down(void *userdata) 1235171095Ssam{ 1236173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)userdata; 1237171095Ssam struct ifnet *ifnetp = lldev->ifnetp; 1238171095Ssam 1239171095Ssam ifnetp->if_flags |= IFF_DRV_OACTIVE; 1240171095Ssam if_link_state_change(ifnetp, LINK_STATE_DOWN); 1241171095Ssam} 1242171095Ssam 1243173139Srwatson/** 1244173139Srwatson * xge_callback_crit_err 1245173139Srwatson * Callback for Critical error indication from HAL 1246173139Srwatson * 1247173139Srwatson * @userdata Per-adapter data 1248173139Srwatson * @type Event type (Enumerated hardware error) 1249173139Srwatson * @serr_data Hardware status 1250173139Srwatson */ 1251171095Ssamvoid 1252173139Srwatsonxge_callback_crit_err(void *userdata, xge_hal_event_e type, u64 serr_data) 1253171095Ssam{ 1254171095Ssam xge_trace(XGE_ERR, "Critical Error"); 1255173139Srwatson xge_reset(userdata); 1256171095Ssam} 1257171095Ssam 1258173139Srwatson/** 1259173139Srwatson * xge_callback_event 1260173139Srwatson * Callback from HAL indicating that some event has been queued 1261173139Srwatson * 1262173139Srwatson * @item Queued event item 1263173139Srwatson */ 1264171095Ssamvoid 1265173139Srwatsonxge_callback_event(xge_queue_item_t *item) 1266171095Ssam{ 1267173139Srwatson xge_lldev_t *lldev = NULL; 1268171095Ssam xge_hal_device_t *hldev = NULL; 1269171095Ssam struct ifnet *ifnetp = NULL; 1270171095Ssam 1271171095Ssam hldev = item->context; 1272171095Ssam lldev = xge_hal_device_private(hldev); 1273171095Ssam ifnetp = lldev->ifnetp; 1274171095Ssam 1275234506Sdim switch((int)item->event_type) { 1276173139Srwatson case XGE_LL_EVENT_TRY_XMIT_AGAIN: 1277173139Srwatson if(lldev->initialized) { 1278173139Srwatson if(xge_hal_channel_dtr_count(lldev->fifo_channel[0]) > 0) { 1279173139Srwatson ifnetp->if_flags &= ~IFF_DRV_OACTIVE; 1280173139Srwatson } 1281173139Srwatson else { 1282173139Srwatson xge_queue_produce_context( 1283173139Srwatson xge_hal_device_queue(lldev->devh), 1284173139Srwatson XGE_LL_EVENT_TRY_XMIT_AGAIN, lldev->devh); 1285173139Srwatson } 1286171095Ssam } 1287173139Srwatson break; 1288173139Srwatson 1289173139Srwatson case XGE_LL_EVENT_DEVICE_RESETTING: 1290173139Srwatson xge_reset(item->context); 1291173139Srwatson break; 1292173139Srwatson 1293173139Srwatson default: 1294173139Srwatson break; 1295171095Ssam } 1296171095Ssam} 1297171095Ssam 1298173139Srwatson/** 1299173139Srwatson * xge_ifmedia_change 1300173139Srwatson * Media change driver callback 1301173139Srwatson * 1302173139Srwatson * @ifnetp Interface Handle 1303173139Srwatson * 1304173139Srwatson * Returns 0 if media is Ether else EINVAL 1305173139Srwatson */ 1306171095Ssamint 1307171095Ssamxge_ifmedia_change(struct ifnet *ifnetp) 1308171095Ssam{ 1309173139Srwatson xge_lldev_t *lldev = ifnetp->if_softc; 1310173139Srwatson struct ifmedia *ifmediap = &lldev->media; 1311171095Ssam 1312171095Ssam return (IFM_TYPE(ifmediap->ifm_media) != IFM_ETHER) ? EINVAL:0; 1313171095Ssam} 1314171095Ssam 1315173139Srwatson/** 1316173139Srwatson * xge_ifmedia_status 1317173139Srwatson * Media status driver callback 1318173139Srwatson * 1319173139Srwatson * @ifnetp Interface Handle 1320173139Srwatson * @ifmr Interface Media Settings 1321173139Srwatson */ 1322171095Ssamvoid 1323171095Ssamxge_ifmedia_status(struct ifnet *ifnetp, struct ifmediareq *ifmr) 1324171095Ssam{ 1325171095Ssam xge_hal_status_e status; 1326171095Ssam u64 regvalue; 1327173139Srwatson xge_lldev_t *lldev = ifnetp->if_softc; 1328171095Ssam xge_hal_device_t *hldev = lldev->devh; 1329171095Ssam 1330171095Ssam ifmr->ifm_status = IFM_AVALID; 1331171095Ssam ifmr->ifm_active = IFM_ETHER; 1332171095Ssam 1333171095Ssam status = xge_hal_mgmt_reg_read(hldev, 0, 1334171095Ssam xge_offsetof(xge_hal_pci_bar0_t, adapter_status), ®value); 1335171095Ssam if(status != XGE_HAL_OK) { 1336173139Srwatson xge_trace(XGE_TRACE, "Getting adapter status failed"); 1337173139Srwatson goto _exit; 1338171095Ssam } 1339171095Ssam 1340171095Ssam if((regvalue & (XGE_HAL_ADAPTER_STATUS_RMAC_REMOTE_FAULT | 1341171095Ssam XGE_HAL_ADAPTER_STATUS_RMAC_LOCAL_FAULT)) == 0) { 1342171095Ssam ifmr->ifm_status |= IFM_ACTIVE; 1343171095Ssam ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1344171095Ssam if_link_state_change(ifnetp, LINK_STATE_UP); 1345171095Ssam } 1346171095Ssam else { 1347171095Ssam if_link_state_change(ifnetp, LINK_STATE_DOWN); 1348171095Ssam } 1349173139Srwatson_exit: 1350173139Srwatson return; 1351173139Srwatson} 1352171095Ssam 1353173139Srwatson/** 1354173139Srwatson * xge_ioctl_stats 1355173139Srwatson * IOCTL to get statistics 1356173139Srwatson * 1357173139Srwatson * @lldev Per-adapter data 1358173139Srwatson * @ifreqp Interface request 1359173139Srwatson */ 1360173139Srwatsonint 1361173139Srwatsonxge_ioctl_stats(xge_lldev_t *lldev, struct ifreq *ifreqp) 1362173139Srwatson{ 1363173139Srwatson xge_hal_status_e status = XGE_HAL_OK; 1364173139Srwatson char *data = (char *)ifreqp->ifr_data; 1365173139Srwatson void *info = NULL; 1366173139Srwatson int retValue = EINVAL; 1367173139Srwatson 1368173139Srwatson switch(*data) { 1369173139Srwatson case XGE_QUERY_STATS: 1370173139Srwatson mtx_lock(&lldev->mtx_drv); 1371173139Srwatson status = xge_hal_stats_hw(lldev->devh, 1372173139Srwatson (xge_hal_stats_hw_info_t **)&info); 1373173139Srwatson mtx_unlock(&lldev->mtx_drv); 1374173139Srwatson if(status == XGE_HAL_OK) { 1375173139Srwatson if(copyout(info, ifreqp->ifr_data, 1376173139Srwatson sizeof(xge_hal_stats_hw_info_t)) == 0) 1377173139Srwatson retValue = 0; 1378173139Srwatson } 1379173139Srwatson else { 1380173139Srwatson xge_trace(XGE_ERR, "Getting statistics failed (Status: %d)", 1381173139Srwatson status); 1382173139Srwatson } 1383173139Srwatson break; 1384173139Srwatson 1385173139Srwatson case XGE_QUERY_PCICONF: 1386173139Srwatson info = xge_os_malloc(NULL, sizeof(xge_hal_pci_config_t)); 1387173139Srwatson if(info != NULL) { 1388173139Srwatson mtx_lock(&lldev->mtx_drv); 1389173139Srwatson status = xge_hal_mgmt_pci_config(lldev->devh, info, 1390173139Srwatson sizeof(xge_hal_pci_config_t)); 1391173139Srwatson mtx_unlock(&lldev->mtx_drv); 1392173139Srwatson if(status == XGE_HAL_OK) { 1393173139Srwatson if(copyout(info, ifreqp->ifr_data, 1394173139Srwatson sizeof(xge_hal_pci_config_t)) == 0) 1395173139Srwatson retValue = 0; 1396173139Srwatson } 1397173139Srwatson else { 1398173139Srwatson xge_trace(XGE_ERR, 1399173139Srwatson "Getting PCI configuration failed (%d)", status); 1400173139Srwatson } 1401173139Srwatson xge_os_free(NULL, info, sizeof(xge_hal_pci_config_t)); 1402173139Srwatson } 1403173139Srwatson break; 1404173139Srwatson 1405173139Srwatson case XGE_QUERY_DEVSTATS: 1406173139Srwatson info = xge_os_malloc(NULL, sizeof(xge_hal_stats_device_info_t)); 1407173139Srwatson if(info != NULL) { 1408173139Srwatson mtx_lock(&lldev->mtx_drv); 1409173139Srwatson status =xge_hal_mgmt_device_stats(lldev->devh, info, 1410173139Srwatson sizeof(xge_hal_stats_device_info_t)); 1411173139Srwatson mtx_unlock(&lldev->mtx_drv); 1412173139Srwatson if(status == XGE_HAL_OK) { 1413173139Srwatson if(copyout(info, ifreqp->ifr_data, 1414173139Srwatson sizeof(xge_hal_stats_device_info_t)) == 0) 1415173139Srwatson retValue = 0; 1416173139Srwatson } 1417173139Srwatson else { 1418173139Srwatson xge_trace(XGE_ERR, "Getting device info failed (%d)", 1419173139Srwatson status); 1420173139Srwatson } 1421173139Srwatson xge_os_free(NULL, info, 1422173139Srwatson sizeof(xge_hal_stats_device_info_t)); 1423173139Srwatson } 1424173139Srwatson break; 1425173139Srwatson 1426173139Srwatson case XGE_QUERY_SWSTATS: 1427173139Srwatson info = xge_os_malloc(NULL, sizeof(xge_hal_stats_sw_err_t)); 1428173139Srwatson if(info != NULL) { 1429173139Srwatson mtx_lock(&lldev->mtx_drv); 1430173139Srwatson status =xge_hal_mgmt_sw_stats(lldev->devh, info, 1431173139Srwatson sizeof(xge_hal_stats_sw_err_t)); 1432173139Srwatson mtx_unlock(&lldev->mtx_drv); 1433173139Srwatson if(status == XGE_HAL_OK) { 1434173139Srwatson if(copyout(info, ifreqp->ifr_data, 1435173139Srwatson sizeof(xge_hal_stats_sw_err_t)) == 0) 1436173139Srwatson retValue = 0; 1437173139Srwatson } 1438173139Srwatson else { 1439173139Srwatson xge_trace(XGE_ERR, 1440173139Srwatson "Getting tcode statistics failed (%d)", status); 1441173139Srwatson } 1442173139Srwatson xge_os_free(NULL, info, sizeof(xge_hal_stats_sw_err_t)); 1443173139Srwatson } 1444173139Srwatson break; 1445173139Srwatson 1446173139Srwatson case XGE_QUERY_DRIVERSTATS: 1447173139Srwatson if(copyout(&lldev->driver_stats, ifreqp->ifr_data, 1448173139Srwatson sizeof(xge_driver_stats_t)) == 0) { 1449173139Srwatson retValue = 0; 1450173139Srwatson } 1451173139Srwatson else { 1452173139Srwatson xge_trace(XGE_ERR, 1453173139Srwatson "Copyout of driver statistics failed (%d)", status); 1454173139Srwatson } 1455173139Srwatson break; 1456173139Srwatson 1457173139Srwatson case XGE_READ_VERSION: 1458173139Srwatson info = xge_os_malloc(NULL, XGE_BUFFER_SIZE); 1459173139Srwatson if(version != NULL) { 1460173139Srwatson strcpy(info, XGE_DRIVER_VERSION); 1461173139Srwatson if(copyout(info, ifreqp->ifr_data, XGE_BUFFER_SIZE) == 0) 1462173139Srwatson retValue = 0; 1463173139Srwatson xge_os_free(NULL, info, XGE_BUFFER_SIZE); 1464173139Srwatson } 1465173139Srwatson break; 1466173139Srwatson 1467173139Srwatson case XGE_QUERY_DEVCONF: 1468173139Srwatson info = xge_os_malloc(NULL, sizeof(xge_hal_device_config_t)); 1469173139Srwatson if(info != NULL) { 1470173139Srwatson mtx_lock(&lldev->mtx_drv); 1471173139Srwatson status = xge_hal_mgmt_device_config(lldev->devh, info, 1472173139Srwatson sizeof(xge_hal_device_config_t)); 1473173139Srwatson mtx_unlock(&lldev->mtx_drv); 1474173139Srwatson if(status == XGE_HAL_OK) { 1475173139Srwatson if(copyout(info, ifreqp->ifr_data, 1476173139Srwatson sizeof(xge_hal_device_config_t)) == 0) 1477173139Srwatson retValue = 0; 1478173139Srwatson } 1479173139Srwatson else { 1480173139Srwatson xge_trace(XGE_ERR, "Getting devconfig failed (%d)", 1481173139Srwatson status); 1482173139Srwatson } 1483173139Srwatson xge_os_free(NULL, info, sizeof(xge_hal_device_config_t)); 1484173139Srwatson } 1485173139Srwatson break; 1486173139Srwatson 1487173139Srwatson case XGE_QUERY_BUFFER_MODE: 1488173139Srwatson if(copyout(&lldev->buffer_mode, ifreqp->ifr_data, 1489173139Srwatson sizeof(int)) == 0) 1490173139Srwatson retValue = 0; 1491173139Srwatson break; 1492173139Srwatson 1493173139Srwatson case XGE_SET_BUFFER_MODE_1: 1494173139Srwatson case XGE_SET_BUFFER_MODE_2: 1495173139Srwatson case XGE_SET_BUFFER_MODE_5: 1496173139Srwatson *data = (*data == XGE_SET_BUFFER_MODE_1) ? 'Y':'N'; 1497173139Srwatson if(copyout(data, ifreqp->ifr_data, sizeof(data)) == 0) 1498173139Srwatson retValue = 0; 1499173139Srwatson break; 1500173139Srwatson default: 1501173139Srwatson xge_trace(XGE_TRACE, "Nothing is matching"); 1502173139Srwatson retValue = ENOTTY; 1503173139Srwatson break; 1504173139Srwatson } 1505173139Srwatson return retValue; 1506171095Ssam} 1507171095Ssam 1508173139Srwatson/** 1509173139Srwatson * xge_ioctl_registers 1510173139Srwatson * IOCTL to get registers 1511173139Srwatson * 1512173139Srwatson * @lldev Per-adapter data 1513173139Srwatson * @ifreqp Interface request 1514173139Srwatson */ 1515171095Ssamint 1516173139Srwatsonxge_ioctl_registers(xge_lldev_t *lldev, struct ifreq *ifreqp) 1517173139Srwatson{ 1518173139Srwatson xge_register_t *data = (xge_register_t *)ifreqp->ifr_data; 1519173139Srwatson xge_hal_status_e status = XGE_HAL_OK; 1520173139Srwatson int retValue = EINVAL, offset = 0, index = 0; 1521173139Srwatson u64 val64 = 0; 1522173139Srwatson 1523173139Srwatson /* Reading a register */ 1524173139Srwatson if(strcmp(data->option, "-r") == 0) { 1525173139Srwatson data->value = 0x0000; 1526173139Srwatson mtx_lock(&lldev->mtx_drv); 1527173139Srwatson status = xge_hal_mgmt_reg_read(lldev->devh, 0, data->offset, 1528173139Srwatson &data->value); 1529173139Srwatson mtx_unlock(&lldev->mtx_drv); 1530173139Srwatson if(status == XGE_HAL_OK) { 1531173139Srwatson if(copyout(data, ifreqp->ifr_data, sizeof(xge_register_t)) == 0) 1532173139Srwatson retValue = 0; 1533173139Srwatson } 1534173139Srwatson } 1535173139Srwatson /* Writing to a register */ 1536173139Srwatson else if(strcmp(data->option, "-w") == 0) { 1537173139Srwatson mtx_lock(&lldev->mtx_drv); 1538173139Srwatson status = xge_hal_mgmt_reg_write(lldev->devh, 0, data->offset, 1539173139Srwatson data->value); 1540173139Srwatson if(status == XGE_HAL_OK) { 1541173139Srwatson val64 = 0x0000; 1542173139Srwatson status = xge_hal_mgmt_reg_read(lldev->devh, 0, data->offset, 1543173139Srwatson &val64); 1544173139Srwatson if(status != XGE_HAL_OK) { 1545173139Srwatson xge_trace(XGE_ERR, "Reading back updated register failed"); 1546173139Srwatson } 1547173139Srwatson else { 1548173139Srwatson if(val64 != data->value) { 1549173139Srwatson xge_trace(XGE_ERR, 1550173139Srwatson "Read and written register values mismatched"); 1551173139Srwatson } 1552173139Srwatson else retValue = 0; 1553173139Srwatson } 1554173139Srwatson } 1555173139Srwatson else { 1556173139Srwatson xge_trace(XGE_ERR, "Getting register value failed"); 1557173139Srwatson } 1558173139Srwatson mtx_unlock(&lldev->mtx_drv); 1559173139Srwatson } 1560173139Srwatson else { 1561173139Srwatson mtx_lock(&lldev->mtx_drv); 1562173139Srwatson for(index = 0, offset = 0; offset <= XGE_OFFSET_OF_LAST_REG; 1563173139Srwatson index++, offset += 0x0008) { 1564173139Srwatson val64 = 0; 1565173139Srwatson status = xge_hal_mgmt_reg_read(lldev->devh, 0, offset, &val64); 1566173139Srwatson if(status != XGE_HAL_OK) { 1567173139Srwatson xge_trace(XGE_ERR, "Getting register value failed"); 1568173139Srwatson break; 1569173139Srwatson } 1570173139Srwatson *((u64 *)((u64 *)data + index)) = val64; 1571173139Srwatson retValue = 0; 1572173139Srwatson } 1573173139Srwatson mtx_unlock(&lldev->mtx_drv); 1574173139Srwatson 1575173139Srwatson if(retValue == 0) { 1576173139Srwatson if(copyout(data, ifreqp->ifr_data, 1577173139Srwatson sizeof(xge_hal_pci_bar0_t)) != 0) { 1578173139Srwatson xge_trace(XGE_ERR, "Copyout of register values failed"); 1579173139Srwatson retValue = EINVAL; 1580173139Srwatson } 1581173139Srwatson } 1582173139Srwatson else { 1583173139Srwatson xge_trace(XGE_ERR, "Getting register values failed"); 1584173139Srwatson } 1585173139Srwatson } 1586173139Srwatson return retValue; 1587173139Srwatson} 1588173139Srwatson 1589173139Srwatson/** 1590173139Srwatson * xge_ioctl 1591173139Srwatson * Callback to control the device - Interface configuration 1592173139Srwatson * 1593173139Srwatson * @ifnetp Interface Handle 1594173139Srwatson * @command Device control command 1595173139Srwatson * @data Parameters associated with command (if any) 1596173139Srwatson */ 1597173139Srwatsonint 1598171095Ssamxge_ioctl(struct ifnet *ifnetp, unsigned long command, caddr_t data) 1599171095Ssam{ 1600173139Srwatson struct ifreq *ifreqp = (struct ifreq *)data; 1601173139Srwatson xge_lldev_t *lldev = ifnetp->if_softc; 1602173139Srwatson struct ifmedia *ifmediap = &lldev->media; 1603173139Srwatson int retValue = 0, mask = 0; 1604171095Ssam 1605171095Ssam if(lldev->in_detach) { 1606171095Ssam return retValue; 1607171095Ssam } 1608171095Ssam 1609171095Ssam switch(command) { 1610171095Ssam /* Set/Get ifnet address */ 1611171095Ssam case SIOCSIFADDR: 1612171095Ssam case SIOCGIFADDR: 1613171095Ssam ether_ioctl(ifnetp, command, data); 1614171095Ssam break; 1615171095Ssam 1616171095Ssam /* Set ifnet MTU */ 1617171095Ssam case SIOCSIFMTU: 1618173139Srwatson retValue = xge_change_mtu(lldev, ifreqp->ifr_mtu); 1619171095Ssam break; 1620171095Ssam 1621171095Ssam /* Set ifnet flags */ 1622171095Ssam case SIOCSIFFLAGS: 1623171095Ssam if(ifnetp->if_flags & IFF_UP) { 1624171095Ssam /* Link status is UP */ 1625171095Ssam if(!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)) { 1626173139Srwatson xge_init(lldev); 1627171095Ssam } 1628171095Ssam xge_disable_promisc(lldev); 1629171095Ssam xge_enable_promisc(lldev); 1630171095Ssam } 1631171095Ssam else { 1632171095Ssam /* Link status is DOWN */ 1633171095Ssam /* If device is in running, make it down */ 1634171095Ssam if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) { 1635171095Ssam xge_stop(lldev); 1636171095Ssam } 1637171095Ssam } 1638171095Ssam break; 1639171095Ssam 1640171095Ssam /* Add/delete multicast address */ 1641171095Ssam case SIOCADDMULTI: 1642171095Ssam case SIOCDELMULTI: 1643171095Ssam if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) { 1644171095Ssam xge_setmulti(lldev); 1645171095Ssam } 1646171095Ssam break; 1647171095Ssam 1648171095Ssam /* Set/Get net media */ 1649171095Ssam case SIOCSIFMEDIA: 1650171095Ssam case SIOCGIFMEDIA: 1651171095Ssam retValue = ifmedia_ioctl(ifnetp, ifreqp, ifmediap, command); 1652171095Ssam break; 1653171095Ssam 1654171095Ssam /* Set capabilities */ 1655171095Ssam case SIOCSIFCAP: 1656173139Srwatson mtx_lock(&lldev->mtx_drv); 1657171095Ssam mask = ifreqp->ifr_reqcap ^ ifnetp->if_capenable; 1658173139Srwatson if(mask & IFCAP_TXCSUM) { 1659173139Srwatson if(ifnetp->if_capenable & IFCAP_TXCSUM) { 1660173139Srwatson ifnetp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TXCSUM); 1661173139Srwatson ifnetp->if_hwassist &= 1662173139Srwatson ~(CSUM_TCP | CSUM_UDP | CSUM_TSO); 1663173139Srwatson } 1664173139Srwatson else { 1665173139Srwatson ifnetp->if_capenable |= IFCAP_TXCSUM; 1666173139Srwatson ifnetp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1667173139Srwatson } 1668173139Srwatson } 1669171095Ssam if(mask & IFCAP_TSO4) { 1670171095Ssam if(ifnetp->if_capenable & IFCAP_TSO4) { 1671171095Ssam ifnetp->if_capenable &= ~IFCAP_TSO4; 1672171095Ssam ifnetp->if_hwassist &= ~CSUM_TSO; 1673173139Srwatson 1674173139Srwatson xge_os_printf("%s: TSO Disabled", 1675173139Srwatson device_get_nameunit(lldev->device)); 1676171095Ssam } 1677173139Srwatson else if(ifnetp->if_capenable & IFCAP_TXCSUM) { 1678171095Ssam ifnetp->if_capenable |= IFCAP_TSO4; 1679171095Ssam ifnetp->if_hwassist |= CSUM_TSO; 1680173139Srwatson 1681173139Srwatson xge_os_printf("%s: TSO Enabled", 1682173139Srwatson device_get_nameunit(lldev->device)); 1683171095Ssam } 1684171095Ssam } 1685173139Srwatson 1686173139Srwatson mtx_unlock(&lldev->mtx_drv); 1687171095Ssam break; 1688171095Ssam 1689173139Srwatson /* Custom IOCTL 0 */ 1690171095Ssam case SIOCGPRIVATE_0: 1691173139Srwatson retValue = xge_ioctl_stats(lldev, ifreqp); 1692171095Ssam break; 1693171095Ssam 1694173139Srwatson /* Custom IOCTL 1 */ 1695171095Ssam case SIOCGPRIVATE_1: 1696173139Srwatson retValue = xge_ioctl_registers(lldev, ifreqp); 1697171095Ssam break; 1698171095Ssam 1699171095Ssam default: 1700171095Ssam retValue = EINVAL; 1701171095Ssam break; 1702171095Ssam } 1703171095Ssam return retValue; 1704171095Ssam} 1705171095Ssam 1706173139Srwatson/** 1707173139Srwatson * xge_init 1708173139Srwatson * Initialize the interface 1709173139Srwatson * 1710173139Srwatson * @plldev Per-adapter Data 1711173139Srwatson */ 1712171095Ssamvoid 1713171095Ssamxge_init(void *plldev) 1714171095Ssam{ 1715173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)plldev; 1716171095Ssam 1717173139Srwatson mtx_lock(&lldev->mtx_drv); 1718173139Srwatson xge_os_memzero(&lldev->driver_stats, sizeof(xge_driver_stats_t)); 1719173139Srwatson xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL); 1720173139Srwatson mtx_unlock(&lldev->mtx_drv); 1721171095Ssam} 1722171095Ssam 1723173139Srwatson/** 1724173139Srwatson * xge_device_init 1725173139Srwatson * Initialize the interface (called by holding lock) 1726173139Srwatson * 1727173139Srwatson * @pdevin Per-adapter Data 1728173139Srwatson */ 1729171095Ssamvoid 1730173139Srwatsonxge_device_init(xge_lldev_t *lldev, xge_hal_channel_reopen_e option) 1731171095Ssam{ 1732173139Srwatson struct ifnet *ifnetp = lldev->ifnetp; 1733173139Srwatson xge_hal_device_t *hldev = lldev->devh; 1734173139Srwatson struct ifaddr *ifaddrp; 1735173139Srwatson unsigned char *macaddr; 1736173139Srwatson struct sockaddr_dl *sockaddrp; 1737173139Srwatson int status = XGE_HAL_OK; 1738171095Ssam 1739173139Srwatson mtx_assert((&lldev->mtx_drv), MA_OWNED); 1740171095Ssam 1741171095Ssam /* If device is in running state, initializing is not required */ 1742173139Srwatson if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) 1743171095Ssam return; 1744171095Ssam 1745171095Ssam /* Initializing timer */ 1746171095Ssam callout_init(&lldev->timer, CALLOUT_MPSAFE); 1747171095Ssam 1748173139Srwatson xge_trace(XGE_TRACE, "Set MTU size"); 1749173139Srwatson status = xge_hal_device_mtu_set(hldev, ifnetp->if_mtu); 1750173139Srwatson if(status != XGE_HAL_OK) { 1751173139Srwatson xge_trace(XGE_ERR, "Setting MTU in HAL device failed"); 1752173139Srwatson goto _exit; 1753173139Srwatson } 1754171095Ssam 1755173139Srwatson /* Enable HAL device */ 1756173139Srwatson xge_hal_device_enable(hldev); 1757173139Srwatson 1758173139Srwatson /* Get MAC address and update in HAL */ 1759173139Srwatson ifaddrp = ifnetp->if_addr; 1760173139Srwatson sockaddrp = (struct sockaddr_dl *)ifaddrp->ifa_addr; 1761173139Srwatson sockaddrp->sdl_type = IFT_ETHER; 1762173139Srwatson sockaddrp->sdl_alen = ifnetp->if_addrlen; 1763173139Srwatson macaddr = LLADDR(sockaddrp); 1764173139Srwatson xge_trace(XGE_TRACE, 1765173139Srwatson "Setting MAC address: %02x:%02x:%02x:%02x:%02x:%02x\n", 1766173139Srwatson *macaddr, *(macaddr + 1), *(macaddr + 2), *(macaddr + 3), 1767173139Srwatson *(macaddr + 4), *(macaddr + 5)); 1768173139Srwatson status = xge_hal_device_macaddr_set(hldev, 0, macaddr); 1769173139Srwatson if(status != XGE_HAL_OK) 1770173139Srwatson xge_trace(XGE_ERR, "Setting MAC address failed (%d)", status); 1771173139Srwatson 1772173139Srwatson /* Opening channels */ 1773173139Srwatson mtx_unlock(&lldev->mtx_drv); 1774173139Srwatson status = xge_channel_open(lldev, option); 1775173139Srwatson mtx_lock(&lldev->mtx_drv); 1776173139Srwatson if(status != XGE_HAL_OK) 1777173139Srwatson goto _exit; 1778173139Srwatson 1779173139Srwatson /* Set appropriate flags */ 1780173139Srwatson ifnetp->if_drv_flags |= IFF_DRV_RUNNING; 1781173139Srwatson ifnetp->if_flags &= ~IFF_DRV_OACTIVE; 1782173139Srwatson 1783173139Srwatson /* Checksum capability */ 1784173139Srwatson ifnetp->if_hwassist = (ifnetp->if_capenable & IFCAP_TXCSUM) ? 1785173139Srwatson (CSUM_TCP | CSUM_UDP) : 0; 1786173139Srwatson 1787173139Srwatson if((lldev->enabled_tso) && (ifnetp->if_capenable & IFCAP_TSO4)) 1788173139Srwatson ifnetp->if_hwassist |= CSUM_TSO; 1789173139Srwatson 1790173139Srwatson /* Enable interrupts */ 1791173139Srwatson xge_hal_device_intr_enable(hldev); 1792173139Srwatson 1793173139Srwatson callout_reset(&lldev->timer, 10*hz, xge_timer, lldev); 1794173139Srwatson 1795173139Srwatson /* Disable promiscuous mode */ 1796173139Srwatson xge_trace(XGE_TRACE, "If opted, enable promiscuous mode"); 1797173139Srwatson xge_enable_promisc(lldev); 1798173139Srwatson 1799173139Srwatson /* Device is initialized */ 1800173139Srwatson lldev->initialized = 1; 1801173139Srwatson xge_os_mdelay(1000); 1802173139Srwatson 1803173139Srwatson_exit: 1804173139Srwatson return; 1805171095Ssam} 1806171095Ssam 1807173139Srwatson/** 1808173139Srwatson * xge_timer 1809173139Srwatson * Timer timeout function to handle link status 1810173139Srwatson * 1811173139Srwatson * @devp Per-adapter Data 1812173139Srwatson */ 1813171095Ssamvoid 1814171095Ssamxge_timer(void *devp) 1815171095Ssam{ 1816173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)devp; 1817171095Ssam xge_hal_device_t *hldev = lldev->devh; 1818171095Ssam 1819171095Ssam /* Poll for changes */ 1820171095Ssam xge_hal_device_poll(hldev); 1821171095Ssam 1822171095Ssam /* Reset timer */ 1823171095Ssam callout_reset(&lldev->timer, hz, xge_timer, lldev); 1824171095Ssam 1825171095Ssam return; 1826171095Ssam} 1827171095Ssam 1828173139Srwatson/** 1829173139Srwatson * xge_stop 1830173139Srwatson * De-activate the interface 1831173139Srwatson * 1832173139Srwatson * @lldev Per-adater Data 1833173139Srwatson */ 1834171095Ssamvoid 1835173139Srwatsonxge_stop(xge_lldev_t *lldev) 1836171095Ssam{ 1837173139Srwatson mtx_lock(&lldev->mtx_drv); 1838173139Srwatson xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL); 1839173139Srwatson mtx_unlock(&lldev->mtx_drv); 1840171095Ssam} 1841171095Ssam 1842173139Srwatson/** 1843173139Srwatson * xge_isr_filter 1844173139Srwatson * ISR filter function - to filter interrupts from other devices (shared) 1845173139Srwatson * 1846173139Srwatson * @handle Per-adapter Data 1847173139Srwatson * 1848173139Srwatson * Returns 1849173139Srwatson * FILTER_STRAY if interrupt is from other device 1850173139Srwatson * FILTER_SCHEDULE_THREAD if interrupt is from Xframe device 1851171095Ssam */ 1852171095Ssamint 1853173139Srwatsonxge_isr_filter(void *handle) 1854171095Ssam{ 1855173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)handle; 1856173139Srwatson xge_hal_pci_bar0_t *bar0 = (xge_hal_pci_bar0_t *)((lldev->devh)->bar0); 1857171095Ssam u16 retValue = FILTER_STRAY; 1858171095Ssam u64 val64 = 0; 1859171095Ssam 1860173139Srwatson XGE_DRV_STATS(isr_filter); 1861171095Ssam 1862173139Srwatson val64 = xge_os_pio_mem_read64(lldev->pdev, (lldev->devh)->regh0, 1863171095Ssam &bar0->general_int_status); 1864171095Ssam retValue = (!val64) ? FILTER_STRAY : FILTER_SCHEDULE_THREAD; 1865171095Ssam 1866171095Ssam return retValue; 1867171095Ssam} 1868171095Ssam 1869173139Srwatson/** 1870173139Srwatson * xge_isr_line 1871173139Srwatson * Interrupt service routine for Line interrupts 1872173139Srwatson * 1873173139Srwatson * @plldev Per-adapter Data 1874173139Srwatson */ 1875171095Ssamvoid 1876173139Srwatsonxge_isr_line(void *plldev) 1877171095Ssam{ 1878171095Ssam xge_hal_status_e status; 1879173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)plldev; 1880171095Ssam xge_hal_device_t *hldev = (xge_hal_device_t *)lldev->devh; 1881171095Ssam struct ifnet *ifnetp = lldev->ifnetp; 1882171095Ssam 1883173139Srwatson XGE_DRV_STATS(isr_line); 1884173139Srwatson 1885171095Ssam if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) { 1886171095Ssam status = xge_hal_device_handle_irq(hldev); 1887173139Srwatson if(!(IFQ_DRV_IS_EMPTY(&ifnetp->if_snd))) 1888173139Srwatson xge_send(ifnetp); 1889171095Ssam } 1890171095Ssam} 1891171095Ssam 1892173139Srwatson/* 1893173139Srwatson * xge_isr_msi 1894173139Srwatson * ISR for Message signaled interrupts 1895173139Srwatson */ 1896173139Srwatsonvoid 1897173139Srwatsonxge_isr_msi(void *plldev) 1898173139Srwatson{ 1899173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)plldev; 1900173139Srwatson XGE_DRV_STATS(isr_msi); 1901173139Srwatson xge_hal_device_continue_irq(lldev->devh); 1902173139Srwatson} 1903173139Srwatson 1904173139Srwatson/** 1905173139Srwatson * xge_rx_open 1906173139Srwatson * Initiate and open all Rx channels 1907173139Srwatson * 1908173139Srwatson * @qid Ring Index 1909173139Srwatson * @lldev Per-adapter Data 1910173139Srwatson * @rflag Channel open/close/reopen flag 1911173139Srwatson * 1912173139Srwatson * Returns 0 or Error Number 1913173139Srwatson */ 1914171095Ssamint 1915173139Srwatsonxge_rx_open(int qid, xge_lldev_t *lldev, xge_hal_channel_reopen_e rflag) 1916171095Ssam{ 1917171095Ssam u64 adapter_status = 0x0; 1918173139Srwatson xge_hal_status_e status = XGE_HAL_FAIL; 1919171095Ssam 1920171095Ssam xge_hal_channel_attr_t attr = { 1921171095Ssam .post_qid = qid, 1922171095Ssam .compl_qid = 0, 1923173139Srwatson .callback = xge_rx_compl, 1924173139Srwatson .per_dtr_space = sizeof(xge_rx_priv_t), 1925171095Ssam .flags = 0, 1926171095Ssam .type = XGE_HAL_CHANNEL_TYPE_RING, 1927171095Ssam .userdata = lldev, 1928173139Srwatson .dtr_init = xge_rx_initial_replenish, 1929173139Srwatson .dtr_term = xge_rx_term 1930171095Ssam }; 1931171095Ssam 1932171095Ssam /* If device is not ready, return */ 1933173139Srwatson status = xge_hal_device_status(lldev->devh, &adapter_status); 1934173139Srwatson if(status != XGE_HAL_OK) { 1935173139Srwatson xge_os_printf("Adapter Status: 0x%llx", (long long) adapter_status); 1936173139Srwatson XGE_EXIT_ON_ERR("Device is not ready", _exit, XGE_HAL_FAIL); 1937171095Ssam } 1938173139Srwatson else { 1939173139Srwatson status = xge_hal_channel_open(lldev->devh, &attr, 1940173139Srwatson &lldev->ring_channel[qid], rflag); 1941171095Ssam } 1942171095Ssam 1943173139Srwatson_exit: 1944173139Srwatson return status; 1945171095Ssam} 1946171095Ssam 1947173139Srwatson/** 1948173139Srwatson * xge_tx_open 1949173139Srwatson * Initialize and open all Tx channels 1950173139Srwatson * 1951173139Srwatson * @lldev Per-adapter Data 1952173139Srwatson * @tflag Channel open/close/reopen flag 1953173139Srwatson * 1954173139Srwatson * Returns 0 or Error Number 1955173139Srwatson */ 1956171095Ssamint 1957173139Srwatsonxge_tx_open(xge_lldev_t *lldev, xge_hal_channel_reopen_e tflag) 1958171095Ssam{ 1959173139Srwatson xge_hal_status_e status = XGE_HAL_FAIL; 1960171095Ssam u64 adapter_status = 0x0; 1961173139Srwatson int qindex, index; 1962171095Ssam 1963171095Ssam xge_hal_channel_attr_t attr = { 1964171095Ssam .compl_qid = 0, 1965173139Srwatson .callback = xge_tx_compl, 1966173139Srwatson .per_dtr_space = sizeof(xge_tx_priv_t), 1967171095Ssam .flags = 0, 1968171095Ssam .type = XGE_HAL_CHANNEL_TYPE_FIFO, 1969171095Ssam .userdata = lldev, 1970173139Srwatson .dtr_init = xge_tx_initial_replenish, 1971173139Srwatson .dtr_term = xge_tx_term 1972171095Ssam }; 1973171095Ssam 1974171095Ssam /* If device is not ready, return */ 1975173139Srwatson status = xge_hal_device_status(lldev->devh, &adapter_status); 1976173139Srwatson if(status != XGE_HAL_OK) { 1977173139Srwatson xge_os_printf("Adapter Status: 0x%llx", (long long) adapter_status); 1978173139Srwatson XGE_EXIT_ON_ERR("Device is not ready", _exit, XGE_HAL_FAIL); 1979171095Ssam } 1980171095Ssam 1981173139Srwatson for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) { 1982173139Srwatson attr.post_qid = qindex, 1983173139Srwatson status = xge_hal_channel_open(lldev->devh, &attr, 1984173139Srwatson &lldev->fifo_channel[qindex], tflag); 1985173139Srwatson if(status != XGE_HAL_OK) { 1986173139Srwatson for(index = 0; index < qindex; index++) 1987173139Srwatson xge_hal_channel_close(lldev->fifo_channel[index], tflag); 1988173139Srwatson } 1989171095Ssam } 1990171095Ssam 1991173139Srwatson_exit: 1992173139Srwatson return status; 1993173139Srwatson} 1994171095Ssam 1995173139Srwatson/** 1996173139Srwatson * xge_enable_msi 1997173139Srwatson * Enables MSI 1998173139Srwatson * 1999173139Srwatson * @lldev Per-adapter Data 2000173139Srwatson */ 2001173139Srwatsonvoid 2002173139Srwatsonxge_enable_msi(xge_lldev_t *lldev) 2003173139Srwatson{ 2004173139Srwatson xge_list_t *item = NULL; 2005173139Srwatson xge_hal_device_t *hldev = lldev->devh; 2006173139Srwatson xge_hal_channel_t *channel = NULL; 2007173139Srwatson u16 offset = 0, val16 = 0; 2008173139Srwatson 2009173139Srwatson xge_os_pci_read16(lldev->pdev, NULL, 2010173139Srwatson xge_offsetof(xge_hal_pci_config_le_t, msi_control), &val16); 2011173139Srwatson 2012173139Srwatson /* Update msi_data */ 2013173139Srwatson offset = (val16 & 0x80) ? 0x4c : 0x48; 2014173139Srwatson xge_os_pci_read16(lldev->pdev, NULL, offset, &val16); 2015173139Srwatson if(val16 & 0x1) 2016173139Srwatson val16 &= 0xfffe; 2017173139Srwatson else 2018173139Srwatson val16 |= 0x1; 2019173139Srwatson xge_os_pci_write16(lldev->pdev, NULL, offset, val16); 2020173139Srwatson 2021173139Srwatson /* Update msi_control */ 2022173139Srwatson xge_os_pci_read16(lldev->pdev, NULL, 2023173139Srwatson xge_offsetof(xge_hal_pci_config_le_t, msi_control), &val16); 2024173139Srwatson val16 |= 0x10; 2025173139Srwatson xge_os_pci_write16(lldev->pdev, NULL, 2026173139Srwatson xge_offsetof(xge_hal_pci_config_le_t, msi_control), val16); 2027173139Srwatson 2028173139Srwatson /* Set TxMAT and RxMAT registers with MSI */ 2029173139Srwatson xge_list_for_each(item, &hldev->free_channels) { 2030173139Srwatson channel = xge_container_of(item, xge_hal_channel_t, item); 2031173139Srwatson xge_hal_channel_msi_set(channel, 1, (u32)val16); 2032173139Srwatson } 2033171095Ssam} 2034171095Ssam 2035173139Srwatson/** 2036173139Srwatson * xge_channel_open 2037173139Srwatson * Open both Tx and Rx channels 2038173139Srwatson * 2039173139Srwatson * @lldev Per-adapter Data 2040173139Srwatson * @option Channel reopen option 2041173139Srwatson */ 2042171095Ssamint 2043173139Srwatsonxge_channel_open(xge_lldev_t *lldev, xge_hal_channel_reopen_e option) 2044171095Ssam{ 2045173139Srwatson xge_lro_entry_t *lro_session = NULL; 2046173139Srwatson xge_hal_status_e status = XGE_HAL_OK; 2047173139Srwatson int index = 0, index2 = 0; 2048171095Ssam 2049173139Srwatson if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) { 2050173139Srwatson xge_msi_info_restore(lldev); 2051173139Srwatson xge_enable_msi(lldev); 2052173139Srwatson } 2053171095Ssam 2054173139Srwatson_exit2: 2055173139Srwatson status = xge_create_dma_tags(lldev->device); 2056173139Srwatson if(status != XGE_HAL_OK) 2057173139Srwatson XGE_EXIT_ON_ERR("DMA tag creation failed", _exit, status); 2058173139Srwatson 2059171095Ssam /* Open ring (Rx) channel */ 2060171095Ssam for(index = 0; index < XGE_RING_COUNT; index++) { 2061173139Srwatson status = xge_rx_open(index, lldev, option); 2062173139Srwatson if(status != XGE_HAL_OK) { 2063173139Srwatson /* 2064173139Srwatson * DMA mapping fails in the unpatched Kernel which can't 2065173139Srwatson * allocate contiguous memory for Jumbo frames. 2066173139Srwatson * Try using 5 buffer mode. 2067173139Srwatson */ 2068173139Srwatson if((lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) && 2069173139Srwatson (((lldev->ifnetp)->if_mtu + XGE_HAL_MAC_HEADER_MAX_SIZE) > 2070173139Srwatson MJUMPAGESIZE)) { 2071173139Srwatson /* Close so far opened channels */ 2072173139Srwatson for(index2 = 0; index2 < index; index2++) { 2073173139Srwatson xge_hal_channel_close(lldev->ring_channel[index2], 2074173139Srwatson option); 2075173139Srwatson } 2076173139Srwatson 2077173139Srwatson /* Destroy DMA tags intended to use for 1 buffer mode */ 2078173139Srwatson if(bus_dmamap_destroy(lldev->dma_tag_rx, 2079173139Srwatson lldev->extra_dma_map)) { 2080173139Srwatson xge_trace(XGE_ERR, "Rx extra DMA map destroy failed"); 2081173139Srwatson } 2082173139Srwatson if(bus_dma_tag_destroy(lldev->dma_tag_rx)) 2083173139Srwatson xge_trace(XGE_ERR, "Rx DMA tag destroy failed"); 2084173139Srwatson if(bus_dma_tag_destroy(lldev->dma_tag_tx)) 2085173139Srwatson xge_trace(XGE_ERR, "Tx DMA tag destroy failed"); 2086173139Srwatson 2087173139Srwatson /* Switch to 5 buffer mode */ 2088173139Srwatson lldev->buffer_mode = XGE_HAL_RING_QUEUE_BUFFER_MODE_5; 2089173139Srwatson xge_buffer_mode_init(lldev, (lldev->ifnetp)->if_mtu); 2090173139Srwatson 2091173139Srwatson /* Restart init */ 2092173139Srwatson goto _exit2; 2093171095Ssam } 2094173139Srwatson else { 2095173139Srwatson XGE_EXIT_ON_ERR("Opening Rx channel failed", _exit1, 2096173139Srwatson status); 2097173139Srwatson } 2098171095Ssam } 2099171095Ssam } 2100171095Ssam 2101173139Srwatson if(lldev->enabled_lro) { 2102173139Srwatson SLIST_INIT(&lldev->lro_free); 2103173139Srwatson SLIST_INIT(&lldev->lro_active); 2104173139Srwatson lldev->lro_num = XGE_LRO_DEFAULT_ENTRIES; 2105173139Srwatson 2106173139Srwatson for(index = 0; index < lldev->lro_num; index++) { 2107173139Srwatson lro_session = (xge_lro_entry_t *) 2108173139Srwatson xge_os_malloc(NULL, sizeof(xge_lro_entry_t)); 2109173139Srwatson if(lro_session == NULL) { 2110173139Srwatson lldev->lro_num = index; 2111173139Srwatson break; 2112173139Srwatson } 2113173139Srwatson SLIST_INSERT_HEAD(&lldev->lro_free, lro_session, next); 2114171095Ssam } 2115171095Ssam } 2116171095Ssam 2117173139Srwatson /* Open FIFO (Tx) channel */ 2118173139Srwatson status = xge_tx_open(lldev, option); 2119173139Srwatson if(status != XGE_HAL_OK) 2120173139Srwatson XGE_EXIT_ON_ERR("Opening Tx channel failed", _exit1, status); 2121173139Srwatson 2122173139Srwatson goto _exit; 2123173139Srwatson 2124173139Srwatson_exit1: 2125173139Srwatson /* 2126173139Srwatson * Opening Rx channel(s) failed (index is <last ring index - 1>) or 2127173139Srwatson * Initialization of LRO failed (index is XGE_RING_COUNT) 2128173139Srwatson * Opening Tx channel failed (index is XGE_RING_COUNT) 2129173139Srwatson */ 2130173139Srwatson for(index2 = 0; index2 < index; index2++) 2131173139Srwatson xge_hal_channel_close(lldev->ring_channel[index2], option); 2132173139Srwatson 2133173139Srwatson_exit: 2134171095Ssam return status; 2135171095Ssam} 2136171095Ssam 2137173139Srwatson/** 2138173139Srwatson * xge_channel_close 2139173139Srwatson * Close both Tx and Rx channels 2140173139Srwatson * 2141173139Srwatson * @lldev Per-adapter Data 2142173139Srwatson * @option Channel reopen option 2143173139Srwatson * 2144173139Srwatson */ 2145173139Srwatsonvoid 2146173139Srwatsonxge_channel_close(xge_lldev_t *lldev, xge_hal_channel_reopen_e option) 2147171095Ssam{ 2148173139Srwatson int qindex = 0; 2149171095Ssam 2150171095Ssam DELAY(1000 * 1000); 2151171095Ssam 2152171095Ssam /* Close FIFO (Tx) channel */ 2153173139Srwatson for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) 2154173139Srwatson xge_hal_channel_close(lldev->fifo_channel[qindex], option); 2155171095Ssam 2156173139Srwatson /* Close Ring (Rx) channels */ 2157173139Srwatson for(qindex = 0; qindex < XGE_RING_COUNT; qindex++) 2158173139Srwatson xge_hal_channel_close(lldev->ring_channel[qindex], option); 2159171095Ssam 2160173139Srwatson if(bus_dmamap_destroy(lldev->dma_tag_rx, lldev->extra_dma_map)) 2161173139Srwatson xge_trace(XGE_ERR, "Rx extra map destroy failed"); 2162173139Srwatson if(bus_dma_tag_destroy(lldev->dma_tag_rx)) 2163173139Srwatson xge_trace(XGE_ERR, "Rx DMA tag destroy failed"); 2164173139Srwatson if(bus_dma_tag_destroy(lldev->dma_tag_tx)) 2165173139Srwatson xge_trace(XGE_ERR, "Tx DMA tag destroy failed"); 2166171095Ssam} 2167171095Ssam 2168173139Srwatson/** 2169173139Srwatson * dmamap_cb 2170173139Srwatson * DMA map callback 2171173139Srwatson * 2172173139Srwatson * @arg Parameter passed from dmamap 2173173139Srwatson * @segs Segments 2174173139Srwatson * @nseg Number of segments 2175173139Srwatson * @error Error 2176173139Srwatson */ 2177171095Ssamvoid 2178171095Ssamdmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 2179171095Ssam{ 2180171095Ssam if(!error) { 2181171095Ssam *(bus_addr_t *) arg = segs->ds_addr; 2182171095Ssam } 2183171095Ssam} 2184171095Ssam 2185173139Srwatson/** 2186173139Srwatson * xge_reset 2187173139Srwatson * Device Reset 2188173139Srwatson * 2189173139Srwatson * @lldev Per-adapter Data 2190173139Srwatson */ 2191171095Ssamvoid 2192173139Srwatsonxge_reset(xge_lldev_t *lldev) 2193171095Ssam{ 2194171095Ssam xge_trace(XGE_TRACE, "Reseting the chip"); 2195171095Ssam 2196171095Ssam /* If the device is not initialized, return */ 2197173139Srwatson if(lldev->initialized) { 2198173139Srwatson mtx_lock(&lldev->mtx_drv); 2199173139Srwatson xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL); 2200173139Srwatson xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL); 2201173139Srwatson mtx_unlock(&lldev->mtx_drv); 2202171095Ssam } 2203171095Ssam 2204171095Ssam return; 2205171095Ssam} 2206171095Ssam 2207173139Srwatson/** 2208173139Srwatson * xge_setmulti 2209173139Srwatson * Set an address as a multicast address 2210173139Srwatson * 2211173139Srwatson * @lldev Per-adapter Data 2212173139Srwatson */ 2213171095Ssamvoid 2214173139Srwatsonxge_setmulti(xge_lldev_t *lldev) 2215171095Ssam{ 2216171095Ssam struct ifmultiaddr *ifma; 2217171095Ssam u8 *lladdr; 2218171095Ssam xge_hal_device_t *hldev = (xge_hal_device_t *)lldev->devh; 2219171095Ssam struct ifnet *ifnetp = lldev->ifnetp; 2220171095Ssam int index = 0; 2221171095Ssam int offset = 1; 2222171095Ssam int table_size = 47; 2223171095Ssam xge_hal_status_e status = XGE_HAL_OK; 2224171095Ssam u8 initial_addr[]= {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; 2225171095Ssam 2226171095Ssam if((ifnetp->if_flags & IFF_MULTICAST) && (!lldev->all_multicast)) { 2227171095Ssam status = xge_hal_device_mcast_enable(hldev); 2228171095Ssam lldev->all_multicast = 1; 2229171095Ssam } 2230171095Ssam else if((ifnetp->if_flags & IFF_MULTICAST) && (lldev->all_multicast)) { 2231171095Ssam status = xge_hal_device_mcast_disable(hldev); 2232171095Ssam lldev->all_multicast = 0; 2233171095Ssam } 2234171095Ssam 2235171095Ssam if(status != XGE_HAL_OK) { 2236173139Srwatson xge_trace(XGE_ERR, "Enabling/disabling multicast failed"); 2237173139Srwatson goto _exit; 2238171095Ssam } 2239171095Ssam 2240171095Ssam /* Updating address list */ 2241195049Srwatson if_maddr_rlock(ifnetp); 2242171095Ssam index = 0; 2243171095Ssam TAILQ_FOREACH(ifma, &ifnetp->if_multiaddrs, ifma_link) { 2244171095Ssam if(ifma->ifma_addr->sa_family != AF_LINK) { 2245171095Ssam continue; 2246171095Ssam } 2247171095Ssam lladdr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2248171095Ssam index += 1; 2249171095Ssam } 2250195049Srwatson if_maddr_runlock(ifnetp); 2251171095Ssam 2252171095Ssam if((!lldev->all_multicast) && (index)) { 2253171095Ssam lldev->macaddr_count = (index + 1); 2254171095Ssam if(lldev->macaddr_count > table_size) { 2255173139Srwatson goto _exit; 2256171095Ssam } 2257171095Ssam 2258171095Ssam /* Clear old addresses */ 2259171095Ssam for(index = 0; index < 48; index++) { 2260171095Ssam xge_hal_device_macaddr_set(hldev, (offset + index), 2261171095Ssam initial_addr); 2262171095Ssam } 2263171095Ssam } 2264171095Ssam 2265171095Ssam /* Add new addresses */ 2266195049Srwatson if_maddr_rlock(ifnetp); 2267171095Ssam index = 0; 2268171095Ssam TAILQ_FOREACH(ifma, &ifnetp->if_multiaddrs, ifma_link) { 2269171095Ssam if(ifma->ifma_addr->sa_family != AF_LINK) { 2270171095Ssam continue; 2271171095Ssam } 2272171095Ssam lladdr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2273171095Ssam xge_hal_device_macaddr_set(hldev, (offset + index), lladdr); 2274171095Ssam index += 1; 2275171095Ssam } 2276195049Srwatson if_maddr_runlock(ifnetp); 2277171095Ssam 2278173139Srwatson_exit: 2279173139Srwatson return; 2280171095Ssam} 2281171095Ssam 2282173139Srwatson/** 2283173139Srwatson * xge_enable_promisc 2284173139Srwatson * Enable Promiscuous Mode 2285173139Srwatson * 2286173139Srwatson * @lldev Per-adapter Data 2287173139Srwatson */ 2288171095Ssamvoid 2289173139Srwatsonxge_enable_promisc(xge_lldev_t *lldev) 2290171095Ssam{ 2291171095Ssam struct ifnet *ifnetp = lldev->ifnetp; 2292171095Ssam xge_hal_device_t *hldev = lldev->devh; 2293171095Ssam xge_hal_pci_bar0_t *bar0 = NULL; 2294171095Ssam u64 val64 = 0; 2295171095Ssam 2296171095Ssam bar0 = (xge_hal_pci_bar0_t *) hldev->bar0; 2297171095Ssam 2298171095Ssam if(ifnetp->if_flags & IFF_PROMISC) { 2299171095Ssam xge_hal_device_promisc_enable(lldev->devh); 2300173139Srwatson 2301171095Ssam /* 2302171095Ssam * When operating in promiscuous mode, don't strip the VLAN tag 2303171095Ssam */ 2304171095Ssam val64 = xge_os_pio_mem_read64(lldev->pdev, hldev->regh0, 2305171095Ssam &bar0->rx_pa_cfg); 2306171095Ssam val64 &= ~XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1); 2307171095Ssam val64 |= XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(0); 2308171095Ssam xge_os_pio_mem_write64(lldev->pdev, hldev->regh0, val64, 2309171095Ssam &bar0->rx_pa_cfg); 2310171095Ssam 2311171095Ssam xge_trace(XGE_TRACE, "Promiscuous mode ON"); 2312171095Ssam } 2313171095Ssam} 2314171095Ssam 2315173139Srwatson/** 2316173139Srwatson * xge_disable_promisc 2317173139Srwatson * Disable Promiscuous Mode 2318173139Srwatson * 2319173139Srwatson * @lldev Per-adapter Data 2320173139Srwatson */ 2321171095Ssamvoid 2322173139Srwatsonxge_disable_promisc(xge_lldev_t *lldev) 2323171095Ssam{ 2324171095Ssam xge_hal_device_t *hldev = lldev->devh; 2325171095Ssam xge_hal_pci_bar0_t *bar0 = NULL; 2326171095Ssam u64 val64 = 0; 2327171095Ssam 2328171095Ssam bar0 = (xge_hal_pci_bar0_t *) hldev->bar0; 2329171095Ssam 2330171095Ssam xge_hal_device_promisc_disable(lldev->devh); 2331171095Ssam 2332171095Ssam /* 2333171095Ssam * Strip VLAN tag when operating in non-promiscuous mode 2334171095Ssam */ 2335171095Ssam val64 = xge_os_pio_mem_read64(lldev->pdev, hldev->regh0, 2336171095Ssam &bar0->rx_pa_cfg); 2337171095Ssam val64 &= ~XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1); 2338171095Ssam val64 |= XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1); 2339171095Ssam xge_os_pio_mem_write64(lldev->pdev, hldev->regh0, val64, 2340171095Ssam &bar0->rx_pa_cfg); 2341171095Ssam 2342171095Ssam xge_trace(XGE_TRACE, "Promiscuous mode OFF"); 2343171095Ssam} 2344171095Ssam 2345173139Srwatson/** 2346173139Srwatson * xge_change_mtu 2347173139Srwatson * Change interface MTU to a requested valid size 2348171095Ssam * 2349173139Srwatson * @lldev Per-adapter Data 2350173139Srwatson * @NewMtu Requested MTU 2351171095Ssam * 2352173139Srwatson * Returns 0 or Error Number 2353173139Srwatson */ 2354173139Srwatsonint 2355173139Srwatsonxge_change_mtu(xge_lldev_t *lldev, int new_mtu) 2356171095Ssam{ 2357173139Srwatson int status = XGE_HAL_OK; 2358171095Ssam 2359173139Srwatson /* Check requested MTU size for boundary */ 2360173139Srwatson if(xge_hal_device_mtu_check(lldev->devh, new_mtu) != XGE_HAL_OK) { 2361173139Srwatson XGE_EXIT_ON_ERR("Invalid MTU", _exit, EINVAL); 2362171095Ssam } 2363171095Ssam 2364173139Srwatson lldev->mtu = new_mtu; 2365173139Srwatson xge_confirm_changes(lldev, XGE_SET_MTU); 2366171095Ssam 2367173139Srwatson_exit: 2368173139Srwatson return status; 2369171095Ssam} 2370171095Ssam 2371173139Srwatson/** 2372173139Srwatson * xge_device_stop 2373171095Ssam * 2374173139Srwatson * Common code for both stop and part of reset. Disables device, interrupts and 2375173139Srwatson * closes channels 2376171095Ssam * 2377173139Srwatson * @dev Device Handle 2378173139Srwatson * @option Channel normal/reset option 2379173139Srwatson */ 2380173139Srwatsonvoid 2381173139Srwatsonxge_device_stop(xge_lldev_t *lldev, xge_hal_channel_reopen_e option) 2382171095Ssam{ 2383171095Ssam xge_hal_device_t *hldev = lldev->devh; 2384171095Ssam struct ifnet *ifnetp = lldev->ifnetp; 2385173139Srwatson u64 val64 = 0; 2386171095Ssam 2387173139Srwatson mtx_assert((&lldev->mtx_drv), MA_OWNED); 2388173139Srwatson 2389173139Srwatson /* If device is not in "Running" state, return */ 2390173139Srwatson if (!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)) 2391173139Srwatson goto _exit; 2392173139Srwatson 2393171095Ssam /* Set appropriate flags */ 2394171095Ssam ifnetp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2395171095Ssam 2396171095Ssam /* Stop timer */ 2397171095Ssam callout_stop(&lldev->timer); 2398171095Ssam 2399171095Ssam /* Disable interrupts */ 2400171095Ssam xge_hal_device_intr_disable(hldev); 2401171095Ssam 2402173139Srwatson mtx_unlock(&lldev->mtx_drv); 2403171095Ssam xge_queue_flush(xge_hal_device_queue(lldev->devh)); 2404173139Srwatson mtx_lock(&lldev->mtx_drv); 2405171095Ssam 2406171095Ssam /* Disable HAL device */ 2407171095Ssam if(xge_hal_device_disable(hldev) != XGE_HAL_OK) { 2408171095Ssam xge_trace(XGE_ERR, "Disabling HAL device failed"); 2409173139Srwatson xge_hal_device_status(hldev, &val64); 2410173139Srwatson xge_trace(XGE_ERR, "Adapter Status: 0x%llx", (long long)val64); 2411171095Ssam } 2412171095Ssam 2413171095Ssam /* Close Tx and Rx channels */ 2414173139Srwatson xge_channel_close(lldev, option); 2415171095Ssam 2416171095Ssam /* Reset HAL device */ 2417171095Ssam xge_hal_device_reset(hldev); 2418171095Ssam 2419171095Ssam xge_os_mdelay(1000); 2420171095Ssam lldev->initialized = 0; 2421171095Ssam 2422171095Ssam if_link_state_change(ifnetp, LINK_STATE_DOWN); 2423171095Ssam 2424173139Srwatson_exit: 2425173139Srwatson return; 2426171095Ssam} 2427171095Ssam 2428173139Srwatson/** 2429173139Srwatson * xge_set_mbuf_cflags 2430173139Srwatson * set checksum flag for the mbuf 2431173139Srwatson * 2432173139Srwatson * @pkt Packet 2433173139Srwatson */ 2434173139Srwatsonvoid 2435173139Srwatsonxge_set_mbuf_cflags(mbuf_t pkt) 2436171095Ssam{ 2437171095Ssam pkt->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 2438171095Ssam pkt->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2439171095Ssam pkt->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 2440171095Ssam pkt->m_pkthdr.csum_data = htons(0xffff); 2441171095Ssam} 2442171095Ssam 2443173139Srwatson/** 2444173139Srwatson * xge_lro_flush_sessions 2445173139Srwatson * Flush LRO session and send accumulated LRO packet to upper layer 2446173139Srwatson * 2447173139Srwatson * @lldev Per-adapter Data 2448173139Srwatson */ 2449173139Srwatsonvoid 2450173139Srwatsonxge_lro_flush_sessions(xge_lldev_t *lldev) 2451171095Ssam{ 2452173139Srwatson xge_lro_entry_t *lro_session = NULL; 2453171095Ssam 2454173139Srwatson while(!SLIST_EMPTY(&lldev->lro_active)) { 2455173139Srwatson lro_session = SLIST_FIRST(&lldev->lro_active); 2456173139Srwatson SLIST_REMOVE_HEAD(&lldev->lro_active, next); 2457173139Srwatson xge_lro_flush(lldev, lro_session); 2458173139Srwatson } 2459173139Srwatson} 2460171095Ssam 2461173139Srwatson/** 2462173139Srwatson * xge_lro_flush 2463173139Srwatson * Flush LRO session. Send accumulated LRO packet to upper layer 2464173139Srwatson * 2465173139Srwatson * @lldev Per-adapter Data 2466173139Srwatson * @lro LRO session to be flushed 2467173139Srwatson */ 2468173139Srwatsonstatic void 2469173139Srwatsonxge_lro_flush(xge_lldev_t *lldev, xge_lro_entry_t *lro_session) 2470173139Srwatson{ 2471173139Srwatson struct ip *header_ip; 2472173139Srwatson struct tcphdr *header_tcp; 2473173139Srwatson u32 *ptr; 2474171095Ssam 2475173139Srwatson if(lro_session->append_cnt) { 2476173139Srwatson header_ip = lro_session->lro_header_ip; 2477173139Srwatson header_ip->ip_len = htons(lro_session->len - ETHER_HDR_LEN); 2478173139Srwatson lro_session->m_head->m_pkthdr.len = lro_session->len; 2479173139Srwatson header_tcp = (struct tcphdr *)(header_ip + 1); 2480173139Srwatson header_tcp->th_ack = lro_session->ack_seq; 2481173139Srwatson header_tcp->th_win = lro_session->window; 2482173139Srwatson if(lro_session->timestamp) { 2483173139Srwatson ptr = (u32 *)(header_tcp + 1); 2484173139Srwatson ptr[1] = htonl(lro_session->tsval); 2485173139Srwatson ptr[2] = lro_session->tsecr; 2486173139Srwatson } 2487171095Ssam } 2488173139Srwatson 2489173139Srwatson (*lldev->ifnetp->if_input)(lldev->ifnetp, lro_session->m_head); 2490173139Srwatson lro_session->m_head = NULL; 2491173139Srwatson lro_session->timestamp = 0; 2492173139Srwatson lro_session->append_cnt = 0; 2493173139Srwatson SLIST_INSERT_HEAD(&lldev->lro_free, lro_session, next); 2494171095Ssam} 2495171095Ssam 2496173139Srwatson/** 2497173139Srwatson * xge_lro_accumulate 2498173139Srwatson * Accumulate packets to form a large LRO packet based on various conditions 2499173139Srwatson * 2500173139Srwatson * @lldev Per-adapter Data 2501173139Srwatson * @m_head Current Packet 2502173139Srwatson * 2503173139Srwatson * Returns XGE_HAL_OK or XGE_HAL_FAIL (failure) 2504173139Srwatson */ 2505173139Srwatsonstatic int 2506173139Srwatsonxge_lro_accumulate(xge_lldev_t *lldev, struct mbuf *m_head) 2507171095Ssam{ 2508173139Srwatson struct ether_header *header_ethernet; 2509173139Srwatson struct ip *header_ip; 2510173139Srwatson struct tcphdr *header_tcp; 2511173139Srwatson u32 seq, *ptr; 2512173139Srwatson struct mbuf *buffer_next, *buffer_tail; 2513173139Srwatson xge_lro_entry_t *lro_session; 2514173139Srwatson xge_hal_status_e status = XGE_HAL_FAIL; 2515173139Srwatson int hlen, ip_len, tcp_hdr_len, tcp_data_len, tot_len, tcp_options; 2516173139Srwatson int trim; 2517171095Ssam 2518173139Srwatson /* Get Ethernet header */ 2519173139Srwatson header_ethernet = mtod(m_head, struct ether_header *); 2520171095Ssam 2521173139Srwatson /* Return if it is not IP packet */ 2522173139Srwatson if(header_ethernet->ether_type != htons(ETHERTYPE_IP)) 2523173139Srwatson goto _exit; 2524171095Ssam 2525173139Srwatson /* Get IP header */ 2526173139Srwatson header_ip = lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1 ? 2527173139Srwatson (struct ip *)(header_ethernet + 1) : 2528173139Srwatson mtod(m_head->m_next, struct ip *); 2529173139Srwatson 2530173139Srwatson /* Return if it is not TCP packet */ 2531173139Srwatson if(header_ip->ip_p != IPPROTO_TCP) 2532173139Srwatson goto _exit; 2533173139Srwatson 2534173139Srwatson /* Return if packet has options */ 2535173139Srwatson if((header_ip->ip_hl << 2) != sizeof(*header_ip)) 2536173139Srwatson goto _exit; 2537173139Srwatson 2538173139Srwatson /* Return if packet is fragmented */ 2539173139Srwatson if(header_ip->ip_off & htons(IP_MF | IP_OFFMASK)) 2540173139Srwatson goto _exit; 2541173139Srwatson 2542173139Srwatson /* Get TCP header */ 2543173139Srwatson header_tcp = (struct tcphdr *)(header_ip + 1); 2544173139Srwatson 2545173139Srwatson /* Return if not ACK or PUSH */ 2546173139Srwatson if((header_tcp->th_flags & ~(TH_ACK | TH_PUSH)) != 0) 2547173139Srwatson goto _exit; 2548173139Srwatson 2549173139Srwatson /* Only timestamp option is handled */ 2550173139Srwatson tcp_options = (header_tcp->th_off << 2) - sizeof(*header_tcp); 2551173139Srwatson tcp_hdr_len = sizeof(*header_tcp) + tcp_options; 2552173139Srwatson ptr = (u32 *)(header_tcp + 1); 2553173139Srwatson if(tcp_options != 0) { 2554173139Srwatson if(__predict_false(tcp_options != TCPOLEN_TSTAMP_APPA) || 2555173139Srwatson (*ptr != ntohl(TCPOPT_NOP << 24 | TCPOPT_NOP << 16 | 2556173139Srwatson TCPOPT_TIMESTAMP << 8 | TCPOLEN_TIMESTAMP))) { 2557173139Srwatson goto _exit; 2558173139Srwatson } 2559171095Ssam } 2560171095Ssam 2561173139Srwatson /* Total length of packet (IP) */ 2562173139Srwatson ip_len = ntohs(header_ip->ip_len); 2563171095Ssam 2564173139Srwatson /* TCP data size */ 2565173139Srwatson tcp_data_len = ip_len - (header_tcp->th_off << 2) - sizeof(*header_ip); 2566171095Ssam 2567173139Srwatson /* If the frame is padded, trim it */ 2568173139Srwatson tot_len = m_head->m_pkthdr.len; 2569173139Srwatson trim = tot_len - (ip_len + ETHER_HDR_LEN); 2570173139Srwatson if(trim != 0) { 2571173139Srwatson if(trim < 0) 2572173139Srwatson goto _exit; 2573173139Srwatson m_adj(m_head, -trim); 2574173139Srwatson tot_len = m_head->m_pkthdr.len; 2575171095Ssam } 2576171095Ssam 2577173139Srwatson buffer_next = m_head; 2578173139Srwatson buffer_tail = NULL; 2579173139Srwatson while(buffer_next != NULL) { 2580173139Srwatson buffer_tail = buffer_next; 2581173139Srwatson buffer_next = buffer_tail->m_next; 2582173139Srwatson } 2583171095Ssam 2584173139Srwatson /* Total size of only headers */ 2585173139Srwatson hlen = ip_len + ETHER_HDR_LEN - tcp_data_len; 2586171095Ssam 2587173139Srwatson /* Get sequence number */ 2588173139Srwatson seq = ntohl(header_tcp->th_seq); 2589173139Srwatson 2590173139Srwatson SLIST_FOREACH(lro_session, &lldev->lro_active, next) { 2591173139Srwatson if(lro_session->source_port == header_tcp->th_sport && 2592173139Srwatson lro_session->dest_port == header_tcp->th_dport && 2593173139Srwatson lro_session->source_ip == header_ip->ip_src.s_addr && 2594173139Srwatson lro_session->dest_ip == header_ip->ip_dst.s_addr) { 2595173139Srwatson 2596173139Srwatson /* Unmatched sequence number, flush LRO session */ 2597173139Srwatson if(__predict_false(seq != lro_session->next_seq)) { 2598173139Srwatson SLIST_REMOVE(&lldev->lro_active, lro_session, 2599173139Srwatson xge_lro_entry_t, next); 2600173139Srwatson xge_lro_flush(lldev, lro_session); 2601173139Srwatson goto _exit; 2602173139Srwatson } 2603173139Srwatson 2604173139Srwatson /* Handle timestamp option */ 2605173139Srwatson if(tcp_options) { 2606173139Srwatson u32 tsval = ntohl(*(ptr + 1)); 2607173139Srwatson if(__predict_false(lro_session->tsval > tsval || 2608173139Srwatson *(ptr + 2) == 0)) { 2609173139Srwatson goto _exit; 2610173139Srwatson } 2611173139Srwatson lro_session->tsval = tsval; 2612173139Srwatson lro_session->tsecr = *(ptr + 2); 2613173139Srwatson } 2614173139Srwatson 2615173139Srwatson lro_session->next_seq += tcp_data_len; 2616173139Srwatson lro_session->ack_seq = header_tcp->th_ack; 2617173139Srwatson lro_session->window = header_tcp->th_win; 2618173139Srwatson 2619173139Srwatson /* If TCP data/payload is of 0 size, free mbuf */ 2620173139Srwatson if(tcp_data_len == 0) { 2621173139Srwatson m_freem(m_head); 2622173139Srwatson status = XGE_HAL_OK; 2623173139Srwatson goto _exit; 2624173139Srwatson } 2625173139Srwatson 2626173139Srwatson lro_session->append_cnt++; 2627173139Srwatson lro_session->len += tcp_data_len; 2628173139Srwatson 2629173139Srwatson /* Adjust mbuf so that m_data points to payload than headers */ 2630173139Srwatson m_adj(m_head, hlen); 2631173139Srwatson 2632173139Srwatson /* Append this packet to LRO accumulated packet */ 2633173139Srwatson lro_session->m_tail->m_next = m_head; 2634173139Srwatson lro_session->m_tail = buffer_tail; 2635173139Srwatson 2636173139Srwatson /* Flush if LRO packet is exceeding maximum size */ 2637173139Srwatson if(lro_session->len > 2638173139Srwatson (XGE_HAL_LRO_DEFAULT_FRM_LEN - lldev->ifnetp->if_mtu)) { 2639173139Srwatson SLIST_REMOVE(&lldev->lro_active, lro_session, 2640173139Srwatson xge_lro_entry_t, next); 2641173139Srwatson xge_lro_flush(lldev, lro_session); 2642173139Srwatson } 2643173139Srwatson status = XGE_HAL_OK; 2644173139Srwatson goto _exit; 2645173139Srwatson } 2646171095Ssam } 2647171095Ssam 2648173139Srwatson if(SLIST_EMPTY(&lldev->lro_free)) 2649173139Srwatson goto _exit; 2650171095Ssam 2651173139Srwatson /* Start a new LRO session */ 2652173139Srwatson lro_session = SLIST_FIRST(&lldev->lro_free); 2653173139Srwatson SLIST_REMOVE_HEAD(&lldev->lro_free, next); 2654173139Srwatson SLIST_INSERT_HEAD(&lldev->lro_active, lro_session, next); 2655173139Srwatson lro_session->source_port = header_tcp->th_sport; 2656173139Srwatson lro_session->dest_port = header_tcp->th_dport; 2657173139Srwatson lro_session->source_ip = header_ip->ip_src.s_addr; 2658173139Srwatson lro_session->dest_ip = header_ip->ip_dst.s_addr; 2659173139Srwatson lro_session->next_seq = seq + tcp_data_len; 2660173139Srwatson lro_session->mss = tcp_data_len; 2661173139Srwatson lro_session->ack_seq = header_tcp->th_ack; 2662173139Srwatson lro_session->window = header_tcp->th_win; 2663173139Srwatson 2664173139Srwatson lro_session->lro_header_ip = header_ip; 2665173139Srwatson 2666173139Srwatson /* Handle timestamp option */ 2667173139Srwatson if(tcp_options) { 2668173139Srwatson lro_session->timestamp = 1; 2669173139Srwatson lro_session->tsval = ntohl(*(ptr + 1)); 2670173139Srwatson lro_session->tsecr = *(ptr + 2); 2671171095Ssam } 2672171095Ssam 2673173139Srwatson lro_session->len = tot_len; 2674173139Srwatson lro_session->m_head = m_head; 2675173139Srwatson lro_session->m_tail = buffer_tail; 2676173139Srwatson status = XGE_HAL_OK; 2677173139Srwatson 2678173139Srwatson_exit: 2679173139Srwatson return status; 2680173139Srwatson} 2681173139Srwatson 2682173139Srwatson/** 2683173139Srwatson * xge_accumulate_large_rx 2684173139Srwatson * Accumulate packets to form a large LRO packet based on various conditions 2685173139Srwatson * 2686173139Srwatson * @lldev Per-adapter Data 2687173139Srwatson * @pkt Current packet 2688173139Srwatson * @pkt_length Packet Length 2689173139Srwatson * @rxd_priv Rx Descriptor Private Data 2690173139Srwatson */ 2691173139Srwatsonvoid 2692173139Srwatsonxge_accumulate_large_rx(xge_lldev_t *lldev, struct mbuf *pkt, int pkt_length, 2693173139Srwatson xge_rx_priv_t *rxd_priv) 2694173139Srwatson{ 2695173139Srwatson if(xge_lro_accumulate(lldev, pkt) != XGE_HAL_OK) { 2696171095Ssam bus_dmamap_sync(lldev->dma_tag_rx, rxd_priv->dmainfo[0].dma_map, 2697171095Ssam BUS_DMASYNC_POSTREAD); 2698173139Srwatson (*lldev->ifnetp->if_input)(lldev->ifnetp, pkt); 2699171095Ssam } 2700171095Ssam} 2701171095Ssam 2702173139Srwatson/** 2703173139Srwatson * xge_rx_compl 2704173139Srwatson * If the interrupt is due to received frame (Rx completion), send it up 2705173139Srwatson * 2706173139Srwatson * @channelh Ring Channel Handle 2707173139Srwatson * @dtr Current Descriptor 2708173139Srwatson * @t_code Transfer Code indicating success or error 2709173139Srwatson * @userdata Per-adapter Data 2710173139Srwatson * 2711173139Srwatson * Returns XGE_HAL_OK or HAL error enums 2712173139Srwatson */ 2713171095Ssamxge_hal_status_e 2714173139Srwatsonxge_rx_compl(xge_hal_channel_h channelh, xge_hal_dtr_h dtr, u8 t_code, 2715171095Ssam void *userdata) 2716171095Ssam{ 2717171095Ssam struct ifnet *ifnetp; 2718173139Srwatson xge_rx_priv_t *rxd_priv = NULL; 2719173139Srwatson mbuf_t mbuf_up = NULL; 2720173139Srwatson xge_hal_status_e status = XGE_HAL_OK; 2721173139Srwatson xge_hal_dtr_info_t ext_info; 2722173139Srwatson int index; 2723173139Srwatson u16 vlan_tag; 2724171095Ssam 2725171095Ssam /*get the user data portion*/ 2726173139Srwatson xge_lldev_t *lldev = xge_hal_channel_userdata(channelh); 2727171095Ssam if(!lldev) { 2728173139Srwatson XGE_EXIT_ON_ERR("Failed to get user data", _exit, XGE_HAL_FAIL); 2729171095Ssam } 2730171095Ssam 2731173139Srwatson XGE_DRV_STATS(rx_completions); 2732171095Ssam 2733171095Ssam /* get the interface pointer */ 2734171095Ssam ifnetp = lldev->ifnetp; 2735171095Ssam 2736171095Ssam do { 2737173139Srwatson XGE_DRV_STATS(rx_desc_compl); 2738173139Srwatson 2739171095Ssam if(!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)) { 2740173139Srwatson status = XGE_HAL_FAIL; 2741173139Srwatson goto _exit; 2742171095Ssam } 2743171095Ssam 2744171095Ssam if(t_code) { 2745171095Ssam xge_trace(XGE_TRACE, "Packet dropped because of %d", t_code); 2746173139Srwatson XGE_DRV_STATS(rx_tcode); 2747171095Ssam xge_hal_device_handle_tcode(channelh, dtr, t_code); 2748171095Ssam xge_hal_ring_dtr_post(channelh,dtr); 2749171095Ssam continue; 2750171095Ssam } 2751171095Ssam 2752171095Ssam /* Get the private data for this descriptor*/ 2753173139Srwatson rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh, 2754171095Ssam dtr); 2755171095Ssam if(!rxd_priv) { 2756173139Srwatson XGE_EXIT_ON_ERR("Failed to get descriptor private data", _exit, 2757173139Srwatson XGE_HAL_FAIL); 2758171095Ssam } 2759171095Ssam 2760173139Srwatson /* 2761173139Srwatson * Prepare one buffer to send it to upper layer -- since the upper 2762173139Srwatson * layer frees the buffer do not use rxd_priv->buffer. Meanwhile 2763173139Srwatson * prepare a new buffer, do mapping, use it in the current 2764173139Srwatson * descriptor and post descriptor back to ring channel 2765173139Srwatson */ 2766171095Ssam mbuf_up = rxd_priv->bufferArray[0]; 2767171095Ssam 2768171095Ssam /* Gets details of mbuf i.e., packet length */ 2769171095Ssam xge_ring_dtr_get(mbuf_up, channelh, dtr, lldev, rxd_priv); 2770171095Ssam 2771173139Srwatson status = 2772171095Ssam (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ? 2773173139Srwatson xge_get_buf(dtr, rxd_priv, lldev, 0) : 2774173139Srwatson xge_get_buf_3b_5b(dtr, rxd_priv, lldev); 2775171095Ssam 2776173139Srwatson if(status != XGE_HAL_OK) { 2777171095Ssam xge_trace(XGE_ERR, "No memory"); 2778173139Srwatson XGE_DRV_STATS(rx_no_buf); 2779171095Ssam 2780171095Ssam /* 2781173139Srwatson * Unable to allocate buffer. Instead of discarding, post 2782173139Srwatson * descriptor back to channel for future processing of same 2783173139Srwatson * packet. 2784171095Ssam */ 2785171095Ssam xge_hal_ring_dtr_post(channelh, dtr); 2786171095Ssam continue; 2787171095Ssam } 2788171095Ssam 2789171095Ssam /* Get the extended information */ 2790171095Ssam xge_hal_ring_dtr_info_get(channelh, dtr, &ext_info); 2791171095Ssam 2792173139Srwatson /* 2793173139Srwatson * As we have allocated a new mbuf for this descriptor, post this 2794173139Srwatson * descriptor with new mbuf back to ring channel 2795173139Srwatson */ 2796173139Srwatson vlan_tag = ext_info.vlan; 2797173139Srwatson xge_hal_ring_dtr_post(channelh, dtr); 2798173139Srwatson if ((!(ext_info.proto & XGE_HAL_FRAME_PROTO_IP_FRAGMENTED) && 2799173139Srwatson (ext_info.proto & XGE_HAL_FRAME_PROTO_TCP_OR_UDP) && 2800173139Srwatson (ext_info.l3_cksum == XGE_HAL_L3_CKSUM_OK) && 2801173139Srwatson (ext_info.l4_cksum == XGE_HAL_L4_CKSUM_OK))) { 2802171095Ssam 2803173139Srwatson /* set Checksum Flag */ 2804173139Srwatson xge_set_mbuf_cflags(mbuf_up); 2805173139Srwatson 2806173139Srwatson if(lldev->enabled_lro) { 2807173139Srwatson xge_accumulate_large_rx(lldev, mbuf_up, mbuf_up->m_len, 2808173139Srwatson rxd_priv); 2809171095Ssam } 2810171095Ssam else { 2811173139Srwatson /* Post-Read sync for buffers*/ 2812173139Srwatson for(index = 0; index < lldev->rxd_mbuf_cnt; index++) { 2813173139Srwatson bus_dmamap_sync(lldev->dma_tag_rx, 2814173139Srwatson rxd_priv->dmainfo[0].dma_map, BUS_DMASYNC_POSTREAD); 2815171095Ssam } 2816173139Srwatson (*ifnetp->if_input)(ifnetp, mbuf_up); 2817171095Ssam } 2818171095Ssam } 2819171095Ssam else { 2820171095Ssam /* 2821173139Srwatson * Packet with erroneous checksum , let the upper layer deal 2822173139Srwatson * with it 2823171095Ssam */ 2824171095Ssam 2825173139Srwatson /* Post-Read sync for buffers*/ 2826173139Srwatson for(index = 0; index < lldev->rxd_mbuf_cnt; index++) { 2827173139Srwatson bus_dmamap_sync(lldev->dma_tag_rx, 2828173139Srwatson rxd_priv->dmainfo[0].dma_map, BUS_DMASYNC_POSTREAD); 2829171095Ssam } 2830171095Ssam 2831173139Srwatson if(vlan_tag) { 2832173139Srwatson mbuf_up->m_pkthdr.ether_vtag = vlan_tag; 2833173139Srwatson mbuf_up->m_flags |= M_VLANTAG; 2834171095Ssam } 2835173139Srwatson 2836173139Srwatson if(lldev->enabled_lro) 2837173139Srwatson xge_lro_flush_sessions(lldev); 2838173139Srwatson 2839173139Srwatson (*ifnetp->if_input)(ifnetp, mbuf_up); 2840171095Ssam } 2841171095Ssam } while(xge_hal_ring_dtr_next_completed(channelh, &dtr, &t_code) 2842171095Ssam == XGE_HAL_OK); 2843171095Ssam 2844173139Srwatson if(lldev->enabled_lro) 2845173139Srwatson xge_lro_flush_sessions(lldev); 2846171095Ssam 2847173139Srwatson_exit: 2848173139Srwatson return status; 2849171095Ssam} 2850171095Ssam 2851173139Srwatson/** 2852173139Srwatson * xge_ring_dtr_get 2853173139Srwatson * Get descriptors 2854173139Srwatson * 2855173139Srwatson * @mbuf_up Packet to send up 2856173139Srwatson * @channelh Ring Channel Handle 2857173139Srwatson * @dtr Descriptor 2858173139Srwatson * @lldev Per-adapter Data 2859173139Srwatson * @rxd_priv Rx Descriptor Private Data 2860173139Srwatson * 2861173139Srwatson * Returns XGE_HAL_OK or HAL error enums 2862173139Srwatson */ 2863171095Ssamint 2864171095Ssamxge_ring_dtr_get(mbuf_t mbuf_up, xge_hal_channel_h channelh, xge_hal_dtr_h dtr, 2865173139Srwatson xge_lldev_t *lldev, xge_rx_priv_t *rxd_priv) 2866171095Ssam{ 2867171095Ssam mbuf_t m; 2868171095Ssam int pkt_length[5]={0,0}, pkt_len=0; 2869171095Ssam dma_addr_t dma_data[5]; 2870171095Ssam int index; 2871171095Ssam 2872171095Ssam m = mbuf_up; 2873171095Ssam pkt_len = 0; 2874171095Ssam 2875171095Ssam if(lldev->buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_1) { 2876171095Ssam xge_os_memzero(pkt_length, sizeof(pkt_length)); 2877171095Ssam 2878171095Ssam /* 2879171095Ssam * Retrieve data of interest from the completed descriptor -- This 2880171095Ssam * returns the packet length 2881171095Ssam */ 2882171095Ssam if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) { 2883171095Ssam xge_hal_ring_dtr_5b_get(channelh, dtr, dma_data, pkt_length); 2884171095Ssam } 2885171095Ssam else { 2886171095Ssam xge_hal_ring_dtr_3b_get(channelh, dtr, dma_data, pkt_length); 2887171095Ssam } 2888171095Ssam 2889171095Ssam for(index = 0; index < lldev->rxd_mbuf_cnt; index++) { 2890171095Ssam m->m_len = pkt_length[index]; 2891171095Ssam 2892171095Ssam if(index < (lldev->rxd_mbuf_cnt-1)) { 2893171095Ssam m->m_next = rxd_priv->bufferArray[index + 1]; 2894171095Ssam m = m->m_next; 2895171095Ssam } 2896171095Ssam else { 2897171095Ssam m->m_next = NULL; 2898171095Ssam } 2899171095Ssam pkt_len+=pkt_length[index]; 2900171095Ssam } 2901171095Ssam 2902171095Ssam /* 2903171095Ssam * Since 2 buffer mode is an exceptional case where data is in 3rd 2904171095Ssam * buffer but not in 2nd buffer 2905171095Ssam */ 2906171095Ssam if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) { 2907171095Ssam m->m_len = pkt_length[2]; 2908171095Ssam pkt_len+=pkt_length[2]; 2909171095Ssam } 2910171095Ssam 2911171095Ssam /* 2912171095Ssam * Update length of newly created buffer to be sent up with packet 2913171095Ssam * length 2914171095Ssam */ 2915171095Ssam mbuf_up->m_pkthdr.len = pkt_len; 2916171095Ssam } 2917171095Ssam else { 2918171095Ssam /* 2919171095Ssam * Retrieve data of interest from the completed descriptor -- This 2920171095Ssam * returns the packet length 2921171095Ssam */ 2922171095Ssam xge_hal_ring_dtr_1b_get(channelh, dtr,&dma_data[0], &pkt_length[0]); 2923171095Ssam 2924171095Ssam /* 2925171095Ssam * Update length of newly created buffer to be sent up with packet 2926171095Ssam * length 2927171095Ssam */ 2928171095Ssam mbuf_up->m_len = mbuf_up->m_pkthdr.len = pkt_length[0]; 2929171095Ssam } 2930171095Ssam 2931173139Srwatson return XGE_HAL_OK; 2932171095Ssam} 2933171095Ssam 2934173139Srwatson/** 2935173139Srwatson * xge_flush_txds 2936173139Srwatson * Flush Tx descriptors 2937173139Srwatson * 2938173139Srwatson * @channelh Channel handle 2939173139Srwatson */ 2940173139Srwatsonstatic void inline 2941173139Srwatsonxge_flush_txds(xge_hal_channel_h channelh) 2942173139Srwatson{ 2943173139Srwatson xge_lldev_t *lldev = xge_hal_channel_userdata(channelh); 2944173139Srwatson xge_hal_dtr_h tx_dtr; 2945173139Srwatson xge_tx_priv_t *tx_priv; 2946173139Srwatson u8 t_code; 2947171095Ssam 2948173139Srwatson while(xge_hal_fifo_dtr_next_completed(channelh, &tx_dtr, &t_code) 2949173139Srwatson == XGE_HAL_OK) { 2950173139Srwatson XGE_DRV_STATS(tx_desc_compl); 2951173139Srwatson if(t_code) { 2952173139Srwatson xge_trace(XGE_TRACE, "Tx descriptor with t_code %d", t_code); 2953173139Srwatson XGE_DRV_STATS(tx_tcode); 2954173139Srwatson xge_hal_device_handle_tcode(channelh, tx_dtr, t_code); 2955173139Srwatson } 2956173139Srwatson 2957173139Srwatson tx_priv = xge_hal_fifo_dtr_private(tx_dtr); 2958173139Srwatson bus_dmamap_unload(lldev->dma_tag_tx, tx_priv->dma_map); 2959173139Srwatson m_freem(tx_priv->buffer); 2960173139Srwatson tx_priv->buffer = NULL; 2961173139Srwatson xge_hal_fifo_dtr_free(channelh, tx_dtr); 2962173139Srwatson } 2963173139Srwatson} 2964173139Srwatson 2965173139Srwatson/** 2966173139Srwatson * xge_send 2967173139Srwatson * Transmit function 2968173139Srwatson * 2969173139Srwatson * @ifnetp Interface Handle 2970173139Srwatson */ 2971171095Ssamvoid 2972171095Ssamxge_send(struct ifnet *ifnetp) 2973171095Ssam{ 2974173139Srwatson int qindex = 0; 2975173139Srwatson xge_lldev_t *lldev = ifnetp->if_softc; 2976171095Ssam 2977173139Srwatson for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) { 2978173139Srwatson if(mtx_trylock(&lldev->mtx_tx[qindex]) == 0) { 2979173139Srwatson XGE_DRV_STATS(tx_lock_fail); 2980173139Srwatson break; 2981173139Srwatson } 2982173139Srwatson xge_send_locked(ifnetp, qindex); 2983173139Srwatson mtx_unlock(&lldev->mtx_tx[qindex]); 2984173139Srwatson } 2985171095Ssam} 2986171095Ssam 2987173139Srwatsonstatic void inline 2988173139Srwatsonxge_send_locked(struct ifnet *ifnetp, int qindex) 2989171095Ssam{ 2990171095Ssam xge_hal_dtr_h dtr; 2991173139Srwatson static bus_dma_segment_t segs[XGE_MAX_SEGS]; 2992173139Srwatson xge_hal_status_e status; 2993171095Ssam unsigned int max_fragments; 2994173139Srwatson xge_lldev_t *lldev = ifnetp->if_softc; 2995173139Srwatson xge_hal_channel_h channelh = lldev->fifo_channel[qindex]; 2996171095Ssam mbuf_t m_head = NULL; 2997171095Ssam mbuf_t m_buf = NULL; 2998173139Srwatson xge_tx_priv_t *ll_tx_priv = NULL; 2999171095Ssam register unsigned int count = 0; 3000171095Ssam unsigned int nsegs = 0; 3001171095Ssam u16 vlan_tag; 3002171095Ssam 3003171095Ssam max_fragments = ((xge_hal_fifo_t *)channelh)->config->max_frags; 3004171095Ssam 3005171095Ssam /* If device is not initialized, return */ 3006173139Srwatson if((!lldev->initialized) || (!(ifnetp->if_drv_flags & IFF_DRV_RUNNING))) 3007199554Sjhb return; 3008171095Ssam 3009173139Srwatson XGE_DRV_STATS(tx_calls); 3010173139Srwatson 3011171095Ssam /* 3012173139Srwatson * This loop will be executed for each packet in the kernel maintained 3013173139Srwatson * queue -- each packet can be with fragments as an mbuf chain 3014171095Ssam */ 3015173139Srwatson for(;;) { 3016171095Ssam IF_DEQUEUE(&ifnetp->if_snd, m_head); 3017199554Sjhb if (m_head == NULL) { 3018199554Sjhb ifnetp->if_drv_flags &= ~(IFF_DRV_OACTIVE); 3019199554Sjhb return; 3020199554Sjhb } 3021171095Ssam 3022173139Srwatson for(m_buf = m_head; m_buf != NULL; m_buf = m_buf->m_next) { 3023173139Srwatson if(m_buf->m_len) count += 1; 3024171095Ssam } 3025171095Ssam 3026171095Ssam if(count >= max_fragments) { 3027243857Sglebius m_buf = m_defrag(m_head, M_NOWAIT); 3028173139Srwatson if(m_buf != NULL) m_head = m_buf; 3029173139Srwatson XGE_DRV_STATS(tx_defrag); 3030171095Ssam } 3031171095Ssam 3032171095Ssam /* Reserve descriptors */ 3033173139Srwatson status = xge_hal_fifo_dtr_reserve(channelh, &dtr); 3034173139Srwatson if(status != XGE_HAL_OK) { 3035173139Srwatson XGE_DRV_STATS(tx_no_txd); 3036173139Srwatson xge_flush_txds(channelh); 3037199554Sjhb break; 3038171095Ssam } 3039171095Ssam 3040173139Srwatson vlan_tag = 3041173139Srwatson (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0; 3042171095Ssam xge_hal_fifo_dtr_vlan_set(dtr, vlan_tag); 3043171095Ssam 3044171095Ssam /* Update Tx private structure for this descriptor */ 3045171095Ssam ll_tx_priv = xge_hal_fifo_dtr_private(dtr); 3046171095Ssam ll_tx_priv->buffer = m_head; 3047171095Ssam 3048171095Ssam /* 3049171095Ssam * Do mapping -- Required DMA tag has been created in xge_init 3050171095Ssam * function and DMA maps have already been created in the 3051171095Ssam * xgell_tx_replenish function. 3052171095Ssam * Returns number of segments through nsegs 3053171095Ssam */ 3054171095Ssam if(bus_dmamap_load_mbuf_sg(lldev->dma_tag_tx, 3055171095Ssam ll_tx_priv->dma_map, m_head, segs, &nsegs, BUS_DMA_NOWAIT)) { 3056173139Srwatson xge_trace(XGE_TRACE, "DMA map load failed"); 3057173139Srwatson XGE_DRV_STATS(tx_map_fail); 3058199554Sjhb break; 3059171095Ssam } 3060171095Ssam 3061173139Srwatson if(lldev->driver_stats.tx_max_frags < nsegs) 3062173139Srwatson lldev->driver_stats.tx_max_frags = nsegs; 3063173139Srwatson 3064171095Ssam /* Set descriptor buffer for header and each fragment/segment */ 3065171095Ssam count = 0; 3066171095Ssam do { 3067171095Ssam xge_hal_fifo_dtr_buffer_set(channelh, dtr, count, 3068171095Ssam (dma_addr_t)htole64(segs[count].ds_addr), 3069171095Ssam segs[count].ds_len); 3070173139Srwatson count++; 3071171095Ssam } while(count < nsegs); 3072171095Ssam 3073171095Ssam /* Pre-write Sync of mapping */ 3074171095Ssam bus_dmamap_sync(lldev->dma_tag_tx, ll_tx_priv->dma_map, 3075171095Ssam BUS_DMASYNC_PREWRITE); 3076171095Ssam 3077173139Srwatson if((lldev->enabled_tso) && 3078173139Srwatson (m_head->m_pkthdr.csum_flags & CSUM_TSO)) { 3079173139Srwatson XGE_DRV_STATS(tx_tso); 3080173139Srwatson xge_hal_fifo_dtr_mss_set(dtr, m_head->m_pkthdr.tso_segsz); 3081173139Srwatson } 3082173139Srwatson 3083171095Ssam /* Checksum */ 3084171095Ssam if(ifnetp->if_hwassist > 0) { 3085171095Ssam xge_hal_fifo_dtr_cksum_set_bits(dtr, XGE_HAL_TXD_TX_CKO_IPV4_EN 3086171095Ssam | XGE_HAL_TXD_TX_CKO_TCP_EN | XGE_HAL_TXD_TX_CKO_UDP_EN); 3087171095Ssam } 3088171095Ssam 3089171095Ssam /* Post descriptor to FIFO channel */ 3090171095Ssam xge_hal_fifo_dtr_post(channelh, dtr); 3091173139Srwatson XGE_DRV_STATS(tx_posted); 3092171095Ssam 3093171095Ssam /* Send the same copy of mbuf packet to BPF (Berkely Packet Filter) 3094171095Ssam * listener so that we can use tools like tcpdump */ 3095171095Ssam ETHER_BPF_MTAP(ifnetp, m_head); 3096171095Ssam } 3097199554Sjhb 3098171095Ssam /* Prepend the packet back to queue */ 3099171095Ssam IF_PREPEND(&ifnetp->if_snd, m_head); 3100173139Srwatson ifnetp->if_drv_flags |= IFF_DRV_OACTIVE; 3101173139Srwatson 3102173139Srwatson xge_queue_produce_context(xge_hal_device_queue(lldev->devh), 3103173139Srwatson XGE_LL_EVENT_TRY_XMIT_AGAIN, lldev->devh); 3104173139Srwatson XGE_DRV_STATS(tx_again); 3105171095Ssam} 3106171095Ssam 3107173139Srwatson/** 3108173139Srwatson * xge_get_buf 3109173139Srwatson * Allocates new mbufs to be placed into descriptors 3110173139Srwatson * 3111173139Srwatson * @dtrh Descriptor Handle 3112173139Srwatson * @rxd_priv Rx Descriptor Private Data 3113173139Srwatson * @lldev Per-adapter Data 3114173139Srwatson * @index Buffer Index (if multi-buffer mode) 3115173139Srwatson * 3116173139Srwatson * Returns XGE_HAL_OK or HAL error enums 3117173139Srwatson */ 3118171095Ssamint 3119173139Srwatsonxge_get_buf(xge_hal_dtr_h dtrh, xge_rx_priv_t *rxd_priv, 3120173139Srwatson xge_lldev_t *lldev, int index) 3121171095Ssam{ 3122171095Ssam register mbuf_t mp = NULL; 3123171095Ssam struct ifnet *ifnetp = lldev->ifnetp; 3124173139Srwatson int status = XGE_HAL_OK; 3125173139Srwatson int buffer_size = 0, cluster_size = 0, count; 3126173139Srwatson bus_dmamap_t map = rxd_priv->dmainfo[index].dma_map; 3127173139Srwatson bus_dma_segment_t segs[3]; 3128171095Ssam 3129173139Srwatson buffer_size = (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ? 3130173139Srwatson ifnetp->if_mtu + XGE_HAL_MAC_HEADER_MAX_SIZE : 3131173139Srwatson lldev->rxd_mbuf_len[index]; 3132173139Srwatson 3133173139Srwatson if(buffer_size <= MCLBYTES) { 3134173139Srwatson cluster_size = MCLBYTES; 3135243857Sglebius mp = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 3136171095Ssam } 3137171095Ssam else { 3138173139Srwatson cluster_size = MJUMPAGESIZE; 3139173139Srwatson if((lldev->buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_5) && 3140173139Srwatson (buffer_size > MJUMPAGESIZE)) { 3141173139Srwatson cluster_size = MJUM9BYTES; 3142171095Ssam } 3143243857Sglebius mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, cluster_size); 3144171095Ssam } 3145171095Ssam if(!mp) { 3146171095Ssam xge_trace(XGE_ERR, "Out of memory to allocate mbuf"); 3147173139Srwatson status = XGE_HAL_FAIL; 3148171095Ssam goto getbuf_out; 3149171095Ssam } 3150171095Ssam 3151171095Ssam /* Update mbuf's length, packet length and receive interface */ 3152173139Srwatson mp->m_len = mp->m_pkthdr.len = buffer_size; 3153171095Ssam mp->m_pkthdr.rcvif = ifnetp; 3154171095Ssam 3155171095Ssam /* Load DMA map */ 3156173139Srwatson if(bus_dmamap_load_mbuf_sg(lldev->dma_tag_rx, lldev->extra_dma_map, 3157173139Srwatson mp, segs, &count, BUS_DMA_NOWAIT)) { 3158173139Srwatson XGE_DRV_STATS(rx_map_fail); 3159171095Ssam m_freem(mp); 3160173139Srwatson XGE_EXIT_ON_ERR("DMA map load failed", getbuf_out, XGE_HAL_FAIL); 3161171095Ssam } 3162171095Ssam 3163171095Ssam /* Update descriptor private data */ 3164171095Ssam rxd_priv->bufferArray[index] = mp; 3165173139Srwatson rxd_priv->dmainfo[index].dma_phyaddr = htole64(segs->ds_addr); 3166173139Srwatson rxd_priv->dmainfo[index].dma_map = lldev->extra_dma_map; 3167173139Srwatson lldev->extra_dma_map = map; 3168171095Ssam 3169171095Ssam /* Pre-Read/Write sync */ 3170173139Srwatson bus_dmamap_sync(lldev->dma_tag_rx, map, BUS_DMASYNC_POSTREAD); 3171171095Ssam 3172173139Srwatson /* Unload DMA map of mbuf in current descriptor */ 3173173139Srwatson bus_dmamap_unload(lldev->dma_tag_rx, map); 3174173139Srwatson 3175171095Ssam /* Set descriptor buffer */ 3176171095Ssam if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) { 3177171095Ssam xge_hal_ring_dtr_1b_set(dtrh, rxd_priv->dmainfo[0].dma_phyaddr, 3178173139Srwatson cluster_size); 3179171095Ssam } 3180171095Ssam 3181171095Ssamgetbuf_out: 3182173139Srwatson return status; 3183171095Ssam} 3184171095Ssam 3185173139Srwatson/** 3186173139Srwatson * xge_get_buf_3b_5b 3187173139Srwatson * Allocates new mbufs to be placed into descriptors (in multi-buffer modes) 3188173139Srwatson * 3189173139Srwatson * @dtrh Descriptor Handle 3190173139Srwatson * @rxd_priv Rx Descriptor Private Data 3191173139Srwatson * @lldev Per-adapter Data 3192173139Srwatson * 3193173139Srwatson * Returns XGE_HAL_OK or HAL error enums 3194173139Srwatson */ 3195171095Ssamint 3196173139Srwatsonxge_get_buf_3b_5b(xge_hal_dtr_h dtrh, xge_rx_priv_t *rxd_priv, 3197173139Srwatson xge_lldev_t *lldev) 3198171095Ssam{ 3199171095Ssam bus_addr_t dma_pointers[5]; 3200171095Ssam int dma_sizes[5]; 3201173139Srwatson int status = XGE_HAL_OK, index; 3202171095Ssam int newindex = 0; 3203171095Ssam 3204171095Ssam for(index = 0; index < lldev->rxd_mbuf_cnt; index++) { 3205173139Srwatson status = xge_get_buf(dtrh, rxd_priv, lldev, index); 3206173139Srwatson if(status != XGE_HAL_OK) { 3207171095Ssam for(newindex = 0; newindex < index; newindex++) { 3208171095Ssam m_freem(rxd_priv->bufferArray[newindex]); 3209171095Ssam } 3210173139Srwatson XGE_EXIT_ON_ERR("mbuf allocation failed", _exit, status); 3211171095Ssam } 3212171095Ssam } 3213171095Ssam 3214171095Ssam for(index = 0; index < lldev->buffer_mode; index++) { 3215171095Ssam if(lldev->rxd_mbuf_len[index] != 0) { 3216171095Ssam dma_pointers[index] = rxd_priv->dmainfo[index].dma_phyaddr; 3217171095Ssam dma_sizes[index] = lldev->rxd_mbuf_len[index]; 3218171095Ssam } 3219171095Ssam else { 3220171095Ssam dma_pointers[index] = rxd_priv->dmainfo[index-1].dma_phyaddr; 3221171095Ssam dma_sizes[index] = 1; 3222171095Ssam } 3223171095Ssam } 3224171095Ssam 3225171095Ssam /* Assigning second buffer to third pointer in 2 buffer mode */ 3226171095Ssam if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) { 3227171095Ssam dma_pointers[2] = dma_pointers[1]; 3228171095Ssam dma_sizes[2] = dma_sizes[1]; 3229171095Ssam dma_sizes[1] = 1; 3230171095Ssam } 3231171095Ssam 3232171095Ssam if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) { 3233171095Ssam xge_hal_ring_dtr_5b_set(dtrh, dma_pointers, dma_sizes); 3234171095Ssam } 3235171095Ssam else { 3236171095Ssam xge_hal_ring_dtr_3b_set(dtrh, dma_pointers, dma_sizes); 3237171095Ssam } 3238171095Ssam 3239173139Srwatson_exit: 3240173139Srwatson return status; 3241171095Ssam} 3242171095Ssam 3243173139Srwatson/** 3244173139Srwatson * xge_tx_compl 3245173139Srwatson * If the interrupt is due to Tx completion, free the sent buffer 3246173139Srwatson * 3247173139Srwatson * @channelh Channel Handle 3248173139Srwatson * @dtr Descriptor 3249173139Srwatson * @t_code Transfer Code indicating success or error 3250173139Srwatson * @userdata Per-adapter Data 3251173139Srwatson * 3252173139Srwatson * Returns XGE_HAL_OK or HAL error enum 3253173139Srwatson */ 3254171095Ssamxge_hal_status_e 3255173139Srwatsonxge_tx_compl(xge_hal_channel_h channelh, 3256171095Ssam xge_hal_dtr_h dtr, u8 t_code, void *userdata) 3257171095Ssam{ 3258173139Srwatson xge_tx_priv_t *ll_tx_priv = NULL; 3259173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)userdata; 3260173139Srwatson struct ifnet *ifnetp = lldev->ifnetp; 3261173139Srwatson mbuf_t m_buffer = NULL; 3262173139Srwatson int qindex = xge_hal_channel_id(channelh); 3263171095Ssam 3264173139Srwatson mtx_lock(&lldev->mtx_tx[qindex]); 3265173139Srwatson 3266173139Srwatson XGE_DRV_STATS(tx_completions); 3267173139Srwatson 3268173139Srwatson /* 3269173139Srwatson * For each completed descriptor: Get private structure, free buffer, 3270173139Srwatson * do unmapping, and free descriptor 3271173139Srwatson */ 3272171095Ssam do { 3273173139Srwatson XGE_DRV_STATS(tx_desc_compl); 3274173139Srwatson 3275171095Ssam if(t_code) { 3276173139Srwatson XGE_DRV_STATS(tx_tcode); 3277171095Ssam xge_trace(XGE_TRACE, "t_code %d", t_code); 3278171095Ssam xge_hal_device_handle_tcode(channelh, dtr, t_code); 3279171095Ssam } 3280171095Ssam 3281171095Ssam ll_tx_priv = xge_hal_fifo_dtr_private(dtr); 3282171095Ssam m_buffer = ll_tx_priv->buffer; 3283171095Ssam bus_dmamap_unload(lldev->dma_tag_tx, ll_tx_priv->dma_map); 3284171095Ssam m_freem(m_buffer); 3285171095Ssam ll_tx_priv->buffer = NULL; 3286171095Ssam xge_hal_fifo_dtr_free(channelh, dtr); 3287171095Ssam } while(xge_hal_fifo_dtr_next_completed(channelh, &dtr, &t_code) 3288171095Ssam == XGE_HAL_OK); 3289173139Srwatson xge_send_locked(ifnetp, qindex); 3290171095Ssam ifnetp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3291171095Ssam 3292173139Srwatson mtx_unlock(&lldev->mtx_tx[qindex]); 3293173139Srwatson 3294171095Ssam return XGE_HAL_OK; 3295171095Ssam} 3296171095Ssam 3297173139Srwatson/** 3298173139Srwatson * xge_tx_initial_replenish 3299173139Srwatson * Initially allocate buffers and set them into descriptors for later use 3300173139Srwatson * 3301173139Srwatson * @channelh Tx Channel Handle 3302173139Srwatson * @dtrh Descriptor Handle 3303173139Srwatson * @index 3304173139Srwatson * @userdata Per-adapter Data 3305173139Srwatson * @reopen Channel open/reopen option 3306173139Srwatson * 3307173139Srwatson * Returns XGE_HAL_OK or HAL error enums 3308173139Srwatson */ 3309171095Ssamxge_hal_status_e 3310173139Srwatsonxge_tx_initial_replenish(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh, 3311171095Ssam int index, void *userdata, xge_hal_channel_reopen_e reopen) 3312171095Ssam{ 3313173139Srwatson xge_tx_priv_t *txd_priv = NULL; 3314173139Srwatson int status = XGE_HAL_OK; 3315171095Ssam 3316171095Ssam /* Get the user data portion from channel handle */ 3317173139Srwatson xge_lldev_t *lldev = xge_hal_channel_userdata(channelh); 3318171095Ssam if(lldev == NULL) { 3319173139Srwatson XGE_EXIT_ON_ERR("Failed to get user data from channel", txinit_out, 3320173139Srwatson XGE_HAL_FAIL); 3321171095Ssam } 3322171095Ssam 3323171095Ssam /* Get the private data */ 3324173139Srwatson txd_priv = (xge_tx_priv_t *) xge_hal_fifo_dtr_private(dtrh); 3325171095Ssam if(txd_priv == NULL) { 3326173139Srwatson XGE_EXIT_ON_ERR("Failed to get descriptor private data", txinit_out, 3327173139Srwatson XGE_HAL_FAIL); 3328171095Ssam } 3329171095Ssam 3330171095Ssam /* Create DMA map for this descriptor */ 3331171095Ssam if(bus_dmamap_create(lldev->dma_tag_tx, BUS_DMA_NOWAIT, 3332171095Ssam &txd_priv->dma_map)) { 3333173139Srwatson XGE_EXIT_ON_ERR("DMA map creation for Tx descriptor failed", 3334173139Srwatson txinit_out, XGE_HAL_FAIL); 3335171095Ssam } 3336171095Ssam 3337171095Ssamtxinit_out: 3338173139Srwatson return status; 3339171095Ssam} 3340171095Ssam 3341173139Srwatson/** 3342173139Srwatson * xge_rx_initial_replenish 3343173139Srwatson * Initially allocate buffers and set them into descriptors for later use 3344173139Srwatson * 3345173139Srwatson * @channelh Tx Channel Handle 3346173139Srwatson * @dtrh Descriptor Handle 3347173139Srwatson * @index Ring Index 3348173139Srwatson * @userdata Per-adapter Data 3349173139Srwatson * @reopen Channel open/reopen option 3350173139Srwatson * 3351173139Srwatson * Returns XGE_HAL_OK or HAL error enums 3352173139Srwatson */ 3353171095Ssamxge_hal_status_e 3354173139Srwatsonxge_rx_initial_replenish(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh, 3355171095Ssam int index, void *userdata, xge_hal_channel_reopen_e reopen) 3356171095Ssam{ 3357173139Srwatson xge_rx_priv_t *rxd_priv = NULL; 3358173139Srwatson int status = XGE_HAL_OK; 3359173139Srwatson int index1 = 0, index2 = 0; 3360171095Ssam 3361171095Ssam /* Get the user data portion from channel handle */ 3362173139Srwatson xge_lldev_t *lldev = xge_hal_channel_userdata(channelh); 3363171095Ssam if(lldev == NULL) { 3364173139Srwatson XGE_EXIT_ON_ERR("Failed to get user data from channel", rxinit_out, 3365173139Srwatson XGE_HAL_FAIL); 3366171095Ssam } 3367171095Ssam 3368171095Ssam /* Get the private data */ 3369173139Srwatson rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh, dtrh); 3370171095Ssam if(rxd_priv == NULL) { 3371173139Srwatson XGE_EXIT_ON_ERR("Failed to get descriptor private data", rxinit_out, 3372173139Srwatson XGE_HAL_FAIL); 3373171095Ssam } 3374171095Ssam 3375173139Srwatson rxd_priv->bufferArray = xge_os_malloc(NULL, 3376173139Srwatson (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt)); 3377171095Ssam 3378171095Ssam if(rxd_priv->bufferArray == NULL) { 3379173139Srwatson XGE_EXIT_ON_ERR("Failed to allocate Rxd private", rxinit_out, 3380173139Srwatson XGE_HAL_FAIL); 3381171095Ssam } 3382171095Ssam 3383171095Ssam if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) { 3384171095Ssam /* Create DMA map for these descriptors*/ 3385171095Ssam if(bus_dmamap_create(lldev->dma_tag_rx , BUS_DMA_NOWAIT, 3386171095Ssam &rxd_priv->dmainfo[0].dma_map)) { 3387173139Srwatson XGE_EXIT_ON_ERR("DMA map creation for Rx descriptor failed", 3388173139Srwatson rxinit_err_out, XGE_HAL_FAIL); 3389171095Ssam } 3390171095Ssam /* Get a buffer, attach it to this descriptor */ 3391173139Srwatson status = xge_get_buf(dtrh, rxd_priv, lldev, 0); 3392171095Ssam } 3393171095Ssam else { 3394171095Ssam for(index1 = 0; index1 < lldev->rxd_mbuf_cnt; index1++) { 3395171095Ssam /* Create DMA map for this descriptor */ 3396171095Ssam if(bus_dmamap_create(lldev->dma_tag_rx , BUS_DMA_NOWAIT , 3397171095Ssam &rxd_priv->dmainfo[index1].dma_map)) { 3398171095Ssam for(index2 = index1 - 1; index2 >= 0; index2--) { 3399171095Ssam bus_dmamap_destroy(lldev->dma_tag_rx, 3400171095Ssam rxd_priv->dmainfo[index2].dma_map); 3401171095Ssam } 3402173139Srwatson XGE_EXIT_ON_ERR( 3403173139Srwatson "Jumbo DMA map creation for Rx descriptor failed", 3404173139Srwatson rxinit_err_out, XGE_HAL_FAIL); 3405171095Ssam } 3406171095Ssam } 3407173139Srwatson status = xge_get_buf_3b_5b(dtrh, rxd_priv, lldev); 3408171095Ssam } 3409171095Ssam 3410173139Srwatson if(status != XGE_HAL_OK) { 3411171095Ssam for(index1 = 0; index1 < lldev->rxd_mbuf_cnt; index1++) { 3412171095Ssam bus_dmamap_destroy(lldev->dma_tag_rx, 3413171095Ssam rxd_priv->dmainfo[index1].dma_map); 3414171095Ssam } 3415171095Ssam goto rxinit_err_out; 3416171095Ssam } 3417171095Ssam else { 3418171095Ssam goto rxinit_out; 3419171095Ssam } 3420171095Ssam 3421171095Ssamrxinit_err_out: 3422173139Srwatson xge_os_free(NULL, rxd_priv->bufferArray, 3423173139Srwatson (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt)); 3424171095Ssamrxinit_out: 3425173139Srwatson return status; 3426171095Ssam} 3427171095Ssam 3428173139Srwatson/** 3429173139Srwatson * xge_rx_term 3430173139Srwatson * During unload terminate and free all descriptors 3431173139Srwatson * 3432173139Srwatson * @channelh Rx Channel Handle 3433173139Srwatson * @dtrh Rx Descriptor Handle 3434173139Srwatson * @state Descriptor State 3435173139Srwatson * @userdata Per-adapter Data 3436173139Srwatson * @reopen Channel open/reopen option 3437173139Srwatson */ 3438171095Ssamvoid 3439173139Srwatsonxge_rx_term(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh, 3440171095Ssam xge_hal_dtr_state_e state, void *userdata, 3441171095Ssam xge_hal_channel_reopen_e reopen) 3442171095Ssam{ 3443173139Srwatson xge_rx_priv_t *rxd_priv = NULL; 3444173139Srwatson xge_lldev_t *lldev = NULL; 3445173139Srwatson int index = 0; 3446171095Ssam 3447171095Ssam /* Descriptor state is not "Posted" */ 3448173139Srwatson if(state != XGE_HAL_DTR_STATE_POSTED) goto rxterm_out; 3449171095Ssam 3450171095Ssam /* Get the user data portion */ 3451171095Ssam lldev = xge_hal_channel_userdata(channelh); 3452171095Ssam 3453171095Ssam /* Get the private data */ 3454173139Srwatson rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh, dtrh); 3455171095Ssam 3456173139Srwatson for(index = 0; index < lldev->rxd_mbuf_cnt; index++) { 3457173139Srwatson if(rxd_priv->dmainfo[index].dma_map != NULL) { 3458171095Ssam bus_dmamap_sync(lldev->dma_tag_rx, 3459171095Ssam rxd_priv->dmainfo[index].dma_map, BUS_DMASYNC_POSTREAD); 3460171095Ssam bus_dmamap_unload(lldev->dma_tag_rx, 3461171095Ssam rxd_priv->dmainfo[index].dma_map); 3462173139Srwatson if(rxd_priv->bufferArray[index] != NULL) 3463173139Srwatson m_free(rxd_priv->bufferArray[index]); 3464171095Ssam bus_dmamap_destroy(lldev->dma_tag_rx, 3465171095Ssam rxd_priv->dmainfo[index].dma_map); 3466171095Ssam } 3467171095Ssam } 3468173139Srwatson xge_os_free(NULL, rxd_priv->bufferArray, 3469173139Srwatson (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt)); 3470171095Ssam 3471171095Ssam /* Free the descriptor */ 3472171095Ssam xge_hal_ring_dtr_free(channelh, dtrh); 3473171095Ssam 3474171095Ssamrxterm_out: 3475171095Ssam return; 3476171095Ssam} 3477171095Ssam 3478173139Srwatson/** 3479173139Srwatson * xge_tx_term 3480173139Srwatson * During unload terminate and free all descriptors 3481173139Srwatson * 3482173139Srwatson * @channelh Rx Channel Handle 3483173139Srwatson * @dtrh Rx Descriptor Handle 3484173139Srwatson * @state Descriptor State 3485173139Srwatson * @userdata Per-adapter Data 3486173139Srwatson * @reopen Channel open/reopen option 3487173139Srwatson */ 3488171095Ssamvoid 3489173139Srwatsonxge_tx_term(xge_hal_channel_h channelh, xge_hal_dtr_h dtr, 3490171095Ssam xge_hal_dtr_state_e state, void *userdata, 3491171095Ssam xge_hal_channel_reopen_e reopen) 3492171095Ssam{ 3493173139Srwatson xge_tx_priv_t *ll_tx_priv = xge_hal_fifo_dtr_private(dtr); 3494173139Srwatson xge_lldev_t *lldev = (xge_lldev_t *)userdata; 3495171095Ssam 3496171095Ssam /* Destroy DMA map */ 3497171095Ssam bus_dmamap_destroy(lldev->dma_tag_tx, ll_tx_priv->dma_map); 3498171095Ssam} 3499171095Ssam 3500173139Srwatson/** 3501171095Ssam * xge_methods 3502171095Ssam * 3503171095Ssam * FreeBSD device interface entry points 3504173139Srwatson */ 3505171095Ssamstatic device_method_t xge_methods[] = { 3506171095Ssam DEVMETHOD(device_probe, xge_probe), 3507171095Ssam DEVMETHOD(device_attach, xge_attach), 3508171095Ssam DEVMETHOD(device_detach, xge_detach), 3509171095Ssam DEVMETHOD(device_shutdown, xge_shutdown), 3510246128Ssbz 3511246128Ssbz DEVMETHOD_END 3512171095Ssam}; 3513171095Ssam 3514171095Ssamstatic driver_t xge_driver = { 3515171095Ssam "nxge", 3516171095Ssam xge_methods, 3517173139Srwatson sizeof(xge_lldev_t), 3518171095Ssam}; 3519171095Ssamstatic devclass_t xge_devclass; 3520171095SsamDRIVER_MODULE(nxge, pci, xge_driver, xge_devclass, 0, 0); 3521173139Srwatson 3522