Commit fdc13979 authored by Lorenzo Bianconi's avatar Lorenzo Bianconi Committed by Daniel Borkmann
Browse files

bpf, devmap: Move drop error path to devmap for XDP_REDIRECT 



We want to change the current ndo_xdp_xmit drop semantics because it will
allow us to implement better queue overflow handling. This is working
towards the larger goal of a XDP TX queue-hook. Move XDP_REDIRECT error
path handling from each XDP ethernet driver to devmap code. According to
the new APIs, the driver running the ndo_xdp_xmit pointer, will break tx
loop whenever the hw reports a tx error and it will just return to devmap
caller the number of successfully transmitted frames. It will be devmap
responsibility to free dropped frames.

Move each XDP ndo_xdp_xmit capable driver to the new APIs:

- veth
- virtio-net
- mvneta
- mvpp2
- socionext
- amazon ena
- bnxt
- freescale (dpaa2, dpaa)
- xen-frontend
- qede
- ice
- igb
- ixgbe
- i40e
- mlx5
- ti (cpsw, cpsw-new)
- tun
- sfc

Signed-off-by: default avatarLorenzo Bianconi <lorenzo@kernel.org>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Reviewed-by: default avatarIoana Ciornei <ioana.ciornei@nxp.com>
Reviewed-by: default avatarIlias Apalodimas <ilias.apalodimas@linaro.org>
Reviewed-by: default avatarCamelia Groza <camelia.groza@nxp.com>
Acked-by: default avatarEdward Cree <ecree.xilinx@gmail.com>
Acked-by: default avatarJesper Dangaard Brouer <brouer@redhat.com>
Acked-by: default avatarShay Agroskin <shayagr@amazon.com>
Link: https://lore.kernel.org/bpf/ed670de24f951cfd77590decf0229a0ad7fd12f6.1615201152.git.lorenzo@kernel.org
parent 6b282765

drivers/net/ethernet/amazon/ena/ena_netdev.c

+9 −12
Original line number Original line Diff line number Diff line
@@ -300,7 +300,7 @@ static int ena_xdp_xmit_frame(struct ena_ring *xdp_ring, 




	rc = ena_xdp_tx_map_frame(xdp_ring, tx_info, xdpf, &push_hdr, &push_len);

	rc = ena_xdp_tx_map_frame(xdp_ring, tx_info, xdpf, &push_hdr, &push_len);

	if (unlikely(rc))

	if (unlikely(rc))

		goto error_drop_packet;

		return rc;





	ena_tx_ctx.ena_bufs = tx_info->bufs;

	ena_tx_ctx.ena_bufs = tx_info->bufs;

	ena_tx_ctx.push_header = push_hdr;

	ena_tx_ctx.push_header = push_hdr;
@@ -330,8 +330,6 @@ static int ena_xdp_xmit_frame(struct ena_ring *xdp_ring, 
error_unmap_dma:

error_unmap_dma:

	ena_unmap_tx_buff(xdp_ring, tx_info);

	ena_unmap_tx_buff(xdp_ring, tx_info);

	tx_info->xdpf = NULL;

	tx_info->xdpf = NULL;

error_drop_packet:

	xdp_return_frame(xdpf);

	return rc;

	return rc;

}

}
@@ -339,8 +337,8 @@ static int ena_xdp_xmit(struct net_device *dev, int n, 
			struct xdp_frame **frames, u32 flags)

			struct xdp_frame **frames, u32 flags)

{

{

	struct ena_adapter *adapter = netdev_priv(dev);

	struct ena_adapter *adapter = netdev_priv(dev);

	int qid, i, err, drops = 0;

	struct ena_ring *xdp_ring;

	struct ena_ring *xdp_ring;

	int qid, i, nxmit = 0;





	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))

	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))

		return -EINVAL;

		return -EINVAL;
@@ -360,12 +358,9 @@ static int ena_xdp_xmit(struct net_device *dev, int n, 
	spin_lock(&xdp_ring->xdp_tx_lock);

	spin_lock(&xdp_ring->xdp_tx_lock);





	for (i = 0; i < n; i++) {

	for (i = 0; i < n; i++) {

		err = ena_xdp_xmit_frame(xdp_ring, dev, frames[i], 0);

		if (ena_xdp_xmit_frame(xdp_ring, dev, frames[i], 0))

		/* The descriptor is freed by ena_xdp_xmit_frame in case

			break;

		 * of an error.

		nxmit++;

		 */

		if (err)

			drops++;

	}

	}





	/* Ring doorbell to make device aware of the packets */

	/* Ring doorbell to make device aware of the packets */
@@ -378,7 +373,7 @@ static int ena_xdp_xmit(struct net_device *dev, int n, 
	spin_unlock(&xdp_ring->xdp_tx_lock);

	spin_unlock(&xdp_ring->xdp_tx_lock);





	/* Return number of packets sent */

	/* Return number of packets sent */

	return n - drops;

	return nxmit;

}

}





static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp)

static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp)
@@ -415,7 +410,9 @@ static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp) 
		/* The XDP queues are shared between XDP_TX and XDP_REDIRECT */

		/* The XDP queues are shared between XDP_TX and XDP_REDIRECT */

		spin_lock(&xdp_ring->xdp_tx_lock);

		spin_lock(&xdp_ring->xdp_tx_lock);





		ena_xdp_xmit_frame(xdp_ring, rx_ring->netdev, xdpf, XDP_XMIT_FLUSH);

		if (ena_xdp_xmit_frame(xdp_ring, rx_ring->netdev, xdpf,

				       XDP_XMIT_FLUSH))

			xdp_return_frame(xdpf);





		spin_unlock(&xdp_ring->xdp_tx_lock);

		spin_unlock(&xdp_ring->xdp_tx_lock);

		xdp_stat = &rx_ring->rx_stats.xdp_tx;

		xdp_stat = &rx_ring->rx_stats.xdp_tx;

drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c

+8 −12
Original line number Original line Diff line number Diff line
@@ -217,7 +217,7 @@ int bnxt_xdp_xmit(struct net_device *dev, int num_frames, 
	struct pci_dev *pdev = bp->pdev;

	struct pci_dev *pdev = bp->pdev;

	struct bnxt_tx_ring_info *txr;

	struct bnxt_tx_ring_info *txr;

	dma_addr_t mapping;

	dma_addr_t mapping;

	int drops = 0;

	int nxmit = 0;

	int ring;

	int ring;

	int i;

	int i;
@@ -233,21 +233,17 @@ int bnxt_xdp_xmit(struct net_device *dev, int num_frames, 
		struct xdp_frame *xdp = frames[i];

		struct xdp_frame *xdp = frames[i];





		if (!txr || !bnxt_tx_avail(bp, txr) ||

		if (!txr || !bnxt_tx_avail(bp, txr) ||

		    !(bp->bnapi[ring]->flags & BNXT_NAPI_FLAG_XDP)) {

		    !(bp->bnapi[ring]->flags & BNXT_NAPI_FLAG_XDP))

			xdp_return_frame_rx_napi(xdp);

			break;

			drops++;

			continue;

		}





		mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len,

		mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len,

					 DMA_TO_DEVICE);

					 DMA_TO_DEVICE);





		if (dma_mapping_error(&pdev->dev, mapping)) {

		if (dma_mapping_error(&pdev->dev, mapping))

			xdp_return_frame_rx_napi(xdp);

			break;

			drops++;



			continue;

		}

		__bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp);

		__bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp);

		nxmit++;

	}

	}





	if (flags & XDP_XMIT_FLUSH) {

	if (flags & XDP_XMIT_FLUSH) {
@@ -256,7 +252,7 @@ int bnxt_xdp_xmit(struct net_device *dev, int num_frames, 
		bnxt_db_write(bp, &txr->tx_db, txr->tx_prod);

		bnxt_db_write(bp, &txr->tx_db, txr->tx_prod);

	}

	}





	return num_frames - drops;

	return nxmit;

}

}





/* Under rtnl_lock */

/* Under rtnl_lock */

drivers/net/ethernet/freescale/dpaa/dpaa_eth.c

+5 −7
Original line number Original line Diff line number Diff line
@@ -3081,7 +3081,7 @@ static int dpaa_xdp_xmit(struct net_device *net_dev, int n, 
			 struct xdp_frame **frames, u32 flags)

			 struct xdp_frame **frames, u32 flags)

{

{

	struct xdp_frame *xdpf;

	struct xdp_frame *xdpf;

	int i, err, drops = 0;

	int i, nxmit = 0;





	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))

	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))

		return -EINVAL;

		return -EINVAL;
@@ -3091,14 +3091,12 @@ static int dpaa_xdp_xmit(struct net_device *net_dev, int n, 




	for (i = 0; i < n; i++) {

	for (i = 0; i < n; i++) {

		xdpf = frames[i];

		xdpf = frames[i];

		err = dpaa_xdp_xmit_frame(net_dev, xdpf);

		if (dpaa_xdp_xmit_frame(net_dev, xdpf))

		if (err) {

			break;

			xdp_return_frame_rx_napi(xdpf);

		nxmit++;

			drops++;

		}

	}

	}





	return n - drops;

	return nxmit;

}

}





static int dpaa_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)

static int dpaa_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)

drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c

+0 −2
Original line number Original line Diff line number Diff line
@@ -2431,8 +2431,6 @@ static int dpaa2_eth_xdp_xmit(struct net_device *net_dev, int n, 
	percpu_stats->tx_packets += enqueued;

	percpu_stats->tx_packets += enqueued;

	for (i = 0; i < enqueued; i++)

	for (i = 0; i < enqueued; i++)

		percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]);

		percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]);

	for (i = enqueued; i < n; i++)

		xdp_return_frame_rx_napi(frames[i]);





	return enqueued;

	return enqueued;

}

}

drivers/net/ethernet/intel/i40e/i40e_txrx.c

+7 −8
Original line number Original line Diff line number Diff line
@@ -3847,8 +3847,8 @@ netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 
 * @frames: array of XDP buffer pointers

 * @frames: array of XDP buffer pointers

 * @flags: XDP extra info

 * @flags: XDP extra info

 *

 *

 * Returns number of frames successfully sent. Frames that fail are

 * Returns number of frames successfully sent. Failed frames

 * free'ed via XDP return API.

 * will be free'ed by XDP core.

 *

 *

 * For error cases, a negative errno code is returned and no-frames

 * For error cases, a negative errno code is returned and no-frames

 * are transmitted (caller must handle freeing frames).

 * are transmitted (caller must handle freeing frames).
@@ -3861,7 +3861,7 @@ int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 
	struct i40e_vsi *vsi = np->vsi;

	struct i40e_vsi *vsi = np->vsi;

	struct i40e_pf *pf = vsi->back;

	struct i40e_pf *pf = vsi->back;

	struct i40e_ring *xdp_ring;

	struct i40e_ring *xdp_ring;

	int drops = 0;

	int nxmit = 0;

	int i;

	int i;





	if (test_bit(__I40E_VSI_DOWN, vsi->state))

	if (test_bit(__I40E_VSI_DOWN, vsi->state))
@@ -3881,14 +3881,13 @@ int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 
		int err;

		int err;





		err = i40e_xmit_xdp_ring(xdpf, xdp_ring);

		err = i40e_xmit_xdp_ring(xdpf, xdp_ring);

		if (err != I40E_XDP_TX) {

		if (err != I40E_XDP_TX)

			xdp_return_frame_rx_napi(xdpf);

			break;

			drops++;

		nxmit++;

		}

	}

	}





	if (unlikely(flags & XDP_XMIT_FLUSH))

	if (unlikely(flags & XDP_XMIT_FLUSH))

		i40e_xdp_ring_update_tail(xdp_ring);

		i40e_xdp_ring_update_tail(xdp_ring);





	return n - drops;

	return nxmit;

}
 
}