|
|
|
@@ -1116,16 +1116,17 @@ static void fbnic_free_napi_vector(struct fbnic_net *fbn,
|
|
|
|
|
fbnic_free_irq(fbd, v_idx, nv);
|
|
|
|
|
page_pool_destroy(nv->page_pool);
|
|
|
|
|
netif_napi_del(&nv->napi);
|
|
|
|
|
list_del(&nv->napis);
|
|
|
|
|
fbn->napi[fbnic_napi_idx(nv)] = NULL;
|
|
|
|
|
kfree(nv);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void fbnic_free_napi_vectors(struct fbnic_net *fbn)
|
|
|
|
|
{
|
|
|
|
|
struct fbnic_napi_vector *nv, *temp;
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry_safe(nv, temp, &fbn->napis, napis)
|
|
|
|
|
fbnic_free_napi_vector(fbn, nv);
|
|
|
|
|
for (i = 0; i < fbn->num_napi; i++)
|
|
|
|
|
if (fbn->napi[i])
|
|
|
|
|
fbnic_free_napi_vector(fbn, fbn->napi[i]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void fbnic_name_napi_vector(struct fbnic_napi_vector *nv)
|
|
|
|
@@ -1222,7 +1223,7 @@ static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn,
|
|
|
|
|
nv->v_idx = v_idx;
|
|
|
|
|
|
|
|
|
|
/* Tie napi to netdev */
|
|
|
|
|
list_add(&nv->napis, &fbn->napis);
|
|
|
|
|
fbn->napi[fbnic_napi_idx(nv)] = nv;
|
|
|
|
|
netif_napi_add(fbn->netdev, &nv->napi, fbnic_poll);
|
|
|
|
|
|
|
|
|
|
/* Record IRQ to NAPI struct */
|
|
|
|
@@ -1307,7 +1308,7 @@ pp_destroy:
|
|
|
|
|
page_pool_destroy(nv->page_pool);
|
|
|
|
|
napi_del:
|
|
|
|
|
netif_napi_del(&nv->napi);
|
|
|
|
|
list_del(&nv->napis);
|
|
|
|
|
fbn->napi[fbnic_napi_idx(nv)] = NULL;
|
|
|
|
|
kfree(nv);
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
@@ -1612,19 +1613,18 @@ free_resources:
|
|
|
|
|
|
|
|
|
|
void fbnic_free_resources(struct fbnic_net *fbn)
|
|
|
|
|
{
|
|
|
|
|
struct fbnic_napi_vector *nv;
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(nv, &fbn->napis, napis)
|
|
|
|
|
fbnic_free_nv_resources(fbn, nv);
|
|
|
|
|
for (i = 0; i < fbn->num_napi; i++)
|
|
|
|
|
fbnic_free_nv_resources(fbn, fbn->napi[i]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int fbnic_alloc_resources(struct fbnic_net *fbn)
|
|
|
|
|
{
|
|
|
|
|
struct fbnic_napi_vector *nv;
|
|
|
|
|
int err = -ENODEV;
|
|
|
|
|
int i, err = -ENODEV;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(nv, &fbn->napis, napis) {
|
|
|
|
|
err = fbnic_alloc_nv_resources(fbn, nv);
|
|
|
|
|
for (i = 0; i < fbn->num_napi; i++) {
|
|
|
|
|
err = fbnic_alloc_nv_resources(fbn, fbn->napi[i]);
|
|
|
|
|
if (err)
|
|
|
|
|
goto free_resources;
|
|
|
|
|
}
|
|
|
|
@@ -1632,8 +1632,8 @@ int fbnic_alloc_resources(struct fbnic_net *fbn)
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
free_resources:
|
|
|
|
|
list_for_each_entry_continue_reverse(nv, &fbn->napis, napis)
|
|
|
|
|
fbnic_free_nv_resources(fbn, nv);
|
|
|
|
|
while (i--)
|
|
|
|
|
fbnic_free_nv_resources(fbn, fbn->napi[i]);
|
|
|
|
|
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
@@ -1670,33 +1670,34 @@ static void fbnic_disable_rcq(struct fbnic_ring *rxr)
|
|
|
|
|
|
|
|
|
|
void fbnic_napi_disable(struct fbnic_net *fbn)
|
|
|
|
|
{
|
|
|
|
|
struct fbnic_napi_vector *nv;
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(nv, &fbn->napis, napis) {
|
|
|
|
|
napi_disable(&nv->napi);
|
|
|
|
|
for (i = 0; i < fbn->num_napi; i++) {
|
|
|
|
|
napi_disable(&fbn->napi[i]->napi);
|
|
|
|
|
|
|
|
|
|
fbnic_nv_irq_disable(nv);
|
|
|
|
|
fbnic_nv_irq_disable(fbn->napi[i]);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void fbnic_disable(struct fbnic_net *fbn)
|
|
|
|
|
{
|
|
|
|
|
struct fbnic_dev *fbd = fbn->fbd;
|
|
|
|
|
struct fbnic_napi_vector *nv;
|
|
|
|
|
int i, j;
|
|
|
|
|
int i, j, t;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < fbn->num_napi; i++) {
|
|
|
|
|
struct fbnic_napi_vector *nv = fbn->napi[i];
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(nv, &fbn->napis, napis) {
|
|
|
|
|
/* Disable Tx queue triads */
|
|
|
|
|
for (i = 0; i < nv->txt_count; i++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[i];
|
|
|
|
|
for (t = 0; t < nv->txt_count; t++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[t];
|
|
|
|
|
|
|
|
|
|
fbnic_disable_twq0(&qt->sub0);
|
|
|
|
|
fbnic_disable_tcq(&qt->cmpl);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Disable Rx queue triads */
|
|
|
|
|
for (j = 0; j < nv->rxt_count; j++, i++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[i];
|
|
|
|
|
for (j = 0; j < nv->rxt_count; j++, t++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[t];
|
|
|
|
|
|
|
|
|
|
fbnic_disable_bdq(&qt->sub0, &qt->sub1);
|
|
|
|
|
fbnic_disable_rcq(&qt->cmpl);
|
|
|
|
@@ -1792,14 +1793,15 @@ int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail)
|
|
|
|
|
|
|
|
|
|
void fbnic_flush(struct fbnic_net *fbn)
|
|
|
|
|
{
|
|
|
|
|
struct fbnic_napi_vector *nv;
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(nv, &fbn->napis, napis) {
|
|
|
|
|
int i, j;
|
|
|
|
|
for (i = 0; i < fbn->num_napi; i++) {
|
|
|
|
|
struct fbnic_napi_vector *nv = fbn->napi[i];
|
|
|
|
|
int j, t;
|
|
|
|
|
|
|
|
|
|
/* Flush any processed Tx Queue Triads and drop the rest */
|
|
|
|
|
for (i = 0; i < nv->txt_count; i++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[i];
|
|
|
|
|
for (t = 0; t < nv->txt_count; t++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[t];
|
|
|
|
|
struct netdev_queue *tx_queue;
|
|
|
|
|
|
|
|
|
|
/* Clean the work queues of unprocessed work */
|
|
|
|
@@ -1823,8 +1825,8 @@ void fbnic_flush(struct fbnic_net *fbn)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Flush any processed Rx Queue Triads and drop the rest */
|
|
|
|
|
for (j = 0; j < nv->rxt_count; j++, i++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[i];
|
|
|
|
|
for (j = 0; j < nv->rxt_count; j++, t++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[t];
|
|
|
|
|
|
|
|
|
|
/* Clean the work queues of unprocessed work */
|
|
|
|
|
fbnic_clean_bdq(nv, 0, &qt->sub0, qt->sub0.tail);
|
|
|
|
@@ -1845,14 +1847,15 @@ void fbnic_flush(struct fbnic_net *fbn)
|
|
|
|
|
|
|
|
|
|
void fbnic_fill(struct fbnic_net *fbn)
|
|
|
|
|
{
|
|
|
|
|
struct fbnic_napi_vector *nv;
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(nv, &fbn->napis, napis) {
|
|
|
|
|
int i, j;
|
|
|
|
|
for (i = 0; i < fbn->num_napi; i++) {
|
|
|
|
|
struct fbnic_napi_vector *nv = fbn->napi[i];
|
|
|
|
|
int j, t;
|
|
|
|
|
|
|
|
|
|
/* Configure NAPI mapping for Tx */
|
|
|
|
|
for (i = 0; i < nv->txt_count; i++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[i];
|
|
|
|
|
for (t = 0; t < nv->txt_count; t++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[t];
|
|
|
|
|
|
|
|
|
|
/* Nothing to do if Tx queue is disabled */
|
|
|
|
|
if (qt->sub0.flags & FBNIC_RING_F_DISABLED)
|
|
|
|
@@ -1866,8 +1869,8 @@ void fbnic_fill(struct fbnic_net *fbn)
|
|
|
|
|
/* Configure NAPI mapping and populate pages
|
|
|
|
|
* in the BDQ rings to use for Rx
|
|
|
|
|
*/
|
|
|
|
|
for (j = 0; j < nv->rxt_count; j++, i++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[i];
|
|
|
|
|
for (j = 0; j < nv->rxt_count; j++, t++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[t];
|
|
|
|
|
|
|
|
|
|
/* Associate Rx queue with NAPI */
|
|
|
|
|
netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx,
|
|
|
|
@@ -2025,21 +2028,23 @@ static void fbnic_enable_rcq(struct fbnic_napi_vector *nv,
|
|
|
|
|
void fbnic_enable(struct fbnic_net *fbn)
|
|
|
|
|
{
|
|
|
|
|
struct fbnic_dev *fbd = fbn->fbd;
|
|
|
|
|
struct fbnic_napi_vector *nv;
|
|
|
|
|
int i, j;
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < fbn->num_napi; i++) {
|
|
|
|
|
struct fbnic_napi_vector *nv = fbn->napi[i];
|
|
|
|
|
int j, t;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(nv, &fbn->napis, napis) {
|
|
|
|
|
/* Setup Tx Queue Triads */
|
|
|
|
|
for (i = 0; i < nv->txt_count; i++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[i];
|
|
|
|
|
for (t = 0; t < nv->txt_count; t++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[t];
|
|
|
|
|
|
|
|
|
|
fbnic_enable_twq0(&qt->sub0);
|
|
|
|
|
fbnic_enable_tcq(nv, &qt->cmpl);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Setup Rx Queue Triads */
|
|
|
|
|
for (j = 0; j < nv->rxt_count; j++, i++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[i];
|
|
|
|
|
for (j = 0; j < nv->rxt_count; j++, t++) {
|
|
|
|
|
struct fbnic_q_triad *qt = &nv->qt[t];
|
|
|
|
|
|
|
|
|
|
fbnic_enable_bdq(&qt->sub0, &qt->sub1);
|
|
|
|
|
fbnic_config_drop_mode_rcq(nv, &qt->cmpl);
|
|
|
|
@@ -2064,10 +2069,11 @@ void fbnic_napi_enable(struct fbnic_net *fbn)
|
|
|
|
|
{
|
|
|
|
|
u32 irqs[FBNIC_MAX_MSIX_VECS / 32] = {};
|
|
|
|
|
struct fbnic_dev *fbd = fbn->fbd;
|
|
|
|
|
struct fbnic_napi_vector *nv;
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(nv, &fbn->napis, napis) {
|
|
|
|
|
for (i = 0; i < fbn->num_napi; i++) {
|
|
|
|
|
struct fbnic_napi_vector *nv = fbn->napi[i];
|
|
|
|
|
|
|
|
|
|
napi_enable(&nv->napi);
|
|
|
|
|
|
|
|
|
|
fbnic_nv_irq_enable(nv);
|
|
|
|
@@ -2096,17 +2102,18 @@ void fbnic_napi_depletion_check(struct net_device *netdev)
|
|
|
|
|
struct fbnic_net *fbn = netdev_priv(netdev);
|
|
|
|
|
u32 irqs[FBNIC_MAX_MSIX_VECS / 32] = {};
|
|
|
|
|
struct fbnic_dev *fbd = fbn->fbd;
|
|
|
|
|
struct fbnic_napi_vector *nv;
|
|
|
|
|
int i, j;
|
|
|
|
|
int i, j, t;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < fbn->num_napi; i++) {
|
|
|
|
|
struct fbnic_napi_vector *nv = fbn->napi[i];
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(nv, &fbn->napis, napis) {
|
|
|
|
|
/* Find RQs which are completely out of pages */
|
|
|
|
|
for (i = nv->txt_count, j = 0; j < nv->rxt_count; j++, i++) {
|
|
|
|
|
for (t = nv->txt_count, j = 0; j < nv->rxt_count; j++, t++) {
|
|
|
|
|
/* Assume 4 pages is always enough to fit a packet
|
|
|
|
|
* and therefore generate a completion and an IRQ.
|
|
|
|
|
*/
|
|
|
|
|
if (fbnic_desc_used(&nv->qt[i].sub0) < 4 ||
|
|
|
|
|
fbnic_desc_used(&nv->qt[i].sub1) < 4)
|
|
|
|
|
if (fbnic_desc_used(&nv->qt[t].sub0) < 4 ||
|
|
|
|
|
fbnic_desc_used(&nv->qt[t].sub1) < 4)
|
|
|
|
|
irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|