From e9a1d3b40c0a09105dde91852a4fc018daee9b54 Mon Sep 17 00:00:00 2001 From: Tianyi Gao Date: Sun, 15 Nov 2026 12:53:24 +0000 Subject: [PATCH] Hand an RPC off at the end of a batch * homa_dispatch_pkts holds the RPC lock while it processes a batch of packets for one RPC, but homa_data_pkt handed the RPC off on the first packet of the batch (homa_rpc_alloc_server did so even before the packet was queued). The woken thread then asked for the RPC lock while SoftIRQ still needed it, so SoftIRQ released the lock in the middle of the batch and waited to get it back. * Now homa_dispatch_pkts hands the RPC off once, after the whole batch and just before homa_grant_check_rpc, if the RPC has packets queued and is not ready yet. homa_data_pkt and homa_rpc_alloc_server no longer hand off. --- homa_incoming.c | 24 ++++++------- homa_rpc.c | 8 +---- test/unit_homa_incoming.c | 73 ++++++++++++++++++++++----------------- test/unit_homa_rpc.c | 37 ++------------------ 4 files changed, 55 insertions(+), 87 deletions(-) diff --git a/homa_incoming.c b/homa_incoming.c index cbf06730..3e542aec 100644 --- a/homa_incoming.c +++ b/homa_incoming.c @@ -673,6 +673,15 @@ void homa_dispatch_pkts(struct sk_buff *skb) kfree_skb(skb); } if (rpc) { + /* Hand off only after the whole batch: a thread woken earlier + * would ask for the RPC lock while the rest of the batch + * still needs it. + */ + if (skb_queue_len(&rpc->msgin.packets) != 0 && + !test_bit(RPC_PKTS_READY, &rpc->flags)) { + set_bit(RPC_PKTS_READY, &rpc->flags); + homa_rpc_handoff(rpc); + } IF_NO_STRIP(homa_grant_check_rpc(rpc)); homa_rpc_put(rpc); homa_rpc_unlock(rpc); @@ -792,20 +801,9 @@ void homa_data_pkt(struct sk_buff *skb, struct homa_rpc *rpc) drop_reason = homa_add_packet(rpc, skb); - if (skb_queue_len(&rpc->msgin.packets) != 0 && - !test_bit(RPC_PKTS_READY, &rpc->flags)) { - set_bit(RPC_PKTS_READY, &rpc->flags); - homa_rpc_handoff(rpc); - } - handle_ack: - /* It's important to handle acks *after* calling homa_add_packet. - * Otherwise we can end up with extraneous handoffs: we have to release - * the RPC lock to handle acks, but that could allow some other thread - * to receive a previous handoff made by homa_rpc_alloc_server, even - * though the packet hasn't been added to the RPC. If this happens - * the call to homa_rpc_handoff above will have to make an additional - * handoff. + /* Handling acks releases the RPC lock, so the packet is queued + * first: a thread that copies from the RPC meanwhile will find it. */ if (h->ack.client_id) { const struct in6_addr saddr = skb_canonical_ipv6_saddr(skb); diff --git a/homa_rpc.c b/homa_rpc.c index bdbe8752..b0636d82 100644 --- a/homa_rpc.c +++ b/homa_rpc.c @@ -97,9 +97,7 @@ struct homa_rpc *homa_rpc_alloc_client(struct homa_sock *hsk, /** * homa_rpc_alloc_server() - Allocate and initialize a server RPC (one that is - * used to manage an incoming request). If appropriate, the RPC will also - * be handed off (we do it here, while we have the socket locked, to avoid - * acquiring the socket lock a second time later for the handoff). + * used to manage an incoming request). * @hsk: Socket that owns this RPC. * @source: IP address (network byte order) of the RPC's client. * @h: Header for the first data packet received for this RPC; used @@ -210,10 +208,6 @@ struct homa_rpc *homa_rpc_alloc_server(struct homa_sock *hsk, hlist_add_head(&srpc->hash_links, &bucket->rpcs); list_add_tail_rcu(&srpc->active_links, &hsk->active_rpcs); homa_sock_unlock(hsk); - if (h->type == DATA && srpc->msgin.num_bpages > 0) { - set_bit(RPC_PKTS_READY, &srpc->flags); - homa_rpc_handoff(srpc); - } INC_METRIC(requests_received, 1); return srpc; diff --git a/test/unit_homa_incoming.c b/test/unit_homa_incoming.c index 56230230..5ca75691 100644 --- a/test/unit_homa_incoming.c +++ b/test/unit_homa_incoming.c @@ -49,6 +49,14 @@ static void wait_hook4(char *id) homa_rpc_handoff(hook_rpc); } +static void count_packets_data_ready(struct sock *sk) +{ + struct homa_rpc *rpc = list_first_entry(&homa_sk(sk)->active_rpcs, + struct homa_rpc, active_links); + + hook_count = skb_queue_len(&rpc->msgin.packets); +} + static void handoff_hook(char *id) { if (strcmp(id, "spin_lock") != 0) @@ -1307,6 +1315,38 @@ TEST_F(homa_incoming, homa_dispatch_pkts__cant_create_server_rpc) EXPECT_EQ(1, homa_metrics_per_cpu()->server_cant_create_rpcs); #endif /* See strip.py */ } +TEST_F(homa_incoming, homa_dispatch_pkts__handoff_after_batch) +{ + struct sk_buff *skb; + + self->data.msg_length = htonl(3000); + skb = mock_skb_alloc(self->client_ip, self->server_ip, + &self->data.common, 1400, 0); + self->data.seg.offset = htonl(1400); + skb->next = mock_skb_alloc(self->client_ip, self->server_ip, + &self->data.common, 1400, 1400); + self->hsk2.sock.sk_data_ready = count_packets_data_ready; + hook_count = 0; + homa_dispatch_pkts(skb); + EXPECT_EQ(1, unit_list_length(&self->hsk2.ready_rpcs)); + EXPECT_EQ(2, hook_count); +} +TEST_F(homa_incoming, homa_dispatch_pkts__no_handoff_if_pending) +{ + self->data.msg_length = htonl(3000); + homa_dispatch_pkts(mock_skb_alloc(self->client_ip, self->server_ip, + &self->data.common, 1400, 0)); + EXPECT_EQ(1, unit_list_length(&self->hsk2.ready_rpcs)); + + /* The RPC is still waiting to be received: no second handoff. */ + self->data.seg.offset = htonl(1400); + self->hsk2.sock.sk_data_ready = count_packets_data_ready; + hook_count = 0; + homa_dispatch_pkts(mock_skb_alloc(self->client_ip, self->server_ip, + &self->data.common, 1400, 1400)); + EXPECT_EQ(0, hook_count); + EXPECT_EQ(1, unit_list_length(&self->hsk2.ready_rpcs)); +} TEST_F(homa_incoming, homa_dispatch_pkts__existing_server_rpc) { struct homa_rpc *srpc = unit_server_rpc(&self->hsk2, UNIT_RCVD_ONE_PKT, @@ -1621,7 +1661,7 @@ TEST_F(homa_incoming, homa_data_pkt__basics) homa_data_pkt(mock_skb_alloc(self->server_ip, self->client_ip, &self->data.common, 1400, 0), crpc); EXPECT_EQ(RPC_INCOMING, crpc->state); - EXPECT_EQ(1, unit_list_length(&self->hsk.ready_rpcs)); + EXPECT_EQ(0, unit_list_length(&self->hsk.ready_rpcs)); EXPECT_EQ(200, crpc->msgin.bytes_remaining); EXPECT_EQ(1, skb_queue_len(&crpc->msgin.packets)); #ifndef __STRIP__ /* See strip.py */ @@ -1767,37 +1807,6 @@ TEST_F(homa_incoming, homa_data_pkt__homa_add_packet_returns_error) EXPECT_EQ(1, mock_num_drop_reasons); EXPECT_EQ(SKB_DROP_REASON_PKT_TOO_BIG, mock_drop_reasons[0]); } -TEST_F(homa_incoming, homa_data_pkt__handoff) -{ - struct homa_rpc *crpc = unit_client_rpc(&self->hsk, - UNIT_OUTGOING, self->client_ip, self->server_ip, - self->server_port, self->client_id, 1000, 3000); - - ASSERT_NE(NULL, crpc); - unit_log_clear(); - crpc->msgout.next_xmit_offset = crpc->msgout.length; - - /* First packet triggers handoff. */ - self->data.msg_length = htonl(3000); - self->data.seg.offset = htonl(1400); - homa_data_pkt(mock_skb_alloc(self->server_ip, self->client_ip, - &self->data.common, 1400, 0), crpc); - EXPECT_EQ(1, unit_list_length(&self->hsk.ready_rpcs)); - EXPECT_TRUE(test_bit(RPC_PKTS_READY, &crpc->flags)); - EXPECT_EQ(1600, crpc->msgin.bytes_remaining); - EXPECT_EQ(1, skb_queue_len(&crpc->msgin.packets)); - EXPECT_STREQ("sk->sk_data_ready invoked", unit_log_get()); - - /* Second packet doesn't trigger a handoff because one is - * already pending. - */ - self->data.msg_length = htonl(3000); - self->data.seg.offset = htonl(2800); - unit_log_clear(); - homa_data_pkt(mock_skb_alloc(self->server_ip, self->client_ip, - &self->data.common, 200, 0), crpc); - EXPECT_STREQ("", unit_log_get()); -} TEST_F(homa_incoming, homa_data_pkt__handle_ack) { struct homa_rpc *srpc1 = unit_server_rpc(&self->hsk, UNIT_OUTGOING, diff --git a/test/unit_homa_rpc.c b/test/unit_homa_rpc.c index d201be6e..a5bb136a 100644 --- a/test/unit_homa_rpc.c +++ b/test/unit_homa_rpc.c @@ -261,10 +261,11 @@ TEST_F(homa_rpc, homa_rpc_alloc_server__socket_shutdown) EXPECT_EQ(0, unit_list_length(&self->hsk.active_rpcs)); self->hsk.shutdown = 0; } -TEST_F(homa_rpc, homa_rpc_alloc_server__handoff_rpc) +TEST_F(homa_rpc, homa_rpc_alloc_server__no_handoff) { struct homa_rpc *srpc; + /* homa_data_pkt hands the message over, not homa_rpc_alloc_server. */ self->data.msg_length = N(1400); srpc = homa_rpc_alloc_server(&self->hsk, self->client_ip, &self->data.common); @@ -272,43 +273,9 @@ TEST_F(homa_rpc, homa_rpc_alloc_server__handoff_rpc) homa_rpc_unlock(srpc); EXPECT_EQ(RPC_INCOMING, srpc->state); EXPECT_EQ(1, unit_list_length(&self->hsk.active_rpcs)); - EXPECT_EQ(1, unit_list_length(&self->hsk.ready_rpcs)); - homa_rpc_end(srpc); -} -TEST_F(homa_rpc, homa_rpc_alloc_server__dont_handoff_no_buffers) -{ - struct homa_rpc *srpc; - - mock_check_bpool_leaks = false; - self->data.msg_length = N(1400); - self->hsk.buffer_pool->free_bpages = 0; - srpc = homa_rpc_alloc_server(&self->hsk, self->client_ip, - &self->data.common); - ASSERT_FALSE(IS_ERR(srpc)); - homa_rpc_unlock(srpc); - EXPECT_EQ(0, unit_list_length(&self->hsk.ready_rpcs)); - homa_rpc_end(srpc); -} -#ifndef __STRIP__ /* See strip.py */ -TEST_F(homa_rpc, homa_rpc_alloc_server__dont_handoff_rpc_start_msg) -{ - struct homa_start_msg_hdr h; - struct homa_rpc *srpc; - - h.common.sport = n(self->client_port); - h.common.dport = n(self->server_port); - h.common.type = START_MSG; - h.common.sender_id = cpu_to_be64(self->client_id); - h.msg_length = N(3000); - srpc = homa_rpc_alloc_server(&self->hsk, self->client_ip, &h.common); - ASSERT_FALSE(IS_ERR(srpc)); - homa_rpc_unlock(srpc); - EXPECT_EQ(RPC_INCOMING, srpc->state); - EXPECT_EQ(1, unit_list_length(&self->hsk.active_rpcs)); EXPECT_EQ(0, unit_list_length(&self->hsk.ready_rpcs)); homa_rpc_end(srpc); } -#endif /* See strip.py */ TEST_F(homa_rpc, homa_rpc_alloc_server__unlink_from_waiting_for_bufs_after_error) { struct homa_rpc *srpc;