diff --git a/homa_incoming.c b/homa_incoming.c index cbf06730..3e542aec 100644 --- a/homa_incoming.c +++ b/homa_incoming.c @@ -673,6 +673,15 @@ void homa_dispatch_pkts(struct sk_buff *skb) kfree_skb(skb); } if (rpc) { + /* Hand off only after the whole batch: a thread woken earlier + * would ask for the RPC lock while the rest of the batch + * still needs it. + */ + if (skb_queue_len(&rpc->msgin.packets) != 0 && + !test_bit(RPC_PKTS_READY, &rpc->flags)) { + set_bit(RPC_PKTS_READY, &rpc->flags); + homa_rpc_handoff(rpc); + } IF_NO_STRIP(homa_grant_check_rpc(rpc)); homa_rpc_put(rpc); homa_rpc_unlock(rpc); @@ -792,20 +801,9 @@ void homa_data_pkt(struct sk_buff *skb, struct homa_rpc *rpc) drop_reason = homa_add_packet(rpc, skb); - if (skb_queue_len(&rpc->msgin.packets) != 0 && - !test_bit(RPC_PKTS_READY, &rpc->flags)) { - set_bit(RPC_PKTS_READY, &rpc->flags); - homa_rpc_handoff(rpc); - } - handle_ack: - /* It's important to handle acks *after* calling homa_add_packet. - * Otherwise we can end up with extraneous handoffs: we have to release - * the RPC lock to handle acks, but that could allow some other thread - * to receive a previous handoff made by homa_rpc_alloc_server, even - * though the packet hasn't been added to the RPC. If this happens - * the call to homa_rpc_handoff above will have to make an additional - * handoff. + /* Handling acks releases the RPC lock, so the packet is queued + * first: a thread that copies from the RPC meanwhile will find it. */ if (h->ack.client_id) { const struct in6_addr saddr = skb_canonical_ipv6_saddr(skb); diff --git a/homa_rpc.c b/homa_rpc.c index bdbe8752..b0636d82 100644 --- a/homa_rpc.c +++ b/homa_rpc.c @@ -97,9 +97,7 @@ struct homa_rpc *homa_rpc_alloc_client(struct homa_sock *hsk, /** * homa_rpc_alloc_server() - Allocate and initialize a server RPC (one that is - * used to manage an incoming request). If appropriate, the RPC will also - * be handed off (we do it here, while we have the socket locked, to avoid - * acquiring the socket lock a second time later for the handoff). + * used to manage an incoming request). * @hsk: Socket that owns this RPC. * @source: IP address (network byte order) of the RPC's client. * @h: Header for the first data packet received for this RPC; used @@ -210,10 +208,6 @@ struct homa_rpc *homa_rpc_alloc_server(struct homa_sock *hsk, hlist_add_head(&srpc->hash_links, &bucket->rpcs); list_add_tail_rcu(&srpc->active_links, &hsk->active_rpcs); homa_sock_unlock(hsk); - if (h->type == DATA && srpc->msgin.num_bpages > 0) { - set_bit(RPC_PKTS_READY, &srpc->flags); - homa_rpc_handoff(srpc); - } INC_METRIC(requests_received, 1); return srpc; diff --git a/test/unit_homa_incoming.c b/test/unit_homa_incoming.c index 56230230..5ca75691 100644 --- a/test/unit_homa_incoming.c +++ b/test/unit_homa_incoming.c @@ -49,6 +49,14 @@ static void wait_hook4(char *id) homa_rpc_handoff(hook_rpc); } +static void count_packets_data_ready(struct sock *sk) +{ + struct homa_rpc *rpc = list_first_entry(&homa_sk(sk)->active_rpcs, + struct homa_rpc, active_links); + + hook_count = skb_queue_len(&rpc->msgin.packets); +} + static void handoff_hook(char *id) { if (strcmp(id, "spin_lock") != 0) @@ -1307,6 +1315,38 @@ TEST_F(homa_incoming, homa_dispatch_pkts__cant_create_server_rpc) EXPECT_EQ(1, homa_metrics_per_cpu()->server_cant_create_rpcs); #endif /* See strip.py */ } +TEST_F(homa_incoming, homa_dispatch_pkts__handoff_after_batch) +{ + struct sk_buff *skb; + + self->data.msg_length = htonl(3000); + skb = mock_skb_alloc(self->client_ip, self->server_ip, + &self->data.common, 1400, 0); + self->data.seg.offset = htonl(1400); + skb->next = mock_skb_alloc(self->client_ip, self->server_ip, + &self->data.common, 1400, 1400); + self->hsk2.sock.sk_data_ready = count_packets_data_ready; + hook_count = 0; + homa_dispatch_pkts(skb); + EXPECT_EQ(1, unit_list_length(&self->hsk2.ready_rpcs)); + EXPECT_EQ(2, hook_count); +} +TEST_F(homa_incoming, homa_dispatch_pkts__no_handoff_if_pending) +{ + self->data.msg_length = htonl(3000); + homa_dispatch_pkts(mock_skb_alloc(self->client_ip, self->server_ip, + &self->data.common, 1400, 0)); + EXPECT_EQ(1, unit_list_length(&self->hsk2.ready_rpcs)); + + /* The RPC is still waiting to be received: no second handoff. */ + self->data.seg.offset = htonl(1400); + self->hsk2.sock.sk_data_ready = count_packets_data_ready; + hook_count = 0; + homa_dispatch_pkts(mock_skb_alloc(self->client_ip, self->server_ip, + &self->data.common, 1400, 1400)); + EXPECT_EQ(0, hook_count); + EXPECT_EQ(1, unit_list_length(&self->hsk2.ready_rpcs)); +} TEST_F(homa_incoming, homa_dispatch_pkts__existing_server_rpc) { struct homa_rpc *srpc = unit_server_rpc(&self->hsk2, UNIT_RCVD_ONE_PKT, @@ -1621,7 +1661,7 @@ TEST_F(homa_incoming, homa_data_pkt__basics) homa_data_pkt(mock_skb_alloc(self->server_ip, self->client_ip, &self->data.common, 1400, 0), crpc); EXPECT_EQ(RPC_INCOMING, crpc->state); - EXPECT_EQ(1, unit_list_length(&self->hsk.ready_rpcs)); + EXPECT_EQ(0, unit_list_length(&self->hsk.ready_rpcs)); EXPECT_EQ(200, crpc->msgin.bytes_remaining); EXPECT_EQ(1, skb_queue_len(&crpc->msgin.packets)); #ifndef __STRIP__ /* See strip.py */ @@ -1767,37 +1807,6 @@ TEST_F(homa_incoming, homa_data_pkt__homa_add_packet_returns_error) EXPECT_EQ(1, mock_num_drop_reasons); EXPECT_EQ(SKB_DROP_REASON_PKT_TOO_BIG, mock_drop_reasons[0]); } -TEST_F(homa_incoming, homa_data_pkt__handoff) -{ - struct homa_rpc *crpc = unit_client_rpc(&self->hsk, - UNIT_OUTGOING, self->client_ip, self->server_ip, - self->server_port, self->client_id, 1000, 3000); - - ASSERT_NE(NULL, crpc); - unit_log_clear(); - crpc->msgout.next_xmit_offset = crpc->msgout.length; - - /* First packet triggers handoff. */ - self->data.msg_length = htonl(3000); - self->data.seg.offset = htonl(1400); - homa_data_pkt(mock_skb_alloc(self->server_ip, self->client_ip, - &self->data.common, 1400, 0), crpc); - EXPECT_EQ(1, unit_list_length(&self->hsk.ready_rpcs)); - EXPECT_TRUE(test_bit(RPC_PKTS_READY, &crpc->flags)); - EXPECT_EQ(1600, crpc->msgin.bytes_remaining); - EXPECT_EQ(1, skb_queue_len(&crpc->msgin.packets)); - EXPECT_STREQ("sk->sk_data_ready invoked", unit_log_get()); - - /* Second packet doesn't trigger a handoff because one is - * already pending. - */ - self->data.msg_length = htonl(3000); - self->data.seg.offset = htonl(2800); - unit_log_clear(); - homa_data_pkt(mock_skb_alloc(self->server_ip, self->client_ip, - &self->data.common, 200, 0), crpc); - EXPECT_STREQ("", unit_log_get()); -} TEST_F(homa_incoming, homa_data_pkt__handle_ack) { struct homa_rpc *srpc1 = unit_server_rpc(&self->hsk, UNIT_OUTGOING, diff --git a/test/unit_homa_rpc.c b/test/unit_homa_rpc.c index d201be6e..a5bb136a 100644 --- a/test/unit_homa_rpc.c +++ b/test/unit_homa_rpc.c @@ -261,10 +261,11 @@ TEST_F(homa_rpc, homa_rpc_alloc_server__socket_shutdown) EXPECT_EQ(0, unit_list_length(&self->hsk.active_rpcs)); self->hsk.shutdown = 0; } -TEST_F(homa_rpc, homa_rpc_alloc_server__handoff_rpc) +TEST_F(homa_rpc, homa_rpc_alloc_server__no_handoff) { struct homa_rpc *srpc; + /* homa_data_pkt hands the message over, not homa_rpc_alloc_server. */ self->data.msg_length = N(1400); srpc = homa_rpc_alloc_server(&self->hsk, self->client_ip, &self->data.common); @@ -272,43 +273,9 @@ TEST_F(homa_rpc, homa_rpc_alloc_server__handoff_rpc) homa_rpc_unlock(srpc); EXPECT_EQ(RPC_INCOMING, srpc->state); EXPECT_EQ(1, unit_list_length(&self->hsk.active_rpcs)); - EXPECT_EQ(1, unit_list_length(&self->hsk.ready_rpcs)); - homa_rpc_end(srpc); -} -TEST_F(homa_rpc, homa_rpc_alloc_server__dont_handoff_no_buffers) -{ - struct homa_rpc *srpc; - - mock_check_bpool_leaks = false; - self->data.msg_length = N(1400); - self->hsk.buffer_pool->free_bpages = 0; - srpc = homa_rpc_alloc_server(&self->hsk, self->client_ip, - &self->data.common); - ASSERT_FALSE(IS_ERR(srpc)); - homa_rpc_unlock(srpc); - EXPECT_EQ(0, unit_list_length(&self->hsk.ready_rpcs)); - homa_rpc_end(srpc); -} -#ifndef __STRIP__ /* See strip.py */ -TEST_F(homa_rpc, homa_rpc_alloc_server__dont_handoff_rpc_start_msg) -{ - struct homa_start_msg_hdr h; - struct homa_rpc *srpc; - - h.common.sport = n(self->client_port); - h.common.dport = n(self->server_port); - h.common.type = START_MSG; - h.common.sender_id = cpu_to_be64(self->client_id); - h.msg_length = N(3000); - srpc = homa_rpc_alloc_server(&self->hsk, self->client_ip, &h.common); - ASSERT_FALSE(IS_ERR(srpc)); - homa_rpc_unlock(srpc); - EXPECT_EQ(RPC_INCOMING, srpc->state); - EXPECT_EQ(1, unit_list_length(&self->hsk.active_rpcs)); EXPECT_EQ(0, unit_list_length(&self->hsk.ready_rpcs)); homa_rpc_end(srpc); } -#endif /* See strip.py */ TEST_F(homa_rpc, homa_rpc_alloc_server__unlink_from_waiting_for_bufs_after_error) { struct homa_rpc *srpc;