diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index cca41c569f37..edd846fee90f 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1739,8 +1739,8 @@ int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
 		if (rcvbuf > INT_MAX >> 1)
 			goto drop;
 
-		/* Always allow at least one packet for small buffer. */
-		if (rmem > rcvbuf)
+		/* Accept the packet if queue is empty. */
+		if (rmem)
 			goto drop;
 	}