diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index a1b7475c05d0..9dfe364d4c0d 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -407,7 +407,7 @@ int __khugepaged_enter(struct mm_struct *mm)
 		return -ENOMEM;
 
 	/* __khugepaged_exit() must not run from under us */
-	VM_BUG_ON_MM(khugepaged_test_exit(mm), mm);
+	VM_BUG_ON_MM(atomic_read(&mm->mm_users) == 0, mm);
 	if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
 		free_mm_slot(mm_slot);
 		return 0;