diff --git a/redirect_nftables_exprs.go b/redirect_nftables_exprs.go
index d29c113..24af90f 100644
--- a/redirect_nftables_exprs.go
+++ b/redirect_nftables_exprs.go
@@ -161,7 +161,7 @@ func nftablesCreateIPSet(
 			IntervalEnd: true,
 		})
 	}
-	if len(prefixList) == 0 && appendDefault {
+	if appendDefault && len(setElements) == 0 {
 		if family == nftables.TableFamilyIPv4 {
 			setElements = append(setElements, nftables.SetElement{
 				Key: netip.IPv4Unspecified().AsSlice(),