diff --git a/src/dualtvdd_surrogate.jl b/src/dualtvdd_surrogate.jl
index dbd2fcf2ee66f3af4ce911e49c35d28924ce5d61..4c23cfc7ff5e6d84e1284dd79489c2ea82b25c00 100644
--- a/src/dualtvdd_surrogate.jl
+++ b/src/dualtvdd_surrogate.jl
@@ -96,6 +96,7 @@ function step!(ctx::DualTVDDSurrogateState)
     cids = chessboard_coloring(size(ctx.subax))
 
     for (color, ids) in enumerate(cids)
+        isempty(ids) && continue
         p_current = extend(zeros(eltype(ctx.p), size(ctx.p)),
             StaticKernels.ExtensionNothing())
         for i in ids