diff --git a/src/libsysprof-ui/egg-resizer.c b/src/libsysprof-ui/egg-resizer.c
index 1292f7ed..c2b427bc 100644
--- a/src/libsysprof-ui/egg-resizer.c
+++ b/src/libsysprof-ui/egg-resizer.c
@@ -195,10 +195,13 @@ egg_resizer_measure (GtkWidget      *widget,
     {
       int handle_min, handle_nat;
 
-      if (self->drag_position > *minimum)
-        *natural = self->drag_position;
-      else if (self->drag_position < *minimum)
-        *natural = *minimum;
+      if (self->drag_position != 0)
+        {
+          if (self->drag_position > *minimum)
+            *natural = self->drag_position;
+          else if (self->drag_position < *minimum)
+            *natural = *minimum;
+        }
 
       if (gtk_widget_get_visible (GTK_WIDGET (self->handle)))
         {