Extend delete shutdown grace period

Author: yummybomb

lib/instances/delete.go

@@ -15,7 +15,7 @@ import (
 	"go.opentelemetry.io/otel/attribute"
 )
 
-const deleteGracefulShutdownTimeout = 2
+const deleteGracefulShutdownTimeout = 5
 
 type deleteInstanceOptions struct {
 	skipGracefulShutdown bool

lib/instances/delete_test.go

@@ -1,14 +1,124 @@
 package instances
 
 import (
+	"context"
+	"fmt"
+	"net"
+	"os"
 	"os/exec"
+	"path/filepath"
+	"sync"
+	"syscall"
 	"testing"
 	"time"
 
+	"github.com/kernel/hypeman/lib/guest"
+	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/test/bufconn"
 )
 
+var deleteTestVsockListeners sync.Map
+
+func init() {
+	hypervisor.RegisterVsockDialerFactory(lifecycleNoopHypervisorType, func(vsockSocket string, _ int64) hypervisor.VsockDialer {
+		return deleteTestVsockDialer{key: vsockSocket}
+	})
+}
+
+type deleteTestVsockDialer struct {
+	key string
+}
+
+func (d deleteTestVsockDialer) DialVsock(ctx context.Context, _ int) (net.Conn, error) {
+	listener, ok := deleteTestVsockListeners.Load(d.key)
+	if !ok {
+		return nil, fmt.Errorf("missing test vsock listener for %s", d.key)
+	}
+	return listener.(*bufconn.Listener).DialContext(ctx)
+}
+
+func (d deleteTestVsockDialer) Key() string {
+	return "delete-test:" + d.key
+}
+
+type delayedShutdownGuestServer struct {
+	guest.UnimplementedGuestServiceServer
+	pid        int
+	delay      time.Duration
+	markerPath string
+	once       sync.Once
+}
+
+func (s *delayedShutdownGuestServer) Shutdown(context.Context, *guest.ShutdownRequest) (*guest.ShutdownResponse, error) {
+	s.once.Do(func() {
+		go func() {
+			time.Sleep(s.delay)
+			if !processExists(s.pid) {
+				return
+			}
+			if err := os.WriteFile(s.markerPath, []byte("graceful"), 0o644); err != nil {
+				return
+			}
+			_ = syscall.Kill(s.pid, syscall.SIGTERM)
+		}()
+	})
+	return &guest.ShutdownResponse{}, nil
+}
+
+func TestDeleteInstanceAllowsGracefulShutdownBeyondTwoSeconds(t *testing.T) {
+	mgr, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC())
+	markerPath := filepath.Join(t.TempDir(), "graceful-shutdown")
+
+	cmd := exec.Command("sleep", "30")
+	require.NoError(t, cmd.Start())
+	t.Cleanup(func() {
+		_ = cmd.Process.Kill()
+		_, _ = cmd.Process.Wait()
+	})
+
+	shutdownDelay := 2200 * time.Millisecond
+	shutdownServer := &delayedShutdownGuestServer{
+		pid:        cmd.Process.Pid,
+		delay:      shutdownDelay,
+		markerPath: markerPath,
+	}
+	listener := bufconn.Listen(1024 * 1024)
+	grpcServer := grpc.NewServer()
+	guest.RegisterGuestServiceServer(grpcServer, shutdownServer)
+	go func() {
+		_ = grpcServer.Serve(listener)
+	}()
+	t.Cleanup(func() {
+		grpcServer.Stop()
+		_ = listener.Close()
+	})
+
+	meta, err := mgr.loadMetadata(id)
+	require.NoError(t, err)
+	meta.HypervisorPID = &shutdownServer.pid
+	meta.StopTimeout = 10
+	meta.VsockSocket = filepath.Join(meta.DataDir, "test-vsock.sock")
+	deleteTestVsockListeners.Store(meta.VsockSocket, listener)
+	t.Cleanup(func() {
+		deleteTestVsockListeners.Delete(meta.VsockSocket)
+		guest.CloseConn(deleteTestVsockDialer{key: meta.VsockSocket}.Key())
+	})
+	require.NoError(t, mgr.saveMetadata(meta))
+
+	start := time.Now()
+	require.NoError(t, mgr.deleteInstance(context.Background(), id))
+	elapsed := time.Since(start)
+
+	marker, err := os.ReadFile(markerPath)
+	require.NoError(t, err, "shutdown should finish before the delete timeout")
+	assert.Equal(t, "graceful", string(marker))
+	assert.GreaterOrEqual(t, elapsed, shutdownDelay)
+	assert.Less(t, elapsed, time.Duration(deleteGracefulShutdownTimeout)*time.Second)
+}
+
 func TestWaitForProcessExit_ReapsZombieChild(t *testing.T) {
 	t.Parallel()
 	cmd := exec.Command("sh", "-c", "exit 0")