[live-migration] Preserve host-side GCS logging across migration

internal/builder/vm/lcow/kernel_args.go

@@ -27,7 +27,6 @@ func buildKernelArgs(
 	kernelDirect bool,
 	hasConsole bool,
 	rootFsFile string,
-	liveMigrationSupportEnabled bool,
 ) (string, error) {
 
 	log.G(ctx).WithField("rootFsFile", rootFsFile).Debug("buildKernelArgs: starting kernel arguments construction")
@@ -83,7 +82,7 @@ func buildKernelArgs(
 
 	// 8. Init arguments (passed after "--" separator)
 	initArgs := buildInitArgs(ctx, opts, annotations,
-		writableOverlayDirs, disableTimeSyncService, processDumpLocation, rootFsFile, hasConsole, liveMigrationSupportEnabled)
+		writableOverlayDirs, disableTimeSyncService, processDumpLocation, rootFsFile, hasConsole)
 	args = append(args, "--", initArgs)
 
 	result := strings.Join(args, " ")
@@ -153,7 +152,6 @@ func buildInitArgs(
 	processDumpLocation string,
 	rootFsFile string,
 	hasConsole bool,
-	liveMigrationSupportEnabled bool,
 ) string {
 	log.G(ctx).WithFields(logrus.Fields{
 		"rootFsFile": rootFsFile,
@@ -163,7 +161,7 @@ func buildInitArgs(
 	entropyArgs := fmt.Sprintf("-e %d", vmutils.LinuxEntropyVsockPort)
 
 	// Build GCS execution command
-	gcsCmd := buildGCSCommand(opts, annotations, disableTimeSyncService, processDumpLocation, liveMigrationSupportEnabled)
+	gcsCmd := buildGCSCommand(ctx, opts, annotations, disableTimeSyncService, processDumpLocation)
 
 	// Construct init arguments
 	var initArgsList []string
@@ -194,12 +192,26 @@ func buildInitArgs(
 
 // buildGCSCommand constructs the GCS (Guest Compute Service) command line.
 func buildGCSCommand(
+	ctx context.Context,
 	opts *runhcsoptions.Options,
 	annotations map[string]string,
 	disableTimeSyncService bool,
 	processDumpLocation string,
-	liveMigrationSupportEnabled bool,
 ) string {
+	// Start with vsockexec wrapper
+	var cmdParts []string
+	cmdParts = append(cmdParts, "/bin/vsockexec")
+
+	// When live migration is enabled, run vsockexec in reconnect mode (-r) so
+	// guest logging tolerates the host log listener being absent at boot and
+	// reconnects to the destination host's listener after a migration.
+	if oci.ParseAnnotationsBool(ctx, annotations, shimannotations.LiveMigrationSupportEnabled, false) {
+		cmdParts = append(cmdParts, "-r")
+	}
+
+	// Add logging vsock port
+	cmdParts = append(cmdParts, fmt.Sprintf("-e %d", vmutils.LinuxLogVsockPort))
+
 	// Determine log level
 	logLevel := "info"
 	if opts != nil && opts.LogLevel != "" {
@@ -232,19 +244,8 @@ func buildGCSCommand(
 		gcsParts = append(gcsParts, s)
 	}
 
-	gcsCmd := strings.Join(gcsParts, " ")
-
-	// Live-migratable pods skip the /bin/vsockexec wrapper. The wrapper exists
-	// solely to forward GCS stderr to the host-side log listener, but that listener
-	// is host-local state that live migration does not transfer, so the host
-	// does not run it for these pods.
-	// Without a listener, vsockexec's outbound connect would block and stall guest init,
-	// so we emit /bin/gcs directly instead.
-	if liveMigrationSupportEnabled {
-		return gcsCmd
-	}
+	// Combine vsockexec and GCS command
+	cmdParts = append(cmdParts, strings.Join(gcsParts, " "))
 
-	// vsockexec `-e <port>` wires gcs's stderr to LinuxLogVsockPort, which
-	// the host listener reads and republishes.
-	return fmt.Sprintf("/bin/vsockexec -e %d %s", vmutils.LinuxLogVsockPort, gcsCmd)
+	return strings.Join(cmdParts, " ")
 }

internal/builder/vm/lcow/sandbox_options.go

@@ -25,10 +25,6 @@ type SandboxOptions struct {
 	// ConfidentialConfig carries confidential computing fields that are not
 	// part of the HCS document but are needed for confidential VM setup.
 	ConfidentialConfig *ConfidentialConfig
-
-	// LiveMigrationSupportEnabled indicates that the live migration feature set is
-	// enabled for the sandbox, constraining it to migration-compatible features.
-	LiveMigrationSupportEnabled bool
 }
 
 // ConfidentialConfig carries confidential computing configuration that is not

internal/builder/vm/lcow/specs.go

@@ -222,7 +222,6 @@ func BuildSandboxConfig(
 			bootOptions.LinuxKernelDirect != nil, // isKernelDirectBoot
 			comPorts != nil,                      // hasConsole
 			filepath.Base(rootFsFullPath),
-			sandboxOptions.LiveMigrationSupportEnabled,
 		)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to build kernel args: %w", err)
@@ -331,11 +330,10 @@ func parseSandboxOptions(ctx context.Context, platform string, annotations map[s
 	log.G(ctx).WithField("platform", platform).Debug("parseSandboxOptions: starting sandbox options parsing")
 	sandboxOptions := &SandboxOptions{
 		// Extract architecture from platform string (e.g., "linux/amd64" -> "amd64")
-		Architecture:                platform[strings.IndexByte(platform, '/')+1:],
-		FullyPhysicallyBacked:       oci.ParseAnnotationsBool(ctx, annotations, shimannotations.FullyPhysicallyBacked, false),
-		PolicyBasedRouting:          oci.ParseAnnotationsBool(ctx, annotations, iannotations.NetworkingPolicyBasedRouting, false),
-		NoWritableFileShares:        oci.ParseAnnotationsBool(ctx, annotations, shimannotations.DisableWritableFileShares, false),
-		LiveMigrationSupportEnabled: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.LiveMigrationSupportEnabled, false),
+		Architecture:          platform[strings.IndexByte(platform, '/')+1:],
+		FullyPhysicallyBacked: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.FullyPhysicallyBacked, false),
+		PolicyBasedRouting:    oci.ParseAnnotationsBool(ctx, annotations, iannotations.NetworkingPolicyBasedRouting, false),
+		NoWritableFileShares:  oci.ParseAnnotationsBool(ctx, annotations, shimannotations.DisableWritableFileShares, false),
 	}
 
 	// Determine if this is a confidential VM early, as it affects boot options parsing

internal/builder/vm/lcow/specs_test.go

@@ -2129,34 +2129,35 @@ func TestBuildSandboxConfig_CPUClamping(t *testing.T) {
 }
 
 // TestBuildSandboxConfig_LiveMigration validates the wiring for the
-// io.microsoft.migration.support-enabled sandbox annotation. The annotation is parsed
-// into SandboxOptions.LiveMigrationSupportEnabled and threaded down into the kernel
-// command line: live-migratable sandboxes must skip the /bin/vsockexec wrapper
-// (which would otherwise stall init waiting for a host log listener that the
-// LM-enabled host does not run), while non-LM sandboxes must continue to use
-// vsockexec so that GCS stderr is forwarded over LinuxLogVsockPort.
+// io.microsoft.migration.support-enabled sandbox annotation. The annotation is
+// parsed directly when building the GCS command: live-migratable sandboxes run
+// the /bin/vsockexec wrapper in reconnect mode (-r) so guest logging tolerates a
+// missing host log listener and reconnects after a migration, while non-LM
+// sandboxes use the plain wrapper to forward GCS stderr over LinuxLogVsockPort.
 func TestBuildSandboxConfig_LiveMigration(t *testing.T) {
 	ctx := context.Background()
 
 	validBootFilesPath := newBootFilesPath(t)
 	defaultOpts := defaultSandboxOpts(validBootFilesPath)
 
-	// Pre-format the vsockexec prefix once so the assertions are obviously
-	// driven by the same constant the production code uses.
+	// Pre-format the vsockexec prefixes once so the assertions are obviously
+	// driven by the same constants the production code uses.
 	vsockexecPrefix := fmt.Sprintf("/bin/vsockexec -e %d", vmutils.LinuxLogVsockPort)
+	// When live migration is enabled the wrapper runs in reconnect mode (-r).
+	vsockexecReconnectPrefix := fmt.Sprintf("/bin/vsockexec -r -e %d", vmutils.LinuxLogVsockPort)
 
 	tests := []specTestCase{
 		{
 			name: "live migration disabled by default",
 			validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) {
 				t.Helper()
-				if sandboxOpts.LiveMigrationSupportEnabled {
-					t.Errorf("expected LiveMigrationSupportEnabled=false by default, got true")
-				}
 				kernelArgs := getKernelArgs(doc)
 				if !strings.Contains(kernelArgs, vsockexecPrefix) {
 					t.Errorf("expected vsockexec wrapper %q in kernel args (LM disabled), got %q", vsockexecPrefix, kernelArgs)
 				}
+				if strings.Contains(kernelArgs, vsockexecReconnectPrefix) {
+					t.Errorf("expected no vsockexec reconnect mode when LM disabled, got %q", kernelArgs)
+				}
 				if !strings.Contains(kernelArgs, "/bin/gcs") {
 					t.Errorf("expected /bin/gcs in kernel args, got %q", kernelArgs)
 				}
@@ -2171,39 +2172,32 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) {
 			},
 			validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) {
 				t.Helper()
-				if sandboxOpts.LiveMigrationSupportEnabled {
-					t.Errorf("expected LiveMigrationSupportEnabled=false when annotation=\"false\", got true")
-				}
 				kernelArgs := getKernelArgs(doc)
 				if !strings.Contains(kernelArgs, vsockexecPrefix) {
 					t.Errorf("expected vsockexec wrapper %q in kernel args, got %q", vsockexecPrefix, kernelArgs)
 				}
+				if strings.Contains(kernelArgs, vsockexecReconnectPrefix) {
+					t.Errorf("expected no vsockexec reconnect mode when LM disabled, got %q", kernelArgs)
+				}
 			},
 		},
 		{
-			name: "live migration enabled drops vsockexec wrapper",
+			name: "live migration enabled uses vsockexec reconnect mode",
 			spec: &vm.Spec{
 				Annotations: map[string]string{
 					shimannotations.LiveMigrationSupportEnabled: "true",
 				},
 			},
 			validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) {
 				t.Helper()
-				if !sandboxOpts.LiveMigrationSupportEnabled {
-					t.Errorf("expected LiveMigrationSupportEnabled=true when annotation=\"true\", got false")
-				}
 				kernelArgs := getKernelArgs(doc)
-				// The vsockexec wrapper must not appear at all when LM is on:
-				// neither the prefix nor the binary path on its own.
-				if strings.Contains(kernelArgs, "vsockexec") {
-					t.Errorf("expected no vsockexec in kernel args when LM enabled, got %q", kernelArgs)
+				// When LM is on the wrapper runs in reconnect mode (-r) so the
+				// guest tolerates a missing or dropped host log listener.
+				if !strings.Contains(kernelArgs, vsockexecReconnectPrefix) {
+					t.Errorf("expected vsockexec reconnect wrapper %q in kernel args when LM enabled, got %q", vsockexecReconnectPrefix, kernelArgs)
 				}
-				if strings.Contains(kernelArgs, fmt.Sprintf("-e %d", vmutils.LinuxLogVsockPort)) {
-					t.Errorf("expected no log vsock port (%d) wiring when LM enabled, got %q", vmutils.LinuxLogVsockPort, kernelArgs)
-				}
-				// /bin/gcs must still be invoked - just without the wrapper.
 				if !strings.Contains(kernelArgs, "/bin/gcs") {
-					t.Errorf("expected /bin/gcs in kernel args even when LM enabled, got %q", kernelArgs)
+					t.Errorf("expected /bin/gcs in kernel args when LM enabled, got %q", kernelArgs)
 				}
 			},
 		},
@@ -2221,22 +2215,19 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) {
 			},
 			validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) {
 				t.Helper()
-				if !sandboxOpts.LiveMigrationSupportEnabled {
-					t.Errorf("expected LiveMigrationSupportEnabled=true, got false")
-				}
 				kernelArgs := getKernelArgs(doc)
-				// Other GCS flags must still be threaded through the command
-				// even when the vsockexec wrapper is removed.
+				// Other GCS flags must still be threaded through the reconnect
+				// wrapper when LM is enabled.
 				if !strings.Contains(kernelArgs, "-loglevel debug") {
 					t.Errorf("expected -loglevel debug in kernel args when LM enabled, got %q", kernelArgs)
 				}
-				if strings.Contains(kernelArgs, "vsockexec") {
-					t.Errorf("expected no vsockexec when LM enabled, got %q", kernelArgs)
+				if !strings.Contains(kernelArgs, vsockexecReconnectPrefix) {
+					t.Errorf("expected vsockexec reconnect wrapper %q when LM enabled, got %q", vsockexecReconnectPrefix, kernelArgs)
 				}
 			},
 		},
 		{
-			name: "live migration with disable time sync still drops vsockexec",
+			name: "live migration with disable time sync uses vsockexec reconnect mode",
 			spec: &vm.Spec{
 				Annotations: map[string]string{
 					shimannotations.LiveMigrationSupportEnabled: "true",
@@ -2245,15 +2236,12 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) {
 			},
 			validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) {
 				t.Helper()
-				if !sandboxOpts.LiveMigrationSupportEnabled {
-					t.Errorf("expected LiveMigrationSupportEnabled=true, got false")
-				}
 				kernelArgs := getKernelArgs(doc)
 				if !strings.Contains(kernelArgs, "-disable-time-sync") {
 					t.Errorf("expected -disable-time-sync flag in kernel args, got %q", kernelArgs)
 				}
-				if strings.Contains(kernelArgs, "vsockexec") {
-					t.Errorf("expected no vsockexec when LM enabled, got %q", kernelArgs)
+				if !strings.Contains(kernelArgs, vsockexecReconnectPrefix) {
+					t.Errorf("expected vsockexec reconnect wrapper %q when LM enabled, got %q", vsockexecReconnectPrefix, kernelArgs)
 				}
 			},
 		},
@@ -2269,13 +2257,13 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) {
 			},
 			validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) {
 				t.Helper()
-				if sandboxOpts.LiveMigrationSupportEnabled {
-					t.Errorf("expected LiveMigrationSupportEnabled=false on invalid annotation value, got true")
-				}
 				kernelArgs := getKernelArgs(doc)
 				if !strings.Contains(kernelArgs, vsockexecPrefix) {
 					t.Errorf("expected vsockexec wrapper %q in kernel args, got %q", vsockexecPrefix, kernelArgs)
 				}
+				if strings.Contains(kernelArgs, vsockexecReconnectPrefix) {
+					t.Errorf("expected no vsockexec reconnect mode on invalid LM annotation, got %q", kernelArgs)
+				}
 			},
 		},
 	}

internal/controller/vm/save_lcow.go

@@ -18,6 +18,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/wclayer"
 
 	"github.com/Microsoft/go-winio"
+	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 )
@@ -222,6 +223,27 @@ func (c *Controller) Resume(ctx context.Context, rebuildBridge bool) error {
 		if err := c.guest.ResumeConnection(ctx); err != nil {
 			return fmt.Errorf("resume guest connection: %w", err)
 		}
+
+		// The blackout also dropped the source's GCS log connection, which tore
+		// down its listener and closed logOutputDone. Install a fresh signal and
+		// re-arm the listener so the resumed guest's reconnect-mode vsockexec can
+		// reconnect and host-side logs resume. The accept runs in the background
+		// (WithoutCancel so it outlives this call; AcceptConnection still returns
+		// on VM exit) so a slow guest re-dial cannot stall resume.
+		c.logOutputDone = make(chan struct{})
+		g, gctx := errgroup.WithContext(ctx)
+		defer func() {
+			_ = g.Wait()
+		}()
+
+		if err := c.setupLoggingListener(gctx, g); err != nil {
+			return fmt.Errorf("re-arm logging listener on resume: %w", err)
+		}
+
+		// Collect any errors from establishing the log connection.
+		if err := g.Wait(); err != nil {
+			return err
+		}
 	default:
 		// Destination: reuse the connection already armed at start.
 		if err := c.guest.CreateConnection(ctx, false); err != nil {
@@ -252,18 +274,6 @@ func (c *Controller) Resume(ctx context.Context, rebuildBridge bool) error {
 
 	c.vmState = StateRunning
 
-	if c.sandboxOptions != nil {
-		c.sandboxOptions.LiveMigrationSupportEnabled = true
-	}
-
-	// Destination never ran setupLoggingListener; close so [Controller.Wait]
-	// does not block. Already closed on source — receive falls through.
-	select {
-	case <-c.logOutputDone:
-	default:
-		close(c.logOutputDone)
-	}
-
 	log.G(ctx).WithField(logfields.UVMID, c.vmID).Debug("resumed VM from migration")
 	return nil
 }