diff --git a/exp_auto_restart/README.md b/exp_auto_restart/README.md new file mode 100644 index 0000000000..954b9b0959 --- /dev/null +++ b/exp_auto_restart/README.md @@ -0,0 +1,26 @@ +# ExpGaming - Auto Restart + +Restarts hosts, and optionally the controller, that need a restart once nobody is playing on them. + +A host needs a restart after a plugin or clusterio update, and after a config change that only applies on restart. The web UI shows this as "Restart Required". This plugin waits until the host has had no players online for a while and then sends it the same restart request the web UI button does. + +Restarting a host stops every instance on it. Once the host is back, the instances that were running are started again. Instances with auto start enabled are started by the host itself. + +## Requirements + +- Hosts must be run with `--can-restart` so their process monitor starts them again. The systemd units and run scripts written by the installer do this. +- `controller.system_metrics_interval` must be above 0. That poll is how the controller learns a host needs a restart. +- Installing a plugin on a host does not yet mark it as needing a restart, only updates and config changes do. Until clusterio does, restart the host by hand after an install. + +## Config + +All fields are on the controller. + +| Field | Default | Description | +|---|---|---| +| `exp_auto_restart.idle_seconds` | 300 | How long a host must have had no players online before it is restarted. | +| `exp_auto_restart.scope` | host | With `cluster`, players on any host hold back every restart. | +| `exp_auto_restart.start_instances` | true | Start the instances that were running again once the host is back. | +| `exp_auto_restart.restart_controller` | false | Also restart the controller when it needs it and no players are online anywhere. | + +Hosts are checked every 10 seconds. A host that refuses the restart, for example because it was not run with `--can-restart`, is logged once and left alone until its process changes. diff --git a/exp_auto_restart/controller.ts b/exp_auto_restart/controller.ts new file mode 100644 index 0000000000..130dd2b06f --- /dev/null +++ b/exp_auto_restart/controller.ts @@ -0,0 +1,295 @@ +import type { + Controller, ControllerPluginContext, HostConnection, HostRecord, InstanceRecord, +} from "@clusterio/controller"; +import * as lib from "@clusterio/lib"; + +/** How often hosts are checked for a pending restart */ +export const CHECK_INTERVAL_MS = 10_000; +/** How long to wait for a restarted host to report its instances back before giving up on starting them */ +export const PENDING_TIMEOUT_MS = 10 * 60_000; + +/** A host that was told to restart and has not reported all of its instances back yet */ +export interface PendingRestart { + requestedAtMs: number; + /** Instances that were running when the restart was requested */ + instanceIds: Set; +} + +export class AutoRestart { + controller: Controller; + logger: lib.Logger; + name: string; + + /** When each host, and the cluster as a whole, last became empty of players. Absent while players are online. */ + idleSince = new Map(); + /** Hosts that were told to restart and are waiting for their instances to come back */ + pending = new Map(); + /** Process start time of each host last asked to restart, so the same process is not asked twice */ + requested = new Map(); + /** Hosts already warned about not being restartable */ + warned = new Set(); + controllerRestarting = false; + private checkInterval?: ReturnType; + + constructor(context: ControllerPluginContext) { + this.controller = context.controller; + this.logger = context.logger; + this.name = context.plugin.name; + } + + /** Attach to the controller hooks and start checking for restarts. */ + start() { + const hooks = this.controller.hooks; + hooks.shutdown.attach(this.name, () => this.stop()); + hooks.playerEvent.attach(this.name, (instance, event) => this.onPlayerEvent(instance, event)); + hooks.instanceStatusChanged.attach(this.name, (instance, prev) => this.onInstanceStatusChanged(instance, prev)); + hooks.hostConnectionEvent.attach(this.name, (connection, event) => this.onHostConnectionEvent(connection, event)); + + if (this.controller.config.get("controller.system_metrics_interval") <= 0) { + this.logger.warn( + "controller.system_metrics_interval is 0, so the controller never learns which hosts need a restart" + ); + } + this.refreshIdle(); + this.checkInterval = setInterval(() => { + this.check().catch((err: any) => { + this.logger.error(`Unexpected error checking for restarts:\n${err.stack ?? err.message}`); + }); + }, CHECK_INTERVAL_MS); + this.checkInterval.unref(); + } + + stop() { + clearInterval(this.checkInterval); + } + + onPlayerEvent(instance: InstanceRecord, event: lib.PlayerEvent) { + if (event.type === "join" || event.type === "leave") { + this.refreshIdle(); + } + } + + onInstanceStatusChanged(instance: InstanceRecord, prev?: lib.InstanceStatus) { + this.refreshIdle(); + // A host reports every instance it has when it connects, which is how + // a restarted host announces it is back + if (prev === "unknown") { + this.instanceReported(instance); + } + } + + onHostConnectionEvent(connection: HostConnection, event: "connect" | "drop" | "resume" | "close") { + if (event === "connect" || event === "close") { + this.refreshIdle(); + } + } + + /** Players online per host, counted from the users online on running instances. */ + onlinePlayers() { + const counts = new Map(); + for (const user of this.controller.users.records.values()) { + if (user.isDeleted) { + continue; + } + for (const instanceId of user.instances) { + const instance = this.controller.instances.get(instanceId); + // Nobody is online on an instance that is not running, whatever + // the user records say after a host crashed + if (!instance || instance.status !== "running") { + continue; + } + const hostId = instance.config.get("instance.assigned_host"); + if (hostId === null) { + continue; + } + counts.set(hostId, (counts.get(hostId) ?? 0) + 1); + } + } + return counts; + } + + /** Record when connected hosts and the cluster became empty, from the players online now. */ + refreshIdle(nowMs = Date.now()) { + const counts = this.onlinePlayers(); + for (const host of this.controller.hosts.values()) { + if (host.isDeleted || !host.connected) { + this.idleSince.delete(host.id); + continue; + } + this.markIdle(host.id, !counts.get(host.id), nowMs); + } + + let total = 0; + for (const count of counts.values()) { + total += count; + } + this.markIdle("cluster", total === 0, nowMs); + } + + private markIdle(key: number | "cluster", idle: boolean, nowMs: number) { + if (!idle) { + this.idleSince.delete(key); + } else if (!this.idleSince.has(key)) { + this.idleSince.set(key, nowMs); + } + } + + /** How long a host has been without players, or undefined while it has some. Uses the whole cluster with scope cluster. */ + idleMs(hostId: number, nowMs: number) { + const scope = this.controller.config.get("exp_auto_restart.scope"); + const since = this.idleSince.get(scope === "cluster" ? "cluster" : hostId); + return since === undefined ? undefined : nowMs - since; + } + + /** Restart every host, and the controller if enabled, that needs it and has been empty for long enough. */ + async check(nowMs = Date.now()) { + this.refreshIdle(nowMs); + this.expirePending(nowMs); + const waitMs = this.controller.config.get("exp_auto_restart.idle_seconds") * 1000; + + for (const system of this.controller.systems.values()) { + if (system.id === "controller" || system.isDeleted || !system.restartRequired) { + continue; + } + const hostId = system.id; + const host = this.controller.hosts.get(hostId); + if (!host || host.isDeleted || !host.connected) { + continue; + } + // Already asked, either waiting for it to come back or it refused + if (this.pending.has(hostId) || this.requested.get(hostId) === system.processStartedAtMs) { + continue; + } + if (!system.canRestart) { + this.warnOnce(hostId, `Host ${host.name} needs a restart but is not run with --can-restart`); + continue; + } + const idleMs = this.idleMs(hostId, nowMs); + if (idleMs === undefined || idleMs < waitMs) { + continue; + } + await this.restartHost(host, system, idleMs, nowMs); + } + + if (this.controller.config.get("exp_auto_restart.restart_controller")) { + await this.checkController(nowMs, waitMs); + } + } + + async restartHost(host: Readonly, system: lib.SystemInfo, idleMs: number, nowMs: number) { + const running = [...this.controller.instances.values()].filter( + instance => instance.status === "running" && instance.config.get("instance.assigned_host") === host.id + ); + this.requested.set(host.id, system.processStartedAtMs); + if (running.length && this.controller.config.get("exp_auto_restart.start_instances")) { + this.pending.set(host.id, { + requestedAtMs: nowMs, + instanceIds: new Set(running.map(instance => instance.id)), + }); + } + this.logger.info( + `Restarting host ${host.name} after ${Math.round(idleMs / 1000)} s without players, ` + + `${running.length} instance(s) running` + ); + + try { + await this.controller.sendTo({ hostId: host.id }, new lib.HostRestartRequest()); + } catch (err: any) { + this.pending.delete(host.id); + // The host refusing is final for this process, anything else is retried + if (!(err instanceof lib.RequestError)) { + this.requested.delete(host.id); + } + this.logger.error(`Failed to restart host ${host.name}: ${err.message}`); + } + } + + /** An instance on a host that reconnected has reported its status. */ + instanceReported(instance: InstanceRecord) { + const hostId = instance.config.get("instance.assigned_host"); + if (hostId === null) { + return; + } + const pending = this.pending.get(hostId); + if (!pending?.instanceIds.delete(instance.id)) { + return; + } + if (!pending.instanceIds.size) { + this.pending.delete(hostId); + } + + // The host starts these itself right after reporting them + if (instance.status !== "stopped" || instance.config.get("instance.auto_start")) { + return; + } + this.startInstance(instance); + } + + async startInstance(instance: InstanceRecord) { + const name = instance.config.get("instance.name"); + this.logger.info(`Starting ${name} again after its host restarted`); + try { + await this.controller.sendTo({ instanceId: instance.id }, new lib.InstanceStartRequest()); + } catch (err: any) { + this.logger.error(`Failed to start ${name} after its host restarted: ${err.message}`); + } + } + + /** Forget hosts that were restarted but never reported their instances back. */ + expirePending(nowMs: number) { + for (const [hostId, pending] of this.pending) { + if (nowMs - pending.requestedAtMs >= PENDING_TIMEOUT_MS) { + this.pending.delete(hostId); + this.logger.warn( + `Host ${hostId} has not reported back since it was restarted, giving up on starting its instances` + ); + } + } + } + + async checkController(nowMs: number, waitMs: number) { + if (this.controllerRestarting) { + return; + } + const system = this.controller.systems.get("controller"); + if (!system?.restartRequired) { + return; + } + if (!system.canRestart) { + this.warnOnce("controller", "The controller needs a restart but is not run with --can-restart"); + return; + } + const since = this.idleSince.get("cluster"); + if (since === undefined || nowMs - since < waitMs) { + return; + } + const downgrade = await this.controller.checkRestartDowngrade(); + if (downgrade) { + this.warnOnce( + "controller", + `Not restarting the controller because installed version ${downgrade.installedVersion} ` + + `is older than running version ${downgrade.runningVersion}` + ); + return; + } + + this.controllerRestarting = true; + this.logger.info(`Restarting controller after ${Math.round((nowMs - since) / 1000)} s without players`); + this.controller.shouldRestart = true; + this.controller.stop().catch((err: any) => { + this.logger.error(`Failed to restart the controller: ${err.message}`); + }); + } + + private warnOnce(key: number | "controller", message: string) { + if (this.warned.has(key)) { + return; + } + this.warned.add(key); + this.logger.warn(message); + } +} + +export default async function (context: ControllerPluginContext) { + new AutoRestart(context).start(); +} diff --git a/exp_auto_restart/index.ts b/exp_auto_restart/index.ts new file mode 100644 index 0000000000..2742041113 --- /dev/null +++ b/exp_auto_restart/index.ts @@ -0,0 +1,50 @@ +import * as lib from "@clusterio/lib"; + +declare module "@clusterio/lib" { + export interface ControllerConfigFields { + "exp_auto_restart.idle_seconds": number; + "exp_auto_restart.scope": "host" | "cluster"; + "exp_auto_restart.start_instances": boolean; + "exp_auto_restart.restart_controller": boolean; + } +} + +export const plugin: lib.PluginDeclaration = { + name: "exp_auto_restart", + title: "ExpGaming - Auto Restart", + description: "Clusterio plugin restarting hosts and the controller once no players are online", + + controllerEntrypoint: "./dist/node/controller.js", + controllerConfigFields: { + "exp_auto_restart.idle_seconds": { + title: "Idle Seconds", + description: "How long a host must have had no players online before it is restarted", + type: "number", + initialValue: 300, + validator: (value: number) => { + if (value < 0) { + throw new Error("Idle seconds cannot be negative"); + } + }, + }, + "exp_auto_restart.scope": { + title: "Scope", + description: "Whether players on other hosts also hold back a restart", + type: "string", + enum: ["host", "cluster"], + initialValue: "host", + }, + "exp_auto_restart.start_instances": { + title: "Start Instances", + description: "Start the instances that were running again once the host is back", + type: "boolean", + initialValue: true, + }, + "exp_auto_restart.restart_controller": { + title: "Restart Controller", + description: "Also restart the controller when it needs it and no players are online anywhere", + type: "boolean", + initialValue: false, + }, + }, +}; diff --git a/exp_auto_restart/package.json b/exp_auto_restart/package.json new file mode 100644 index 0000000000..57df55ac10 --- /dev/null +++ b/exp_auto_restart/package.json @@ -0,0 +1,37 @@ +{ + "name": "@expcluster/auto-restart", + "version": "7.0.1", + "description": "Clusterio plugin restarting hosts and the controller once no players are online.", + "author": "Cooldude2606 ", + "license": "MIT", + "repository": "explosivegaming/ExpCluster", + "type": "module", + "main": "dist/node/index.js", + "scripts": { + "prepare": "tsc --build", + "test": "tap --type-strip-only --disable-coverage --allow-empty-coverage test/*.test.js", + "coverage": "tap --coverage-report=text test/*.test.js" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@clusterio/controller": "workspace:^", + "@clusterio/lib": "workspace:^" + }, + "devDependencies": { + "@clusterio/controller": "workspace:^", + "@clusterio/lib": "workspace:^", + "@types/node": "catalog:", + "tap": "^21.1.0", + "typescript": "catalog:" + }, + "publishConfig": { + "access": "public" + }, + "keywords": [ + "clusterio", + "clusterio-plugin", + "factorio" + ] +} diff --git a/exp_auto_restart/test/controller.test.js b/exp_auto_restart/test/controller.test.js new file mode 100644 index 0000000000..e7461f5382 --- /dev/null +++ b/exp_auto_restart/test/controller.test.js @@ -0,0 +1,353 @@ +import t from "tap"; +import * as lib from "@clusterio/lib"; +import { Controller, HostRecord, InstanceRecord } from "@clusterio/controller"; +import entrypoint, { AutoRestart, PENDING_TIMEOUT_MS } from "../dist/node/controller.js"; +import { plugin as pluginDeclaration } from "../dist/node/index.js"; + +// The plugin's config fields must be defined before a ControllerConfig can set them +lib.addPluginConfigFields([pluginDeclaration]); + +// Silence the singleton logger to avoid polluting the test output +lib.logger.silent = true; +t.after(() => { + lib.logger.silent = false; +}); + +/** Build the plugin around a real controller, which is side effect free while not started. */ +async function startPlugin(t2, { config = {}, canRestart = false } = {}) { + const logs = []; + const logger = { + child: () => logger, + info: message => logs.push(["info", message]), + warn: message => logs.push(["warn", message]), + error: message => logs.push(["error", message]), + verbose: () => {}, + }; + const controllerConfig = new lib.ControllerConfig("controller", { + "controller.database_directory": t2.testdir(), + ...config, + }); + const controller = new Controller(logger, [], controllerConfig, canRestart); + + // Record what would be sent to hosts and instances instead of sending it + const sent = []; + controller.sendTo = async (address, message) => { + sent.push({ address, message }); + }; + + const plugin = new AutoRestart({ controller, logger, plugin: pluginDeclaration, metrics: undefined }); + plugin.start(); + t2.teardown(() => plugin.stop()); + return { plugin, controller, sent, logs }; +} + +function addHost(controller, id, connected = true) { + controller.hosts.set(new HostRecord(id, `host-${id}`, "2.0.0", new Map(), connected)); +} + +function addInstance(controller, id, hostId, status = "running", fields = {}) { + const config = new lib.InstanceConfig("controller", { + "instance.id": id, + "instance.name": `instance-${id}`, + "instance.assigned_host": hostId, + ...fields, + }); + const record = new InstanceRecord(config, status); + controller.instances.records.set(record); + return record; +} + +function addSystem(controller, id, { restartRequired = true, canRestart = true, processStartedAtMs = 1000 } = {}) { + controller.systems.set(new lib.SystemInfo( + id, "box", "v22", "Linux", "x64", "cpu", [], 0, 0, 0, 0, + canRestart, restartRequired, 0, processStartedAtMs, Date.now(), false, + )); +} + +const join = (controller, name, instanceId) => controller.users.getOrCreateUser(name).notifyJoin(instanceId); +const leave = (controller, name, instanceId) => controller.users.getOrCreateUser(name).notifyLeave(instanceId); +const restartsSent = sent => sent.filter(({ message }) => message instanceof lib.HostRestartRequest); +const startsSent = sent => sent.filter(({ message }) => message instanceof lib.InstanceStartRequest); + +t.test("entrypoint", async t2 => { + const logger = { child: () => logger, info: () => {}, warn: () => {}, error: () => {}, verbose: () => {} }; + const controllerConfig = new lib.ControllerConfig("controller", { "controller.database_directory": t2.testdir() }); + const controller = new Controller(logger, [], controllerConfig); + await entrypoint({ controller, logger, plugin: pluginDeclaration, metrics: undefined }); + for (const hook of ["shutdown", "playerEvent", "instanceStatusChanged", "hostConnectionEvent"]) { + t2.ok([...controller.hooks[hook].attached].includes("exp_auto_restart"), `attached to ${hook}`); + } + await controller.hooks.shutdown.invoke(); +}); + +t.test("class AutoRestart", t2 => { + t2.test(".onlinePlayers() counts players on running instances per host", async t3 => { + const { plugin, controller } = await startPlugin(t3); + addHost(controller, 1); + addHost(controller, 2); + addInstance(controller, 10, 1); + addInstance(controller, 11, 1, "stopped"); + addInstance(controller, 20, 2); + join(controller, "alice", 10); + join(controller, "bob", 10); + join(controller, "carol", 11); + join(controller, "dave", 20); + + t3.strictSame(plugin.onlinePlayers(), new Map([[1, 2], [2, 1]]), "players on stopped instances are not counted"); + }); + + t2.test(".refreshIdle() tracks when hosts and the cluster became empty", async t3 => { + const { plugin, controller } = await startPlugin(t3); + addHost(controller, 1); + addHost(controller, 2); + addHost(controller, 3, false); + addInstance(controller, 10, 1); + join(controller, "alice", 10); + + plugin.refreshIdle(1000); + t3.notOk(plugin.idleSince.has(1), "a host with players is not idle"); + t3.strictSame(plugin.idleSince.get(2), 1000, "an empty host is idle from now"); + t3.notOk(plugin.idleSince.has(3), "a disconnected host is not tracked"); + t3.notOk(plugin.idleSince.has("cluster"), "the cluster is not idle while anyone is online"); + + leave(controller, "alice", 10); + plugin.refreshIdle(2000); + t3.strictSame(plugin.idleSince.get(1), 2000, "a host becomes idle when its last player leaves"); + t3.strictSame(plugin.idleSince.get(2), 1000, "an idle host keeps its original time"); + t3.strictSame(plugin.idleSince.get("cluster"), 2000, "the cluster becomes idle with the last player"); + + join(controller, "alice", 10); + plugin.refreshIdle(3000); + t3.notOk(plugin.idleSince.has(1), "a join clears the idle time"); + t3.notOk(plugin.idleSince.has("cluster")); + }); + + t2.test(".check() restarts a host once it has been empty for long enough", async t3 => { + const { plugin, controller, sent } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 60 }, + }); + const now = Date.now(); + addHost(controller, 1); + addInstance(controller, 10, 1); + addInstance(controller, 11, 1, "stopped"); + addSystem(controller, 1); + join(controller, "alice", 10); + + await plugin.check(now); + t3.strictSame(restartsSent(sent).length, 0, "not restarted while a player is online"); + + leave(controller, "alice", 10); + await plugin.check(now + 10_000); + t3.strictSame(restartsSent(sent).length, 0, "not restarted before the idle time has passed"); + + await plugin.check(now + 70_000); + t3.strictSame(restartsSent(sent).length, 1, "restarted once the idle time has passed"); + t3.strictSame(restartsSent(sent)[0].address, { hostId: 1 }, "the restart is sent to the host"); + t3.strictSame([...plugin.pending.get(1).instanceIds], [10], "only the running instances are remembered"); + + await plugin.check(now + 80_000); + t3.strictSame(restartsSent(sent).length, 1, "not restarted again while waiting for it to come back"); + }); + + t2.test(".check() asks each host process only once", async t3 => { + const { plugin, controller, sent } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0 }, + }); + const now = Date.now(); + addHost(controller, 1); + addSystem(controller, 1, { processStartedAtMs: 1000 }); + + await plugin.check(now); + t3.strictSame(restartsSent(sent).length, 1, "restarted with no running instances"); + t3.notOk(plugin.pending.has(1), "nothing to wait for without running instances"); + + await plugin.check(now + 10_000); + t3.strictSame(restartsSent(sent).length, 1, "the same process is not asked again"); + + addSystem(controller, 1, { restartRequired: false, processStartedAtMs: 2000 }); + await plugin.check(now + 20_000); + t3.strictSame(restartsSent(sent).length, 1, "the new process does not need a restart"); + + addSystem(controller, 1, { processStartedAtMs: 2000 }); + await plugin.check(now + 30_000); + t3.strictSame(restartsSent(sent).length, 2, "the new process is restarted when it needs it"); + }); + + t2.test(".check() leaves hosts alone that cannot or need not restart", async t3 => { + const { plugin, controller, sent, logs } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0 }, + }); + const now = Date.now(); + addHost(controller, 1); + addSystem(controller, 1, { canRestart: false }); + addHost(controller, 2); + addSystem(controller, 2, { restartRequired: false }); + addHost(controller, 3, false); + addSystem(controller, 3); + + await plugin.check(now); + await plugin.check(now + 10_000); + t3.strictSame(restartsSent(sent).length, 0, "no host is restarted"); + const warnings = logs.filter(([level, message]) => level === "warn" && message.includes("--can-restart")); + t3.strictSame(warnings.length, 1, "a host without a process monitor is warned about once"); + }); + + t2.test(".check() with scope cluster waits for the whole cluster to be empty", async t3 => { + const { plugin, controller, sent } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0, "exp_auto_restart.scope": "cluster" }, + }); + const now = Date.now(); + addHost(controller, 1); + addSystem(controller, 1); + addHost(controller, 2); + addInstance(controller, 20, 2); + join(controller, "alice", 20); + + await plugin.check(now); + t3.strictSame(restartsSent(sent).length, 0, "a player on another host holds back the restart"); + + leave(controller, "alice", 20); + await plugin.check(now + 10_000); + t3.strictSame(restartsSent(sent).length, 1, "restarted once the cluster is empty"); + }); + + t2.test(".restartHost() retries transport errors but not refusals", async t3 => { + const { plugin, controller, sent, logs } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0 }, + }); + const now = Date.now(); + addHost(controller, 1); + addInstance(controller, 10, 1); + addSystem(controller, 1); + + controller.sendTo = async () => { throw new lib.RequestError("Cannot restart"); }; + await plugin.check(now); + await plugin.check(now + 10_000); + t3.strictSame(logs.filter(([level]) => level === "error").length, 1, "a refusal is logged once"); + t3.notOk(plugin.pending.has(1), "a refused restart is not waited on"); + + addSystem(controller, 1, { processStartedAtMs: 2000 }); + let attempts = 0; + controller.sendTo = async () => { attempts += 1; throw new Error("Connection lost"); }; + await plugin.check(now + 20_000); + await plugin.check(now + 30_000); + t3.strictSame(attempts, 2, "a transport error is retried on the next check"); + }); + + t2.test(".instanceReported() starts the instances that were running", async t3 => { + const { plugin, controller, sent } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0 }, + }); + const now = Date.now(); + addHost(controller, 1); + const manual = addInstance(controller, 10, 1); + const auto = addInstance(controller, 11, 1, "running", { "instance.auto_start": true }); + const stopped = addInstance(controller, 12, 1, "stopped"); + addSystem(controller, 1); + await plugin.check(now); + t3.strictSame([...plugin.pending.get(1).instanceIds], [10, 11]); + + // The host comes back and reports every instance as stopped + for (const instance of [manual, auto, stopped]) { + instance.status = "stopped"; + await plugin.onInstanceStatusChanged(instance, "unknown"); + } + t3.strictSame(startsSent(sent).map(({ address }) => address), [{ instanceId: 10 }], "only the instance the host will not start itself is started"); + t3.notOk(plugin.pending.has(1), "the host is no longer waited on"); + + // An instance reporting in without a pending restart is ignored + await plugin.onInstanceStatusChanged(manual, "unknown"); + t3.strictSame(startsSent(sent).length, 1); + }); + + t2.test(".restartHost() does not wait on instances when they are not to be started", async t3 => { + const { plugin, controller, sent } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0, "exp_auto_restart.start_instances": false }, + }); + addHost(controller, 1); + const instance = addInstance(controller, 10, 1); + addSystem(controller, 1); + await plugin.check(Date.now()); + t3.strictSame(restartsSent(sent).length, 1); + t3.notOk(plugin.pending.has(1), "nothing is waited on"); + + instance.status = "stopped"; + await plugin.onInstanceStatusChanged(instance, "unknown"); + t3.strictSame(startsSent(sent).length, 0, "nothing is started"); + }); + + t2.test(".expirePending() gives up on hosts that never come back", async t3 => { + const { plugin, controller, logs } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0 }, + }); + const now = Date.now(); + addHost(controller, 1); + addInstance(controller, 10, 1); + addSystem(controller, 1); + await plugin.check(now); + t3.ok(plugin.pending.has(1)); + + await plugin.check(now + PENDING_TIMEOUT_MS - 1); + t3.ok(plugin.pending.has(1), "still waited on before the timeout"); + await plugin.check(now + PENDING_TIMEOUT_MS); + t3.notOk(plugin.pending.has(1), "no longer waited on after the timeout"); + t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("giving up")), "giving up is logged"); + }); + + t2.test(".checkController() restarts the controller when the cluster is empty", async t3 => { + const { plugin, controller } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 60, "exp_auto_restart.restart_controller": true }, + canRestart: true, + }); + const now = Date.now(); + let stopped = 0; + controller.stop = async () => { stopped += 1; }; + controller.checkRestartDowngrade = async () => null; + addHost(controller, 1); + addInstance(controller, 10, 1); + addSystem(controller, "controller"); + join(controller, "alice", 10); + + await plugin.check(now); + t3.strictSame(stopped, 0, "not restarted while a player is online"); + + leave(controller, "alice", 10); + await plugin.check(now + 10_000); + t3.strictSame(stopped, 0, "not restarted before the idle time has passed"); + + await plugin.check(now + 70_000); + t3.strictSame(stopped, 1, "restarted once the cluster has been empty for long enough"); + t3.ok(controller.shouldRestart, "stopped with the intent to restart"); + + await plugin.check(now + 80_000); + t3.strictSame(stopped, 1, "not restarted twice"); + }); + + t2.test(".checkController() refuses downgrades and missing process monitors", async t3 => { + const { plugin, controller, logs } = await startPlugin(t3, { + config: { "exp_auto_restart.idle_seconds": 0, "exp_auto_restart.restart_controller": true }, + canRestart: true, + }); + let stopped = 0; + controller.stop = async () => { stopped += 1; }; + controller.checkRestartDowngrade = async () => ({ installedVersion: "2.0.0", runningVersion: "2.0.1" }); + addSystem(controller, "controller"); + + await plugin.check(Date.now()); + t3.strictSame(stopped, 0, "not restarted into an older version"); + t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("older")), "the downgrade is logged"); + + plugin.warned.clear(); + addSystem(controller, "controller", { canRestart: false }); + await plugin.check(Date.now()); + t3.strictSame(stopped, 0, "not restarted without a process monitor"); + t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("--can-restart"))); + }); + + t2.test(".start() warns when system metrics are disabled", async t3 => { + const { logs } = await startPlugin(t3, { config: { "controller.system_metrics_interval": 0 } }); + t3.ok(logs.some(([level, message]) => level === "warn" && message.includes("system_metrics_interval"))); + }); + + t2.end(); +}); diff --git a/exp_auto_restart/tsconfig.json b/exp_auto_restart/tsconfig.json new file mode 100644 index 0000000000..a0473e7523 --- /dev/null +++ b/exp_auto_restart/tsconfig.json @@ -0,0 +1,6 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.node.json" } + ] +} diff --git a/exp_auto_restart/tsconfig.node.json b/exp_auto_restart/tsconfig.node.json new file mode 100644 index 0000000000..3218f2e757 --- /dev/null +++ b/exp_auto_restart/tsconfig.node.json @@ -0,0 +1,5 @@ +{ + "extends": "../tsconfig.node.json", + "include": ["./**/*.ts"], + "exclude": ["test/*", "./dist/*"], +} diff --git a/tsconfig.json b/tsconfig.json index 8afb8e7c2e..94818dee53 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,7 @@ { "files": [], "references": [ + { "path": "./exp_auto_restart/" }, { "path": "./exp_commands/" }, { "path": "./exp_groups/" }, { "path": "./exp_gui/" },