diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa70651d0..81027169d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -314,7 +314,9 @@ jobs: - { PGVERSION: 19, schedule: quick } # node: create_standby_with_pgdata, maintenance_and_drop, auth, # monitor_disabled, replace_monitor, extension_update, - # debian_clusters, tablespaces + # debian_clusters, tablespaces, fsm_step_report_advance, + # replication_stall/demote_timeout/timeline_fork deadlocks + # — see tests/tap/schedules/node.sch for the full list - { PGVERSION: 14, schedule: node } - { PGVERSION: 15, schedule: node } - { PGVERSION: 16, schedule: node } @@ -332,6 +334,7 @@ jobs: - { PGVERSION: 17, schedule: multi-alternate } - { PGVERSION: 17, schedule: multi-misc } - { PGVERSION: 17, schedule: multi-async } + - { PGVERSION: 17, schedule: node-fsm-gaps } - { PGVERSION: 17, schedule: citus-1 } - { PGVERSION: 17, schedule: citus-2 } # citus on PG18 (supported); allow failure until officially validated diff --git a/docs/failover-state-machine.rst b/docs/failover-state-machine.rst index 37806b851..da3130bc2 100644 --- a/docs/failover-state-machine.rst +++ b/docs/failover-state-machine.rst @@ -630,3 +630,157 @@ node reacting to the other side of that removal. the full graph, ``join_primary`` included, as one Graphviz file (e.g. to pipe into their own tooling), but it is no longer the documented way to visualize the FSM. + +The monitor's FSM: ``pgautofailover.fsm`` +------------------------------------------ + +The diagrams above are rendered from the *keeper's* side of the FSM +(``KeeperFSM[]``, ``src/bin/pg_autoctl/fsm.c``): the transitions a node +knows how to perform. The *monitor's* side is a separate, matching +declarative table, ``MonitorFSM[]`` (``src/monitor/group_state_machine.c``): +the rules deciding, for every combination of reported states and cluster +conditions, which goal state to assign next. It's exposed read-only via the +``pgautofailover.fsm`` view, one row per rule, ordered by ``pos``: + +:: + + =# SELECT pos, section, comment FROM pgautofailover.fsm + WHERE pos BETWEEN 301 AND 305; + pos | section | comment + -----+--------------------+--------------------------------------------------------------------------------- + 301 | reporting_node | converged secondary, reportedTLI not an ancestor of reference -> catchingup + 303 | reporting_node | converged secondary/catchingup, primary reachable, primary not in the primary + | | states -> catchingup + 305 | reporting_node | multi-standby cascade resume point (see MonitorFSM_MultiStandbyCascadeResumeAfterPos) + (3 rows) + +Every column of a single rule, expanded (``\x on``): + +:: + + =# \x on + =# SELECT * FROM pgautofailover.fsm WHERE pos = 301; + -[ RECORD 1 ]-----------------+---------------------------------------------------------------- + pos | 301 + section | reporting_node + comment | converged secondary, reportedTLI not an ancestor of reference -> catchingup + active_node_current_state | secondary + other_node_current_state | + candidate_node_current_state | + active_node_conditions | isComparableToReferenceTli=false + other_node_conditions | + candidate_node_conditions | + group_conditions | + active_node_assigned_state | catchingup + other_node_assigned_state | + has_extra_action | f + section_path | reporting_node.from_context + +``section_path`` is an ``ltree`` column, so the table's rows can be queried +hierarchically instead of by exact section name -- for example, every rule +belonging to the multi-standby candidate-election machinery, regardless of +how deep its own sub-leaf goes:: + + =# SELECT pos, comment FROM pgautofailover.fsm + WHERE section_path <@ 'reporting_node.ms_failover'::ltree + ORDER BY pos; + +Like the keeper diagrams above, this view is generated straight from the +compiled-in ``MonitorFSM[]`` table -- it's the same on every fresh monitor of +a given pg_auto_failover version, unaffected by any node or formation state, +and changes only when a rule is added, removed, or edited in a new release. + +Cross-checking the monitor and keeper FSMs +------------------------------------------- + +The monitor and the keeper are two different programs (the monitor extension +runs inside Postgres, the keeper is the ``pg_autoctl run`` process on each +node) with two independently-maintained tables: ``MonitorFSM[]`` decides +*what* goal state to assign, ``KeeperFSM[]`` decides whether a node *can +execute* the transition it's just been assigned. If a monitor rule is +changed or added without a matching keeper edge, the keeper has no way to +perform what it's told and fails at runtime with an error like +``pg_autoctl does not know how to reach state "X" from "Y"`` -- historically +only ever discovered when an operator's cluster actually reached that +specific combination of states in production. + +Three building blocks turn that from a runtime surprise into something +checked ahead of time: + +``pgautofailover.dump_fsm_edges()`` + Resolves every ``MonitorFSM[]`` row's state pattern into its concrete + ``(pos, current_state, assigned_state)`` edges -- the same information + ``pgautofailover.fsm`` shows as a pattern, fully expanded one row per + reachable current state. Reflexive edges (current state == assigned + state) and the ``api_triggered`` section are deliberately excluded: the + former are no-ops, and the latter resolves which node plays which role + via hand-written C ahead of dispatch, so its own state pattern was never + meant to double as a full reachability precondition. + +``pgautofailover.check_fsm_reachability(keeper_edges jsonb)`` + Takes a JSON array of ``{"current": ..., "assigned": ...}`` edges -- the + transitions *some* keeper knows how to perform -- and returns every + ``dump_fsm_edges()`` edge missing from it: every transition the monitor + could assign that this particular keeper has no edge for. + +``pg_autoctl inspect fsm check`` + The live, end-to-end version of the same check, run from a node against + its own monitor: it serializes the *real*, compiled-in ``KeeperFSM[]`` + (``KeeperFSMToJSON()``) and passes it straight to + ``check_fsm_reachability()`` above -- no synthetic input, no assumptions + about what the keeper can do. A clean cluster reports:: + + $ pg_autoctl inspect fsm check + 12:00:00 1 INFO OK: every monitor FSM transition has a matching keeper edge + + A gap reports one line per missing edge and exits non-zero, so it can be + used as a build gate. For illustration, here is what running an + *older* keeper binary against a *newer* monitor -- one that has since + learned a transition the old keeper predates -- would report (this is a + hypothetical mismatch for illustration, not a gap that exists in the + current tables):: + + $ pg_autoctl inspect fsm check + 12:00:00 1 ERROR pos 381: draining -> single has no matching keeper edge + (other node was forcibly removed, now single) + $ echo $? + 1 + + ``--json`` is also available, returning ``{"ok": false, "mismatches": [...]}`` + for scripting. + +How this is tested in CI +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Three regress tests exercise this mechanism on every build, for every +supported Postgres version, as part of the ``make -C src/monitor +installcheck`` step run while building each ``pgaf:run-pgN`` Docker image +(see the ``build_run_images`` job) -- so a gap fails CI directly, without +needing a live two-process cluster: + +- ``fsm.sql`` -- a plain dump of the whole ``pgautofailover.fsm`` view. + Since the table is compile-time-fixed, its expected output changes only + when a rule is added, removed, or edited, giving that change an explicit, + reviewable diff. +- ``check_fsm_reachability.sql`` -- exercises the SQL-side mechanism itself + against small, synthetic keeper-edge inputs (an edge present drops out of + the mismatch list, an edge absent stays in, an unrecognized state name + fails loudly). It doesn't touch the real ``KeeperFSM[]``, which lives in + the ``pg_autoctl`` binary, not the database -- it only proves the + comparison logic itself is correct. +- ``keeper_fsm_edges.sql`` -- the real end-to-end static check, without + needing a live cluster. It loads ``keeper_fsm_edges.json``, a fixture + generated from the actual ``KeeperFSM[]`` via ``pg_autoctl inspect fsm + list --json`` and committed alongside the test (regenerated by hand + whenever ``KeeperFSM[]`` changes), then cross-references it against + ``dump_fsm_edges()`` in both directions: every monitor edge with no + matching keeper row (a real, actionable gap), and every keeper row the + monitor never actually dispatches to (dead weight worth a second look, + not a build failure). + +``pg_autoctl inspect fsm check`` itself -- talking to a real monitor over +the network -- is exercised live rather than in the regress suite: it's +part of the ``fsm_step_report_advance`` pgaftest spec (see +:ref:`pg_autoctl_manual_fsm_step`) and is also the tool to reach for by hand +after any manual edit to either FSM table, or when investigating a report +that looks like a reachability gap. diff --git a/docs/ref/pg_autoctl_inspect.rst b/docs/ref/pg_autoctl_inspect.rst index a9c7c009d..239e2e37f 100644 --- a/docs/ref/pg_autoctl_inspect.rst +++ b/docs/ref/pg_autoctl_inspect.rst @@ -34,9 +34,18 @@ variable is required. All commands in this group are safe to run while tune Compute and log some Postgres tuning options pg_autoctl inspect fsm - state Read the keeper's state from disk and display it - list List reachable FSM states from current state - gv Output the FSM as a .gv program suitable for graphviz/dot + state Read the keeper's state from disk and display it + list List reachable FSM states from current state + check Check that every monitor FSM transition has a matching keeper edge + gv Output the FSM as a .gv program suitable for graphviz/dot + mermaid Output the FSM as Mermaid stateDiagram-v2 programs, split by phase for readability + + pg_autoctl inspect fsm mermaid + init Mermaid diagram: how a node comes into existence or rejoins + steady-state Mermaid diagram: normal operation, no failure + failover Mermaid diagram: primary failover/promotion, including multi-standby candidate election + maintenance Mermaid diagram: planned maintenance + removal Mermaid diagram: node removal/drop pg_autoctl inspect show ipaddr Print this node's IP address information diff --git a/src/bin/common/pgsql.c b/src/bin/common/pgsql.c index 105c4f96c..3807f41d2 100644 --- a/src/bin/common/pgsql.c +++ b/src/bin/common/pgsql.c @@ -1051,6 +1051,15 @@ pgsql_execute(PGSQL *pgsql, const char *sql) } +/* + * Cap on how much of a single query parameter's own value gets printed in + * pgsql_execute_with_params' debug trace (see its own comment at the one + * call site that truncates). Keeps that trace's fixed BUFSIZE buffer from + * overflowing on a large parameter, without limiting what's actually sent + * to Postgres (PQexecParams always gets the real, untruncated value). + */ +#define DEBUG_PARAM_VALUE_MAX_LEN 200 + /* * pgsql_execute_with_params opens a connection, runs a given SQL command, * and closes the connection again. @@ -1081,7 +1090,9 @@ pgsql_execute_with_params(PGSQL *pgsql, const char *sql, int paramCount, int remainingBytes = BUFSIZE; char *writePointer = (char *) debugParameters; - for (paramIndex = 0; paramIndex < paramCount; paramIndex++) + for (paramIndex = 0; + paramIndex < paramCount && remainingBytes > 0; + paramIndex++) { int bytesWritten = 0; const char *value = paramValues[paramIndex]; @@ -1091,12 +1102,37 @@ pgsql_execute_with_params(PGSQL *pgsql, const char *sql, int paramCount, bytesWritten = sformat(writePointer, remainingBytes, ", "); remainingBytes -= bytesWritten; writePointer += bytesWritten; + + if (remainingBytes <= 0) + { + break; + } } if (value == NULL) { bytesWritten = sformat(writePointer, remainingBytes, "NULL"); } + else if (strlen(value) > DEBUG_PARAM_VALUE_MAX_LEN) + { + /* + * A parameter can be arbitrarily large (e.g. the JSON payload + * "pg_autoctl inspect fsm check" sends to + * pgautofailover.check_fsm_reachability(), several KB) -- + * printing it here in full would overflow debugParameters' + * own fixed BUFSIZE and make sformat() itself log a "BUG:" + * about it on every single call, which is exactly what + * happened the first time that command ran for real. This is + * a debug-only trace, not the actual query (PQexecParams + * below always gets the real, untruncated paramValues), so + * truncating what gets logged here changes nothing about + * query correctness. + */ + bytesWritten = sformat(writePointer, remainingBytes, + "'%.*s...' (%zu bytes total)", + DEBUG_PARAM_VALUE_MAX_LEN, value, + strlen(value)); + } else { bytesWritten = diff --git a/src/bin/pg_autoctl/cli_common.c b/src/bin/pg_autoctl/cli_common.c index 915d5ab08..709504b7c 100644 --- a/src/bin/pg_autoctl/cli_common.c +++ b/src/bin/pg_autoctl/cli_common.c @@ -1208,6 +1208,140 @@ cli_getopt_pgdata(int argc, char **argv) } +/* + * cli_getopt_pgdata_or_json is a variant of cli_getopt_pgdata used only by + * "pg_autoctl inspect fsm list": with --json, that command dumps the + * keeper's own static KeeperFSM[] table (cli_do_fsm_list, cli_do_fsm.c), + * which has no dependency on any node's actual config or on-disk state, so + * it must be able to run with neither --pgdata nor an existing + * configuration file. Without --json it behaves exactly like + * cli_getopt_pgdata, because that mode reports on the keeper's current + * state and does need a real config. This parses the identical option set + * as cli_getopt_pgdata; only the decision of whether to call + * prepare_keeper_options differs, so as not to change cli_getopt_pgdata + * itself (used, unconditionally, by every other terminal command). + */ +int +cli_getopt_pgdata_or_json(int argc, char **argv) +{ + KeeperConfig options = { 0 }; + int c, option_index = 0, errors = 0; + int verboseCount = 0; + bool printVersion = false; + + static struct option long_options[] = { + { "pgdata", required_argument, NULL, 'D' }, + { "json", no_argument, NULL, 'J' }, + { "version", no_argument, NULL, 'V' }, + { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, 'q' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + optind = 0; + + unsetenv("POSIXLY_CORRECT"); + + while ((c = getopt_long(argc, argv, "D:JVvqh", + long_options, &option_index)) != -1) + { + switch (c) + { + case 'D': + { + strlcpy(options.pgSetup.pgdata, optarg, MAXPGPATH); + log_trace("--pgdata %s", options.pgSetup.pgdata); + break; + } + + case 'J': + { + outputJSON = true; + log_trace("--json"); + break; + } + + case 'V': + { + printVersion = true; + break; + } + + case 'v': + { + ++verboseCount; + switch (verboseCount) + { + case 1: + { + log_set_level(LOG_INFO); + break; + } + + case 2: + { + log_set_level(LOG_DEBUG); + break; + } + + default: + { + log_set_level(LOG_TRACE); + break; + } + } + break; + } + + case 'q': + { + log_set_level(LOG_ERROR); + break; + } + + case 'h': + { + commandline_help(stderr); + exit(EXIT_CODE_QUIT); + break; + } + + default: + { + /* getopt_long already wrote an error message */ + errors++; + break; + } + } + } + + if (errors > 0) + { + commandline_help(stderr); + exit(EXIT_CODE_BAD_ARGS); + } + + if (printVersion) + { + keeper_cli_print_version(argc, argv); + } + + /* + * --json needs neither --pgdata nor an existing config file (see this + * function's own comment); anything else still needs the full + * pgdata/config validation cli_getopt_pgdata itself always applies. + */ + if (!outputJSON) + { + (void) prepare_keeper_options(&options); + } + + keeperOptions = options; + + return optind; +} + + /* * prepare_keeper_options finishes the preparation of the keeperOptions that * hosts the command line options. diff --git a/src/bin/pg_autoctl/cli_common.h b/src/bin/pg_autoctl/cli_common.h index dce65b2a6..45bfca490 100644 --- a/src/bin/pg_autoctl/cli_common.h +++ b/src/bin/pg_autoctl/cli_common.h @@ -182,6 +182,7 @@ int cli_create_node_getopts(int argc, char **argv, KeeperConfig *options); int cli_getopt_pgdata(int argc, char **argv); +int cli_getopt_pgdata_or_json(int argc, char **argv); void prepare_keeper_options(KeeperConfig *options); void set_first_pgctl(PostgresSetup *pgSetup); diff --git a/src/bin/pg_autoctl/cli_do_fsm.c b/src/bin/pg_autoctl/cli_do_fsm.c index fd32f930b..63b92e614 100644 --- a/src/bin/pg_autoctl/cli_do_fsm.c +++ b/src/bin/pg_autoctl/cli_do_fsm.c @@ -15,6 +15,8 @@ #include "postgres_fe.h" +#include "parson.h" + #include "cli_common.h" #include "commandline.h" #include "defaults.h" @@ -23,6 +25,7 @@ #include "fsm_mermaid.h" #include "keeper_config.h" #include "keeper.h" +#include "monitor.h" #include "parsing.h" #include "pgctl.h" #include "state.h" @@ -33,6 +36,7 @@ static void cli_do_fsm_init(int argc, char **argv); static void cli_do_fsm_state(int argc, char **argv); static void cli_do_fsm_list(int argc, char **argv); +static void cli_do_fsm_check(int argc, char **argv); static void cli_do_fsm_gv(int argc, char **argv); static void cli_do_fsm_mermaid_init(int argc, char **argv); static void cli_do_fsm_mermaid_steady_state(int argc, char **argv); @@ -66,9 +70,17 @@ CommandLine fsm_list = "List reachable FSM states from current state", CLI_PGDATA_USAGE, CLI_PGDATA_OPTION, - cli_getopt_pgdata, + cli_getopt_pgdata_or_json, cli_do_fsm_list); +CommandLine fsm_check = + make_command("check", + "Check that every monitor FSM transition has a matching keeper edge", + CLI_PGDATA_USAGE, + CLI_PGDATA_OPTION, + cli_getopt_pgdata, + cli_do_fsm_check); + CommandLine fsm_gv = make_command("gv", "Output the FSM as a .gv program suitable for graphviz/dot", @@ -320,11 +332,39 @@ cli_do_fsm_state(int argc, char **argv) /* - * cli_do_fsm_list lists reachable states from the current one. + * cli_do_fsm_list lists reachable states from the current one, or (with + * --json) dumps the full KeeperFSM[] edge set as JSON -- the design doc's + * own proposed standalone use of KeeperFSMToJSON() ("a human or another + * tool may want the raw edge list without a monitor round trip at all"), + * and the source this project's own keeper_fsm_edges.json regress fixture + * (src/monitor/) is regenerated from. + * + * --json needs neither a keeper config nor an on-disk state file at all: + * KeeperFSM[] is pure static data with no dependency on any node's actual + * config or reported state (unlike the ordinary, current-state-filtered + * list output below, which needs keeperState.current_role) -- so it's + * checked first and short-circuits before either read, meaning this mode + * can run with zero setup: no --pgdata, no live cluster, just the binary. + * This is only reachable because fsm_list's own CommandLine (above) uses + * cli_getopt_pgdata_or_json rather than the shared cli_getopt_pgdata: the + * latter unconditionally requires an existing config file + * (prepare_keeper_options, cli_common.c) before this function is ever + * called, regardless of --json. */ static void cli_do_fsm_list(int argc, char **argv) { + if (outputJSON) + { + char *keeperEdgesJSON = KeeperFSMToJSON(); + + fformat(stdout, "%s\n", keeperEdgesJSON); + + json_free_serialized_string(keeperEdgesJSON); + + return; + } + KeeperStateData keeperState = { 0 }; KeeperConfig config = keeperOptions; @@ -348,13 +388,105 @@ cli_do_fsm_list(int argc, char **argv) exit(EXIT_CODE_BAD_STATE); } + print_reachable_states(&keeperState); + fformat(stdout, "\n"); +} + + +/* + * cli_do_fsm_check serializes this node's own KeeperFSM[] to JSON + * (KeeperFSMToJSON(), fsm.c) and sends it to the monitor's + * pgautofailover.check_fsm_reachability(jsonb) (via + * monitor_check_fsm_reachability(), monitor.c) to confirm every transition + * the monitor's own MonitorFSM[] (group_state_machine.c) can ever assign + * has a matching edge here. Unlike cli_do_fsm_list/cli_do_fsm_gv (purely + * local, no monitor connection at all), this genuinely needs one -- mirrors + * cli_do_monitor_get_primary_node's own shape in cli_do_monitor.c. + */ +static void +cli_do_fsm_check(int argc, char **argv) +{ + KeeperConfig config = keeperOptions; + Monitor monitor = { 0 }; + FsmReachabilityResult result = { 0 }; + + bool missingPgdataIsOk = true; + bool pgIsNotRunningIsOk = true; + bool monitorDisabledIsOk = false; + + if (!keeper_config_read_file(&config, + missingPgdataIsOk, + pgIsNotRunningIsOk, + monitorDisabledIsOk)) + { + /* errors have already been logged */ + exit(EXIT_CODE_BAD_CONFIG); + } + + if (!monitor_init(&monitor, config.monitor_pguri)) + { + log_fatal("Failed to contact the monitor because its URL is invalid, " + "see above for details"); + exit(EXIT_CODE_BAD_CONFIG); + } + + char *keeperEdgesJSON = KeeperFSMToJSON(); + + if (!monitor_check_fsm_reachability(&monitor, keeperEdgesJSON, &result)) + { + log_error("Failed to check FSM reachability against the monitor, " + "see above for details"); + json_free_serialized_string(keeperEdgesJSON); + exit(EXIT_CODE_MONITOR); + } + + json_free_serialized_string(keeperEdgesJSON); + if (outputJSON) { - log_warn("This command does not support JSON output at the moment"); + JSON_Value *js = json_value_init_object(); + JSON_Object *root = json_value_get_object(js); + JSON_Value *jsMismatches = json_value_init_array(); + JSON_Array *mismatches = json_value_get_array(jsMismatches); + + for (int i = 0; i < result.count; i++) + { + FsmReachabilityMismatch *mismatch = &(result.mismatches[i]); + JSON_Value *jsEntry = json_value_init_object(); + JSON_Object *jsObj = json_value_get_object(jsEntry); + + json_object_set_number(jsObj, "pos", (double) mismatch->pos); + json_object_set_string(jsObj, "current", mismatch->currentState); + json_object_set_string(jsObj, "assigned", mismatch->assignedState); + json_object_set_string(jsObj, "comment", mismatch->comment); + + json_array_append_value(mismatches, jsEntry); + } + + json_object_set_boolean(root, "ok", result.count == 0); + json_object_set_value(root, "mismatches", jsMismatches); + + cli_pprint_json(js); + } + else if (result.count == 0) + { + log_info("OK: every monitor FSM transition has a matching keeper edge"); } + else + { + for (int i = 0; i < result.count; i++) + { + FsmReachabilityMismatch *mismatch = &(result.mismatches[i]); - print_reachable_states(&keeperState); - fformat(stdout, "\n"); + log_error("pos %d: %s -> %s has no matching keeper edge (%s)", + mismatch->pos, + mismatch->currentState, + mismatch->assignedState, + mismatch->comment); + } + } + + exit(result.count == 0 ? EXIT_CODE_QUIT : EXIT_CODE_INTERNAL_ERROR); } diff --git a/src/bin/pg_autoctl/cli_do_root.h b/src/bin/pg_autoctl/cli_do_root.h index 0706b95ef..c2443e35a 100644 --- a/src/bin/pg_autoctl/cli_do_root.h +++ b/src/bin/pg_autoctl/cli_do_root.h @@ -20,6 +20,7 @@ extern CommandLine do_fsm_commands; extern CommandLine fsm_state; extern CommandLine fsm_node_state; extern CommandLine fsm_list; +extern CommandLine fsm_check; extern CommandLine fsm_gv; extern CommandLine fsm_mermaid; diff --git a/src/bin/pg_autoctl/cli_inspect.c b/src/bin/pg_autoctl/cli_inspect.c index 3bb441b80..c599f7f23 100644 --- a/src/bin/pg_autoctl/cli_inspect.c +++ b/src/bin/pg_autoctl/cli_inspect.c @@ -18,12 +18,14 @@ /* * Read-only FSM sub-commands: display the current state, list reachable - * transitions, or dump the full FSM as a graphviz .gv file. + * transitions, check that every monitor transition has a matching keeper + * edge, or dump the full FSM as a graphviz .gv file. * Mutating operations (init, assign, step, nodes set) live under "manual fsm". */ static CommandLine *inspect_fsm_subcommands[] = { &fsm_state, &fsm_list, + &fsm_check, &fsm_gv, &fsm_mermaid, NULL @@ -31,7 +33,8 @@ static CommandLine *inspect_fsm_subcommands[] = { static CommandLine inspect_fsm_commands = make_command_set("fsm", - "Display keeper FSM state and transitions (read-only)", + "Display keeper FSM state and transitions, and check " + "monitor/keeper FSM reachability (read-only)", NULL, NULL, NULL, inspect_fsm_subcommands); /* diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index edc4af972..fb66b060d 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -18,6 +18,7 @@ #include "defaults.h" #include "keeper.h" +#include "parson.h" #include "pgctl.h" #include "fsm.h" #include "log.h" @@ -80,6 +81,10 @@ #define COMMENT_DEMOTE_TIMEOUT_TO_DEMOTED \ "Demote timeout expired" +#define COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED \ + "A different node is taking over as primary, " \ + "stopping Postgres in case it's still running" + #define COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY \ "Confirmed promotion with the monitor" @@ -160,6 +165,14 @@ #define COMMENT_REPORT_LSN_TO_SINGLE \ "There is no other node anymore, promote this node" +#define COMMENT_WAIT_MAINTENANCE_TO_SINGLE \ + "Was waiting to be sent to maintenance, but the primary vanished, " \ + "promote this node" + +#define COMMENT_FAST_FORWARD_TO_SINGLE \ + "Was fetching missing WAL from another standby, but every other node " \ + "vanished, promote this node with whatever it has" + #define COMMENT_FOLLOW_NEW_PRIMARY \ "Switch replication to the new primary" @@ -388,6 +401,191 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_FAILOVER }, + /* + * A node resolved by the monitor as "the primary" (GetPrimaryOrDemoted + * NodeInGroupFromList(), group_state_machine.c) is not always reporting + * one of the ordinary primary-track states above: its own goalState can + * be bumped to a writable state by an unrelated later event (e.g. pos + * 209's "alone in group -> single", group_state_machine.c) well before + * its own reportedState has had a chance to converge, so a genuinely + * dead/partitioned node can present almost any reportedState by the + * time the monitor assigns it demoted/demote_timeout (see + * PrimaryNodeReportedStateCanBeResolved's own comment, + * group_state_machine.c, for the full reachability argument -- this is + * exactly the set of states it proves reachable). fsm_stop_postgres is + * the same role-agnostic "make sure Postgres is stopped" action used by + * every primary-track source state above: safe regardless of what this + * node's own Postgres instance was actually doing when it stopped + * reporting. + */ + { + INIT_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + SINGLE_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + CATCHINGUP_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + SECONDARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + PREP_PROMOTION_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + STOP_REPLICATION_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + MAINTENANCE_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + PREPARE_MAINTENANCE_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + WAIT_MAINTENANCE_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + REPORT_LSN_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + FAST_FORWARD_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + /* + * Same reasoning as the DEMOTED_STATE cluster just above, targeting + * DEMOTE_TIMEOUT_STATE instead: MonitorFSM[] pos 339's own sibling row + * assigns demote_timeout (not demoted) from the same 11 states, plus + * DEMOTED_STATE itself (not a reflexive self-loop against this + * different target). + */ + { + INIT_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + SINGLE_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + DEMOTED_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + CATCHINGUP_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + SECONDARY_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + PREP_PROMOTION_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + STOP_REPLICATION_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + MAINTENANCE_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + PREPARE_MAINTENANCE_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + WAIT_MAINTENANCE_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + REPORT_LSN_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + FAST_FORWARD_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + /* * was demoted after a failure, but standby was forcibly removed */ @@ -516,6 +714,43 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_REMOVAL }, + /* + * was waiting for the primary to disable sync replication before going + * to maintenance (a converged, actively-streaming standby -- entering + * WAIT_MAINTENANCE_STATE itself runs no transition function, see its own + * comment below), but the primary was forcibly removed instead: reuse + * fsm_promote_standby exactly like every other converged-standby source + * state above (SECONDARY/CATCHINGUP/PREP_PROMOTION/STOP_REPLICATION/ + * REPORT_LSN), since Postgres here is already running and replicating, + * with no special handling wait_maintenance itself needs. + */ + { + WAIT_MAINTENANCE_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_WAIT_MAINTENANCE_TO_SINGLE, + &fsm_promote_standby, + FSM_PHASE_REMOVAL + }, + + /* + * was fetching missing WAL from another standby to catch up before + * promotion (a converged-enough standby -- Postgres is already running + * and replicating, same shape as the other converged-standby source + * states above), but every other node vanished, including the peer we + * were fetching from: fsm_fast_forward's own "no upstream found" branch + * already accepts this (skips the fetch, treats local data as the best + * available -- there is nothing more advanced left anywhere to lose), + * so promoting with whatever this node already has is exactly as safe + * as it gets. Reuse fsm_promote_standby exactly like every other + * converged-standby source state -- no separate WAL fetch is attempted + * or needed here, matching PREP_PROMOTION_STATE/STOP_REPLICATION_STATE's + * own direct-to-SINGLE shortcut for "was mid-promotion, peer vanished". + */ + { + FAST_FORWARD_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_FAST_FORWARD_TO_SINGLE, + &fsm_promote_standby, + FSM_PHASE_REMOVAL + }, /* * On the Primary, wait for a standby to be ready: WAIT_PRIMARY @@ -859,6 +1094,152 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_MAINTENANCE }, + /* + * was waiting for the primary to disable sync replication before going + * to maintenance (a converged, actively-streaming standby, same as the + * other reused source states just above), but the primary was forcibly + * removed and this node's own candidate-priority is 0 -- reuse + * fsm_report_lsn exactly like SECONDARY/CATCHINGUP/MAINTENANCE/ + * PREPARE_MAINTENANCE above. + */ + { + WAIT_MAINTENANCE_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_MAINTENANCE + }, + + /* + * was fetching missing WAL from another standby to catch up before + * promotion (same converged-enough shape as the source states above), + * but every other node vanished and this node's own candidate-priority + * is 0 -- reuse fsm_report_lsn exactly like SECONDARY/CATCHINGUP/ + * MAINTENANCE/PREPARE_MAINTENANCE/WAIT_MAINTENANCE above. No WAL fetch + * is attempted here either, same reasoning as FAST_FORWARD_STATE -> + * SINGLE_STATE's own comment. + */ + { + FAST_FORWARD_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_FAILOVER + }, + + /* + * was mid-promotion (selected as the MS-failover candidate, assigned + * prepare_promotion) but every other node vanished and this node's own + * candidate-priority is 0 -- reuse fsm_report_lsn exactly like every + * other converged-standby source state above. Safe to reuse directly, + * with no intermediate hop: entering prepare_promotion itself runs + * fsm_prepare_standby_for_promotion, which is a no-op (see its own + * comment, fsm_transition.c) -- Postgres is still running, still an + * ordinary streaming standby, completely untouched. fsm_report_lsn's + * own restart (standby_restart_with_current_replication_source, + * primary_standby.c) writes a fresh disconnected-standby recovery + * config and restarts Postgres itself, so it doesn't matter that there + * is no live upstream to reach -- it never tries to reach one. + */ + { + PREP_PROMOTION_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_FAILOVER + }, + + /* + * was one step further into that same promotion (prepare_promotion's own + * no-op already gave way to fsm_stop_replication, which really did + * promote Postgres onto a new timeline this time) but every other node + * vanished before this node could finish becoming the new primary, and + * its own candidate-priority is 0 -- reuse fsm_report_lsn exactly like + * every other converged-standby source state above. + * + * Unlike prepare_promotion, Postgres here is no longer an ordinary + * streaming standby -- fsm_stop_replication already promoted it, so it's + * a genuinely writable, disconnected primary on its own new timeline. + * That's still safe to hand to fsm_report_lsn unmodified: its own + * restart (standby_restart_with_current_replication_source, + * primary_standby.c) never contacts any peer at all -- it's a purely + * local stop/reconfigure/restart sequence, entirely oblivious to + * whether this instance was ever promoted. Concretely: it calls + * standby_init_replication_source with an all-zeroed upstream (no + * host), so the subsequent restart's own primaryNode.host check + * (IS_EMPTY_STRING_BUFFER) skips the identify-system connection + * attempt entirely, and pg_setup_standby_mode's own identical check + * skips it too -- neither ever needs a reachable primary. It just + * stops Postgres, writes a fresh standby.signal with no + * primary_conninfo, and restarts -- exactly what turns an ordinary + * disconnected standby into a report_lsn candidate elsewhere in this + * table, and Postgres itself doesn't care that this data directory's + * own history includes a promotion: recovery-on-restart is decided + * purely by standby.signal's presence, not by promotion history. + * + * This was previously left as a documented, unfixed gap based on a + * mistaken assumption that reaching report_lsn from here would need + * the same live-primary-dependent rewind/basebackup machinery + * fsm_restart_standby/fsm_rewind_or_init uses elsewhere (for reaching + * catchingup, a materially different target that actually does need to + * stream from someone) -- it doesn't; fsm_report_lsn was already the + * right tool for this row the whole time. See + * keeper_fsm_gap_stop_replication_report_lsn.pgaf for the live + * reproduction, including both ways out documented on pos 211's own + * comment (group_state_machine.c): raising candidate-priority back + * above 0 (report_lsn -> single, already an existing edge, via pos + * 209), or a new node registering and RegisterNode's own existing + * report_lsn-candidate-priority-0 special case (node_active_protocol.c) + * basebackupping from this node and taking over as primary while this + * node follows it back in as a secondary. + */ + { + STOP_REPLICATION_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_FAILOVER + }, + + /* + * was demoting after losing the primary role (draining timed out, or a + * manual failover put another node in charge) but every other node + * vanished and this node's own candidate-priority is 0 -- reuse + * fsm_report_lsn exactly like every other converged-standby source + * state above. Safe for the same reason as prepare_promotion just + * above: fsm_stop_replication's own default_transaction_read_only=on + * already blocks this node from taking writes while in demote_timeout, + * so no writes have landed here that a real primary elsewhere + * wouldn't also have; fsm_report_lsn's own restart reconfigures and + * restarts Postgres unconditionally, with no live peer required. + */ + { + DEMOTE_TIMEOUT_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_FAILOVER + }, + + /* + * was following a newly-elected primary (report_lsn -> join_secondary, + * Postgres cleanly checkpointed and stopped by + * fsm_checkpoint_and_stop_postgres while switching allegiance) but + * every other node vanished before it could finish, including the new + * primary it was about to follow, and this node's own candidate- + * priority is 0 -- reuse fsm_report_lsn exactly like every other + * converged-standby source state above. Safe for the same reason as + * prepare_promotion above despite Postgres currently being stopped: + * fsm_report_lsn's own restart (standby_restart_with_current_ + * replication_source) stops Postgres if running, reconfigures it as a + * disconnected standby, and starts it back up -- it handles "already + * stopped" and "still running" identically, and never needs to reach + * any peer to do it. The data itself is trustworthy: the checkpoint + * that stopped Postgres happened before any new primary could have + * taken a single write this node doesn't already have. + */ + { + JOIN_SECONDARY_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_FAILOVER + }, + { REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, COMMENT_REPORT_LSN_TO_PREP_PROMOTION, @@ -1017,6 +1398,22 @@ keeper_fsm_step(Keeper *keeper) return false; } + /* + * Repair our own groupId/replication slot name if the monitor's view of + * who we are has drifted from ours -- see + * keeper_maybe_update_group_and_slot's own comment for why this can't + * be skipped here: unlike the autonomous tick body + * (service_keeper_node_active), this is also the code path a suspended + * (step-mode) node's every "fsm step"/"fsm step report" goes through, + * and it's the only place such a node ever gets a fresh + * MonitorAssignedState to check against. + */ + if (!keeper_maybe_update_group_and_slot(keeper, &assignedState)) + { + /* errors have already been logged */ + return false; + } + /* * Assign the new state. We skip writing the state file here since we can * (and should) always get the assigned state from the monitor. @@ -1100,6 +1497,13 @@ keeper_fsm_step_report(Keeper *keeper) return false; } + /* see keeper_fsm_step's own identical call for why this is here too */ + if (!keeper_maybe_update_group_and_slot(keeper, &assignedState)) + { + /* errors have already been logged */ + return false; + } + keeperState->assigned_role = assignedState.state; if (!keeper_update_state(keeper, assignedState.nodeId, assignedState.groupId, @@ -1315,3 +1719,85 @@ print_fsm_for_graphviz(void) } fformat(stdout, "}\n"); } + + +/* + * KeeperFSMToJSONAppendEdge appends one {"current": ..., "assigned": ...} + * object to array for a single KeeperFSMTransition row. current is rendered + * as the literal string "any" for ANY_STATE (state_matches()'s wildcard, + * e.g. fsm.c's "drop node from any state" rows) instead of + * NodeStateToString(ANY_STATE)'s own "#any state#" -- see KeeperFSMToJSON's + * own comment for why this sentinel exists and how the SQL side matches it. + */ +static void +KeeperFSMToJSONAppendEdge(JSON_Array *array, NodeState current, NodeState assigned) +{ + JSON_Value *jsEntry = json_value_init_object(); + JSON_Object *jsObj = json_value_get_object(jsEntry); + + json_object_set_string(jsObj, "current", + current == ANY_STATE ? "any" : NodeStateToString(current)); + json_object_set_string(jsObj, "assigned", NodeStateToString(assigned)); + + json_array_append_value(array, jsEntry); +} + + +/* + * KeeperFSMToJSON serializes KeeperFSM[] into a JSON array of + * {"current": ..., "assigned": ...} objects -- one per KeeperFSMTransition + * row, walked the same way print_fsm_for_graphviz/print_reachable_states + * already do. This is the keeper-side half of the monitor/keeper FSM + * reachability cross-check: "pg_autoctl inspect fsm check" sends this + * verbatim to the monitor's pgautofailover.check_fsm_reachability(jsonb), + * which anti-joins it against every edge the monitor's own MonitorFSM[] + * table (group_state_machine.c) can produce (see dump_fsm_edges()'s own + * comment there) and reports any monitor transition with no matching entry + * here. + * + * A row's own .current can be ANY_STATE -- rendered here as the literal + * sentinel "any" (see KeeperFSMToJSONAppendEdge), not expanded into one + * edge per concrete state: an earlier version of this function expanded it + * against a hand-maintained list of "every real NodeState", which silently + * under-covers the real ANY_STATE semantics the moment that list drifts out + * of sync with the actual NodeState enum -- a false negative baked + * permanently into the fixture, with nothing to ever catch it. Emitting the + * wildcard literally instead lets the SQL comparison (check_fsm_reachability(), + * and this project's own keeper_fsm_edges.sql regress test) match it against + * whatever current_state values pgautofailover.dump_fsm_edges() actually + * produces, so it can never drift out of sync with the real state universe. + * .assigned is never ANY_STATE in any current KeeperFSM[] row (an + * assignment target wildcard has no sensible meaning), so it's passed + * through as-is. + * + * Deliberately omits pgKind and comment: pgKind is not modeled on the + * monitor side of this check at all (every Citus-specific KeeperFSM[] edge + * already has a NODE_KIND_ANY counterpart with the same (current, assigned) + * shape -- see this same file's fsm_mermaid.c-referencing comment), and + * comment is purely descriptive, never part of the comparison. + * + * Returns a malloc'd string (via json_serialize_to_string) the caller must + * free with json_free_serialized_string(). + */ +char * +KeeperFSMToJSON(void) +{ + JSON_Value *jsArray = json_value_init_array(); + JSON_Array *array = json_value_get_array(jsArray); + + KeeperFSMTransition transition = KeeperFSM[0]; + int transitionIndex = 0; + + while (transition.current != NO_STATE) + { + KeeperFSMToJSONAppendEdge(array, transition.current, transition.assigned); + + transition = KeeperFSM[++transitionIndex]; + } + + char *serialized = json_serialize_to_string(jsArray); + + json_value_free(jsArray); + + return serialized; +} diff --git a/src/bin/pg_autoctl/fsm.h b/src/bin/pg_autoctl/fsm.h index 3a8cc6f08..2ab182fc1 100644 --- a/src/bin/pg_autoctl/fsm.h +++ b/src/bin/pg_autoctl/fsm.h @@ -129,6 +129,7 @@ bool prepare_replication(Keeper *keeper, NodeState otherNodeState); */ void print_reachable_states(KeeperStateData *keeperState); void print_fsm_for_graphviz(void); +char * KeeperFSMToJSON(void); bool keeper_fsm_step(Keeper *keeper); bool keeper_fsm_step_report(Keeper *keeper); bool keeper_fsm_step_advance(Keeper *keeper); diff --git a/src/bin/pg_autoctl/keeper.c b/src/bin/pg_autoctl/keeper.c index 887b9178a..c4461f894 100644 --- a/src/bin/pg_autoctl/keeper.c +++ b/src/bin/pg_autoctl/keeper.c @@ -863,6 +863,76 @@ keeper_create_self_signed_cert(Keeper *keeper) } +/* + * keeper_maybe_update_group_and_slot compares what the monitor just told us + * in assignedState (our own nodeId and groupId, as returned by this exact + * node_active() contact) against our own persisted groupId and replication + * slot name, and repairs the on-disk configuration file when they've + * drifted. + * + * This must run on every contact with the monitor, autonomous or step-mode + * alike: keeper_suspended_loop() used to skip it entirely (it called + * keeper_fsm_step()/keeper_fsm_step_report() directly, bypassing + * service_keeper_node_active() where this check used to live inline), which + * left a suspended node permanently unable to notice or repair a + * replication-slot mismatch. In practice this stayed invisible for as long + * as a suspended node kept following the same primary it registered + * against -- the bug surfaces specifically the first time it's later + * assigned to follow a *different* primary (report_lsn -> secondary via + * fsm_follow_new_primary): with no self-heal ever having run for this node, + * its config's replication_slot_name can be empty by that point, so + * prepare_recovery_settings() writes no primary_slot_name at all, Postgres + * streams without a slot, and the new primary's own quorum-candidate query + * (pgsql_get_postgres_metadata, an inner join from pg_replication_slots to + * pg_stat_replication on active_pid) can never match a slot-less + * connection -- so it retries "wait_primary -> primary" forever, logging + * "we don't have a quorum candidate yet" (fsm_enable_sync_rep, + * fsm_transition.c) even though a healthy, caught-up standby is right + * there. + * + * We deliberately only call keeper_config_update() here, not the heavier + * keeper_ensure_configuration(): that function also reconfigures live + * standby settings (primary_conninfo) based on state->current_role, via + * keeper_get_primary(). Calling it from this drift check is unsafe, because + * this check can run *before* the FSM transition that would bring + * current_role up to date -- e.g. keeper_fsm_step_report() reports without + * transitioning, so a node that just lost its primary can still have + * current_role reflecting the old, now-gone primary, and + * keeper_ensure_configuration() would then try to reconnect Postgres to + * that stale/nonexistent primary. All that fsm_follow_new_primary() and + * friends actually need from this self-heal is a correct + * config->replication_slot_name the next time they read it, which a plain + * config-file update already provides. + */ +bool +keeper_maybe_update_group_and_slot(Keeper *keeper, MonitorAssignedState *assignedState) +{ + KeeperConfig *config = &(keeper->config); + + char expectedSlotName[BUFSIZE] = { 0 }; + + (void) postgres_sprintf_replicationSlotName(assignedState->nodeId, + expectedSlotName, + sizeof(expectedSlotName)); + + if (assignedState->groupId != config->groupId || + strneq(config->replication_slot_name, expectedSlotName)) + { + if (!keeper_config_update(config, + assignedState->nodeId, + assignedState->groupId)) + { + log_error("Failed to update the configuration file " + "with groupId %d and replication.slot \"%s\"", + assignedState->groupId, expectedSlotName); + return false; + } + } + + return true; +} + + /* * keeper_ensure_configuration updates the Postgres settings to match the * pg_autoctl configuration file, if necessary. diff --git a/src/bin/pg_autoctl/keeper.h b/src/bin/pg_autoctl/keeper.h index 2139876b7..d136998e7 100644 --- a/src/bin/pg_autoctl/keeper.h +++ b/src/bin/pg_autoctl/keeper.h @@ -61,6 +61,8 @@ bool keeper_maintain_replication_slots(Keeper *keeper); bool keeper_ensure_current_state(Keeper *keeper); bool keeper_create_self_signed_cert(Keeper *keeper); bool keeper_ensure_configuration(Keeper *keeper, bool postgresNotRunningIsOk); +bool keeper_maybe_update_group_and_slot(Keeper *keeper, + MonitorAssignedState *assignedState); bool keeper_update_pg_state(Keeper *keeper, int logLevel); bool keeper_node_active(Keeper *keeper, bool doInit, MonitorAssignedState *assignedState); diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index adc0ce661..f24c34643 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -115,6 +115,13 @@ typedef struct MonitorExtensionVersionParseContext bool parsedOK; } MonitorExtensionVersionParseContext; +typedef struct FsmReachabilityParseContext +{ + char sqlstate[SQLSTATE_LENGTH]; + FsmReachabilityResult *result; + bool parsedOK; +} FsmReachabilityParseContext; + static bool parseNode(PGresult *result, int rowNumber, NodeAddress *node); static void parseNodeResult(void *ctx, PGresult *result); @@ -134,6 +141,7 @@ static void printFormationSettings(void *ctx, PGresult *result); static void printFormationURI(void *ctx, PGresult *result); static void parseCoordinatorNode(void *ctx, PGresult *result); static void parseExtensionVersion(void *ctx, PGresult *result); +static void parseFsmReachabilityResult(void *ctx, PGresult *result); static bool prepare_connection_to_current_system_user(Monitor *source, Monitor *target); @@ -1232,6 +1240,102 @@ monitor_report_timeline_history(Monitor *monitor, int64_t nodeId, } +/* + * monitor_check_fsm_reachability sends this node's own KeeperFSM[] edges + * (as encoded by KeeperFSMToJSON()) to the monitor's + * pgautofailover.check_fsm_reachability(jsonb), and fills in result with + * every monitor FSM transition (MonitorFSM[], group_state_machine.c) that + * has no matching keeper edge. An empty result (result->count == 0) means + * every transition the monitor can ever assign has somewhere for this + * keeper to go. Called from "pg_autoctl inspect fsm check" + * (cli_do_fsm.c). + */ +bool +monitor_check_fsm_reachability(Monitor *monitor, + const char *keeperEdgesJSON, + FsmReachabilityResult *result) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pos, current_state, assigned_state, comment " + " FROM pgautofailover.check_fsm_reachability($1::jsonb)"; + + int paramCount = 1; + Oid paramTypes[1] = { TEXTOID }; + const char *paramValues[1]; + + FsmReachabilityParseContext context = { { 0 }, result, false }; + + paramValues[0] = keeperEdgesJSON; + + if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, + paramValues, + &context, parseFsmReachabilityResult)) + { + log_error("Failed to check FSM reachability against the monitor"); + + return false; + } + + return context.parsedOK; +} + + +/* + * parseFsmReachabilityResult parses the result of + * pgautofailover.check_fsm_reachability(jsonb) into the given + * FsmReachabilityResult. + */ +static void +parseFsmReachabilityResult(void *ctx, PGresult *result) +{ + FsmReachabilityParseContext *context = (FsmReachabilityParseContext *) ctx; + FsmReachabilityResult *fsmResult = context->result; + + int nTuples = PQntuples(result); + + if (nTuples > FSM_REACHABILITY_MISMATCH_MAX_COUNT) + { + log_error("Query returned %d rows, pg_auto_failover supports only up " + "to %d FSM reachability mismatches at the moment", + nTuples, FSM_REACHABILITY_MISMATCH_MAX_COUNT); + context->parsedOK = false; + return; + } + + if (PQnfields(result) != 4) + { + log_error("Query returned %d columns, expected 4", PQnfields(result)); + context->parsedOK = false; + return; + } + + fsmResult->count = nTuples; + + for (int i = 0; i < nTuples; i++) + { + FsmReachabilityMismatch *mismatch = &(fsmResult->mismatches[i]); + + if (!stringToInt(PQgetvalue(result, i, 0), &mismatch->pos)) + { + log_error("Failed to parse FSM reachability pos \"%s\"", + PQgetvalue(result, i, 0)); + context->parsedOK = false; + return; + } + + strlcpy(mismatch->currentState, PQgetvalue(result, i, 1), + sizeof(mismatch->currentState)); + strlcpy(mismatch->assignedState, PQgetvalue(result, i, 2), + sizeof(mismatch->assignedState)); + strlcpy(mismatch->comment, PQgetvalue(result, i, 3), + sizeof(mismatch->comment)); + } + + context->parsedOK = true; +} + + /* * monitor_accept_timeline pins the accepted timeline for a (formation, * group) after an operator has resolved a detected fork, by calling @@ -2804,7 +2908,7 @@ monitor_print_last_events(Monitor *monitor, char *formation, int group, int coun { sql = "SELECT eventTime, nodeid, groupid, " - " reportedstate, goalState, description " + " reportedstate, goalState, description, rule_pos " " FROM pgautofailover.last_events($1, count => $2)"; countStr = intToString(count); @@ -2822,7 +2926,7 @@ monitor_print_last_events(Monitor *monitor, char *formation, int group, int coun { sql = "SELECT eventTime, nodeid, groupid, " - " reportedstate, goalState, description " + " reportedstate, goalState, description, rule_pos " " FROM pgautofailover.last_events($1,$2,$3)"; countStr = intToString(count); @@ -2900,7 +3004,7 @@ monitor_print_last_events_as_json(Monitor *monitor, { sql = "SELECT jsonb_pretty(" "coalesce(jsonb_agg(row_to_json(event)), '[]'))" - " FROM * FROM pgautofailover.last_events($1,$2,$3) as event"; + " FROM pgautofailover.last_events($1,$2,$3) as event"; countStr = intToString(count); groupStr = intToString(group); @@ -2958,20 +3062,20 @@ printLastEvents(void *ctx, PGresult *result) log_trace("printLastEvents: %d tuples", nTuples); - if (PQnfields(result) != 6) + if (PQnfields(result) != 7) { - log_error("Query returned %d columns, expected 6", PQnfields(result)); + log_error("Query returned %d columns, expected 7", PQnfields(result)); context->parsedOK = false; return; } - fformat(stdout, "%30s | %6s | %19s | %19s | %s\n", + fformat(stdout, "%30s | %6s | %19s | %19s | %6s | %s\n", "Event Time", "Node", - "Current State", "Assigned State", "Comment"); - fformat(stdout, "%30s-+-%6s-+-%19s-+-%19s-+-%10s\n", + "Current State", "Assigned State", "Rule", "Comment"); + fformat(stdout, "%30s-+-%6s-+-%19s-+-%19s-+-%6s-+-%10s\n", "------------------------------", "------", "-------------------", - "-------------------", "----------"); + "-------------------", "------", "----------"); for (currentTupleIndex = 0; currentTupleIndex < nTuples; currentTupleIndex++) { @@ -2981,14 +3085,16 @@ printLastEvents(void *ctx, PGresult *result) char *currentState = PQgetvalue(result, currentTupleIndex, 3); char *goalState = PQgetvalue(result, currentTupleIndex, 4); char *description = PQgetvalue(result, currentTupleIndex, 5); + bool rulePosIsNull = PQgetisnull(result, currentTupleIndex, 6); + char *rulePos = rulePosIsNull ? "" : PQgetvalue(result, currentTupleIndex, 6); char node[BUFSIZE]; /* for our grid alignment output it's best to have a single col here */ sformat(node, BUFSIZE, "%s/%s", groupId, nodeId); - fformat(stdout, "%30s | %6s | %19s | %19s | %s\n", + fformat(stdout, "%30s | %6s | %19s | %19s | %6s | %s\n", eventTime, node, - currentState, goalState, description); + currentState, goalState, rulePos, description); } fformat(stdout, "\n"); @@ -3028,7 +3134,7 @@ monitor_get_last_events(Monitor *monitor, char *formation, int group, int count, " reportedstate, goalState, " " reportedrepstate, reportedtli, reportedlsn, " " candidatepriority, replicationquorum, " - " description " + " description, rule_pos, rule_section " " FROM pgautofailover.last_events($1, count => $2)"; countStr = intToString(count); @@ -3047,10 +3153,11 @@ monitor_get_last_events(Monitor *monitor, char *formation, int group, int count, sql = "SELECT eventId, to_char(eventTime, 'YYYY-MM-DD HH24:MI:SS'), " " formationId, nodeid, groupid, " + " nodename, nodehost, nodeport, " " reportedstate, goalState, " " reportedrepstate, reportedtli, reportedlsn, " " candidatepriority, replicationquorum, " - " description " + " description, rule_pos, rule_section " " FROM pgautofailover.last_events($1,$2,$3)"; countStr = intToString(count); @@ -3115,9 +3222,9 @@ getLastEvents(void *ctx, PGresult *result) return; } - if (PQnfields(result) != 16) + if (PQnfields(result) != 18) { - log_error("Query returned %d columns, expected 16", PQnfields(result)); + log_error("Query returned %d columns, expected 18", PQnfields(result)); context->parsedOK = false; return; } @@ -3234,6 +3341,27 @@ getLastEvents(void *ctx, PGresult *result) value = PQgetvalue(result, currentTupleIndex, 15); strlcpy(event->description, value, sizeof(event->description)); + /* rule_pos: NULL means "no rule attributed", represented as 0 */ + if (PQgetisnull(result, currentTupleIndex, 16)) + { + event->rulePos = 0; + event->ruleSection[0] = '\0'; + } + else + { + value = PQgetvalue(result, currentTupleIndex, 16); + + if (!stringToInt(value, &(event->rulePos))) + { + log_error("Invalid rule_pos \"%s\" returned by monitor", value); + ++errors; + } + + /* rule_section, only meaningful alongside a real rule_pos */ + value = PQgetvalue(result, currentTupleIndex, 17); + strlcpy(event->ruleSection, value, sizeof(event->ruleSection)); + } + if (errors > 0) { context->parsedOK = false; @@ -5214,6 +5342,37 @@ monitor_extension_update(Monitor *monitor, const char *targetVersion) } } + /* + * Same story as btree_gist above, this time for ltree: version 2.3 added + * it to control's "requires" (pgautofailover.fsm's section_path column + * is cast to ltree). ALTER EXTENSION ... UPDATE checks "requires" against + * what's already installed before ever running the upgrade script body, + * so putting a "CREATE EXTENSION IF NOT EXISTS ltree" inside + * pgautofailover--2.2--2.3.sql itself is too late: the ALTER EXTENSION + * statement already failed by the time that script would run. + */ + if (targetVersionNum >= 203) + { + char *ltreeExtName = "ltree"; + + if (!find_extension_control_file(monitor->config.pgSetup.pg_ctl, + ltreeExtName)) + { + log_warn("Failed to find extension control file for \"%s\"", + ltreeExtName); + log_info("You might have to install a PostgreSQL contrib package"); + } + + if (!pgsql_create_extension(pgsql, ltreeExtName)) + { + log_error("Failed to create extension \"%s\" " + "required by \"%s\" extension version 2.3", + ltreeExtName, + PG_AUTOCTL_MONITOR_EXTENSION_NAME); + return false; + } + } + return pgsql_alter_extension_update_to(pgsql, PG_AUTOCTL_MONITOR_EXTENSION_NAME, targetVersion); diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index dec18add9..3b0fbe770 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -68,6 +68,17 @@ typedef struct MonitorEvent int candidatePriority; bool replicationQuorum; char description[BUFSIZE]; + + /* + * Which MonitorFSM[] row (if any) produced this event -- see + * pgautofailover.event's own rule_pos/rule_section comment + * (pgautofailover.sql) for exactly what NULL does and doesn't mean. + * rulePos 0 here represents SQL NULL (matches the monitor-side + * CurrentMonitorFSMRulePos convention: 0 is never a real row position), + * in which case ruleSection is left empty. + */ + int rulePos; + char ruleSection[NAMEDATALEN]; } MonitorEvent; #define EVENTS_ARRAY_MAX_COUNT 1024 @@ -84,6 +95,29 @@ typedef struct MonitorExtensionVersion char installedVersion[BUFSIZE]; } MonitorExtensionVersion; +/* + * One row per monitor FSM transition (MonitorFSM[], group_state_machine.c) + * with no matching edge in this node's own KeeperFSM[] -- the result of + * "pg_autoctl inspect fsm check" (see monitor_check_fsm_reachability() and + * pgautofailover.check_fsm_reachability(jsonb)). Fixed-capacity, same + * pattern as MonitorEventsArray above. + */ +#define FSM_REACHABILITY_MISMATCH_MAX_COUNT 1024 + +typedef struct FsmReachabilityMismatch +{ + int pos; + char currentState[NAMEDATALEN]; + char assignedState[NAMEDATALEN]; + char comment[BUFSIZE]; +} FsmReachabilityMismatch; + +typedef struct FsmReachabilityResult +{ + int count; + FsmReachabilityMismatch mismatches[FSM_REACHABILITY_MISMATCH_MAX_COUNT]; +} FsmReachabilityResult; + typedef struct CoordinatorNodeAddress { bool found; @@ -167,6 +201,9 @@ bool monitor_report_postgres_version(Monitor *monitor, int64_t nodeId, PostgresVersionInfo *pgVersion); bool monitor_report_timeline_history(Monitor *monitor, int64_t nodeId, const char *historyJSON); +bool monitor_check_fsm_reachability(Monitor *monitor, + const char *keeperEdgesJSON, + FsmReachabilityResult *result); bool monitor_accept_timeline(Monitor *monitor, char *formation, int group, int tli, char *decidedBy); bool monitor_print_timeline(Monitor *monitor, char *formation, int group); diff --git a/src/bin/pg_autoctl/service_keeper.c b/src/bin/pg_autoctl/service_keeper.c index 3b0569e81..f67a7d715 100644 --- a/src/bin/pg_autoctl/service_keeper.c +++ b/src/bin/pg_autoctl/service_keeper.c @@ -1305,7 +1305,6 @@ keeper_suspended_loop(Keeper *keeper, pid_t start_pid) static bool service_keeper_node_active(Keeper *keeper, bool doInit) { - KeeperConfig *config = &(keeper->config); KeeperStateData *keeperState = &(keeper->state); MonitorAssignedState assignedState = { 0 }; @@ -1362,36 +1361,13 @@ service_keeper_node_active(Keeper *keeper, bool doInit) /* * Also update the groupId and replication slot name in the - * configuration file. + * configuration file, if the monitor's own view of who we are has + * drifted from ours. */ - char expectedSlotName[BUFSIZE] = { 0 }; - - (void) postgres_sprintf_replicationSlotName(assignedState.nodeId, - expectedSlotName, - sizeof(expectedSlotName)); - - if (assignedState.groupId != config->groupId || - strneq(config->replication_slot_name, expectedSlotName)) + if (!keeper_maybe_update_group_and_slot(keeper, &assignedState)) { - bool postgresNotRunningIsOk = false; - - if (!keeper_config_update(config, - assignedState.nodeId, - assignedState.groupId)) - { - log_error("Failed to update the configuration file " - "with groupId %d and replication.slot \"%s\"", - assignedState.groupId, expectedSlotName); - return false; - } - - if (!keeper_ensure_configuration(keeper, postgresNotRunningIsOk)) - { - log_error("Failed to update our Postgres configuration " - "after a change of groupId or " - "replication slot name, see above for details"); - return false; - } + /* errors have already been logged */ + return false; } return true; diff --git a/src/bin/pg_autoctl/watch.c b/src/bin/pg_autoctl/watch.c index 3fccaf927..9426d107d 100644 --- a/src/bin/pg_autoctl/watch.c +++ b/src/bin/pg_autoctl/watch.c @@ -1265,6 +1265,10 @@ compute_events_sizes(WatchContext *context) int timeSize = 19; /* "YYYY-MM-DD HH:MI:SS" is 19 chars long */ int descSize = 60; /* desc. has horizontal scrolling */ + /* rulePos 0 means "no rule attributed" (blank), not a real width */ + int ruleSize = + event->rulePos > 0 ? ((int) log10(event->rulePos) + 1) : 1; + if (headers->maxEventIdSize < idSize) { headers->maxEventIdSize = idSize; @@ -1280,6 +1284,17 @@ compute_events_sizes(WatchContext *context) headers->maxEventNodeNameSize = nameSize; } + /* "Rule" (the column header) is 4 chars, never shrink below that */ + if (headers->maxEventRulePosSize < 4) + { + headers->maxEventRulePosSize = 4; + } + + if (headers->maxEventRulePosSize < ruleSize) + { + headers->maxEventRulePosSize = ruleSize; + } + if (headers->maxEventDescSize < descSize) { headers->maxEventDescSize = descSize; @@ -1315,6 +1330,11 @@ compute_event_column_size(EventColumnType type, MonitorEventsHeaders *headers) return headers->maxEventNodeNameSize; } + case EVENT_COLUMN_TYPE_RULE_POS: + { + return headers->maxEventRulePosSize; + } + case EVENT_COLUMN_TYPE_DESCRIPTION: { return headers->maxEventDescSize; @@ -1449,6 +1469,19 @@ print_event(WatchContext *context, EventColPolicy *policy, int index, int r, int break; } + case EVENT_COLUMN_TYPE_RULE_POS: + { + if (event->rulePos > 0) + { + mvprintw(r, cc, "%*d", len, event->rulePos); + } + else + { + mvprintw(r, cc, "%*s", len, ""); + } + break; + } + case EVENT_COLUMN_TYPE_DESCRIPTION: { char *text = event->description; diff --git a/src/bin/pg_autoctl/watch.h b/src/bin/pg_autoctl/watch.h index f5c119548..0a2e1e748 100644 --- a/src/bin/pg_autoctl/watch.h +++ b/src/bin/pg_autoctl/watch.h @@ -38,6 +38,7 @@ typedef struct MonitorEventsHeaders int maxEventIdSize; int maxEventTimeSize; int maxEventNodeNameSize; + int maxEventRulePosSize; int maxEventDescSize; } MonitorEventsHeaders; diff --git a/src/bin/pg_autoctl/watch_colspecs.h b/src/bin/pg_autoctl/watch_colspecs.h index f3726581f..7b411e108 100644 --- a/src/bin/pg_autoctl/watch_colspecs.h +++ b/src/bin/pg_autoctl/watch_colspecs.h @@ -226,6 +226,7 @@ typedef enum EVENT_COLUMN_TYPE_LSN, EVENT_COLUMN_TYPE_CANDIDATE_PRIORITY, EVENT_COLUMN_TYPE_REPLICATION_QUORUM, + EVENT_COLUMN_TYPE_RULE_POS, EVENT_COLUMN_TYPE_DESCRIPTION, EVENT_COLUMN_TYPE_LAST @@ -291,6 +292,7 @@ EventColPolicy EventColumnPolicies[] = { { EVENT_COLUMN_TYPE_ID, "Id", 0 }, { EVENT_COLUMN_TYPE_TIME, "Event Time", 0 }, { EVENT_COLUMN_TYPE_NODE_NAME, "Name", 0 }, + { EVENT_COLUMN_TYPE_RULE_POS, "Rule", 0 }, { EVENT_COLUMN_TYPE_DESCRIPTION, "Description", 0 }, { EVENT_COLUMN_TYPE_LAST, "", 0 } } diff --git a/src/monitor/Makefile b/src/monitor/Makefile index 43f1f85c8..a86986656 100644 --- a/src/monitor/Makefile +++ b/src/monitor/Makefile @@ -14,7 +14,7 @@ MODULE_big = $(EXTENSION) OBJS = $(patsubst ${SRC_DIR}%.c,%.o,$(wildcard ${SRC_DIR}*.c)) PG_CPPFLAGS = -Wall -Werror -Wno-unused-parameter -Iinclude -I$(libpq_srcdir) -g SHLIB_LINK = $(libpq) -REGRESS = create_extension monitor workers node_active_protocol guard_data_loss fast_forward drop_node stale_primary_report lock_and_fetch_migration timeline_fork_detection dummy_update drop_extension upgrade +REGRESS = create_extension monitor workers node_active_protocol guard_data_loss fast_forward drop_node stale_primary_report candidate_count_gate lock_and_fetch_migration timeline_fork_detection dummy_update drop_extension upgrade ISOLATION = concurrent_remove_node concurrent_remove_standby_and_primary_report concurrent_remove_standby_and_standby_report concurrent_second_primary_death_report concurrent_health_check_and_report concurrent_candidate_priority_and_quorum PG_CONFIG ?= pg_config diff --git a/src/monitor/expected/candidate_count_gate.out b/src/monitor/expected/candidate_count_gate.out new file mode 100644 index 000000000..9cfa786d7 --- /dev/null +++ b/src/monitor/expected/candidate_count_gate.out @@ -0,0 +1,390 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Regression test for ProceedGroupStateForMSFailover's candidateCount == 0 +-- gate (reporting_node.ms_failover.promotion_outcome.candidate_count_gate in +-- MonitorFSM[]): the window where the primary has gone unhealthy but NOT A +-- SINGLE standby has yet reported reaching report_lsn. +-- +-- Unlike the other two counting gates -- missingNodesCount (guard_data_loss.sql) +-- and quorumCandidateCount (stale_primary_report.sql), both named explicitly +-- in those files' own header comments -- no existing test exercises this one +-- by name. It has no dedicated log message either (the original hand-written +-- code silently `return`s false here, and the declarative row that now +-- matches this same condition carries no extraAction, matching that exactly) +-- so there is no pgautofailover.event row to check for it; the only +-- observable effect is what does NOT happen: neither standby should reach +-- prepare_promotion/fast_forward this round. +-- +-- guard_data_loss is set to false: with the default (true), the +-- missingNodesCount > 0 gate above this one in ProceedGroupStateForMSFailover +-- would itself decline and return before ever reaching this gate, since both +-- standbys are still SECONDARY/CATCHINGUP (each counted as missing, per +-- BuildCandidateList's own fan-out branch) at the moment this test polls +-- them. +-- +-- startup_grace_period is also lowered to 1, same as guard_data_loss.sql/ +-- fast_forward.sql/stale_primary_report.sql: NodeIsUnhealthy() only honors a +-- BAD health reading once at least this many seconds have passed since the +-- monitor process itself started (PgStartTime), to avoid spurious failovers +-- right after the monitor restarts. The default (10s) is longer than this +-- whole schedule takes to reach this test file when run automated, which +-- would silently make p never register as unhealthy and this test's own +-- ProceedGroupStateForMSFailover call never even fire. +\x on +-- ── formation and node registration ───────────────────────────────────────── +SELECT pgautofailover.create_formation('ccg_test', 'pgsql', 'postgres', true, 1); +-[ RECORD 1 ]----+------------------------------ +create_formation | (ccg_test,pgsql,postgres,t,1) + +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_p', 5432, + 'postgres', 'ccg_p', 1); +-[ RECORD 1 ]---------------+------- +assigned_node_id | 21 +assigned_group_id | 0 +assigned_group_state | single +assigned_candidate_priority | 100 +assigned_replication_quorum | t +assigned_node_name | ccg_p + +SELECT nodeid AS np FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p' \gset +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_s1', 5432, + 'postgres', 'ccg_s1', 1); +-[ RECORD 1 ]---------------+------------- +assigned_node_id | 22 +assigned_group_id | 0 +assigned_group_state | wait_standby +assigned_candidate_priority | 100 +assigned_replication_quorum | t +assigned_node_name | ccg_s1 + +SELECT nodeid AS ns1 FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_s1' \gset +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_s2', 5432, + 'postgres', 'ccg_s2', 1); +-[ RECORD 1 ]---------------+------------- +assigned_node_id | 23 +assigned_group_id | 0 +assigned_group_state | wait_standby +assigned_candidate_priority | 100 +assigned_replication_quorum | t +assigned_node_name | ccg_s2 + +SELECT nodeid AS ns2 FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_s2' \gset +-- ── bootstrap: drive the FSM to primary + secondary + secondary ───────────── +-- Same sequence as guard_data_loss.sql's own bootstrap. +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'single'); +-[ RECORD 1 ]--------+------- +assigned_group_state | single + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'wait_standby'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_standby + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'single', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'wait_standby'); +-[ RECORD 1 ]--------+----------- +assigned_group_state | catchingup + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+-------- +assigned_group_state | primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+-------- +assigned_group_state | primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'wait_standby'); +-[ RECORD 1 ]--------+----------- +assigned_group_state | catchingup + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+----------- +assigned_group_state | catchingup + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+----------- +assigned_group_state | catchingup + +-- p: primary (refresh to pick up second secondary) +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+--------------- +assigned_group_state | apply_settings + +-- Verify bootstrap: p=primary, s1=secondary, s2=secondary. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; +-[ RECORD 1 ]-+--------------- +nodename | ccg_p +goalstate | apply_settings +reportedstate | primary +-[ RECORD 2 ]-+--------------- +nodename | ccg_s1 +goalstate | secondary +reportedstate | secondary +-[ RECORD 3 ]-+--------------- +nodename | ccg_s2 +goalstate | catchingup +reportedstate | secondary + +-- ── manufacture: primary unhealthy, NEITHER standby has reported yet ──────── +-- +-- p goes unhealthy and is demoted to draining/draining (same manufactured +-- shape as guard_data_loss.sql/stale_primary_report.sql). s1/s2 are left +-- exactly as bootstrap left them -- secondary/secondary, neither has been +-- assigned report_lsn yet -- so candidateCount is 0 for both of them: no +-- standby has reached report_lsn, the exact window this gate covers. +SET pgautofailover.startup_grace_period = 1; +SET pgautofailover.guard_data_loss TO false; +UPDATE pgautofailover.node + SET health = 0, + healthchecktime = now(), + reporttime = now() - interval '60 seconds' + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p'; +UPDATE pgautofailover.node + SET goalstate = 'draining', reportedstate = 'draining' + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p'; +-- Verify the manufactured state before the test call. +SELECT nodename, goalstate, reportedstate, health + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; +-[ RECORD 1 ]-+----------- +nodename | ccg_p +goalstate | draining +reportedstate | draining +health | 0 +-[ RECORD 2 ]-+----------- +nodename | ccg_s1 +goalstate | secondary +reportedstate | secondary +health | -1 +-[ RECORD 3 ]-+----------- +nodename | ccg_s2 +goalstate | catchingup +reportedstate | secondary +health | -1 + +-- ── test: poll exactly one secondary, candidateCount == 0 for both ────────── +-- +-- s1 reports secondary/0-5000 again (nothing new from its own point of +-- view). This drives ProceedGroupState(s1) -> ActionRunMultiStandbyFailover +-- Cascade -> ProceedGroupStateForMSFailover, which runs BuildCandidateList +-- over the WHOLE group (not just s1): both s1 and s2 are still SECONDARY/ +-- CATCHINGUP, so BOTH get fanned out to report_lsn in this same call (the +-- fan-out rows, already covered elsewhere) -- but candidateCount is 0 (no +-- node's reportedState is report_lsn yet), so the candidate_count_gate row +-- matches and the function returns false: no candidate is selected. +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +-- s1 and s2 must both have been fanned out to report_lsn (goalstate), but +-- NEITHER may have reached prepare_promotion/fast_forward: the +-- candidate_count_gate declined before any candidate could be selected. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; +-[ RECORD 1 ]-+----------- +nodename | ccg_p +goalstate | draining +reportedstate | draining +-[ RECORD 2 ]-+----------- +nodename | ccg_s1 +goalstate | report_lsn +reportedstate | secondary +-[ RECORD 3 ]-+----------- +nodename | ccg_s2 +goalstate | report_lsn +reportedstate | secondary + +RESET pgautofailover.guard_data_loss; +RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('ccg_test', count => 100); +-[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "single" +-[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "wait_standby" +-[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "wait_primary" +-[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "catchingup" +-[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "secondary" +-[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "primary" +-[ RECORD 12 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "wait_standby" +-[ RECORD 13 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "catchingup" +-[ RECORD 16 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "secondary" +-[ RECORD 17 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | report_lsn +rule_pos | 367 +rule_section | reporting_node +description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) +-[ RECORD 18 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | report_lsn +rule_pos | 367 +rule_section | reporting_node +description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) + diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out new file mode 100644 index 000000000..467daebb8 --- /dev/null +++ b/src/monitor/expected/check_fsm_reachability.out @@ -0,0 +1,111 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Exercises the mechanics of pgautofailover.check_fsm_reachability(jsonb) +-- (and the pgautofailover.dump_fsm_edges() it's built on) against small, +-- synthetic keeper-edge inputs -- not the real KeeperFSM[] table, which +-- lives in the pg_autoctl binary, not this database. The real, end-to-end +-- completeness check (does the real KeeperFSM[] cover every real +-- MonitorFSM[] edge) is run separately, live, via +-- "pg_autoctl inspect fsm check" against a real monitor+keeper pair -- this +-- test only confirms the SQL-side comparison mechanism itself behaves +-- correctly: an edge present in the keeper_edges parameter drops out of the +-- mismatch list, an edge absent from it stays in, and an unrecognized state +-- name fails loudly rather than silently never matching. +-- Every edge dump_fsm_edges() can produce, fully resolved: a fixed, +-- reviewable count for the current MonitorFSM[] -- changes only when a row +-- is added, removed, or edited there, same as fsm.sql's own row count. +SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); + total_edge_count +------------------ + 177 +(1 row) + +-- pos 301 ("converged secondary, reportedTLI not an ancestor of reference +-- -> catchingup", single edge) and pos 343 ("stop_replication, primary +-- converged prepare_maintenance -> wait_primary + maintenance", two edges, +-- one per role) are stable, non-reflexive, non-api_triggered rows -- good, +-- deterministic targets to check both single- and dual-edge rows against. +SELECT pos, current_state, assigned_state + FROM pgautofailover.dump_fsm_edges() + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + pos | current_state | assigned_state +-----+---------------------+---------------- + 301 | secondary | catchingup + 343 | stop_replication | wait_primary + 343 | prepare_maintenance | maintenance +(3 rows) + +-- Two categories of edges dump_fsm_edges() deliberately never produces, see +-- its own comment for why: reflexive (current == assigned) edges, and the +-- whole api_triggered section (every row there resolves activeNode to a +-- specific role via hand-written C before dispatch, so its own +-- NodeStatePattern was never meant to double as a full reachability +-- precondition). pos 403 ("all nodes async, zero secondaries -> +-- wait_primary") has both an ordinary edge (primary/apply_settings -> +-- wait_primary) and a reflexive one (wait_primary -> wait_primary) in its +-- own source pattern -- only the former should appear. pos 105 (api +-- triggered: perform_failover) should produce no edges at all. +SELECT pos, current_state, assigned_state + FROM pgautofailover.dump_fsm_edges() + WHERE pos = 403 + ORDER BY current_state; + pos | current_state | assigned_state +-----+----------------+---------------- + 403 | primary | wait_primary + 403 | apply_settings | wait_primary +(2 rows) + +SELECT count(*) AS api_triggered_edge_count + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE f.section LIKE 'api_triggered%'; + api_triggered_edge_count +-------------------------- + 0 +(1 row) + +-- An empty keeper_edges: every single edge dump_fsm_edges() produces comes +-- back as a mismatch, so this count must equal total_edge_count above. +SELECT count(*) AS missing_with_empty_keeper_edges + FROM pgautofailover.check_fsm_reachability('[]'::jsonb); + missing_with_empty_keeper_edges +--------------------------------- + 177 +(1 row) + +-- Providing exactly pos 301's own edge, plus one of pos 343's two edges +-- (leaving its other edge, prepare_maintenance->maintenance, still +-- unmatched): pos 301 must disappear entirely from the mismatch list, pos +-- 343 must still appear, but only once. +SELECT pos, current_state, assigned_state + FROM pgautofailover.check_fsm_reachability( + '[{"current":"secondary","assigned":"catchingup"}, + {"current":"stop_replication","assigned":"wait_primary"}]'::jsonb) + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + pos | current_state | assigned_state +-----+---------------------+---------------- + 343 | prepare_maintenance | maintenance +(1 row) + +-- Providing both of pos 343's edges too: it must now disappear as well. +SELECT pos, current_state, assigned_state + FROM pgautofailover.check_fsm_reachability( + '[{"current":"secondary","assigned":"catchingup"}, + {"current":"stop_replication","assigned":"wait_primary"}, + {"current":"prepare_maintenance","assigned":"maintenance"}]'::jsonb) + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + pos | current_state | assigned_state +-----+---------------+---------------- +(0 rows) + +-- An unrecognized state name in keeper_edges must fail loudly (a real cast +-- error), not silently never match -- the same "fail loudly on drift" +-- instinct as AssignDeclaredGoalState's own trust-check. +SELECT * FROM pgautofailover.check_fsm_reachability( + '[{"current":"not_a_real_state","assigned":"catchingup"}]'::jsonb); +ERROR: invalid input value for enum pgautofailover.replication_state: "not_a_real_state" +CONTEXT: SQL function "check_fsm_reachability" statement 1 diff --git a/src/monitor/expected/cluster_init_failover_rule_attribution.out b/src/monitor/expected/cluster_init_failover_rule_attribution.out new file mode 100644 index 000000000..95c2c3e11 --- /dev/null +++ b/src/monitor/expected/cluster_init_failover_rule_attribution.out @@ -0,0 +1,277 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- End-to-end demonstration of the rule_pos/rule_section attribution +-- mechanism (notifications.c's CurrentMonitorFSMRulePos/RuleSection, +-- InsertEvent()): registers a two-node formation, drives it through the +-- heartbeat-only bootstrap to primary + secondary, triggers a manual +-- perform_failover(), and then joins pgautofailover.event against +-- pgautofailover.fsm on rule_pos = pos to show, for every state +-- transition the monitor produced, exactly which MonitorFSM[] row was +-- selected and executed -- both for the ordinary heartbeat-driven +-- bootstrap rows (api_triggered = false in this join, since rule_pos is +-- only set for rows actually reached through the declarative dispatch +-- table) and for the operator-triggered perform_failover call itself. +\x on +-- ── formation and node registration ───────────────────────────────────────── +SELECT pgautofailover.create_formation('cifra_test', 'pgsql', 'postgres', true, 0); +-[ RECORD 1 ]----+-------------------------------- +create_formation | (cifra_test,pgsql,postgres,t,0) + +SELECT * + FROM pgautofailover.register_node('cifra_test', 'cifra_p', 5432, + 'postgres', 'cifra_p', 1); +-[ RECORD 1 ]---------------+-------- +assigned_node_id | 36 +assigned_group_id | 0 +assigned_group_state | single +assigned_candidate_priority | 100 +assigned_replication_quorum | t +assigned_node_name | cifra_p + +SELECT nodeid AS np FROM pgautofailover.node + WHERE formationid = 'cifra_test' AND nodename = 'cifra_p' \gset +SELECT * + FROM pgautofailover.register_node('cifra_test', 'cifra_s', 5432, + 'postgres', 'cifra_s', 1); +-[ RECORD 1 ]---------------+------------- +assigned_node_id | 37 +assigned_group_id | 0 +assigned_group_state | wait_standby +assigned_candidate_priority | 100 +assigned_replication_quorum | t +assigned_node_name | cifra_s + +SELECT nodeid AS ns FROM pgautofailover.node + WHERE formationid = 'cifra_test' AND nodename = 'cifra_s' \gset +-- ── bootstrap: drive the FSM to primary + secondary ───────────────────────── +-- +-- Mirrors drop_node.sql's bootstrap sequence (register -> single -> +-- wait_primary -> [standby: wait_standby -> catchingup -> secondary] -> +-- primary), including its "confirm" round-trips. +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'single'); +-[ RECORD 1 ]--------+------- +assigned_group_state | single + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'wait_standby'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_standby + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'single', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'wait_standby'); +-[ RECORD 1 ]--------+----------- +assigned_group_state | catchingup + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+-------- +assigned_group_state | primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+-------- +assigned_group_state | primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +-- Verify bootstrap: p=primary, s=secondary. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'cifra_test' + ORDER BY nodename; +-[ RECORD 1 ]-+---------- +nodename | cifra_p +goalstate | primary +reportedstate | primary +-[ RECORD 2 ]-+---------- +nodename | cifra_s +goalstate | secondary +reportedstate | secondary + +-- ── manual failover ────────────────────────────────────────────────────── +-- +-- Two-node group: dispatches through MonitorFSM[]'s API_TRIGGERED section +-- (pos 105, "manual failover, 2-node group, primary+standby both converged +-- -> standby prepare_promotion, primary draining"), attributing both +-- resulting event rows to that one rule. +SELECT pgautofailover.perform_failover('cifra_test', 0); +-[ RECORD 1 ]----+- +perform_failover | + +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'cifra_test' + ORDER BY nodename; +-[ RECORD 1 ]-+------------------ +nodename | cifra_p +goalstate | draining +reportedstate | primary +-[ RECORD 2 ]-+------------------ +nodename | cifra_s +goalstate | prepare_promotion +reportedstate | secondary + +-- ── which rule fired for which event? ─────────────────────────────────────── +-- +-- Every event row this formation produced, joined against the FSM table on +-- rule_pos = pos: rule_pos/rule_section are NULL for the ordinary +-- heartbeat-driven bootstrap transitions above whenever the matched row +-- happens to be identified only by array position in earlier sessions' +-- tests -- here every one of them was reached through the same declarative +-- MonitorFSM[] dispatch table, so each carries its own attribution too. The +-- final two rows (both attributed to pos 105) are the perform_failover() +-- call's own dual assignment (standby -> prepare_promotion, primary -> +-- draining), selected and executed from the API_TRIGGERED section. +SELECT e.eventid, e.nodename, e.reportedstate, e.goalstate, + e.rule_pos, e.rule_section, f.comment AS rule_comment + FROM pgautofailover.event e + LEFT JOIN pgautofailover.fsm f ON f.pos = e.rule_pos + WHERE e.formationid = 'cifra_test' + ORDER BY e.eventid; +-[ RECORD 1 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 204 +nodename | cifra_p +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +rule_comment | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 205 +nodename | cifra_p +reportedstate | single +goalstate | single +rule_pos | +rule_section | +rule_comment | +-[ RECORD 3 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 206 +nodename | cifra_s +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +rule_comment | +-[ RECORD 4 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 207 +nodename | cifra_p +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +rule_comment | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 208 +nodename | cifra_p +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +rule_comment | +-[ RECORD 6 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 209 +nodename | cifra_s +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +rule_comment | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 210 +nodename | cifra_s +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +rule_comment | +-[ RECORD 8 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 211 +nodename | cifra_s +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +rule_comment | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 212 +nodename | cifra_s +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +rule_comment | +-[ RECORD 10 ]+------------------------------------------------------------------------------------------------------------- +eventid | 213 +nodename | cifra_p +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +rule_comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+------------------------------------------------------------------------------------------------------------- +eventid | 214 +nodename | cifra_p +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +rule_comment | +-[ RECORD 12 ]+------------------------------------------------------------------------------------------------------------- +eventid | 215 +nodename | cifra_s +reportedstate | secondary +goalstate | prepare_promotion +rule_pos | 105 +rule_section | api_triggered +rule_comment | manual failover, 2-node group, primary+standby both converged -> standby prepare_promotion, primary draining +-[ RECORD 13 ]+------------------------------------------------------------------------------------------------------------- +eventid | 216 +nodename | cifra_p +reportedstate | primary +goalstate | draining +rule_pos | 105 +rule_section | api_triggered +rule_comment | manual failover, 2-node group, primary+standby both converged -> standby prepare_promotion, primary draining + diff --git a/src/monitor/expected/create_extension.out b/src/monitor/expected/create_extension.out index 083b3077a..63e2e4d17 100644 --- a/src/monitor/expected/create_extension.out +++ b/src/monitor/expected/create_extension.out @@ -2,3 +2,4 @@ -- Licensed under the PostgreSQL License. create extension pgautofailover cascade; NOTICE: installing required extension "btree_gist" +NOTICE: installing required extension "ltree" diff --git a/src/monitor/expected/drop_node.out b/src/monitor/expected/drop_node.out index 5bf508602..38eaeda90 100644 --- a/src/monitor/expected/drop_node.out +++ b/src/monitor/expected/drop_node.out @@ -238,3 +238,99 @@ nodename | dn_p goalstate | single reportedstate | primary +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('dn_test', count => 100); +-[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 16 "dn_p" (dn_p:5432): "single" +-[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 17 "dn_s" (dn_s:5432): "wait_standby" +-[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 16 "dn_p" (dn_p:5432): "wait_primary" +-[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 17 "dn_s" (dn_s:5432): "catchingup" +-[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 17 "dn_s" (dn_s:5432): "secondary" +-[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 16 "dn_p" (dn_p:5432): "primary" +-[ RECORD 12 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | dropped +rule_pos | 103 +rule_section | api_triggered +description | remove_node, removed node cannot take writes -> dropped +-[ RECORD 13 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 14 ]+-------------------------------------------------------------------------------------------- +reportedstate | dropped +goalstate | dropped +rule_pos | +rule_section | +description | New state is reported by node 17 "dn_s" (dn_s:5432): "dropped" + diff --git a/src/monitor/expected/failover_candidate_leaves_secondary.out b/src/monitor/expected/failover_candidate_leaves_secondary.out index 10798ed36..3d7895ca2 100644 --- a/src/monitor/expected/failover_candidate_leaves_secondary.out +++ b/src/monitor/expected/failover_candidate_leaves_secondary.out @@ -52,7 +52,7 @@ SELECT * FROM pgautofailover.register_node('fclma_test', 'fclma_p', 5432, 'postgres', 'fclma_p', 1); -[ RECORD 1 ]---------------+-------- -assigned_node_id | 31 +assigned_node_id | 34 assigned_group_id | 0 assigned_group_state | single assigned_candidate_priority | 100 @@ -65,7 +65,7 @@ SELECT * FROM pgautofailover.register_node('fclma_test', 'fclma_s', 5432, 'postgres', 'fclma_s', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 32 +assigned_node_id | 35 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -182,3 +182,81 @@ nodename | fclma_s reportedstate | prepare_promotion goalstate | prepare_promotion +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('fclma_test', count => 100); +-[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 34 "fclma_p" (fclma_p:5432): "single" +-[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 35 "fclma_s" (fclma_s:5432): "wait_standby" +-[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 34 "fclma_p" (fclma_p:5432): "wait_primary" +-[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 35 "fclma_s" (fclma_s:5432): "catchingup" +-[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 35 "fclma_s" (fclma_s:5432): "secondary" +-[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 34 "fclma_p" (fclma_p:5432): "primary" + diff --git a/src/monitor/expected/fast_forward.out b/src/monitor/expected/fast_forward.out index e39f554ed..6b6bd878b 100644 --- a/src/monitor/expected/fast_forward.out +++ b/src/monitor/expected/fast_forward.out @@ -385,3 +385,117 @@ SELECT node_name, node_lsn, node_is_primary -- ── cleanup ─────────────────────────────────────────────────────────────────── RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('ff_test', count => 100); +-[ RECORD 1 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "single" +-[ RECORD 3 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "wait_standby" +-[ RECORD 4 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "wait_primary" +-[ RECORD 6 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "catchingup" +-[ RECORD 8 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "secondary" +-[ RECORD 10 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "primary" +-[ RECORD 12 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "wait_standby" +-[ RECORD 13 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "catchingup" +-[ RECORD 16 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "secondary" +-[ RECORD 17 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | report_lsn +goalstate | report_lsn +rule_pos | 363 +rule_section | reporting_node +description | MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, guard_data_loss=true -> report_lsn (retry once a source recovers) + diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out new file mode 100644 index 000000000..f0040eb11 --- /dev/null +++ b/src/monitor/expected/fsm.out @@ -0,0 +1,1238 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Plain dump of the monitor's declarative dispatch table, via the +-- pgautofailover.fsm view (pgautofailover.dump_fsm() ordered by pos). This +-- is a static, compile-time-fixed table -- unaffected by any node/formation +-- state -- so its expected output changes only when a row is added, +-- removed, or edited in MonitorFSM[] (group_state_machine.c), giving that +-- change an explicit, reviewable regression diff. +-- +-- \x on: with the *_conditions columns added, a plain tabular row is far +-- wider than a terminal (or this file's own diff-ability), and reads far +-- worse than one field-per-line. +\x on +SELECT pos, section, section_path, + active_node_current_state, other_node_current_state, candidate_node_current_state, + active_node_conditions, other_node_conditions, candidate_node_conditions, + group_conditions, + active_node_assigned_state, other_node_assigned_state, has_extra_action, + comment + FROM pgautofailover.fsm + ORDER BY pos; +-[ RECORD 1 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 101 +section | api_triggered: remove_node +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | canTakeWrites=true +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | dropped +other_node_assigned_state | report_lsn +has_extra_action | f +comment | remove_node, removed node can take writes -> dropped, every surviving non-maintenance standby joins report_lsn +-[ RECORD 2 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 103 +section | api_triggered: remove_node +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | dropped +other_node_assigned_state | +has_extra_action | f +comment | remove_node, removed node cannot take writes -> dropped +-[ RECORD 3 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 105 +section | api_triggered: perform_failover +section_path | api_triggered +active_node_current_state | secondary +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyTwoNodes=true +active_node_assigned_state | prepare_promotion +other_node_assigned_state | draining +has_extra_action | f +comment | manual failover, 2-node group, primary+standby both converged -> standby prepare_promotion, primary draining +-[ RECORD 4 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 107 +section | api_triggered: perform_failover +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isInPrimaryState=true +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasMoreThanTwoNodes=true +active_node_assigned_state | draining +other_node_assigned_state | +has_extra_action | f +comment | manual failover, >2-node group -> primary drains, election proceeds via the heartbeat-driven MS-failover cluster rows +-[ RECORD 5 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 109 +section | api_triggered: start_maintenance +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isInPrimaryState=true +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyTwoNodes=true +active_node_assigned_state | prepare_maintenance +other_node_assigned_state | +has_extra_action | f +comment | start_maintenance, primary, 2-node group -> prepare_maintenance (standby separately assigned prepare_promotion) +-[ RECORD 6 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 111 +section | api_triggered: start_maintenance +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isInPrimaryState=true +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasMoreThanTwoNodes=true +active_node_assigned_state | prepare_maintenance +other_node_assigned_state | +has_extra_action | f +comment | start_maintenance, primary, >2-node group -> prepare_maintenance, election proceeds via the heartbeat-driven MS-failover cluster rows +-[ RECORD 7 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 113 +section | api_triggered: start_maintenance +section_path | api_triggered +active_node_current_state | secondary, catchingup +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | lastHealthySyncStandbyGoingToMaintenance=true +active_node_assigned_state | wait_maintenance +other_node_assigned_state | wait_primary +has_extra_action | f +comment | start_maintenance, secondary, last healthy sync standby -> wait_maintenance, primary wait_primary (disables sync rep) +-[ RECORD 8 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 115 +section | api_triggered: start_maintenance +section_path | api_triggered +active_node_current_state | secondary, catchingup +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | maintenance +other_node_assigned_state | +has_extra_action | f +comment | start_maintenance, secondary, ordinary case -> maintenance +-[ RECORD 9 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 117 +section | api_triggered: stop_maintenance +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | exists=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | stop_maintenance, no primary -> report_lsn +-[ RECORD 10 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 119 +section | api_triggered: stop_maintenance +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isDemotedPrimary=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | stop_maintenance, primary demoted -> report_lsn +-[ RECORD 11 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 121 +section | api_triggered: stop_maintenance +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | failoverInProgress=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | stop_maintenance, failover in progress -> report_lsn (source's own log message says "catchingup" here, but the actual call assigns REPORT_LSN -- see this row's own comment above) +-[ RECORD 12 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 123 +section | api_triggered: stop_maintenance +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | stop_maintenance, ordinary case -> catchingup +-[ RECORD 13 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 125 +section | api_triggered: set_node_candidate_priority +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | apply_settings +other_node_assigned_state | +has_extra_action | f +comment | set_node_candidate_priority, primary not already apply_settings -> apply_settings +-[ RECORD 14 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 127 +section | api_triggered: set_node_replication_quorum +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | apply_settings +other_node_assigned_state | +has_extra_action | f +comment | set_node_replication_quorum, primary not already apply_settings -> apply_settings +-[ RECORD 15 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 129 +section | api_triggered: set_formation_number_sync_standbys +section_path | api_triggered +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | goal=primary|wait_primary +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | apply_settings +other_node_assigned_state | +has_extra_action | f +comment | set_formation_number_sync_standbys, primary in primary/wait_primary -> apply_settings +-[ RECORD 16 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 201 +section | early_checks +section_path | early_checks +active_node_current_state | dropped +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | converged to dropped -> remove the node from the catalog +-[ RECORD 17 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 203 +section | early_checks +section_path | early_checks +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | goal=dropped +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | f +comment | goal already dropped -> no-op +-[ RECORD 18 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 205 +section | early_checks +section_path | early_checks +active_node_current_state | maintenance +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | f +comment | converged to maintenance -> no-op, frozen until stop_maintenance() +-[ RECORD 19 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 207 +section | early_checks +section_path | early_checks +active_node_current_state | demote_timeout +other_node_current_state | +candidate_node_current_state | +active_node_conditions | unreachableFromDemoteTimeout=true +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | demoted +other_node_assigned_state | +has_extra_action | f +comment | reported demote_timeout, assigned goal can't reach it -> demoted +-[ RECORD 20 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 208 +section | early_checks +section_path | early_checks +active_node_current_state | prepare_maintenance +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyOneNode=true +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | f +comment | alone in group, still preparing for maintenance -> no-op +-[ RECORD 21 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 209 +section | early_checks +section_path | early_checks +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | candidateEligible=true, reportedIsWaitStandby=false, reportedIsJoinSecondary=false, reportedIsPrepareMaintenance=false +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyOneNode=true +active_node_assigned_state | single +other_node_assigned_state | +has_extra_action | f +comment | alone in group, candidate-eligible -> single +-[ RECORD 22 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 210 +section | early_checks +section_path | early_checks +active_node_current_state | primary, wait_primary, join_primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | candidateEligible=false +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyOneNode=true +active_node_assigned_state | single +other_node_assigned_state | +has_extra_action | f +comment | alone in group, already primary despite candidatePriority zero -> single +-[ RECORD 23 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 211 +section | early_checks +section_path | early_checks +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | candidateEligible=false, reportedCanTakeWrites=false, reportedIsWaitStandby=false +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyOneNode=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | alone in group, candidatePriority zero -> report_lsn +-[ RECORD 24 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 301 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | secondary +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isComparableToReferenceTli=false +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | converged secondary, reportedTLI not an ancestor of reference -> catchingup +-[ RECORD 25 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 303 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isHealthy=true, isInPrimaryState=true +candidate_node_conditions | +group_conditions | replicationStallExceeded=true +active_node_assigned_state | +other_node_assigned_state | wait_primary +has_extra_action | f +comment | primary healthy, no standby past replication_stall_timeout -> wait_primary +-[ RECORD 26 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 305 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isUnhealthy=true +candidate_node_conditions | +group_conditions | groupHasMoreThanTwoNodes=true +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade +-[ RECORD 27 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 307 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | report_lsn +other_node_current_state | wait_primary, join_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | secondary +other_node_assigned_state | +has_extra_action | f +comment | report_lsn, primary converged wait/join_primary, healthy -> secondary +-[ RECORD 28 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 309 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | report_lsn +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | secondary +other_node_assigned_state | +has_extra_action | f +comment | report_lsn, primary converged primary, healthy -> secondary +-[ RECORD 29 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 311 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | fast_forward +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | prepare_promotion +other_node_assigned_state | +has_extra_action | f +comment | fast_forward done -> prepare_promotion +-[ RECORD 30 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 313 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | report_lsn, fast_forward +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | report_lsn or fast_forward, continuing an already-started failover -> MS-failover cascade +-[ RECORD 31 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 315 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | wait_standby +other_node_current_state | wait_primary, join_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 32 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 317 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | wait_standby +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | replicationQuorum=true +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | apply_settings +has_extra_action | f +comment | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 33 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 319 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | wait_standby +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | replicationQuorum=false +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | wait_standby (not a quorum member), primary converged primary -> catchingup +-[ RECORD 34 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 321 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | catchingup +other_node_current_state | wait_primary, join_primary, primary +candidate_node_current_state | +active_node_conditions | isHealthy=true +other_node_conditions | +candidate_node_conditions | +group_conditions | walWithinSyncThreshold=true, activeAndPrimaryTliMatch=true +active_node_assigned_state | secondary +other_node_assigned_state | +has_extra_action | f +comment | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 35 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 323 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | secondary +other_node_current_state | wait_primary +candidate_node_current_state | +active_node_conditions | isHealthy=true, candidateEligible=true +other_node_conditions | isUnhealthy=true +candidate_node_conditions | +group_conditions | walWithinPromoteThreshold=true +active_node_assigned_state | prepare_promotion +other_node_assigned_state | +has_extra_action | f +comment | primary fails, already converged wait_primary (issue #1168) -> secondary -> prepare_promotion only (1 of 2) +-[ RECORD 36 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 325 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | secondary +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isHealthy=true, candidateEligible=true +other_node_conditions | isUnhealthy=true, isInPrimaryState=true +candidate_node_conditions | +group_conditions | groupHasExactlyOneNode=false, walWithinPromoteThreshold=true +active_node_assigned_state | prepare_promotion +other_node_assigned_state | draining +has_extra_action | f +comment | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) +-[ RECORD 37 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 327 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | wait_maintenance +other_node_current_state | wait_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | maintenance +other_node_assigned_state | +has_extra_action | f +comment | wait_maintenance, primary converged wait_primary -> maintenance +-[ RECORD 38 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 329 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | wait_maintenance +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | goal!=wait_primary +candidate_node_conditions | +group_conditions | +active_node_assigned_state | maintenance +other_node_assigned_state | +has_extra_action | f +comment | wait_maintenance, primary's goal no longer wait_primary -> maintenance +-[ RECORD 39 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 331 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | prepare_promotion +other_node_current_state | prepare_maintenance +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | stop_replication +other_node_assigned_state | +has_extra_action | f +comment | prepare_promotion, primary converged prepare_maintenance -> stop_replication +-[ RECORD 40 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 333 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | prepare_promotion +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isCitusWorkerGroup=true +other_node_conditions | exists=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | demoted +has_extra_action | f +comment | Citus worker prepare_promotion, primary present -> wait_primary + demoted +-[ RECORD 41 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 335 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | prepare_promotion +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isCitusWorkerGroup=true +other_node_conditions | exists=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | +has_extra_action | f +comment | Citus worker prepare_promotion, primary removed -> wait_primary +-[ RECORD 42 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 337 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | prepare_promotion +other_node_current_state | wait_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | exists=true, isInMaintenance=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | stop_replication +other_node_assigned_state | +has_extra_action | f +comment | prepare_promotion, primary already converged wait_primary (issue #1168) -> stop_replication only (1 of 2) +-[ RECORD 43 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 339 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | prepare_promotion +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | exists=true, isInMaintenance=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | stop_replication +other_node_assigned_state | demote_timeout +has_extra_action | f +comment | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) +-[ RECORD 44 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 341 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | prepare_promotion +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | exists=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | +has_extra_action | f +comment | prepare_promotion, primary removed -> wait_primary +-[ RECORD 45 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 343 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | stop_replication +other_node_current_state | prepare_maintenance +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | maintenance +has_extra_action | f +comment | stop_replication, primary converged prepare_maintenance -> wait_primary + maintenance +-[ RECORD 46 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 345 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | stop_replication +other_node_current_state | demote_timeout +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | demoted +has_extra_action | f +comment | stop_replication, primary converged demote_timeout -> wait_primary + demoted (1 of 3) +-[ RECORD 47 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 347 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | stop_replication +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | drainTimeExpired=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | demoted +has_extra_action | f +comment | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) +-[ RECORD 48 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 349 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | stop_replication +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | primaryIsWaitPrimaryPresumedDead=true +active_node_assigned_state | wait_primary +other_node_assigned_state | demoted +has_extra_action | f +comment | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) +-[ RECORD 49 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 351 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | stop_replication +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isCitusWorkerGroup=true +other_node_conditions | exists=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | demoted +has_extra_action | f +comment | Citus worker stop_replication, primary present -> wait_primary + demoted +-[ RECORD 50 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 353 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | stop_replication +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isCitusWorkerGroup=true +other_node_conditions | exists=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | +has_extra_action | f +comment | Citus worker stop_replication, primary removed -> wait_primary +-[ RECORD 51 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 355 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | demoted +other_node_current_state | wait_primary, join_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | demoted, primary reported wait/join_primary with goal primary -> catchingup +-[ RECORD 52 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 357 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | demoted +other_node_current_state | wait_primary, join_primary, primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | demoted, primary converged wait/join_primary/primary, healthy -> catchingup +-[ RECORD 53 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 359 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | join_secondary +other_node_current_state | wait_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | secondary +other_node_assigned_state | +has_extra_action | t +comment | join_secondary, primary reported wait_primary with goal wait/primary -> secondary +-[ RECORD 54 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 361 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | join_secondary +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | secondary +other_node_assigned_state | +has_extra_action | f +comment | join_secondary, primary converged primary -> secondary +-[ RECORD 55 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 363 +section | reporting_node +section_path | reporting_node.ms_failover.retry_reset +active_node_current_state | report_lsn +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | activeNodeAllWalSourcesUnhealthy=true, guardDataLossEnabled=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, guard_data_loss=true -> report_lsn (retry once a source recovers) +-[ RECORD 56 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 365 +section | reporting_node +section_path | reporting_node.ms_failover.candidate_join +active_node_current_state | report_lsn +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | isReadyToStreamWAL=true +group_conditions | candidatePromotionInProgress=true +active_node_assigned_state | join_secondary +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: activeNode in report_lsn, failover candidate ready to stream WAL -> join_secondary +-[ RECORD 57 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 367 +section | reporting_node +section_path | reporting_node.ms_failover.candidate_fanout +active_node_current_state | secondary, catchingup +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | inMSFailoverCluster=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) +-[ RECORD 58 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 369 +section | reporting_node +section_path | reporting_node.ms_failover.candidate_fanout +active_node_current_state | maintenance +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | inMSFailoverCluster=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | MS-failover fan-out: rejoining from maintenance -> report_lsn (2 of 4) +-[ RECORD 59 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 371 +section | reporting_node +section_path | reporting_node.ms_failover.candidate_fanout +active_node_current_state | draining, demoted +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | inMSFailoverCluster=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | MS-failover fan-out: old primary converged draining or demoted -> report_lsn (3 of 4) +-[ RECORD 60 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 373 +section | reporting_node +section_path | reporting_node.ms_failover.candidate_fanout +active_node_current_state | demoted +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | inMSFailoverCluster=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | MS-failover fan-out: old primary demoted, was rejoining a now-failed primary -> report_lsn (4 of 4) +-[ RECORD 61 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 375 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome +active_node_current_state | report_lsn +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | candidatePromotionInProgress=false, mostAdvancedCandidateWithinPromoteThreshold=true, inMSFailoverCluster=true +active_node_assigned_state | prepare_promotion +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate already has all WAL -> prepare_promotion (1 of 2 -- see this row's own comment on why both are listed) +-[ RECORD 62 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 377 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome +active_node_current_state | report_lsn +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | candidatePromotionInProgress=false, mostAdvancedCandidateWithinPromoteThreshold=true, inMSFailoverCluster=true +active_node_assigned_state | fast_forward +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate is lagging -> fast_forward (2 of 2) +-[ RECORD 63 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 379 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.missing_nodes_gate +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | guardDataLossEnabled=true, inMSFailoverCluster=true, inMSFailoverCandidateGate=true, missingNodesCount>=1 +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=true -> decline, wait for more reports (1 of 2) +-[ RECORD 64 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 381 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.missing_nodes_gate +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | guardDataLossEnabled=false, inMSFailoverCluster=true, inMSFailoverCandidateGate=true, missingNodesCount>=1 +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=false -> proceed despite possible data loss (2 of 2) +-[ RECORD 65 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 383 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.candidate_count_gate +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | inMSFailoverCluster=true, inMSFailoverCandidateGate=true, candidateCount=0 +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: zero candidates have reported their LSN yet -> silent decline +-[ RECORD 66 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 385 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.quorum_candidate_gate +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | guardDataLossEnabled=true, inMSFailoverCluster=true, inMSFailoverCandidateGate=true, sufficientQuorumCandidates=false +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | MS-failover: not enough quorum candidates reported yet, guard_data_loss=true -> decline (1 of 2) +-[ RECORD 67 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 387 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.quorum_candidate_gate +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | guardDataLossEnabled=false, inMSFailoverCluster=true, inMSFailoverCandidateGate=true, sufficientQuorumCandidates=false +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | MS-failover: not enough quorum candidates reported yet, guard_data_loss=false -> proceed with fewer than required (2 of 2) +-[ RECORD 68 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 389 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.no_candidate_yet +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | candidatePromotionInProgress=false, inMSFailoverCluster=true +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: no promotion in flight, not enough (or not safe enough) candidates yet -> no-op besides the fan-out above +-[ RECORD 69 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 391 +section | reporting_node +section_path | reporting_node.ms_failover.draining_or_maintenance +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isUnhealthy=true, isInPrimaryState=true +candidate_node_conditions | +group_conditions | groupHasMoreThanTwoNodes=true, atLeastOneHealthyCandidate=true +active_node_assigned_state | +other_node_assigned_state | draining +has_extra_action | f +comment | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining +-[ RECORD 70 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 393 +section | reporting_node +section_path | reporting_node.ms_failover.draining_or_maintenance +active_node_current_state | +other_node_current_state | prepare_maintenance +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isUnhealthy=true +candidate_node_conditions | +group_conditions | groupHasMoreThanTwoNodes=true +active_node_assigned_state | +other_node_assigned_state | maintenance +has_extra_action | f +comment | nodesCount>2, primary unhealthy, converged prepare_maintenance -> primary maintenance +-[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 401 +section | primary_node +section_path | primary_node +active_node_current_state | single +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | anyOtherNodeWaitingStandby=true +active_node_assigned_state | wait_primary +other_node_assigned_state | +has_extra_action | f +comment | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 403 +section | primary_node +section_path | primary_node +active_node_current_state | primary, wait_primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | replicationQuorumCountIsZero=true, secondaryNodesCountIsZero=true +active_node_assigned_state | wait_primary +other_node_assigned_state | catchingup +has_extra_action | f +comment | all nodes async, zero secondaries -> wait_primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 405 +section | primary_node +section_path | primary_node +active_node_current_state | primary, wait_primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | replicationQuorumCountIsZero=true, secondaryNodesCountIsZero=false +active_node_assigned_state | primary +other_node_assigned_state | catchingup +has_extra_action | f +comment | all nodes async, >=1 secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 407 +section | primary_node +section_path | primary_node +active_node_current_state | primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNodesCountIsZero=true, failoverInProgress=false +active_node_assigned_state | wait_primary +other_node_assigned_state | catchingup +has_extra_action | f +comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 75 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 409 +section | primary_node +section_path | primary_node +active_node_current_state | primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | numberSyncStandbysIsZero=false, secondaryQuorumNodesCountIsZero=true, failoverInProgress=false +active_node_assigned_state | primary +other_node_assigned_state | catchingup +has_extra_action | f +comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 411 +section | primary_node +section_path | primary_node +active_node_current_state | wait_primary +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | secondaryQuorumNodesCountIsZero=false +active_node_assigned_state | primary +other_node_assigned_state | catchingup +has_extra_action | f +comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 413 +section | primary_node +section_path | primary_node +active_node_current_state | apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNodesCountIsZero=true +active_node_assigned_state | wait_primary +other_node_assigned_state | catchingup +has_extra_action | f +comment | apply_settings, both zero -> wait_primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 415 +section | primary_node +section_path | primary_node +active_node_current_state | apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | numberSyncStandbysIsZero=false +active_node_assigned_state | primary +other_node_assigned_state | catchingup +has_extra_action | f +comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 417 +section | primary_node +section_path | primary_node +active_node_current_state | apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNodesCountIsZero=false +active_node_assigned_state | primary +other_node_assigned_state | catchingup +has_extra_action | f +comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 80 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 419 +section | primary_node +section_path | primary_node +active_node_current_state | primary, wait_primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | +other_node_assigned_state | catchingup +has_extra_action | f +comment | converged primary/wait_primary/apply_settings, no other condition applies -> no-op besides the unhealthy-secondary fan-out to catchingup +-[ RECORD 81 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 421 +section | primary_node +section_path | primary_node +active_node_current_state | join_primary +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | primary +other_node_assigned_state | +has_extra_action | f +comment | backwards-compat: join_primary -> primary + diff --git a/src/monitor/expected/guard_data_loss.out b/src/monitor/expected/guard_data_loss.out index 98b07b2f7..903a66666 100644 --- a/src/monitor/expected/guard_data_loss.out +++ b/src/monitor/expected/guard_data_loss.out @@ -308,3 +308,117 @@ reportedstate | secondary RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('gdl_test', count => 100); +-[ RECORD 1 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 10 "p" (p:5432): "single" +-[ RECORD 3 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 11 "s1" (s1:5432): "wait_standby" +-[ RECORD 4 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 10 "p" (p:5432): "wait_primary" +-[ RECORD 6 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 11 "s1" (s1:5432): "catchingup" +-[ RECORD 8 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 11 "s1" (s1:5432): "secondary" +-[ RECORD 10 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 10 "p" (p:5432): "primary" +-[ RECORD 12 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 12 "s2" (s2:5432): "wait_standby" +-[ RECORD 13 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 12 "s2" (s2:5432): "catchingup" +-[ RECORD 16 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 12 "s2" (s2:5432): "secondary" +-[ RECORD 17 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | report_lsn +goalstate | prepare_promotion +rule_pos | 375 +rule_section | reporting_node +description | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate already has all WAL -> prepare_promotion (1 of 2 -- see this row's own comment on why both are listed) + diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out new file mode 100644 index 000000000..90060dbeb --- /dev/null +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -0,0 +1,254 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Two-step check of the real keeper FSM (KeeperFSM[], src/bin/pg_autoctl/ +-- fsm.c) against the monitor's own MonitorFSM[] table, using the +-- keeper_fsm_edges.json fixture committed alongside this file. That fixture +-- is generated from the real KeeperFSM[] via +-- "pg_autoctl inspect fsm list --json" (KeeperFSMToJSON(), fsm.c) -- see +-- cli_do_fsm_list's own comment for how that command runs with zero setup +-- (no --pgdata, no live cluster) -- and must be regenerated by hand +-- whenever KeeperFSM[] changes; this test only reads the fixture, it never +-- runs pg_autoctl itself. +-- +-- Step 1: load the fixture client-side (psql's own backtick file +-- embedding, not server-side pg_read_file() -- the latter is +-- superuser/pg_read_server_files-gated and resolves relative paths against +-- $PGDATA, not this test's own directory) into a real table, one row per +-- distinct keeper edge, so this test's own expected/keeper_fsm_edges.out +-- shows the whole keeper FSM line by line, human-reviewable, with a diff on +-- every change to KeeperFSM[] -- the same discipline fsm.sql's own dump +-- already gives MonitorFSM[]. +-- +-- current_state is plain text, not pgautofailover.replication_state: a row +-- whose real KeeperFSM[] .current is ANY_STATE (state_matches()'s wildcard) +-- is serialized by KeeperFSMToJSON() as the literal string "any" (see its +-- own comment, fsm.c), which is not a legal enum value by design -- it's a +-- sentinel Step 2a/2b below match structurally, not a real reported state. +-- Every other value is still round-tripped through the enum (CASE ... ELSE +-- ... ::pgautofailover.replication_state ... END) so a typo'd or unrecognized +-- state name in the fixture still fails loudly here, same as +-- check_fsm_reachability()'s own cast does for the live-cluster path. +-- +-- DISTINCT: nothing in KeeperFSM[] guarantees two different rows can't +-- resolve to the exact same (current, assigned) pair, and the JSON has no +-- per-row provenance to tell such duplicates apart by, so they carry no +-- extra information here and would only clutter the reviewable list. +\set keeper_json `cat keeper_fsm_edges.json` +CREATE TABLE keeper_fsm_edges AS +SELECT DISTINCT + CASE WHEN (edge ->> 'current') = 'any' + THEN 'any' + ELSE ((edge ->> 'current')::pgautofailover.replication_state)::text + END AS current_state, + (edge ->> 'assigned')::pgautofailover.replication_state AS assigned_state + FROM jsonb_array_elements(:'keeper_json'::jsonb) AS edge; +SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; + current_state | assigned_state +---------------------+--------------------- + any | dropped + apply_settings | single + apply_settings | wait_primary + apply_settings | primary + apply_settings | draining + apply_settings | demote_timeout + apply_settings | demoted + apply_settings | join_primary + catchingup | single + catchingup | demote_timeout + catchingup | demoted + catchingup | secondary + catchingup | prepare_promotion + catchingup | maintenance + catchingup | wait_maintenance + catchingup | report_lsn + demote_timeout | single + demote_timeout | primary + demote_timeout | demoted + demote_timeout | report_lsn + demoted | single + demoted | demote_timeout + demoted | catchingup + demoted | report_lsn + draining | single + draining | demote_timeout + draining | demoted + draining | report_lsn + dropped | single + dropped | wait_standby + dropped | report_lsn + fast_forward | single + fast_forward | demote_timeout + fast_forward | demoted + fast_forward | prepare_promotion + fast_forward | report_lsn + init | single + init | demote_timeout + init | demoted + init | wait_standby + init | report_lsn + join_primary | single + join_primary | wait_primary + join_primary | primary + join_primary | draining + join_primary | demote_timeout + join_primary | demoted + join_secondary | secondary + join_secondary | report_lsn + maintenance | demote_timeout + maintenance | demoted + maintenance | catchingup + maintenance | report_lsn + prepare_maintenance | demote_timeout + prepare_maintenance | demoted + prepare_maintenance | catchingup + prepare_maintenance | maintenance + prepare_maintenance | report_lsn + prepare_promotion | single + prepare_promotion | wait_primary + prepare_promotion | demote_timeout + prepare_promotion | demoted + prepare_promotion | stop_replication + prepare_promotion | report_lsn + primary | single + primary | wait_primary + primary | draining + primary | demote_timeout + primary | demoted + primary | maintenance + primary | join_primary + primary | apply_settings + primary | prepare_maintenance + report_lsn | single + report_lsn | demote_timeout + report_lsn | demoted + report_lsn | secondary + report_lsn | prepare_promotion + report_lsn | fast_forward + report_lsn | join_secondary + secondary | single + secondary | demote_timeout + secondary | demoted + secondary | catchingup + secondary | prepare_promotion + secondary | wait_standby + secondary | maintenance + secondary | wait_maintenance + secondary | report_lsn + single | wait_primary + single | demote_timeout + single | demoted + stop_replication | single + stop_replication | wait_primary + stop_replication | demote_timeout + stop_replication | demoted + stop_replication | report_lsn + wait_maintenance | single + wait_maintenance | demote_timeout + wait_maintenance | demoted + wait_maintenance | maintenance + wait_maintenance | report_lsn + wait_primary | single + wait_primary | primary + wait_primary | demoted + wait_primary | join_primary + wait_primary | apply_settings + wait_standby | catchingup +(108 rows) + +-- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() +-- edge the keeper_fsm_edges table above has no matching row for. A +-- non-empty result here is a real, actionable gap: the monitor can assign a +-- transition the keeper has no KeeperFSM[] row to perform. See +-- dump_fsm_edges()'s own comment (group_state_machine.c) for exactly what +-- it resolves and what it deliberately excludes (e.g. the api_triggered +-- section, resolved via hand-written C rather than a NodeStatePattern). +-- +-- k.current_state = 'any' matches every e.current_state -- a keeper row +-- covering every current state also covers this specific one. +-- +-- GROUPING SETS adds one summary row per rule (pos, assigned_state, comment) +-- -- current_state NULL, n = how many current_states that single +-- MonitorFSM[] rule fans out to -- alongside the ordinary per-current_state +-- detail rows, so a rule using a broad NodeStatePattern (matching many +-- states at once) is immediately visible as one big number instead of +-- having to count its own detail rows by hand. NULLS FIRST puts each rule's +-- summary row right before its own detail rows, as a header. +-- +-- Expected result: empty. Every MonitorFSM[] rule currently has a matching +-- KeeperFSM[] row for every current_state it can assign a transition from. +SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE NOT EXISTS ( + SELECT 1 + FROM keeper_fsm_edges k + WHERE k.assigned_state = e.assigned_state + AND (k.current_state = 'any' OR k.current_state = e.current_state::text) + ) + GROUP BY GROUPING SETS ( + (e.pos, e.assigned_state, f.comment, e.current_state), + (e.pos, e.assigned_state, f.comment) + ) + ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; + rule | n | current_state | assigned_state | comment +------+---+---------------+----------------+--------- +(0 rows) + +-- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper +-- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the +-- full, fully-resolved edge set the monitor's table can reach, see its own +-- comment). A non-empty result here means the keeper is prepared to +-- transition through a (current, assigned) pair the monitor itself would +-- never assign -- either genuinely dead keeper code, or a real coverage +-- gap on the monitor side, same "investigate before assuming which" caveat +-- as step 2a's own comment. +-- +-- For a k.current_state = 'any' row, "matches" is existential: at least one +-- current_state for which the monitor assigns this same target is enough +-- to say the target is implementable at all, so a gap here means the +-- monitor can NEVER produce this assigned_state from ANY current state -- +-- e.g. "any -> dropped" (KeeperFSM[]'s two ANY_STATE -> DROPPED rows +-- collapse to this single row) is a standing, expected exception: the +-- monitor's own DROPPED assignment (remove_node(), pos 101/103) lives +-- entirely in the api_triggered section, which dump_fsm_edges() +-- deliberately excludes (see its own comment), so it can never appear +-- here. Same "investigate before assuming which" caveat as the rest of +-- this file applies to every other row below. +SELECT k.current_state, k.assigned_state + FROM keeper_fsm_edges k + WHERE NOT EXISTS ( + SELECT 1 + FROM pgautofailover.dump_fsm_edges() e + WHERE e.assigned_state = k.assigned_state + AND (k.current_state = 'any' OR k.current_state = e.current_state::text) + ) + ORDER BY k.current_state, k.assigned_state; + current_state | assigned_state +---------------------+--------------------- + any | dropped + apply_settings | join_primary + catchingup | prepare_promotion + catchingup | maintenance + catchingup | wait_maintenance + demote_timeout | primary + dropped | single + dropped | wait_standby + dropped | report_lsn + init | wait_standby + maintenance | catchingup + prepare_maintenance | catchingup + primary | maintenance + primary | join_primary + primary | prepare_maintenance + report_lsn | prepare_promotion + report_lsn | fast_forward + report_lsn | join_secondary + secondary | wait_standby + secondary | maintenance + secondary | wait_maintenance + wait_primary | join_primary + wait_primary | apply_settings +(23 rows) + +DROP TABLE keeper_fsm_edges; diff --git a/src/monitor/expected/lock_and_fetch_migration.out b/src/monitor/expected/lock_and_fetch_migration.out index c48cad9f1..5ab44389a 100644 --- a/src/monitor/expected/lock_and_fetch_migration.out +++ b/src/monitor/expected/lock_and_fetch_migration.out @@ -32,15 +32,15 @@ create_formation | (lafm_test,pgsql,postgres,t,1) SELECT pgautofailover.register_node('lafm_test', 'lafm_p', 5432, 'postgres', 'lafm_p', 1); -[ RECORD 1 ]-+--------------------------- -register_node | (21,0,single,100,t,lafm_p) +register_node | (24,0,single,100,t,lafm_p) SELECT pgautofailover.register_node('lafm_test', 'lafm_s1', 5432, 'postgres', 'lafm_s1', 1); -[ RECORD 1 ]-+---------------------------------- -register_node | (22,0,wait_standby,100,t,lafm_s1) +register_node | (25,0,wait_standby,100,t,lafm_s1) SELECT pgautofailover.register_node('lafm_test', 'lafm_s2', 5432, 'postgres', 'lafm_s2', 1); -[ RECORD 1 ]-+---------------------------------- -register_node | (23,0,wait_standby,100,t,lafm_s2) +register_node | (26,0,wait_standby,100,t,lafm_s2) SELECT nodeid AS np FROM pgautofailover.node WHERE formationid = 'lafm_test' AND nodename = 'lafm_p' \gset @@ -267,7 +267,7 @@ SELECT assigned_group_state FROM pgautofailover.node_active('lafm_test', :np, 0, assigned_group_state | primary SELECT pgautofailover.start_maintenance(:ns1); -WARNING: Starting maintenance on node 22 "lafm_s1" (lafm_s1:5432) will block writes on the primary node 21 "lafm_p" (lafm_p:5432) +WARNING: Starting maintenance on node 25 "lafm_s1" (lafm_s1:5432) will block writes on the primary node 24 "lafm_p" (lafm_p:5432) DETAIL: we now have 0 healthy node(s) left in the "secondary" state and formation "lafm_test" number-sync-standbys requires 1 sync standbys -[ RECORD 1 ]-----+-- start_maintenance | t @@ -316,3 +316,195 @@ nodename | lafm_s2_renamed goalstate | catchingup reportedstate | secondary +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('lafm_test', count => 100); +-[ RECORD 1 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "single" +-[ RECORD 3 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 25 "lafm_s1" (lafm_s1:5432): "wait_standby" +-[ RECORD 4 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "wait_primary" +-[ RECORD 6 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 25 "lafm_s1" (lafm_s1:5432): "catchingup" +-[ RECORD 8 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 25 "lafm_s1" (lafm_s1:5432): "secondary" +-[ RECORD 10 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "primary" +-[ RECORD 12 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 26 "lafm_s2" (lafm_s2:5432): "wait_standby" +-[ RECORD 13 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 26 "lafm_s2" (lafm_s2:5432): "catchingup" +-[ RECORD 16 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 26 "lafm_s2" (lafm_s2:5432): "secondary" +-[ RECORD 17 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | apply_settings +goalstate | apply_settings +rule_pos | +rule_section | +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "apply_settings" +-[ RECORD 18 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | apply_settings +goalstate | primary +rule_pos | 415 +rule_section | primary_node +description | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 19 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "primary" +-[ RECORD 20 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 125 +rule_section | api_triggered +description | set_node_candidate_priority, primary not already apply_settings -> apply_settings +-[ RECORD 21 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 125 +rule_section | api_triggered +description | set_node_candidate_priority, primary not already apply_settings -> apply_settings +-[ RECORD 22 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 125 +rule_section | api_triggered +description | set_node_candidate_priority, primary not already apply_settings -> apply_settings +-[ RECORD 23 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 125 +rule_section | api_triggered +description | set_node_candidate_priority, primary not already apply_settings -> apply_settings +-[ RECORD 24 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 127 +rule_section | api_triggered +description | set_node_replication_quorum, primary not already apply_settings -> apply_settings +-[ RECORD 25 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | apply_settings +goalstate | apply_settings +rule_pos | +rule_section | +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "apply_settings" +-[ RECORD 26 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | apply_settings +goalstate | primary +rule_pos | 415 +rule_section | primary_node +description | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 27 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "primary" +-[ RECORD 28 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | maintenance +rule_pos | 115 +rule_section | api_triggered +description | start_maintenance, secondary, ordinary case -> maintenance +-[ RECORD 29 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | maintenance +goalstate | maintenance +rule_pos | +rule_section | +description | New state is reported by node 25 "lafm_s1" (lafm_s1:5432): "maintenance" +-[ RECORD 30 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | maintenance +goalstate | catchingup +rule_pos | 123 +rule_section | api_triggered +description | stop_maintenance, ordinary case -> catchingup + diff --git a/src/monitor/expected/monitor.out b/src/monitor/expected/monitor.out index 6b937bf9f..dc0253af1 100644 --- a/src/monitor/expected/monitor.out +++ b/src/monitor/expected/monitor.out @@ -247,3 +247,26 @@ node_port | 9877 -- should fail as there's no primary at this point select pgautofailover.perform_failover(); ERROR: couldn't find the primary node in formation "default", group 0 +-- last_events() (all three overloads) returns SETOF pgautofailover.event, so +-- its own SELECT list must match that composite type's full column set -- +-- including rule_pos/rule_section -- or the call errors at parse time +-- ("Final statement returns too few columns") before ever running. Not +-- exercised anywhere else in this test suite, so a regression here (e.g. a +-- future column added to pgautofailover.event without updating these three +-- function bodies) would otherwise go unnoticed until a live +-- "pg_autoctl show events"/"pg_autoctl watch" call broke in production. +select count(*) >= 0 as last_events_count_ok + from pgautofailover.last_events(10); +-[ RECORD 1 ]--------+-- +last_events_count_ok | t + +select count(*) >= 0 as last_events_by_formation_count_ok + from pgautofailover.last_events(formation_id => 'default', count => 10); +-[ RECORD 1 ]---------------------+-- +last_events_by_formation_count_ok | t + +select count(*) >= 0 as last_events_by_formation_and_group_count_ok + from pgautofailover.last_events('default', 0, 10); +-[ RECORD 1 ]-------------------------------+-- +last_events_by_formation_and_group_count_ok | t + diff --git a/src/monitor/expected/node_active_protocol.out b/src/monitor/expected/node_active_protocol.out index 1fb5b73c6..40ec2c8d9 100644 --- a/src/monitor/expected/node_active_protocol.out +++ b/src/monitor/expected/node_active_protocol.out @@ -670,3 +670,291 @@ SELECT pgautofailover.report_postgres_version(-1, 170003); -[ RECORD 1 ]-----------+- report_postgres_version | +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events, for each of the two formations used +-- above. Exercises pgautofailover.last_events() against a real scenario -- +-- its own SELECT list didn't match pgautofailover.event's column set for a +-- long time, breaking it outright, and nothing in this suite ever called +-- it to notice (see monitor.sql's own minimal-repro coverage). Filtering +-- by formationid isolates each summary from the other, and from every +-- other test in this schedule sharing the same event table -- safe +-- regardless of where in this file (or the whole schedule) it runs. +-- eventid/eventtime omitted: eventid is a database-wide sequence shared by +-- every test in this schedule (see regress_schedule's own comment) and +-- eventtime is a live timestamp -- neither is a stable value to pin here. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('fsm_test', count => 100); +-[ RECORD 1 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "single" +-[ RECORD 3 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "wait_standby" +-[ RECORD 4 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "wait_primary" +-[ RECORD 6 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "catchingup" +-[ RECORD 8 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "secondary" +-[ RECORD 10 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "primary" +-[ RECORD 12 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | prepare_promotion +rule_pos | 325 +rule_section | reporting_node +description | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) +-[ RECORD 13 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | draining +rule_pos | 325 +rule_section | reporting_node +description | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) +-[ RECORD 14 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | prepare_promotion +goalstate | prepare_promotion +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "prepare_promotion" +-[ RECORD 15 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | prepare_promotion +goalstate | stop_replication +rule_pos | 339 +rule_section | reporting_node +description | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) +-[ RECORD 16 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | demote_timeout +rule_pos | 339 +rule_section | reporting_node +description | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) +-[ RECORD 17 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | demote_timeout +goalstate | demote_timeout +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "demote_timeout" +-[ RECORD 18 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | stop_replication +goalstate | stop_replication +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "stop_replication" +-[ RECORD 19 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | stop_replication +goalstate | wait_primary +rule_pos | 345 +rule_section | reporting_node +description | stop_replication, primary converged demote_timeout -> wait_primary + demoted (1 of 3) +-[ RECORD 20 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | demote_timeout +goalstate | demoted +rule_pos | 345 +rule_section | reporting_node +description | stop_replication, primary converged demote_timeout -> wait_primary + demoted (1 of 3) +-[ RECORD 21 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "wait_primary" +-[ RECORD 22 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | demoted +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "primary" +-[ RECORD 23 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | demoted +goalstate | demoted +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "demoted" +-[ RECORD 24 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | demoted +goalstate | catchingup +rule_pos | 357 +rule_section | reporting_node +description | demoted, primary converged wait/join_primary/primary, healthy -> catchingup +-[ RECORD 25 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "catchingup" +-[ RECORD 26 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 27 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "secondary" +-[ RECORD 28 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 29 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "primary" +-[ RECORD 30 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | prepare_maintenance +rule_pos | 109 +rule_section | api_triggered +description | start_maintenance, primary, 2-node group -> prepare_maintenance (standby separately assigned prepare_promotion) +-[ RECORD 31 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | prepare_promotion +rule_pos | +rule_section | +description | Setting goal state of node 6 "node1" (node1:5432) to prepare_promotion after a user-initiated start_maintenance call. +-[ RECORD 32 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | prepare_promotion +rule_pos | +rule_section | +description | Updating region to "dc2" for node 6 "node1" (node1:5432) + +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('killed_test', count => 100); +-[ RECORD 1 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 8 "ka" (ka:5432): "single" +-[ RECORD 3 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 9 "kb" (kb:5432): "wait_standby" +-[ RECORD 4 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 8 "ka" (ka:5432): "wait_primary" +-[ RECORD 6 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 9 "kb" (kb:5432): "catchingup" +-[ RECORD 8 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 9 "kb" (kb:5432): "secondary" +-[ RECORD 10 ]+-------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+-------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 8 "ka" (ka:5432): "primary" +-[ RECORD 12 ]+-------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | prepare_promotion +rule_pos | 325 +rule_section | reporting_node +description | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) +-[ RECORD 13 ]+-------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | draining +rule_pos | 325 +rule_section | reporting_node +description | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + diff --git a/src/monitor/expected/pg19/expected/fast_forward.out b/src/monitor/expected/pg19/expected/fast_forward.out index be1069382..e96755c8b 100644 --- a/src/monitor/expected/pg19/expected/fast_forward.out +++ b/src/monitor/expected/pg19/expected/fast_forward.out @@ -385,3 +385,117 @@ SELECT node_name, node_lsn, node_is_primary -- ── cleanup ─────────────────────────────────────────────────────────────────── RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('ff_test', count => 100); +-[ RECORD 1 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "single" +-[ RECORD 3 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "wait_standby" +-[ RECORD 4 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "wait_primary" +-[ RECORD 6 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "catchingup" +-[ RECORD 8 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "secondary" +-[ RECORD 10 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "primary" +-[ RECORD 12 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "wait_standby" +-[ RECORD 13 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "catchingup" +-[ RECORD 16 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "secondary" +-[ RECORD 17 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | report_lsn +goalstate | report_lsn +rule_pos | 363 +rule_section | reporting_node +description | MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, guard_data_loss=true -> report_lsn (retry once a source recovers) + diff --git a/src/monitor/expected/pg19/expected/monitor.out b/src/monitor/expected/pg19/expected/monitor.out index 1fa8dd8c0..a39d5767d 100644 --- a/src/monitor/expected/pg19/expected/monitor.out +++ b/src/monitor/expected/pg19/expected/monitor.out @@ -247,3 +247,26 @@ node_port | 9877 -- should fail as there's no primary at this point select pgautofailover.perform_failover(); ERROR: couldn't find the primary node in formation "default", group 0 +-- last_events() (all three overloads) returns SETOF pgautofailover.event, so +-- its own SELECT list must match that composite type's full column set -- +-- including rule_pos/rule_section -- or the call errors at parse time +-- ("Final statement returns too few columns") before ever running. Not +-- exercised anywhere else in this test suite, so a regression here (e.g. a +-- future column added to pgautofailover.event without updating these three +-- function bodies) would otherwise go unnoticed until a live +-- "pg_autoctl show events"/"pg_autoctl watch" call broke in production. +select count(*) >= 0 as last_events_count_ok + from pgautofailover.last_events(10); +-[ RECORD 1 ]--------+-- +last_events_count_ok | t + +select count(*) >= 0 as last_events_by_formation_count_ok + from pgautofailover.last_events(formation_id => 'default', count => 10); +-[ RECORD 1 ]---------------------+-- +last_events_by_formation_count_ok | t + +select count(*) >= 0 as last_events_by_formation_and_group_count_ok + from pgautofailover.last_events('default', 0, 10); +-[ RECORD 1 ]-------------------------------+-- +last_events_by_formation_and_group_count_ok | t + diff --git a/src/monitor/expected/pg19/expected/timeline_fork_detection.out b/src/monitor/expected/pg19/expected/timeline_fork_detection.out index 23aac6f12..baf10a384 100644 --- a/src/monitor/expected/pg19/expected/timeline_fork_detection.out +++ b/src/monitor/expected/pg19/expected/timeline_fork_detection.out @@ -21,7 +21,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-p', 5432, 'postgres', 'p', 1); -[ RECORD 1 ]---------------+------- -assigned_node_id | 24 +assigned_node_id | 27 assigned_group_id | 0 assigned_group_state | single assigned_candidate_priority | 100 @@ -34,7 +34,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-s1', 5432, 'postgres', 's1', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 25 +assigned_node_id | 28 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -47,7 +47,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-s2', 5432, 'postgres', 's2', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 26 +assigned_node_id | 29 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -88,27 +88,27 @@ SELECT nodeid, tli, parenttli, switchpoint_lsn FROM pgautofailover.node_timeline_history ORDER BY nodeid, tli; -[ RECORD 1 ]---+----------- -nodeid | 24 +nodeid | 27 tli | 1 parenttli | 0 switchpoint_lsn | 0/00000000 -[ RECORD 2 ]---+----------- -nodeid | 25 +nodeid | 28 tli | 1 parenttli | 0 switchpoint_lsn | 0/00000000 -[ RECORD 3 ]---+----------- -nodeid | 25 +nodeid | 28 tli | 2 parenttli | 1 switchpoint_lsn | 0/00006000 -[ RECORD 4 ]---+----------- -nodeid | 26 +nodeid | 29 tli | 1 parenttli | 0 switchpoint_lsn | 0/00000000 -[ RECORD 5 ]---+----------- -nodeid | 26 +nodeid | 29 tli | 3 parenttli | 1 switchpoint_lsn | 0/00006000 @@ -227,7 +227,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-p', 5432, 'postgres', 'p', 1); -[ RECORD 1 ]---------------+------- -assigned_node_id | 27 +assigned_node_id | 30 assigned_group_id | 0 assigned_group_state | single assigned_candidate_priority | 100 @@ -240,7 +240,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s1', 5432, 'postgres', 's1', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 28 +assigned_node_id | 31 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -253,7 +253,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s2', 5432, 'postgres', 's2', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 29 +assigned_node_id | 32 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -266,7 +266,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s3', 5432, 'postgres', 's3', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 30 +assigned_node_id | 33 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -540,3 +540,144 @@ resolved | t RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events, for each of the two formations used +-- above. Exercises pgautofailover.last_events() against a real scenario -- +-- its own SELECT list didn't match pgautofailover.event's column set for a +-- long time, breaking it outright, and nothing in this suite ever called +-- it to notice (see monitor.sql's own minimal-repro coverage). eventid/ +-- eventtime omitted: eventid is a database-wide sequence shared by every +-- test in this schedule (see regress_schedule's own comment) and +-- eventtime is a live timestamp -- neither is a stable value to pin here. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_unit', count => 100); +-[ RECORD 1 ]-+--------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single + +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_election', count => 100); +-[ RECORD 1 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 30 "p" (tlfe-p:5432): "single" +-[ RECORD 3 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "wait_standby" +-[ RECORD 4 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 30 "p" (tlfe-p:5432): "wait_primary" +-[ RECORD 6 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "catchingup" +-[ RECORD 8 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "secondary" +-[ RECORD 10 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 30 "p" (tlfe-p:5432): "primary" +-[ RECORD 12 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "wait_standby" +-[ RECORD 13 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "catchingup" +-[ RECORD 16 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "secondary" +-[ RECORD 17 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "wait_standby" +-[ RECORD 18 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "catchingup" +-[ RECORD 19 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "secondary" +-[ RECORD 20 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | report_lsn +goalstate | prepare_promotion +rule_pos | 375 +rule_section | reporting_node +description | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate already has all WAL -> prepare_promotion (1 of 2 -- see this row's own comment on why both are listed) + diff --git a/src/monitor/expected/stale_primary_report.out b/src/monitor/expected/stale_primary_report.out index f6e216d4f..cc40f4d87 100644 --- a/src/monitor/expected/stale_primary_report.out +++ b/src/monitor/expected/stale_primary_report.out @@ -308,3 +308,123 @@ nodename | spr_s2 goalstate | report_lsn reportedstate | secondary +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('spr_test', count => 100); +-[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 18 "spr_p" (spr_p:5432): "single" +-[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "wait_standby" +-[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 18 "spr_p" (spr_p:5432): "wait_primary" +-[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "catchingup" +-[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "secondary" +-[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 18 "spr_p" (spr_p:5432): "primary" +-[ RECORD 12 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "wait_standby" +-[ RECORD 13 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "catchingup" +-[ RECORD 16 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "secondary" +-[ RECORD 17 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "primary" +-[ RECORD 18 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | report_lsn +rule_pos | 367 +rule_section | reporting_node +description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) + diff --git a/src/monitor/expected/timeline_fork_detection.out b/src/monitor/expected/timeline_fork_detection.out index 0cb535d8a..d8fbce08f 100644 --- a/src/monitor/expected/timeline_fork_detection.out +++ b/src/monitor/expected/timeline_fork_detection.out @@ -21,7 +21,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-p', 5432, 'postgres', 'p', 1); -[ RECORD 1 ]---------------+------- -assigned_node_id | 24 +assigned_node_id | 27 assigned_group_id | 0 assigned_group_state | single assigned_candidate_priority | 100 @@ -34,7 +34,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-s1', 5432, 'postgres', 's1', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 25 +assigned_node_id | 28 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -47,7 +47,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-s2', 5432, 'postgres', 's2', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 26 +assigned_node_id | 29 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -88,27 +88,27 @@ SELECT nodeid, tli, parenttli, switchpoint_lsn FROM pgautofailover.node_timeline_history ORDER BY nodeid, tli; -[ RECORD 1 ]---+------- -nodeid | 24 +nodeid | 27 tli | 1 parenttli | 0 switchpoint_lsn | 0/0 -[ RECORD 2 ]---+------- -nodeid | 25 +nodeid | 28 tli | 1 parenttli | 0 switchpoint_lsn | 0/0 -[ RECORD 3 ]---+------- -nodeid | 25 +nodeid | 28 tli | 2 parenttli | 1 switchpoint_lsn | 0/6000 -[ RECORD 4 ]---+------- -nodeid | 26 +nodeid | 29 tli | 1 parenttli | 0 switchpoint_lsn | 0/0 -[ RECORD 5 ]---+------- -nodeid | 26 +nodeid | 29 tli | 3 parenttli | 1 switchpoint_lsn | 0/6000 @@ -227,7 +227,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-p', 5432, 'postgres', 'p', 1); -[ RECORD 1 ]---------------+------- -assigned_node_id | 27 +assigned_node_id | 30 assigned_group_id | 0 assigned_group_state | single assigned_candidate_priority | 100 @@ -240,7 +240,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s1', 5432, 'postgres', 's1', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 28 +assigned_node_id | 31 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -253,7 +253,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s2', 5432, 'postgres', 's2', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 29 +assigned_node_id | 32 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -266,7 +266,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s3', 5432, 'postgres', 's3', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 30 +assigned_node_id | 33 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -540,3 +540,144 @@ resolved | t RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events, for each of the two formations used +-- above. Exercises pgautofailover.last_events() against a real scenario -- +-- its own SELECT list didn't match pgautofailover.event's column set for a +-- long time, breaking it outright, and nothing in this suite ever called +-- it to notice (see monitor.sql's own minimal-repro coverage). eventid/ +-- eventtime omitted: eventid is a database-wide sequence shared by every +-- test in this schedule (see regress_schedule's own comment) and +-- eventtime is a live timestamp -- neither is a stable value to pin here. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_unit', count => 100); +-[ RECORD 1 ]-+--------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single + +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_election', count => 100); +-[ RECORD 1 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 30 "p" (tlfe-p:5432): "single" +-[ RECORD 3 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "wait_standby" +-[ RECORD 4 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 30 "p" (tlfe-p:5432): "wait_primary" +-[ RECORD 6 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "catchingup" +-[ RECORD 8 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "secondary" +-[ RECORD 10 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 30 "p" (tlfe-p:5432): "primary" +-[ RECORD 12 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "wait_standby" +-[ RECORD 13 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "catchingup" +-[ RECORD 16 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "secondary" +-[ RECORD 17 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "wait_standby" +-[ RECORD 18 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "catchingup" +-[ RECORD 19 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "secondary" +-[ RECORD 20 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | report_lsn +goalstate | prepare_promotion +rule_pos | 375 +rule_section | reporting_node +description | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate already has all WAL -> prepare_promotion (1 of 2 -- see this row's own comment on why both are listed) + diff --git a/src/monitor/expected/workers.out b/src/monitor/expected/workers.out index 0cd3a6ef9..49dea7e3f 100644 --- a/src/monitor/expected/workers.out +++ b/src/monitor/expected/workers.out @@ -59,3 +59,33 @@ assigned_candidate_priority | 100 assigned_replication_quorum | t assigned_node_name | worker1a +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +select reportedstate, goalstate, rule_pos, rule_section, description + from pgautofailover.last_events('citus', count => 100); +-[ RECORD 1 ]-+--------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+--------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 4 "coord0a" (localhost:9876): "single" +-[ RECORD 3 ]-+--------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single + diff --git a/src/monitor/formation_metadata.c b/src/monitor/formation_metadata.c index d4974fb6b..10dd0d790 100644 --- a/src/monitor/formation_metadata.c +++ b/src/monitor/formation_metadata.c @@ -15,6 +15,7 @@ #include "funcapi.h" #include "miscadmin.h" +#include "group_state_machine.h" #include "health_check.h" #include "metadata.h" #include "formation_metadata.h" @@ -550,9 +551,6 @@ set_formation_number_sync_standbys(PG_FUNCTION_ARGS) int groupId = 0; int standbyCount = 0; - - char message[BUFSIZE] = { 0 }; - if (formation == NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -626,17 +624,14 @@ set_formation_number_sync_standbys(PG_FUNCTION_ARGS) /* SetFormationNumberSyncStandbys reports ERROR when returning false */ bool success = SetFormationNumberSyncStandbys(formationId, number_sync_standbys); - /* and now ask the primary to change its settings */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to apply_settings " - "after updating number_sync_standbys to %d for formation %s.", - NODE_FORMAT_ARGS(primaryNode), - formation->number_sync_standbys, - formation->formationId); - - SetNodeGoalState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS, message); + /* + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary -> + * apply_settings. Kept as this row's own condition too (see that row's + * comment), in addition to the ereport(ERROR) guard above -- belt and + * suspenders. + */ + (void) ProceedGroupStateForApiTrigger( + API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS, primaryNode, NULL); PG_RETURN_BOOL(success); } diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 85ed020a2..33310e12d 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -35,6 +35,7 @@ #include "utils/rel.h" #include "utils/syscache.h" #include "utils/timestamp.h" +#include "utils/tuplestore.h" /* @@ -54,11 +55,9 @@ typedef struct CandidateList /* private function forward declarations */ -static bool ProceedGroupStateForPrimaryNode(GroupStateContext *ctx, - AutoFailoverNode *primaryNode); static bool ProceedGroupStateForMSFailover(GroupStateContext *ctx, AutoFailoverNode *primaryNode); -static bool ProceedWithMSFailover(AutoFailoverNode *activeNode, +static bool ProceedWithMSFailover(GroupStateContext *ctx, AutoFailoverNode *activeNode, AutoFailoverNode *candidateNode); static bool BuildCandidateList(GroupStateContext *ctx, @@ -69,7 +68,8 @@ static AutoFailoverNode * SelectFailoverCandidateNode(GroupStateContext *ctx, CandidateList *candidateList, AutoFailoverNode *primaryNode); -static bool PromoteSelectedNode(AutoFailoverNode *selectedNode, +static bool PromoteSelectedNode(GroupStateContext *ctx, + AutoFailoverNode *selectedNode, AutoFailoverNode *primaryNode, CandidateList *candidateList); @@ -78,6 +78,1483 @@ static void AssignGoalState(AutoFailoverNode *pgAutoFailoverNode, static bool WalDifferenceWithin(AutoFailoverNode *secondaryNode, AutoFailoverNode *primaryNode, int64 delta); +static void AssertMonitorFSMWellFormed(void); + +/* + * --------------------------------------------------------------------- + * FSM dispatch is table-driven: MonitorFSM[] (below) holds an ordered list + * of MonitorFSMTransition rows, and RuleMatches() picks the first row whose + * conditions hold for the current NodeActiveContext. ProceedGroupStateFrom + * Context() and the primary-role dispatch path both work this way -- see + * the "WHY THESE CONSTANTS EXIST AT ALL" comment above the + * MonitorFSMTransition typedef for how each entry point bounds its search + * to the right rows. + * + * The MS-failover candidate-selection algorithm -- BuildCandidateList, + * SelectFailoverCandidateNode, PromoteSelectedNode, ProceedWithMSFailover, + * WalSourceNodesAreAllUnhealthy -- is hand-written C, reached from the + * table via a row's own extraAction: priority sorting, LSN comparison, and + * WAL-fetch orchestration aren't expressible as a plain set of matched + * conditions. Only the plain goal-state assignments at that algorithm's own + * tail end -- BuildCandidateList's own fan-out to REPORT_LSN, and + * PromoteSelectedNode's own PREPARE_PROMOTION/FAST_FORWARD choice -- are + * dispatched through the table too, via TryFanOutReportLsnRow/ + * DispatchMonitorFSMRuleByPos (see their own comments), each falling back + * to a plain AssignGoalState call when no row matches. + * --------------------------------------------------------------------- + */ + +typedef enum BoolPattern +{ + BOOL_ANY = 0, + BOOL_FALSE, + BOOL_TRUE +} BoolPattern; + +static bool +BoolMatchesPattern(bool actual, BoolPattern pattern) +{ + switch (pattern) + { + case BOOL_FALSE: + { + return !actual; + } + + case BOOL_TRUE: + { + return actual; + } + + case BOOL_ANY: + default: + { + return true; + } + } +} + + +/* + * IntPattern: the integer-valued analogue of BoolPattern, for facts that are + * counts rather than booleans -- BuildCandidateList's own missingNodesCount/ + * candidateCount/quorumCandidateCount (see NodeActiveContext's own fields of + * the same names), matched here instead of pre-flattened into one hand-named + * boolean per threshold the way atLeastOneHealthyCandidate etc. already are. + * INT_PATTERN_ANY is the same "omitted means don't-care" default every other + * pattern kind in this file uses. + * + * No array member (unlike ReplicationStateSet/NodeStatePattern -- see + * STATES()'s own comment on why those stay bare-brace): structurally + * identical to ApiTriggerPattern, so EXACTLY/AT_LEAST/AT_MOST's + * compound-literal cast below is exactly as safe as API_TRIGGER(fn)'s own + * cast (gcc's "initializer element is not constant" rejection is + * specifically about a nested array member, which neither struct has). + */ +typedef enum IntPatternKind +{ + INT_PATTERN_ANY = 0, + INT_PATTERN_EXACTLY, + INT_PATTERN_AT_LEAST, + INT_PATTERN_AT_MOST +} IntPatternKind; + +typedef struct IntPattern +{ + IntPatternKind kind; + int value; /* meaningful only when kind != INT_PATTERN_ANY */ +} IntPattern; + +#define EXACTLY(n) ((IntPattern) { .kind = INT_PATTERN_EXACTLY, .value = (n) }) +#define AT_LEAST(n) ((IntPattern) { .kind = INT_PATTERN_AT_LEAST, .value = (n) }) +#define AT_MOST(n) ((IntPattern) { .kind = INT_PATTERN_AT_MOST, .value = (n) }) + +static bool +IntMatchesPattern(int actual, IntPattern pattern) +{ + switch (pattern.kind) + { + case INT_PATTERN_EXACTLY: + { + return actual == pattern.value; + } + + case INT_PATTERN_AT_LEAST: + { + return actual >= pattern.value; + } + + case INT_PATTERN_AT_MOST: + { + return actual <= pattern.value; + } + + case INT_PATTERN_ANY: + default: + { + return true; + } + } +} + + +typedef enum NodeStatePatternKind +{ + NODE_STATE_ANY = 0, + NODE_STATE_STABLE, + NODE_STATE_NOT_STABLE, + NODE_STATE_REPORTED, + NODE_STATE_ASSIGNED, + NODE_STATE_NOT_ASSIGNED, + NODE_STATE_TRANSITIONING +} NodeStatePatternKind; + +/* Fixed-size (not pointer-based): a pointer initialized from the address of + * a compound literal nested inside another compound literal is not a + * constant expression by the C standard -- clang accepts it as an extension + * in file-scope initializers, but gcc (used by the project's Debian/CI + * build) rejects it with "initializer element is not constant". A fixed-size + * value member sidesteps the whole address-of-compound-literal question: + * no STATES() call in this file passes more than 3 states. + */ +typedef struct ReplicationStateSet +{ + ReplicationState states[4]; + int count; +} ReplicationStateSet; + +#define STATES_NARG_(_1, _2, _3, _4, N, ...) N +#define STATES_NARG(...) STATES_NARG_(__VA_ARGS__, 4, 3, 2, 1) + +/* Builds a { states[4], count } pair inline -- no NONE-terminated array to + * remember to close, no separate _STATES[] declaration to name. Deliberately + * NOT wrapped in a "(ReplicationStateSet) { ... }" compound-literal cast: + * every use is as a designated-initializer value nested inside another + * static aggregate (a MonitorFSMTransition table row, or one of the named + * FSM_* pattern constants below), and a bare brace-list is a plain + * sub-object initializer there -- fully portable ISO C. A compound-literal + * cast would make it a distinct object in its own right, and gcc (unlike + * clang) rejects using one of those, even by value, to initialize part of + * another object with static storage duration ("initializer element is not + * constant") -- see the note on ReplicationStateSet above for the same + * problem one level down. + */ +#define STATES(...) \ + { \ + .states = { __VA_ARGS__ }, \ + .count = STATES_NARG(__VA_ARGS__) \ + } + +typedef struct NodeStatePattern +{ + NodeStatePatternKind kind; + ReplicationStateSet reportedStates; + ReplicationStateSet assignedStates; +} NodeStatePattern; + +static bool +MatchStateSet(ReplicationState actual, ReplicationStateSet declared) +{ + for (int i = 0; i < declared.count; i++) + { + if (declared.states[i] == actual) + { + return true; + } + } + return false; +} + + +/* + * Same reasoning as STATES() above: no compound-literal cast, since every use + * is nested inside another static aggregate's designated initializer (e.g. + * ".statePattern = FSM_STATE(x)" inside a MonitorFSMTransition row). + */ +#define FSM_STATE(x) \ + { .kind = NODE_STATE_STABLE, .reportedStates = STATES(x) } + +/* WAIT_PRIMARY, JOIN_PRIMARY, or PRIMARY -- the "primary is up in some form" set */ +static const NodeStatePattern FSM_PRIMARY_OR_WAIT_OR_JOIN = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_JOIN_PRIMARY, + REPLICATION_STATE_PRIMARY), +}; + +/* + * WAIT_PRIMARY/JOIN_PRIMARY only, not PRIMARY -- a distinct, narrower set from + * the one above + */ +static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_JOIN_PRIMARY), +}; + +/* + * the "primary role" states MONITOR_FSM_SECTION_PRIMARY_NODE's own rows + * match against -- a different three-element set from + * FSM_PRIMARY_OR_WAIT_OR_JOIN above (no JOIN_PRIMARY, has APPLY_SETTINGS) + */ +static const NodeStatePattern FSM_PRIMARY_ROLE_STATES = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_PRIMARY, + REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_APPLY_SETTINGS), +}; + +/* + * same "primary role" scope, minus WAIT_PRIMARY -- a narrower enumerated STABLE + * set + */ +static const NodeStatePattern FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_PRIMARY, + REPLICATION_STATE_APPLY_SETTINGS), +}; + +/* + * reportedState alone (goalState irrelevant) is one of the roles + * CanTakeWritesInState() recognizes as "serving as primary", minus SINGLE -- + * NODE_STATE_REPORTED, deliberately not NODE_STATE_STABLE: a STABLE pattern + * requires reportedState == goalState, which breaks the moment a row using + * this pattern reassigns goalState itself (see pos 210's own comment for the + * real oscillation this caused when it instead relied on IsInPrimaryState(), + * which has the same reported == goal requirement built in). Matching on + * reportedState only means the match survives across the row's own + * extraAction, since nothing here ever touches reportedState directly. + */ +static const NodeStatePattern FSM_REPORTED_PRIMARY_ROLE_STATES = { + .kind = NODE_STATE_REPORTED, + .reportedStates = STATES(REPLICATION_STATE_PRIMARY, + REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_JOIN_PRIMARY, + REPLICATION_STATE_APPLY_SETTINGS), +}; + +/* + * reported WAIT_PRIMARY, goal in {WAIT_PRIMARY, PRIMARY} -- join_secondary's + * cascade row + */ +static const NodeStatePattern FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY = { + .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY), + .assignedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, REPLICATION_STATE_PRIMARY), +}; + +/* + * reported in {WAIT_PRIMARY,JOIN_PRIMARY}, goal PRIMARY -- demoted->catchingup, + * first disjunct + */ +static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY = { + .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_JOIN_PRIMARY), + .assignedStates = STATES(REPLICATION_STATE_PRIMARY), +}; + +/* + * goalState != WAIT_PRIMARY, reportedState irrelevant -- wait_maintenance's + * second row + */ +static const NodeStatePattern FSM_NOT_ASSIGNED_WAIT_PRIMARY = { + .kind = NODE_STATE_NOT_ASSIGNED, + .assignedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY), +}; + +/* + * !IsCurrentState(node, WAIT_PRIMARY): not converged to wait_primary, for any + * reason + */ +static const NodeStatePattern FSM_NOT_STABLE_WAIT_PRIMARY = { + .kind = NODE_STATE_NOT_STABLE, + .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY), +}; + +/* !IsCurrentState(node, SINGLE) */ +static const NodeStatePattern FSM_NOT_STABLE_SINGLE = { + .kind = NODE_STATE_NOT_STABLE, + .reportedStates = STATES(REPLICATION_STATE_SINGLE), +}; + +/* goalState == DROPPED, reportedState irrelevant */ +static const NodeStatePattern FSM_DROPPED_GOAL = { + .kind = NODE_STATE_ASSIGNED, + .assignedStates = STATES(REPLICATION_STATE_DROPPED), +}; + +/* + * reportedState == DEMOTE_TIMEOUT, goalState irrelevant. NOT + * FSM_STATE(DEMOTE_TIMEOUT), which would also require goalState == + * DEMOTE_TIMEOUT -- the opposite of what this self-fence guard needs: it must + * catch a node whose goalState is still whatever was assigned before the + * self-fence fired (see the real comment on this guard, preserved below). + */ +static const NodeStatePattern FSM_REPORTED_DEMOTE_TIMEOUT = { + .kind = NODE_STATE_REPORTED, + .reportedStates = STATES(REPLICATION_STATE_DEMOTE_TIMEOUT), +}; + +/* + * reportedState in {REPORT_LSN, FAST_FORWARD}, converged -- "continue an + * already started failover" guard, the direct (non-cascading) entry into + * ProceedGroupStateForMSFailover + */ +static const NodeStatePattern FSM_REPORT_LSN_OR_FAST_FORWARD = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_REPORT_LSN, + REPLICATION_STATE_FAST_FORWARD), +}; + + +static bool +NodeStateMatchesPattern(const AutoFailoverNode *node, const NodeStatePattern *pattern) +{ + if (node == NULL) + { + /* + * No node this round (no primary). Only NODE_STATE_ANY can still match + * -- every other kind needs a real reportedState/goalState to compare, + * which a nonexistent node simply doesn't have. Existence itself is + * checked separately via NodeStatusPattern.exists. + */ + return pattern->kind == NODE_STATE_ANY; + } + + ReplicationState reported = node->reportedState; + ReplicationState goal = node->goalState; + + switch (pattern->kind) + { + case NODE_STATE_ANY: + { + return true; + } + + case NODE_STATE_STABLE: + { + return (reported == goal) && MatchStateSet(reported, pattern->reportedStates); + } + + case NODE_STATE_NOT_STABLE: + { + return !((reported == goal) && MatchStateSet(reported, + pattern->reportedStates)); + } + + case NODE_STATE_REPORTED: + { + return MatchStateSet(reported, pattern->reportedStates); + } + + case NODE_STATE_ASSIGNED: + { + return MatchStateSet(goal, pattern->assignedStates); + } + + case NODE_STATE_NOT_ASSIGNED: + { + return !MatchStateSet(goal, pattern->assignedStates); + } + + case NODE_STATE_TRANSITIONING: + { + return MatchStateSet(reported, pattern->reportedStates) && + MatchStateSet(goal, pattern->assignedStates); + } + + default: + { + return false; + } + } +} + + +/* + * NodeStatus: every per-node fact this table's conditions need, computed once + * per role (activeNode, primaryNode) at the top of dispatch by + * BuildNodeStatus(). + */ +typedef struct NodeStatus +{ + AutoFailoverNode *node; + GroupStateContext *ctx; + bool isHealthy; + bool isUnhealthy; + bool candidateEligible; + bool isCitusWorkerGroup; + bool replicationQuorum; + bool isComparableToReferenceTli; +} NodeStatus; + +typedef struct NodeStatusPattern +{ + BoolPattern exists; + NodeStatePattern statePattern; + BoolPattern isHealthy; + BoolPattern isUnhealthy; + BoolPattern candidateEligible; + BoolPattern isInPrimaryState; + BoolPattern isInMaintenance; + BoolPattern isDemotedPrimary; + BoolPattern canTakeWrites; + BoolPattern reportedCanTakeWrites; + BoolPattern reportedIsWaitStandby; + BoolPattern reportedIsJoinSecondary; + BoolPattern reportedIsPrepareMaintenance; + BoolPattern isReadyToStreamWAL; + BoolPattern drainTimeExpired; + BoolPattern isCitusWorkerGroup; + BoolPattern replicationQuorum; + BoolPattern isComparableToReferenceTli; + BoolPattern unreachableFromDemoteTimeout; +} NodeStatusPattern; + +static void +BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *status) +{ + memset(status, 0, sizeof(NodeStatus)); + status->node = node; + status->ctx = ctx; + + if (node == NULL) + { + /* + * NodeIsUnhealthy(NULL, ctx) returns true: a nonexistent node is + * "unhealthy" by definition, so every row gated on isUnhealthy stays + * correct when there's no node to check. + */ + status->isUnhealthy = true; + return; + } + + status->isHealthy = NodeIsHealthy(node, ctx); + status->isUnhealthy = NodeIsUnhealthy(node, ctx); + status->candidateEligible = node->candidatePriority > 0; + status->isCitusWorkerGroup = IsCitusFormation(ctx->formation) && node->groupId > 0; + status->replicationQuorum = node->replicationQuorum; +} + + +/* + * isInPrimaryState/isInMaintenance/isDemotedPrimary/canTakeWrites/ + * drainTimeExpired/unreachableFromDemoteTimeout are deliberately NOT cached + * in NodeStatus and computed live here from status->node instead: unlike + * isHealthy/isUnhealthy/candidateEligible (health/priority facts that can't + * change mid-dispatch), these are pure functions of + * node->goalState/reportedState, and a matched row's extraAction can + * reassign the very node being matched (e.g. DRAINING on primaryNode) in + * the SAME node_active() call, before dispatch continues to a later row -- + * exactly like NodeStateMatchesPattern below, which reads status->node's + * fields live for the same reason. Caching these as snapshot booleans would + * leave later rows in the same call matching against a stale "still in + * primary state" fact even after primaryNode had just been moved to + * DRAINING -- concurrent_health_check_and_report requires the "secondary -> + * prepare_promotion" row to correctly stop matching once primaryNode is no + * longer IsInPrimaryState(). + * + * canTakeWrites is distinct from isInPrimaryState: it's CanTakeWritesInState + * (node_metadata.c) applied to goalState alone, with no requirement that + * reportedState has converged to match -- exactly node_active_protocol.c's + * RemoveNode() own "bool currentNodeIsPrimary = + * CanTakeWritesInState(currentNode->goalState);" check, which a mid- + * transition node (goalState allows writes, reportedState hasn't caught up + * yet) still satisfies where isInPrimaryState (which additionally requires + * stability) would not. + * + * reportedCanTakeWrites is CanTakeWritesInState applied to reportedState + * instead of goalState -- the reported-only counterpart pos 210's own + * comment explains the need for: a row whose extraAction reassigns + * goalState (any GOAL(...) assignment) can't gate its own match on anything + * that reads goalState (canTakeWrites, isInPrimaryState) without risking the + * exact self-undermining oscillation documented there, since the row's own + * action changes the very fact its condition is testing. reportedState is + * never written by this row's own action, so a condition built purely from + * it stays stable across dispatches until the keeper itself converges. + * + * reportedIsWaitStandby is a plain reportedState equality check, for the + * same reported-only reason as reportedCanTakeWrites above, but forced by a + * DIFFERENT row's action this time, not this row's own: pos 101's own + * remove_node() fan-out (.otherNodeAssignedState = GOAL(REPORT_LSN), + * unconditional on every surviving non-maintenance standby) can rewrite a + * lone wait_standby node's own goalState to report_lsn synchronously, + * before pos 209/211's own early_checks evaluation ever runs on that node's + * next heartbeat -- a goalState-dependent exclusion (e.g. NOT_STABLE) would + * see reported != goal at that point and treat wait_standby as "no longer + * stable", reviving a match it was supposed to permanently exclude. A + * NOT_STABLE-based exclusion here would look correct in dump_fsm_edges()'s + * own static analysis (which never sees pos 101's cross-row goalState + * write) but would still let pos 209/211 fire for a real wait_standby node, + * exactly because of this -- confirmed live in a pgaftest run. Matching on + * reportedState alone sidesteps it entirely. + * + * reportedIsJoinSecondary, same reported-only shape as reportedIsWaitStandby + * just above, excludes JOIN_SECONDARY_STATE from pos 209's own "alone in + * group, candidate-eligible -> single" match. A node reporting + * join_secondary has already had Postgres cleanly checkpointed and stopped + * (fsm_checkpoint_and_stop_postgres, fsm.c) as part of switching its + * replication target to a newly-elected primary -- its on-disk data is a + * trustworthy copy, but only of *this node's own* last moment as the old + * primary, frozen before that new primary ever took a single write. + * Promoting it straight to SINGLE if it ends up alone risks silently + * discarding whatever the new primary committed in the meantime -- a real + * split-brain/data-loss risk, unlike every other source state pos 209 + * matches, where the reporting node's own data is either already the most + * advanced available or was safely fetched from whichever node was. + * + * reportedIsPrepareMaintenance, same reported-only shape and same pos 209 + * exclusion, for the primary-role counterpart of join_secondary's own risk. + * start_maintenance() assigns PREPARE_MAINTENANCE_STATE to the primary and + * PREPARE_PROMOTION_STATE to its chosen standby in the very same call (see + * pos 109/111's own comment) -- and pos 343 lets that standby advance all + * the way to WAIT_PRIMARY/PRIMARY the moment the old primary's own + * reportedState merely *converges* to prepare_maintenance (Postgres + * cleanly stopped, fsm_stop_postgres_for_primary_maintenance), with no + * requirement that the old primary's row ever be removed first. So a node + * can sit in prepare_maintenance indefinitely while a different, already + * fully-promoted primary is live and taking writes elsewhere -- if that new + * primary later also vanishes, promoting the OLD node straight to SINGLE + * would discard everything the new primary committed in between, the exact + * same split-brain risk reportedIsJoinSecondary guards against, just + * reached one step earlier in the handoff. + */ +static bool +NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) +{ + bool unreachableFromDemoteTimeout = + status->node != NULL && + status->node->goalState != REPLICATION_STATE_DEMOTE_TIMEOUT && + status->node->goalState != REPLICATION_STATE_DEMOTED && + status->node->goalState != REPLICATION_STATE_PRIMARY && + status->node->goalState != REPLICATION_STATE_SINGLE; + + return BoolMatchesPattern(status->node != NULL, pattern->exists) && + NodeStateMatchesPattern(status->node, &pattern->statePattern) && + BoolMatchesPattern(status->isHealthy, pattern->isHealthy) && + BoolMatchesPattern(status->isUnhealthy, pattern->isUnhealthy) && + BoolMatchesPattern(status->candidateEligible, pattern->candidateEligible) && + BoolMatchesPattern(IsInPrimaryState(status->node), + pattern->isInPrimaryState) && + BoolMatchesPattern(IsInMaintenance(status->node), pattern->isInMaintenance) && + BoolMatchesPattern(IsDemotedPrimary(status->node), + pattern->isDemotedPrimary) && + BoolMatchesPattern(status->node != NULL && + CanTakeWritesInState(status->node->goalState), + pattern->canTakeWrites) && + BoolMatchesPattern(status->node != NULL && + CanTakeWritesInState(status->node->reportedState), + pattern->reportedCanTakeWrites) && + BoolMatchesPattern(status->node != NULL && + status->node->reportedState == + REPLICATION_STATE_WAIT_STANDBY, + pattern->reportedIsWaitStandby) && + BoolMatchesPattern(status->node != NULL && + status->node->reportedState == + REPLICATION_STATE_JOIN_SECONDARY, + pattern->reportedIsJoinSecondary) && + BoolMatchesPattern(status->node != NULL && + status->node->reportedState == + REPLICATION_STATE_PREPARE_MAINTENANCE, + pattern->reportedIsPrepareMaintenance) && + BoolMatchesPattern(CandidateNodeIsReadyToStreamWAL(status->node), + pattern->isReadyToStreamWAL) && + BoolMatchesPattern(NodeIsDrainTimeExpired(status->node, status->ctx), + pattern->drainTimeExpired) && + BoolMatchesPattern(status->isCitusWorkerGroup, pattern->isCitusWorkerGroup) && + BoolMatchesPattern(status->replicationQuorum, pattern->replicationQuorum) && + BoolMatchesPattern(status->isComparableToReferenceTli, + pattern->isComparableToReferenceTli) && + BoolMatchesPattern(unreachableFromDemoteTimeout, + pattern->unreachableFromDemoteTimeout); +} + + +/* + * MonitorApiFunction itself is declared in group_state_machine.h, not here: + * ProceedGroupStateForApiTrigger() (below the array) is called from + * node_active_protocol.c/formation_metadata.c with a MonitorApiFunction + * value, so the enum needs to be visible outside this file the same way + * MonitorFSMSection already is. ApiTriggerPattern/ApiTriggerKind/ + * MatchApiTrigger/API_TRIGGER() stay private to this file: only the enum + * itself crosses the module boundary, every caller just picks a tag and + * hands it to ProceedGroupStateForApiTrigger, never builds a pattern + * itself. + * + * Every builder in this file (BuildFromContextNodeActiveContext, + * BuildForPrimaryNodeNodeActiveContext) memset()s its NodeActiveContext to + * zero before filling it in, so apiFunction defaults to API_FUNCTION_NONE + * unless a caller explicitly sets it: no row that omits .conditions.apiTrigger + * changes meaning just because this field exists. + */ + +typedef enum ApiTriggerKind +{ + /* + * The safe default for an omitted .conditions.apiTrigger field: matches + * only an ordinary node_active() heartbeat call. Every row written + * before this mechanism existed implicitly assumed exactly this, so + * omission has to keep meaning "heartbeat only", not "any trigger" -- + * the opposite of BOOL_ANY/NODE_STATE_ANY's "omission matches + * everything" convention elsewhere in this file, and deliberately so: + * here the safe default is the narrower match, since none of the 48 + * existing rows should start matching a manual perform_failover() call + * just because they don't mention this field at all. + */ + API_TRIGGER_NODE_ACTIVE = 0, + API_TRIGGER_SPECIFIC +} ApiTriggerKind; + +typedef struct ApiTriggerPattern +{ + ApiTriggerKind kind; + MonitorApiFunction function; /* meaningful only when kind == API_TRIGGER_SPECIFIC */ +} ApiTriggerPattern; + +#define API_TRIGGER(fn) ((ApiTriggerPattern) { .kind = API_TRIGGER_SPECIFIC, .function = \ + (fn) }) + +static bool +MatchApiTrigger(MonitorApiFunction actual, ApiTriggerPattern pattern) +{ + switch (pattern.kind) + { + case API_TRIGGER_SPECIFIC: + { + return actual == pattern.function; + } + + case API_TRIGGER_NODE_ACTIVE: + default: + { + return actual == API_FUNCTION_NONE; + } + } +} + + +/* + * NodeActiveContext: group-level facts, computed once per dispatch call + * alongside the two NodeStatus roles above. + */ +typedef struct NodeActiveContext +{ + NodeStatus activeNode; + NodeStatus primaryNode; + + /* + * otherNode is who otherNodeAssignedState actually assigns to (see + * DispatchMonitorFSMRule). Set to a plain copy of primaryNode by every + * builder that populates a real primaryNode (BuildFromContextNodeActive + * Context, BuildApiTriggerNodeActiveContext) -- today, "the other node + * in this transition" and "the group's primary" are always the same + * node, so otherNode.node == primaryNode.node everywhere. Kept as its + * own field rather than reusing .primaryNode directly so that role + * stays conceptually activeNode/otherNode independently of whichever + * node the monitor's own domain concepts (primary, candidate, ...) say + * it happens to be. A row can instead resolve a dynamically-sized + * target list via otherNodesFn (see MonitorOtherNodesResolverFunction + * below) -- e.g. remove_node()'s fan-out to every surviving standby -- + * without disturbing every other row's own primaryNode-shaped + * conditions, which simply read .primaryNode directly. + */ + NodeStatus otherNode; + + /* + * candidateNode is only populated by BuildMSFailoverNodeActiveContext + * (the MS-failover cluster's own nested-dispatch context builder, + * mirroring BuildForPrimaryNodeNodeActiveContext's role-specific + * pattern): FindCandidateNodeBeingPromoted(ctx->groupNodeList)'s result, + * i.e. the node currently mid-promotion, if any. .node == NULL if no + * failover candidate has been selected yet this round. Left at its + * memset-zero default (.node == NULL, .isUnhealthy == true) for every + * other dispatch pass -- no row outside the MS-failover sub-section + * references it. + */ + NodeStatus candidateNode; + + MonitorApiFunction apiFunction; + + bool groupHasExactlyOneNode; + bool groupHasExactlyTwoNodes; + bool groupHasMoreThanTwoNodes; + bool anyOtherNodeWaitingStandby; + + bool numberSyncStandbysIsZero; + bool replicationQuorumCountIsZero; + bool secondaryNodesCountIsZero; + bool secondaryQuorumNodesCountIsZero; + bool atLeastOneHealthyCandidate; + + bool walWithinPromoteThreshold; + bool walWithinSyncThreshold; + bool activeAndPrimaryTliMatch; + + bool primaryIsWaitPrimaryPresumedDead; + bool failoverInProgress; + bool replicationStallExceeded; + + /* + * lastHealthySyncStandbyGoingToMaintenance is set only inside + * ProceedGroupStateForApiTrigger's own API_FUNCTION_START_MAINTENANCE + * branch (see its own comment there): true when activeNode is the last + * remaining healthy synchronous standby and formation->number_sync_ + * standbys is zero, meaning putting it into maintenance would otherwise + * block writes on the primary until the monitor separately assigns it + * wait_primary -- mirrors node_active_protocol.c's start_maintenance() + * own condition verbatim. Left false (the memset default) for every + * other dispatch pass; not a general-purpose fact reused elsewhere. + */ + bool lastHealthySyncStandbyGoingToMaintenance; + + /* + * The following four facts are set only by + * BuildMSFailoverNodeActiveContext, for the MS-failover cluster's own + * declarative rows (see group_state_machine.c's "MS-failover / + * candidate-selection cluster" section) -- left false (the memset + * default) for every other dispatch pass. + */ + + /* + * WalSourceNodesAreAllUnhealthy(ctx, ctx->groupNodeList, activeNode) -- + * activeNode-specific despite living at this level (not on NodeStatus): it + * depends on the whole group's other REPORT_LSN peers, not on activeNode's + * own state alone. + */ + bool activeNodeAllWalSourcesUnhealthy; + + /* + * mirrors whether ProceedGroupStateForMSFailover's own "nodeBeingPromoted + * != NULL" branch will actually drive that candidate via + * ProceedWithMSFailover this round, rather than falling through to + * BuildCandidateList/selection instead -- see this file's own + * ActionRunMultiStandbyFailoverCascade comment for the full derivation. + */ + bool candidatePromotionInProgress; + + /* + * WalDifferenceWithin(mostAdvancedNode, ctx->primaryNode, + * PromoteXlogThreshold) for the current MS-failover candidate pool's own + * most-advanced member -- SelectFailoverCandidateNode's own data-loss + * guard. + */ + bool mostAdvancedCandidateWithinPromoteThreshold; + + /* + * GuardDataLoss (metadata.c), the pgautofailover.guard_data_loss GUC -- a + * single global process-wide bool, grouped here (not read directly inside + * an extraAction) so any row gated on it says so in its own conditions. + */ + bool guardDataLossEnabled; + + /* + * Set unconditionally true by BuildMSFailoverNodeActiveContext, false + * (the memset default) everywhere else -- the MS-failover cluster's own + * "am I actually being dispatched from inside the MS-failover cluster's + * own bounded nested search" marker. Every row under SectionMSFailover + * that doesn't already carry a condition guaranteed false outside that + * nested search (as pos 363/365 do, via + * activeNodeAllWalSourcesUnhealthy/candidatePromotionInProgress) must + * require this true: the top-level driver's own ordinary lookup + * (ProceedGroupStateFromContext) scans under SectionReportingNode, which + * -- since every MS-failover row's sectionPath[0] is also + * REPORTING_NODE -- scans straight through this whole cluster too. + * Without this guard, pos 367-373's bare activeNode-state patterns (no + * distinguishing condition of their own) would fire for any ordinary, + * non-MS-failover heartbeat whose reported/goal states happened to line + * up, hijacking normal secondary/catchingup/maintenance convergence -- + * node_active_protocol.out/guard_data_loss.out/ etc. depend on this + * guard to keep that from happening. + */ + bool inMSFailoverCluster; + + /* + * Set unconditionally true by + * BuildMSFailoverCandidateGateNodeActiveContext, false (the memset default) + * everywhere else -- narrows the 5 counting-gate rows + * (reporting_node.ms_failover.promotion_outcome.*_gate) to only ever match + * when dispatched from ProceedGroupStateForMSFailover's own gate checks. + * Without this, TryFanOutReportLsnRow's own candidateNode=NULL call + * (BuildMSFailoverNodeActiveContext) leaves + * candidateCount/missingNodesCount at their memset-0 default and + * candidatePromotionInProgress false -- the exact values the missing-nodes + * and candidate-count gate rows themselves match on -- so its own + * SectionMSFailover-wide scan could spuriously hit one of these gate rows + * instead of correctly falling through to its own plain-AssignGoalState + * fallback whenever pos 367-373 somehow didn't match. + */ + bool inMSFailoverCandidateGate; + + /* + * candidateCount/quorumCandidateCount/missingNodesCount mirror + * CandidateList's own fields of the same names (BuildCandidateList) -- + * set only by BuildMSFailoverCandidateGateNodeActiveContext, left at + * their memset-0 default everywhere else. Matched via IntPattern + * (EXACTLY/AT_LEAST/AT_MOST) instead of being pre-flattened into a + * one-off boolean per threshold, since these are the same reusable + * counts BuildCandidateList already computes once, not new facts. + */ + int candidateCount; + int quorumCandidateCount; + int missingNodesCount; + + /* + * quorumCandidateCount >= (ctx->formation->number_sync_standbys + 1), + * computed once by the same builder as the three counts above. A + * per-formation runtime threshold, not a literal -- IntPattern can only + * compare against a fixed value written into a row, so this one + * comparison is precomputed into a plain bool instead, exactly like + * mostAdvancedCandidateWithinPromoteThreshold already does for its own + * GUC-relative comparison (PromoteXlogThreshold) above. + */ + bool sufficientQuorumCandidates; +} NodeActiveContext; + +typedef struct NodeActiveContextPattern +{ + ApiTriggerPattern apiTrigger; /* omitted -> {0} -> API_TRIGGER_NODE_ACTIVE, + * matching every row's existing meaning with + * no changes required elsewhere */ + + BoolPattern groupHasExactlyOneNode; + BoolPattern groupHasExactlyTwoNodes; + BoolPattern groupHasMoreThanTwoNodes; + BoolPattern anyOtherNodeWaitingStandby; + + BoolPattern numberSyncStandbysIsZero; + BoolPattern replicationQuorumCountIsZero; + BoolPattern secondaryNodesCountIsZero; + BoolPattern secondaryQuorumNodesCountIsZero; + BoolPattern atLeastOneHealthyCandidate; + + BoolPattern walWithinPromoteThreshold; + BoolPattern walWithinSyncThreshold; + BoolPattern activeAndPrimaryTliMatch; + + BoolPattern primaryIsWaitPrimaryPresumedDead; + BoolPattern failoverInProgress; + BoolPattern replicationStallExceeded; + BoolPattern lastHealthySyncStandbyGoingToMaintenance; + + BoolPattern activeNodeAllWalSourcesUnhealthy; + BoolPattern candidatePromotionInProgress; + BoolPattern mostAdvancedCandidateWithinPromoteThreshold; + BoolPattern guardDataLossEnabled; + BoolPattern inMSFailoverCluster; + BoolPattern inMSFailoverCandidateGate; + + IntPattern candidateCount; + IntPattern quorumCandidateCount; + IntPattern missingNodesCount; + BoolPattern sufficientQuorumCandidates; +} NodeActiveContextPattern; + + +typedef enum GoalStateAssignmentKind +{ + GOAL_STATE_NONE = 0, + GOAL_STATE_SET +} GoalStateAssignmentKind; + +typedef struct GoalStateAssignment +{ + GoalStateAssignmentKind kind; + ReplicationState state; +} GoalStateAssignment; + +/* + * No compound-literal cast -- see STATES()'s comment: every use nests inside + * another static aggregate's designated initializer. + */ +#define GOAL(x) { .kind = GOAL_STATE_SET, .state = (x) } + +/* + * A row's otherNodesFn, when set, resolves a dynamically-sized list of + * nodes -- rather than the single nac->otherNode.node target -- that its + * own otherNodeAssignedState gets assigned to. DispatchMonitorFSMRule loops + * over the resolved list and calls AssignDeclaredGoalState once per node, so + * each of those assignments gets exactly the same rule_pos/rule_section + * attribution any other row's own single-target assignment already gets -- + * e.g. OtherNodesNotInMaintenance (pos 101's own remove_node() fan-out) and + * OtherNodesDueForCatchingUp (pos 419's own catchup fan-out). A row sets at + * most one of "otherNodesFn resolves the target list" (this) or + * "nac->otherNode.node is the single, already-resolved target" (omitted, + * the more common case) -- never both. + */ +typedef List *(*MonitorOtherNodesResolverFunction) (GroupStateContext *ctx, + NodeActiveContext *nac); + +/* + * A row's extraAction runs before its own activeNodeAssignedState/ + * otherNodeAssignedState are applied. When a transition needs to keep + * going after its own assignment -- the MS-failover cascade, the + * join_secondary -> nested primary pass -- the action itself performs one + * bounded, explicitly-named nested search+dispatch over MonitorFSM[] -- see + * ActionRunMultiStandbyFailoverCascade and ActionRunPrimaryNodeTransition + * below -- rather than signaling the top-level driver to keep scanning. + * There is deliberately no generic "continue dispatch" flag here: if + * extraAction could return a bool meaning "keep scanning from here", a + * sibling row in the same "family" could match a second time after the + * intended row declined (see ActionRunMultiStandbyFailoverCascade's + * comment). A bounded, named jump cannot have that problem, because it can + * only ever land on one specific row family, not + * wander into whichever row happens to be next. + */ +typedef void (*MonitorExtraActionFunction) (GroupStateContext *ctx, + NodeActiveContext *nac, + char *message); + +/* + * MonitorFSMSection itself is declared in group_state_machine.h, not here: + * it needs to be visible to pg_auto_failover.c (to register + * pgautofailover.fsm_section and wire up dump_fsm()) the same way + * ReplicationState is. See its own comment there for the top-level-vs-leaf + * distinction and why expanding it with fine-grained values changed nothing + * SQL-visible. + * + * MonitorFSMSectionPath is a small, fixed-depth array of MonitorFSMSection + * values -- a row's own place in the section hierarchy (e.g. { REPORTING_NODE, + * MS_FAILOVER, MS_FAILOVER_RETRY_RESET }). A search bounds itself to every + * row whose sectionPath is under a given prefix (SectionPathIsUnderPrefix, + * below) instead of an array-index range, so a row's section membership is a + * fact carried on the row itself, not an implication of where it happens to + * sit in the array -- inserting, removing, or reordering rows within a + * section needs no separate bookkeeping. Trailing unused slots default to + * MONITOR_FSM_SECTION_NONE via ordinary aggregate initialization -- a plain + * array member needs no compound-literal cast any more than + * ReplicationStateSet's own states[4] does (see STATES()'s own comment on + * why that stays bare-brace too). + */ +#define MONITOR_FSM_SECTION_PATH_MAX_DEPTH 4 +typedef MonitorFSMSection MonitorFSMSectionPath[MONITOR_FSM_SECTION_PATH_MAX_DEPTH]; + +/* + * SectionPathIsUnderPrefix returns whether path is prefix (an ancestor of it, + * or itself) -- pure C, no parsing, no cache, no dependency on anything outside + * this file: comparing two small fixed-size enum arrays element by element. + * This is the replacement for FindMatchingMonitorFSMRuleIndexFrom's old + * [startIndex, endIndex) bound (see FindMatchingMonitorFSMRuleIndexUnderPath + * below): a search is now "every row whose sectionPath is under this prefix", + * not "every row between these two array indices". + */ +static bool +SectionPathIsUnderPrefix(const MonitorFSMSectionPath path, const MonitorFSMSectionPath + prefix) +{ + for (int i = 0; i < MONITOR_FSM_SECTION_PATH_MAX_DEPTH; i++) + { + if (prefix[i] == MONITOR_FSM_SECTION_NONE) + { + return true; + } + + if (path[i] != prefix[i]) + { + return false; + } + } + + return true; +} + + +typedef struct MonitorFSMTransition +{ + /* + * pos and sectionPath are metadata, not match inputs: RuleMatches() never + * reads either. pos is a human-facing row number -- purely so a comment, + * a bug report, or a SQL query against dump_fsm() can say "row 203" and + * a reader can find it without counting braces up from the top of the + * array. Each top-level MonitorFSMSection gets its own hundred-block, + * starting at *01 rather than *00 so the position within the block reads + * as an ordinary 1-based count (101 is the section's 1st row, 103 its + * 2nd, ...) instead of an off-by-one 0th/1st/2nd. Rows within a section + * are numbered every 2 (101, 103, 105, ...) rather than consecutively -- + * so a new row can be inserted between two existing ones (e.g. 102 + * between 101 and 103) without renumbering anything else in the file. + * sectionPath records this row's own place in the section hierarchy (see + * MonitorFSMSectionPath above); AssertMonitorFSMWellFormed() uses both to + * confirm each row's sectionPath[0] agrees with what its pos's 100-block + * implies, so a mismatch fails at first use, not by accident months + * later. Both are also exposed to SQL via dump_fsm() (pos, section, and + * the new section_path) and attributed to the pgautofailover.event row a + * matched rule produces (see rule_pos/rule_section below) -- rule_section + * is always derived from sectionPath[0] alone, so it stays exactly the + * same 4-value enum it always was. + */ + int pos; + MonitorFSMSectionPath sectionPath; + + NodeStatusPattern activeNode; + NodeStatusPattern primaryNode; + + /* + * otherNode is the role otherNodeAssignedState actually targets (see + * that field's own comment) when the target is the single, already- + * resolved nac->otherNode.node -- a genuinely distinct role from + * primaryNode, even though every row using it has nac->otherNode.node == + * nac->primaryNode.node (see NodeActiveContext's own comment on + * .otherNode for why). Kept separate from primaryNode in this struct -- + * rather than just reusing .primaryNode wherever a row wants to + * constrain otherNodeAssignedState's target -- so a role exists to write + * conditions against without a name that falsely implies it's always the + * primary. Doesn't apply to a row using otherNodesFn instead (see that + * field's own comment): such a row's target is a dynamically resolved + * list, not this single node, so .otherNode stays at its + * NodeStatusPattern default (omitted, don't-care) there too. Every row + * written before this field existed omits it, which matches + * NodeStatusPattern's own "omitted means don't-care" default -- exactly + * the same zero-risk-to-existing-rows guarantee apiFunction/inMSFailover + * Cluster/etc. already established when each was added. + */ + NodeStatusPattern otherNode; + + NodeStatusPattern candidateNode; /* MS-failover sub-section rows only; see + * NodeActiveContext's own comment on + * .candidateNode */ + NodeActiveContextPattern conditions; + + GoalStateAssignment activeNodeAssignedState; + + /* + * otherNodeAssignedState targets nac->otherNode.node (see + * NodeActiveContext's own comment) -- not nac->primaryNode.node + * directly, even though today the two are always the same pointer -- + * unless otherNodesFn (below) is set, in which case it targets every + * node otherNodesFn resolves instead. + */ + GoalStateAssignment otherNodeAssignedState; + + /* + * When set, otherNodeAssignedState (above) is assigned to every node + * this resolves, not to the single nac->otherNode.node -- see + * MonitorOtherNodesResolverFunction's own comment. NULL (the default) + * for every row using the single-target otherNode mechanism instead. + */ + MonitorOtherNodesResolverFunction otherNodesFn; + + MonitorExtraActionFunction extraAction; + + const char *comment; +} MonitorFSMTransition; + +/* + * MonitorFSM[] is one array holding every FSM transition: operator-triggered + * rows, heartbeat rows for a reporting non-primary node, the MS-failover + * cluster, and primary-role rows all live here, disambiguated by + * .sectionPath and reached via bounded searches (see "WHY THESE CONSTANTS + * EXIST AT ALL" below). Forward-declared here so the extraActions defined + * above it (which each perform one bounded, named nested search over it) + * can reference it by name; the section-path constants below are + * forward-declared the same way, for the same reason -- both those actions + * and the top-level driver need them to bound their searches. + * + * WHY THESE CONSTANTS EXIST AT ALL: a single flat "first match wins over + * the whole array" search can't work here, for two reasons. First, + * .activeNode means something different depending on which section a row + * belongs to: in the reporting-node sections it's the node that just called + * node_active(), while in SectionPrimaryNode it's the group's primary, + * substituted into the activeNode role (see BuildForPrimaryNodeNodeActive + * Context) -- a row from one section matched against the wrong section's + * NodeActiveContext would test the wrong node entirely. Second, one + * fallback (the MS-failover cascade declining) needs to resume scanning + * from a specific *later* point, not from the top. Each named + * MonitorFSMSectionPath constant below is the section a search should be + * bounded to, so it only ever considers rows semantically valid for the + * situation at hand. A row's section membership is a fact carried on the + * row itself (.sectionPath, see MonitorFSMTransition + * above) rather than an implication of where it happens to sit in the + * array, so inserting, removing, or reordering rows within a section + * requires touching no constant at all: + * + * SectionApiTriggered = { MONITOR_FSM_SECTION_API_TRIGGERED } + * Rows whose sectionPath[0] is API_TRIGGERED (pos 101-1xx) -- the + * operator-triggered rows (perform_failover, remove_node, + * start/stop_maintenance, set_node_candidate_priority, + * set_node_replication_quorum, set_formation_number_sync_standbys) -- + * reached only via ProceedGroupStateForApiTrigger(), never via the + * ordinary node_active() heartbeat path (nac->apiFunction stays + * API_FUNCTION_NONE there, and every one of these 15 rows' + * .conditions.apiTrigger requires a specific non-NONE value -- see + * MatchApiTrigger). + * + * SectionEarlyChecks = { MONITOR_FSM_SECTION_EARLY_CHECKS } + * Rows whose sectionPath[0] is EARLY_CHECKS (pos 201-211) -- the six + * checks (DROPPED, goal-DROPPED, MAINTENANCE, the demote_timeout + * self-fence, both "alone in group" rows) that + * ProceedGroupStateFromContext() runs unconditionally, before its one + * real branch point (IsInPrimaryState(activeNode)) -- so they're tried + * first regardless of which way that branch goes. + * + * SectionReportingNode = { MONITOR_FSM_SECTION_REPORTING_NODE } + * Rows whose sectionPath[0] is REPORTING_NODE (pos 301-38x) -- reached + * only when activeNode is confirmed NOT currently primary-role. Covers + * both the ordinary FromContext rows (sectionPath[1] == + * MONITOR_FSM_SECTION_FROM_CONTEXT) and the MS-failover cluster + * (sectionPath[1] == MONITOR_FSM_SECTION_MS_FAILOVER) as one prefix, + * matching the old MonitorFSM_FromContextStart..MonitorFSM_ + * PrimaryNodeSectionStart span exactly. + * + * MonitorFSM_MultiStandbyCascadeResumeAfterPos = 305 + * Not a section at all: a resume point *inside* SectionReportingNode, + * used only by ActionRunMultiStandbyFailoverCascade -- the pos of the + * merged nodesCount>2-unhealthy-primary row itself ("nodesCount>2, + * primary unhealthy -> draining/maintenance + MS-failover cascade"). + * When ProceedGroupStateForMSFailover() declines, dispatch needs to + * keep evaluating the reporting-node rows that come after this point in + * the table, for the same activeNode, in the same node_active() call. + * Section-path containment alone can't express "resume after this + * specific row" (it answers "is this row under X?", not "in array + * order, after row Y") -- pos is already the row's own stable, + * human-facing identity, and AssertMonitorFSMWellFormed() confirms pos + * is strictly increasing == array order, so "pos > afterPos" is exactly + * the resume semantics needed. Conceptually similar to, but NOT the + * same bound as, SectionMSFailover below -- see + * ActionRunMultiStandbyFailoverCascade's own comment for the + * distinction. + * + * SectionMSFailover = { MONITOR_FSM_SECTION_REPORTING_NODE, + * MONITOR_FSM_SECTION_MS_FAILOVER } + * Rows under the MS-failover / candidate-selection cluster's own + * declarative transitions (see that section's own comment below) -- + * two (retry-reset, join_secondary) reached through + * TryMSFailoverDeclarativeRow's bounded nested dispatch, gated by the + * exact same hand-written C condition ProceedGroupStateForMSFailover/ + * ProceedWithMSFailover already evaluate; four (BuildCandidateList's + * own fan-out) through TryFanOutReportLsnRow's bounded nested dispatch, + * one per node the fan-out loop touches; two (PromoteSelectedNode's own + * two outcomes) through DispatchMonitorFSMRuleByPos, since first-match- + * wins can't distinguish between them (see their own comment); three + * for the counting gates' own outcomes -- ProceedGroupStateForMSFailover + * still tests missingNodesCount/candidateCount/quorumCandidateCount as + * plain hand-written ifs (see BuildMSFailoverCandidateGateNodeActiveContext), + * dispatching through one of these rows only for the resulting message + * and rule_pos attribution -- and one more (the "still gathering + * candidates" catch-all) purely for + * dump_fsm() completeness/fallback, not reached through any other + * bounded search. None reached through the ordinary top-level driver. + * + * SectionPrimaryNode = { MONITOR_FSM_SECTION_PRIMARY_NODE } + * Rows whose sectionPath[0] is PRIMARY_NODE (pos 401-421): the + * declarative replacement for ProceedGroupStateForPrimaryNode()'s own + * if-chain, in which .activeNode means the *primary* node, not the + * reporting node. Used both from the top-level driver (activeNode + * already primary-role) and from ActionRunPrimaryNodeTransition's + * nested pass on primaryNode (join_secondary's cascade row). + * + * Table end + * MonitorFSM[] ends with a terminator row (.pos left at its zero + * default, which no real row ever has) rather than a separately + * maintained count -- every bounded search's own linear scan (see + * FindMatchingMonitorFSMRuleIndexUnderPath) and every full-table walk + * (dump_fsm()/dump_fsm_edges()/AssertMonitorFSMWellFormed()) stops there + * instead. + * + * What makes a row's own sectionPath safe to hand-write (rather than a + * foot-gun): every row also carries its own .pos (see MonitorFSMTransition + * above), and AssertMonitorFSMWellFormed() (below the array) walks the + * whole table once and asserts .pos is strictly increasing and that each + * row's sectionPath[0] agrees with what its pos's own 100-block implies + * (API_TRIGGERED for 1xx, EARLY_CHECKS for 2xx, REPORTING_NODE for 3xx, + * PRIMARY_NODE for 4xx) -- a row whose sectionPath drifts out of sync with + * its own pos fails loudly at first use (an assertion), not silently as a + * row from the wrong section matching unexpectedly or a search that scans + * zero rows and never matches. + */ +static const MonitorFSMTransition MonitorFSM[]; + +/* + * Forward-declared for the same reason as MonitorFSM[] just above: these are + * defined far below (near BuildMSFailoverNodeActiveContext), but the + * counting-gate rows inside MonitorFSM[] itself reference them by name as + * their own .extraAction. + */ +static void ActionLogMSFailoverMissingNodesDecline(GroupStateContext *ctx, + NodeActiveContext *nac, + char *message); +static void ActionLogMSFailoverMissingNodesContinue(GroupStateContext *ctx, + NodeActiveContext *nac, + char *message); +static void ActionLogMSFailoverQuorumDecline(GroupStateContext *ctx, + NodeActiveContext *nac, + char *message); +static void ActionLogMSFailoverQuorumContinue(GroupStateContext *ctx, + NodeActiveContext *nac, + char *message); + +#define MonitorFSM_MultiStandbyCascadeResumeAfterPos 305 + +static const MonitorFSMSectionPath SectionApiTriggered = +{ MONITOR_FSM_SECTION_API_TRIGGERED }; +static const MonitorFSMSectionPath SectionEarlyChecks = +{ MONITOR_FSM_SECTION_EARLY_CHECKS }; +static const MonitorFSMSectionPath SectionReportingNode = +{ MONITOR_FSM_SECTION_REPORTING_NODE }; +static const MonitorFSMSectionPath SectionPrimaryNode = +{ MONITOR_FSM_SECTION_PRIMARY_NODE }; +static const MonitorFSMSectionPath SectionMSFailover = +{ MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER }; + +/* + * Leaf prefixes for the 3 MS-failover counting gates (missingNodesCount/ + * candidateCount/quorumCandidateCount, see ProceedGroupStateForMSFailover): + * exact 4-deep paths, each used only by that function's own dedicated + * dispatch call at the matching hand-written `if`, never via the broader + * SectionMSFailover scan other callers use (see inMSFailoverCandidateGate's + * own comment on NodeActiveContext for why that distinction matters). + */ +static const MonitorFSMSectionPath SectionMSFailoverMissingNodesGate = +{ + MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE +}; +static const MonitorFSMSectionPath SectionMSFailoverQuorumCandidateGate = +{ + MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE +}; + +/* + * Forward-declared for the same reason as MonitorFSM[] above: used by + * extraActions (ActionRunPrimaryNodeTransition) defined before its real + * definition further down. + */ +static void BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, + AutoFailoverNode *primaryNode, + NodeActiveContext *nac); + + +static bool +RuleMatches(const NodeActiveContext *nac, const MonitorFSMTransition *rule) +{ + const NodeActiveContextPattern *cond = &rule->conditions; + + return MatchApiTrigger(nac->apiFunction, cond->apiTrigger) && + + NodeMatchesPattern(&nac->activeNode, &rule->activeNode) && + NodeMatchesPattern(&nac->primaryNode, &rule->primaryNode) && + NodeMatchesPattern(&nac->otherNode, &rule->otherNode) && + NodeMatchesPattern(&nac->candidateNode, &rule->candidateNode) && + + BoolMatchesPattern(nac->groupHasExactlyOneNode, + cond->groupHasExactlyOneNode) && + BoolMatchesPattern(nac->groupHasExactlyTwoNodes, + cond->groupHasExactlyTwoNodes) && + BoolMatchesPattern(nac->groupHasMoreThanTwoNodes, + cond->groupHasMoreThanTwoNodes) && + BoolMatchesPattern(nac->anyOtherNodeWaitingStandby, + cond->anyOtherNodeWaitingStandby) && + BoolMatchesPattern(nac->numberSyncStandbysIsZero, + cond->numberSyncStandbysIsZero) && + BoolMatchesPattern(nac->replicationQuorumCountIsZero, + cond->replicationQuorumCountIsZero) && + BoolMatchesPattern(nac->secondaryNodesCountIsZero, + cond->secondaryNodesCountIsZero) && + BoolMatchesPattern(nac->secondaryQuorumNodesCountIsZero, + cond->secondaryQuorumNodesCountIsZero) && + BoolMatchesPattern(nac->atLeastOneHealthyCandidate, + cond->atLeastOneHealthyCandidate) && + BoolMatchesPattern(nac->walWithinPromoteThreshold, + cond->walWithinPromoteThreshold) && + BoolMatchesPattern(nac->walWithinSyncThreshold, + cond->walWithinSyncThreshold) && + BoolMatchesPattern(nac->activeAndPrimaryTliMatch, + cond->activeAndPrimaryTliMatch) && + BoolMatchesPattern(nac->primaryIsWaitPrimaryPresumedDead, + cond->primaryIsWaitPrimaryPresumedDead) && + BoolMatchesPattern(nac->failoverInProgress, cond->failoverInProgress) && + BoolMatchesPattern(nac->replicationStallExceeded, + cond->replicationStallExceeded) && + BoolMatchesPattern(nac->lastHealthySyncStandbyGoingToMaintenance, + cond->lastHealthySyncStandbyGoingToMaintenance) && + BoolMatchesPattern(nac->activeNodeAllWalSourcesUnhealthy, + cond->activeNodeAllWalSourcesUnhealthy) && + BoolMatchesPattern(nac->candidatePromotionInProgress, + cond->candidatePromotionInProgress) && + BoolMatchesPattern(nac->mostAdvancedCandidateWithinPromoteThreshold, + cond->mostAdvancedCandidateWithinPromoteThreshold) && + BoolMatchesPattern(nac->guardDataLossEnabled, cond->guardDataLossEnabled) && + BoolMatchesPattern(nac->inMSFailoverCluster, cond->inMSFailoverCluster) && + BoolMatchesPattern(nac->inMSFailoverCandidateGate, + cond->inMSFailoverCandidateGate) && + + IntMatchesPattern(nac->missingNodesCount, cond->missingNodesCount) && + IntMatchesPattern(nac->candidateCount, cond->candidateCount) && + IntMatchesPattern(nac->quorumCandidateCount, cond->quorumCandidateCount) && + BoolMatchesPattern(nac->sufficientQuorumCandidates, + cond->sufficientQuorumCandidates); +} + + +/* + * FindMatchingMonitorFSMRuleIndexUnderPath scans the whole table in array + * order, considering only rows whose own sectionPath is under prefix (see + * SectionPathIsUnderPrefix) and whose pos is greater than afterPos -- replacing + * the old [startIndex, endIndex) index-range bound with a section-membership + * one. pos is strictly increasing == array order (see + * AssertMonitorFSMWellFormed), so "pos > afterPos" is exactly "resume scanning + * after this row" -- afterPos = 0 (no real row has pos <= 0) means "from the + * very start of whichever rows are under prefix", the ordinary case; a specific + * pos is only ever passed for the one genuine mid-section resume point this + * table has (see MonitorFSM_MultiStandbyCascadeResumeAfterPos). + * + * table is always MonitorFSM[] itself, terminated by a sentinel row whose + * .pos is left at its zero default (no real row ever has pos <= 0) -- the + * loop below stops there instead of at a separately hand-maintained count, + * which is exactly the mechanism that once let a row silently fall out of + * every bounded search in this file when it drifted out of sync (see the + * MonitorFSM[] array's own trailing comment, right after its last real row). + */ +static int +FindMatchingMonitorFSMRuleIndexUnderPath(const MonitorFSMTransition table[], + const MonitorFSMSectionPath prefix, int afterPos, + const NodeActiveContext *nac) +{ + for (int i = 0; table[i].pos != 0; i++) + { + if (table[i].pos <= afterPos) + { + continue; + } + + if (!SectionPathIsUnderPrefix(table[i].sectionPath, prefix)) + { + continue; + } + + if (RuleMatches(nac, &table[i])) + { + return i; + } + } + return -1; +} + + +/* + * AssignDeclaredGoalState asserts that a rule only ever assigns a state it + * actually declared: drift between a row's own assignment slots and what it + * assigns fails loudly (an Assert) instead of silently rotting. + */ +static void +AssignDeclaredGoalState(const MonitorFSMTransition *rule, AutoFailoverNode *node, + ReplicationState state, char *message) +{ +#ifdef USE_ASSERT_CHECKING + bool declared = + (rule->activeNodeAssignedState.kind == GOAL_STATE_SET && + rule->activeNodeAssignedState.state == state) || + (rule->otherNodeAssignedState.kind == GOAL_STATE_SET && + rule->otherNodeAssignedState.state == state); + + Assert(declared); +#endif + + AssignGoalState(node, state, message); +} + + +static void +DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, + const MonitorFSMTransition *rule) +{ + char message[BUFSIZE] = { 0 }; + + /* + * Attribute every event this row's dispatch produces -- its own + * extraAction's side effects included -- to this rule's .pos/.section + * (see CurrentMonitorFSMRulePos/Section in notifications.h). Saved and + * restored rather than cleared to 0 on the way out: a nested dispatch + * (ActionRunMultiStandbyFailoverCascade's/ActionRunPrimaryNodeTransition's + * own bounded search, run from inside extraAction below) sets these to + * the *inner* row's own pos/section for its own duration, and once it + * returns, this row's own subsequent activeNodeAssignedState/ + * otherNodeAssignedState calls need to see this (outer) row's values + * again, not 0. + */ + int savedRulePos = CurrentMonitorFSMRulePos; + int savedRuleSection = CurrentMonitorFSMRuleSection; + + CurrentMonitorFSMRulePos = rule->pos; + CurrentMonitorFSMRuleSection = (int) rule->sectionPath[0]; + + if (rule->comment != NULL) + { + strlcpy(message, rule->comment, BUFSIZE); + } + + if (rule->extraAction != NULL) + { + rule->extraAction(ctx, nac, message); + } + + if (rule->activeNodeAssignedState.kind == GOAL_STATE_SET) + { + AssignDeclaredGoalState(rule, nac->activeNode.node, + rule->activeNodeAssignedState.state, message); + } + + if (rule->otherNodeAssignedState.kind == GOAL_STATE_SET) + { + if (rule->otherNodesFn != NULL) + { + List *otherNodesList = rule->otherNodesFn(ctx, nac); + ListCell *otherNodeCell = NULL; + + foreach(otherNodeCell, otherNodesList) + { + AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(otherNodeCell); + + AssignDeclaredGoalState(rule, otherNode, + rule->otherNodeAssignedState.state, message); + } + } + else + { + AssignDeclaredGoalState(rule, nac->otherNode.node, + rule->otherNodeAssignedState.state, message); + } + } + + CurrentMonitorFSMRulePos = savedRulePos; + CurrentMonitorFSMRuleSection = savedRuleSection; +} + + +/* + * FindAndDispatchMonitorFSMRuleUnderPath bounds a search over MonitorFSM[] to + * every row under prefix with pos > afterPos, and dispatches the first + * match, if any -- the one building block every call site in this file + * needs: the top-level driver's own two straight-line lookups (early checks, + * then either the primary-role section or the rest of + * ProceedGroupStateFromContext()'s rows -- see its comment for why two + * separate lookups), plus the two extraActions that each perform one + * further, separate bounded nested search of their own when their row's own + * cascade declines: ActionRunMultiStandbyFailoverCascade and + * ActionRunPrimaryNodeTransition below. Returns whether a row matched, so + * callers that need to distinguish "matched and handled" from "nothing under + * this prefix applied" can. + */ +static bool +FindAndDispatchMonitorFSMRuleUnderPath(GroupStateContext *ctx, NodeActiveContext *nac, + const MonitorFSMSectionPath prefix, int afterPos) +{ + int index = FindMatchingMonitorFSMRuleIndexUnderPath(MonitorFSM, prefix, afterPos, + nac); + + if (index < 0) + { + return false; + } + + DispatchMonitorFSMRule(ctx, nac, &MonitorFSM[index]); + + return true; +} + /* GUC variables */ int EnableSyncXlogThreshold = DEFAULT_XLOG_SEG_SIZE; @@ -86,1413 +1563,4051 @@ int ReplicationStallTimeoutMs = 10 * 1000; /* - * BuildGroupStateContext loads everything from the database that the - * node_active FSM needs, capturing a single timestamp snapshot and copying - * the current GUC values. Call this once at the top of NodeActive() and pass - * the resulting context to ProceedGroupStateFromContext(). - * - * Returns false (and raises an ereport ERROR) when the formation cannot be - * found. + * BuildGroupStateContext loads everything from the database that the + * node_active FSM needs, capturing a single timestamp snapshot and copying + * the current GUC values. Call this once at the top of NodeActive() and pass + * the resulting context to ProceedGroupStateFromContext(). + * + * Returns false (and raises an ereport ERROR) when the formation cannot be + * found. + */ +bool +BuildGroupStateContext(GroupStateContext *ctx, AutoFailoverNode *activeNode) +{ + /* + * A no-op in a non-assert build (see AssertMonitorFSMWellFormed's own + * #ifdef USE_ASSERT_CHECKING), so this stays an unconditional call -- + * matching AssignDeclaredGoalState's own pattern -- rather than an + * #ifdef'd call site, which would make the function itself look unused + * (and fail -Werror=unused-function) in a non-assert build. + */ + static bool monitorFSMChecked = false; + + if (!monitorFSMChecked) + { + AssertMonitorFSMWellFormed(); + monitorFSMChecked = true; + } + + ctx->formationId = activeNode->formationId; + ctx->groupId = activeNode->groupId; + ctx->activeNode = activeNode; + ctx->formation = GetFormation(activeNode->formationId); + ctx->groupNodeList = + AutoFailoverNodeGroup(activeNode->formationId, activeNode->groupId); + ctx->groupNodeCount = list_length(ctx->groupNodeList); + ctx->now = GetCurrentTimestamp(); + ctx->unhealthyTimeoutMs = UnhealthyTimeoutMs; + ctx->drainTimeoutMs = DrainTimeoutMs; + ctx->startupGracePeriodMs = StartupGracePeriodMs; + ctx->replicationStallTimeoutMs = ReplicationStallTimeoutMs; + + if (ctx->formation == NULL) + { + ereport(ERROR, + (errmsg("Formation for %s could not be found", + activeNode->formationId))); + } + + return true; +} + + +/* + * ProceedGroupState proceeds the state machines of the group of which + * the given node is part. It builds a GroupStateContext from the database and + * delegates to ProceedGroupStateFromContext. + */ +bool +ProceedGroupState(AutoFailoverNode *activeNode) +{ + GroupStateContext ctx; + + BuildGroupStateContext(&ctx, activeNode); + + return ProceedGroupStateFromContext(&ctx); +} + + +/* + * OtherNodeIsDueForCatchingUp is shared between the count computation in + * BuildForPrimaryNodeNodeActiveContext() and the fan-out resolution in + * OtherNodesDueForCatchingUp() below, so the two can never drift apart on + * which nodes they mean by "unhealthy secondary" -- both need to agree, + * since the counts drive which row matches and the fan-out is that row's + * own side effect. + */ +static bool +OtherNodeIsDueForCatchingUp(GroupStateContext *ctx, AutoFailoverNode *otherNode) +{ + return otherNode->goalState == REPLICATION_STATE_SECONDARY && + otherNode->reportedState != REPLICATION_STATE_REPORT_LSN && + otherNode->reportedState != REPLICATION_STATE_JOIN_SECONDARY && + NodeIsUnhealthy(otherNode, ctx); +} + + +static void +ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *message) +{ + RemoveAutoFailoverNode(ctx->activeNode); +} + + +/* + * ActionRunMultiStandbyFailoverCascade implements the whole + * nodesCount>2-unhealthy-primary transition as a single extraAction: it makes + * the DRAINING/MAINTENANCE/nothing decision, then unconditionally calls + * ProceedGroupStateForMSFailover(). The DRAINING/MAINTENANCE decision is + * dispatched through MonitorFSM[]'s own pos 381/383 rows first (see their own + * comment) -- the hand-written if/else-if below only runs as a fallback if + * neither row's own conditions somehow line up with the ones just checked + * (should never happen). This function never returns after assigning + * DRAINING/MAINTENANCE to the primary -- it always falls through to try + * ProceedGroupStateForMSFailover next, and if THAT declines (returns false), + * falls through further still to a bounded resume search over the rest of + * MonitorFSM[]'s REPORTING_NODE section (the report_lsn/prepare_promotion/ + * stop_replication/... rows further down, for this SAME activeNode). + * + * This has to be ONE row/action, not three separate declarative rows sharing + * this action: splitting the DRAINING/MAINTENANCE/catch-all outcomes into + * three rows would let dispatch, after a declined row, keep scanning forward + * and match a later, broader row against the same outer "nodesCount>2, + * primary unhealthy" condition (the catch-all "neither DRAINING nor + * MAINTENANCE applies" case) -- re-invoking ProceedGroupStateForMSFailover a + * *second* time in the same node_active() call. concurrent_second_primary_ + * death_report and concurrent_health_check_and_report both depend on this not + * happening: splitting the action would get the former stuck and make the + * latter produce a spurious second cascade invocation that changes the + * outcome. + * + * When ProceedGroupStateForMSFailover() declines, the fallthrough to "the rest + * of ProceedGroupStateFromContext" is a single bounded nested search over + * SectionReportingNode starting after + * MonitorFSM_MultiStandbyCascadeResumeAfterPos, not a flag back to the + * top-level driver: FindAndDispatchMonitorFSMRuleUnderPath's own internal loop + * already finds whichever row is the correct next match, however many rows + * down that is, in one call -- no repeated re-dispatch needed to walk past + * intervening non-matches. + * + * NOTE on naming: this resume search is NOT the same bound as + * SectionMSFailover, despite both marking a conceptually similar "resume + * point" -- they bound two different things. The resume search (used here) + * just marks "resume scanning ordinary REPORTING_NODE rows after + * ProceedGroupStateForMSFailover declines" -- ProceedGroupStateForMSFailover() + * itself, and the BuildCandidateList/SelectFailoverCandidateNode/ + * PromoteSelectedNode functions it calls, stay hand-written C, called + * wholesale from here (the candidate-selection algorithm itself doesn't + * reduce to declarative conditions any more cleanly than a table row can + * express). SectionMSFailover, by contrast, bounds the *separate* MS-failover + * cluster (pos 363-383, "MS-failover / candidate-selection cluster" section + * below) that those same hand-written functions reach *into*, at their own + * tail end, via + * TryMSFailoverDeclarativeRow/ + * TryFanOutReportLsnRow/DispatchMonitorFSMRuleByPos -- covering the plain + * per-node goal assignments (retry-reset, join_secondary, BuildCandidateList's + * own report_lsn fan-out, PromoteSelectedNode's prepare_promotion/fast_forward + * choice), with a raw AssignGoalState call kept as an unconditional fallback + * on no match. The last two rows in that same cluster (pos 381/383) are a + * fourth, unrelated caller reusing the same bounded range: + * ActionRunMultiStandbyFailoverCascade's own DRAINING/MAINTENANCE outcomes + * (see that function's own comment) -- not part of the candidate-selection + * machinery at all, just sharing the same "safe + * to scan with the ordinary nac" bound. The candidate-selection algorithm's own + * logic (priority sort, LSN comparison, WAL-fetch orchestration) is not part of + * either bounded range. + */ +static void +ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + AutoFailoverNode *primaryNode = nac->primaryNode.node; + + /* + * pos 391/393 are the only two rows FindAndDispatchMonitorFSMRuleUnderPath + * can reach here (nac->inMSFailoverCluster is false for this nac, so + * every other row under SectionMSFailover is unreachable -- see + * inMSFailoverCluster's own comment). A false return means neither + * row's own condition held: a genuine, expected no-op (the primary is + * unhealthy but neither draining nor prepare_maintenance applies yet), + * not a bug -- nothing further to do this round. + */ + (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, nac, SectionMSFailover, 0); + + if (!ProceedGroupStateForMSFailover(ctx, primaryNode)) + { + (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, nac, SectionReportingNode, + MonitorFSM_MultiStandbyCascadeResumeAfterPos); + } +} + + +/* + * ActionRunPlainMSFailoverCascade is the "continue an already-started + * failover" call site: activeNode itself is REPORT_LSN or FAST_FORWARD, so + * there's no DRAINING/MAINTENANCE decision to make and nothing to fall + * through to afterward -- just call ProceedGroupStateForMSFailover() and + * discard its return value. + */ +static void +ActionRunPlainMSFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + (void) ProceedGroupStateForMSFailover(ctx, nac->primaryNode.node); +} + + +/* + * ActionRunPrimaryNodeTransition is the join_secondary cascade: the real + * source's tail call into ProceedGroupStateForPrimaryNode(ctx, primaryNode) + * substitutes primaryNode for activeNode and re-enters that function's own + * dispatch from scratch -- modeled here as one bounded nested search over + * MonitorFSM[]'s ForPrimaryNode section, built with primaryNode playing the + * activeNode role (see BuildForPrimaryNodeNodeActiveContext). + */ +static void +ActionRunPrimaryNodeTransition(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + NodeActiveContext primaryNac; + + BuildForPrimaryNodeNodeActiveContext(ctx, nac->primaryNode.node, &primaryNac); + + (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, &primaryNac, SectionPrimaryNode, + 0); +} + + +/* + * OtherNodesDueForCatchingUp is pos 419's own otherNodesFn: every other node + * in the group that OtherNodeIsDueForCatchingUp() says has come back + * healthy after being sent to catchingup earlier. Replaces the former + * ActionCatchupUnhealthySecondaries extraAction -- the loop-and- + * AssignGoalState body is unchanged, just returning the list instead of + * assigning inline, so DispatchMonitorFSMRule's own otherNodesFn loop (see + * its own comment) does the assignment and gets the same rule_pos/ + * rule_section attribution any other row's assignment gets. + */ +static List * +OtherNodesDueForCatchingUp(GroupStateContext *ctx, NodeActiveContext *nac) +{ + AutoFailoverNode *primaryNode = nac->activeNode.node; + List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); + List *dueNodesList = NIL; + ListCell *nodeCell = NULL; + + foreach(nodeCell, otherNodesGroupList) + { + AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); + + if (OtherNodeIsDueForCatchingUp(ctx, otherNode)) + { + dueNodesList = lappend(dueNodesList, otherNode); + } + } + + return dueNodesList; +} + + +/* + * BuildFromContextNodeActiveContext computes every fact the + * reporting_node.from_context rows (sectionPath[1] == + * MONITOR_FSM_SECTION_FROM_CONTEXT) need. primaryNode may be NULL (failover + * already in progress, primary removed). + */ +static void +BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *primaryNode, + NodeActiveContext *nac) +{ + AutoFailoverNode *activeNode = ctx->activeNode; + + memset(nac, 0, sizeof(NodeActiveContext)); + + BuildNodeStatus(ctx, activeNode, &nac->activeNode); + BuildNodeStatus(ctx, primaryNode, &nac->primaryNode); + nac->otherNode = nac->primaryNode; /* see NodeActiveContext's own comment on .otherNode */ + + /* + * isComparableToReferenceTli defaults to true -- a node that hasn't + * reported a timeline yet (reportedTLI == 0) has nothing to check, so + * the timeline-fork rows below don't fire for it. + */ + nac->activeNode.isComparableToReferenceTli = true; + if (activeNode->reportedTLI > 0) + { + int referenceTli = 0; + List *comparableNodeList = + FilterNodesByTimelineAncestry(ctx->groupNodeList, ctx->formationId, + ctx->groupId, &referenceTli); + + if (referenceTli > 0) + { + bool comparable = false; + ListCell *cell = NULL; + + foreach(cell, comparableNodeList) + { + AutoFailoverNode *node = (AutoFailoverNode *) lfirst(cell); + + if (node->nodeId == activeNode->nodeId) + { + comparable = true; + break; + } + } + + nac->activeNode.isComparableToReferenceTli = comparable; + } + } + + nac->groupHasExactlyOneNode = (ctx->groupNodeCount == 1); + nac->groupHasExactlyTwoNodes = (ctx->groupNodeCount == 2); + nac->groupHasMoreThanTwoNodes = (ctx->groupNodeCount > 2); + nac->failoverInProgress = IsFailoverInProgress(ctx->groupNodeList); + + nac->activeAndPrimaryTliMatch = + primaryNode != NULL && activeNode->reportedTLI == primaryNode->reportedTLI; + + nac->walWithinPromoteThreshold = + WalDifferenceWithin(activeNode, primaryNode, PromoteXlogThreshold); + nac->walWithinSyncThreshold = + WalDifferenceWithin(activeNode, primaryNode, EnableSyncXlogThreshold); + + nac->primaryIsWaitPrimaryPresumedDead = + NodeIsWaitPrimaryPresumedDead(primaryNode, activeNode, ctx); + + nac->replicationStallExceeded = + primaryNode != NULL && + primaryNode->replicationStallSince != 0 && + TimestampDifferenceExceeds(primaryNode->replicationStallSince, + ctx->now, ctx->replicationStallTimeoutMs); + + if (ctx->groupNodeCount > 2 && nac->primaryNode.isUnhealthy) + { + List *candidateNodesList = + AutoFailoverOtherNodesListInState(primaryNode, REPLICATION_STATE_SECONDARY); + + nac->atLeastOneHealthyCandidate = CountHealthyCandidates(candidateNodesList) >= 1; + } +} + + +/* + * BuildForPrimaryNodeNodeActiveContext computes every fact SectionPrimaryNode + * (MonitorFSM[]'s pos 401-421 rows) needs: it loops over every other node in + * the primary's group, using the same OtherNodeIsDueForCatchingUp() test + * OtherNodesDueForCatchingUp() (above) uses for its own fan-out, to derive + * the group-level counts (replicationQuorumCount, secondaryNodesCount, + * secondaryQuorumNodesCount) and the anyOtherNodeWaitingStandby flag those + * rows match against. */ -bool -BuildGroupStateContext(GroupStateContext *ctx, AutoFailoverNode *activeNode) +static void +BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, + AutoFailoverNode *primaryNode, + NodeActiveContext *nac) { - ctx->formationId = activeNode->formationId; - ctx->groupId = activeNode->groupId; - ctx->activeNode = activeNode; - ctx->formation = GetFormation(activeNode->formationId); - ctx->groupNodeList = - AutoFailoverNodeGroup(activeNode->formationId, activeNode->groupId); - ctx->groupNodeCount = list_length(ctx->groupNodeList); - ctx->now = GetCurrentTimestamp(); - ctx->unhealthyTimeoutMs = UnhealthyTimeoutMs; - ctx->drainTimeoutMs = DrainTimeoutMs; - ctx->startupGracePeriodMs = StartupGracePeriodMs; - ctx->replicationStallTimeoutMs = ReplicationStallTimeoutMs; + memset(nac, 0, sizeof(NodeActiveContext)); - if (ctx->formation == NULL) + BuildNodeStatus(ctx, primaryNode, &nac->activeNode); + + /* + * .primaryNode role is unused by every row in MonitorFSM[]'s + * ForPrimaryNode section -- primaryNode IS activeNode here, so every + * condition is expressed against .activeNode directly. + */ + + List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); + int otherNodesCount = list_length(otherNodesGroupList); + + int replicationQuorumCount = otherNodesCount; + int secondaryNodesCount = otherNodesCount; + int secondaryQuorumNodesCount = otherNodesCount; + + ListCell *nodeCell = NULL; + + foreach(nodeCell, otherNodesGroupList) { - ereport(ERROR, - (errmsg("Formation for %s could not be found", - activeNode->formationId))); + AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); + + if (OtherNodeIsDueForCatchingUp(ctx, otherNode)) + { + --secondaryNodesCount; + --secondaryQuorumNodesCount; + } + else if (!IsCurrentState(otherNode, REPLICATION_STATE_SECONDARY)) + { + --secondaryNodesCount; + --secondaryQuorumNodesCount; + } + else if (IsCurrentState(otherNode, REPLICATION_STATE_SECONDARY) && + !otherNode->replicationQuorum) + { + --secondaryQuorumNodesCount; + } + + if (!otherNode->replicationQuorum) + { + --replicationQuorumCount; + } + + if (IsCurrentState(otherNode, REPLICATION_STATE_WAIT_STANDBY)) + { + nac->anyOtherNodeWaitingStandby = true; + } } - return true; + nac->replicationQuorumCountIsZero = (replicationQuorumCount == 0); + nac->secondaryNodesCountIsZero = (secondaryNodesCount == 0); + nac->secondaryQuorumNodesCountIsZero = (secondaryQuorumNodesCount == 0); + nac->numberSyncStandbysIsZero = (ctx->formation->number_sync_standbys == 0); + nac->failoverInProgress = IsFailoverInProgress(ctx->groupNodeList); } /* - * ProceedGroupState proceeds the state machines of the group of which - * the given node is part. It builds a GroupStateContext from the database and - * delegates to ProceedGroupStateFromContext. + * OtherNodesNotInMaintenance is pos 101's own otherNodesFn: RemoveNode's own + * primary-removal fan-out (node_active_protocol.c's RemoveNode(), + * "if (currentNodeIsPrimary) { foreach other node not in maintenance -> + * report_lsn }"). Assigned via this row's own otherNodeAssignedState = + * GOAL(REPORT_LSN) instead of an extraAction -- DispatchMonitorFSMRule's own + * otherNodesFn loop (see its own comment) now runs after this row's + * activeNodeAssignedState = DROPPED, same order as every other row using + * both slots together (e.g. the perform_failover/reporting_node rows that + * already assign both roles). The real source's own order (fan out to the + * survivors, then mark the removed node dropped) doesn't matter here: which + * nodes qualify for the fan-out, and what they're assigned, is entirely + * independent of the removed node's own goalState. + */ +static List * +OtherNodesNotInMaintenance(GroupStateContext *ctx, NodeActiveContext *nac) +{ + List *otherNodesGroupList = AutoFailoverOtherNodesList(nac->activeNode.node); + List *eligibleNodesList = NIL; + ListCell *nodeCell = NULL; + + foreach(nodeCell, otherNodesGroupList) + { + AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); + + if (IsInMaintenance(otherNode)) + { + continue; + } + + eligibleNodesList = lappend(eligibleNodesList, otherNode); + } + + return eligibleNodesList; +} + + +/* + * BuildApiTriggerNodeActiveContext computes every fact the API_TRIGGERED + * section of MonitorFSM[] needs, for one particular apiFunction. + * + * activeNode is whichever single node the call is most fundamentally about: + * the standby being promoted for perform_failover's 2-node row, the primary + * itself for perform_failover's >2-node row and every + * set_node_candidate_priority/set_node_replication_quorum/ + * set_formation_number_sync_standbys row (mirroring + * BuildForPrimaryNodeNodeActiveContext's own "primaryNode IS activeNode" + * convention), the node being removed for remove_node, the node being + * (re)moved into/out of maintenance for start/stop_maintenance. + * + * primaryNode is the group's real primary, only when a row for this + * apiFunction genuinely needs to reference "the primary" as a role distinct + * from activeNode (perform_failover's 2-node row, start_maintenance's + * secondary rows); NULL otherwise, exactly like the heartbeat side already + * passes NULL for primaryNode in the analogous "primaryNode IS activeNode" + * case. + */ +static void +BuildApiTriggerNodeActiveContext(GroupStateContext *ctx, MonitorApiFunction apiFunction, + AutoFailoverNode *activeNode, + AutoFailoverNode *primaryNode, + NodeActiveContext *nac) +{ + memset(nac, 0, sizeof(NodeActiveContext)); + + nac->apiFunction = apiFunction; + + BuildNodeStatus(ctx, activeNode, &nac->activeNode); + BuildNodeStatus(ctx, primaryNode, &nac->primaryNode); + nac->otherNode = nac->primaryNode; /* see NodeActiveContext's own comment on .otherNode */ + + nac->groupHasExactlyOneNode = (ctx->groupNodeCount == 1); + nac->groupHasExactlyTwoNodes = (ctx->groupNodeCount == 2); + nac->groupHasMoreThanTwoNodes = (ctx->groupNodeCount > 2); + nac->failoverInProgress = IsFailoverInProgress(ctx->groupNodeList); + + if (apiFunction == API_FUNCTION_START_MAINTENANCE && primaryNode != NULL) + { + /* + * Mirrors node_active_protocol.c's start_maintenance() own + * pre-dispatch computation verbatim (its own secondaryNodesCount, + * formation->number_sync_standbys, and IsHealthySyncStandby( + * currentNode) triple): true when activeNode is the only remaining + * healthy synchronous standby and number_sync_standbys is zero -- + * putting it into maintenance would otherwise block writes on the + * primary until the monitor separately assigns it wait_primary. + */ + List *secondaryNodesList = + AutoFailoverOtherNodesListInState(primaryNode, REPLICATION_STATE_SECONDARY); + int secondaryNodesCount = CountHealthySyncStandbys(secondaryNodesList); + + nac->lastHealthySyncStandbyGoingToMaintenance = + ctx->formation->number_sync_standbys == 0 && + secondaryNodesCount == 1 && + IsHealthySyncStandby(activeNode); + } +} + + +/* + * ProceedGroupStateForApiTrigger dispatches a single operator-triggered + * transition through MonitorFSM[]'s API_TRIGGERED section (pos 101-1xx). + * Every SQL-callable wrapper in node_active_protocol.c/formation_metadata.c + * that assigns a goal state as a direct consequence of an operator call -- + * perform_failover, remove_node, start/stop_maintenance, + * set_node_candidate_priority, set_node_replication_quorum, + * set_formation_number_sync_standbys -- keeps its own imperative shape + * (argument parsing, locking, resolving which node(s) are involved, and + * every existing validation ereport(ERROR)/WARNING/NOTICE, whose exact + * message text stays test-stable), and calls this once in the middle to + * make the actual state assignment(s), then continues with whatever POST + * side effect the wrapper still needs (a continuation ProceedGroupState() + * call, a candidatePriority trick, number_sync_standbys bookkeeping) as + * further hand-written code -- none of that surrounding code becomes a row; + * only the state assignment itself is declarative. + * + * Unlike the heartbeat side (ProceedGroupStateFromContext, which silently + * no-ops on no match -- see its own comment for why that's the right call + * there), a genuine no-match here is always ereport(ERROR)'d as a bug: by + * the time this is called, the wrapper has already validated, via its own + * preserved pre-checks, that the operation IS valid for the resolved + * node's current state -- reaching no-match here means this table's own + * conditions have drifted out of sync with the wrapper's pre-checks, a + * real gap to fix, not a normal "operator asked for something invalid" + * outcome (that case is already rejected earlier, with its own specific, + * pre-existing ereport(ERROR) message, before this function is ever + * called). */ bool -ProceedGroupState(AutoFailoverNode *activeNode) +ProceedGroupStateForApiTrigger(MonitorApiFunction apiFunction, + AutoFailoverNode *activeNode, + AutoFailoverNode *primaryNode) { GroupStateContext ctx; + NodeActiveContext nac; BuildGroupStateContext(&ctx, activeNode); + BuildApiTriggerNodeActiveContext(&ctx, apiFunction, activeNode, primaryNode, &nac); - return ProceedGroupStateFromContext(&ctx); + int index = FindMatchingMonitorFSMRuleIndexUnderPath(MonitorFSM, + SectionApiTriggered, 0, &nac); + + if (index < 0) + { + ereport(ERROR, + (errmsg("BUG: no MonitorFSM[] row matches api-triggered call for " + NODE_FORMAT " in state \"%s\"", + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->reportedState)))); + } + + DispatchMonitorFSMRule(&ctx, &nac, &MonitorFSM[index]); + + return true; } /* - * ProceedGroupStateFromContext is the core FSM logic, operating entirely on - * the pre-built GroupStateContext. It does not touch the database for reads; - * writes (AssignGoalState, NotifyStateChange) still go to the DB. + * MonitorFSM[]: one array, not several -- see the "WHY THESE CONSTANTS EXIST + * AT ALL" comment above the MonitorFSMTransition typedef for the section + * layout and why a search is bounded by section rather than scanning the + * whole array unconditionally. Row order matters within a bounded search: + * RuleMatches() picks the first matching row it finds scanning in array + * order, so a row earlier in the array takes precedence over a later, + * broader row that would otherwise also match. * - * This separation lets test code inject a synthetic context and exercise the - * FSM without a live database connection. + * --- SectionApiTriggered: pos 101-1xx. Reached only via + * ProceedGroupStateForApiTrigger(), never via the ordinary node_active() + * heartbeat path -- every row here requires a specific non-NONE + * .conditions.apiTrigger, which an ordinary heartbeat call's implicit + * API_FUNCTION_NONE can never match (MatchApiTrigger). See that function's + * own comment (just above this array) for the operator side's dispatch + * shape and why a no-match there is always ereport(ERROR), unlike the + * heartbeat side below. + * + * --- SectionEarlyChecks: pos 201-211, the six checks + * ProceedGroupStateFromContext() runs BEFORE its IsInPrimaryState(activeNode) + * branch point -- DROPPED, goal==DROPPED, MAINTENANCE, the demote_timeout + * self-fence, and both "alone in group" rows. These fire regardless of + * whether activeNode is currently the primary (a primary that just lost its + * only standby must still reach SINGLE here, before ever redirecting into + * SectionPrimaryNode) -- the drop_node regression test depends on this + * ordering: it fails if the primary-state redirect is checked ahead of + * these six checks instead of after them. None of these six rows reference + * .primaryNode at all, so they can be matched against a NodeActiveContext + * built with primaryNode == NULL, before primaryNode is even resolved. */ -bool -ProceedGroupStateFromContext(GroupStateContext *ctx) -{ - AutoFailoverNode *activeNode = ctx->activeNode; - char *formationId = ctx->formationId; - int groupId = ctx->groupId; - int nodesCount = ctx->groupNodeCount; +static const MonitorFSMTransition MonitorFSM[] = { + /* + * remove_node(), node_active_protocol.c's RemoveNode() -- + * primary being removed: fan out report_lsn to every surviving, + * non-maintenance standby (extraAction, runs first), then mark the + * removed node itself dropped (activeNodeAssignedState, runs after). + * canTakeWrites, not isInPrimaryState: RemoveNode's own guard is + * CanTakeWritesInState(currentNode->goalState) with no requirement that + * reportedState has converged -- see canTakeWrites's own comment on + * NodeMatchesPattern above. Must come before the catchall row below: + * first-match-wins dispatch means whichever row is tried first wins + * when both could apply, and only a node that canTakeWrites should get + * the fan-out. + */ + { .pos = 101, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_REMOVE_NODE) }, + .activeNode = { .canTakeWrites = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_DROPPED), + .otherNodesFn = OtherNodesNotInMaintenance, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "remove_node, removed node can take writes -> dropped, " + "every surviving non-maintenance standby joins report_lsn" }, + + /* + * remove_node(), the removed node itself when it's NOT the primary. + * RemoveNode()'s own pre-checks (kept hand-written) already reject the + * "already DROPPED" idempotency case before dispatch is ever called, so + * this row is unconditional here. It must stay a separate row from the + * fan-out row above, not a shared alternative for the DROPPED + * assignment: removing a primary needs BOTH the fan-out AND the DROPPED + * assignment in the same call, so the row above does both itself under + * first-match-wins; this row only needs to cover the non-primary case, + * which the row above's own canTakeWrites condition never matches. + */ + { .pos = 103, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_REMOVE_NODE) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_DROPPED), + .comment = "remove_node, removed node cannot take writes -> dropped" }, + + /* + * perform_failover(), 2-node group -- node_active_protocol.c's + * perform_failover(), 2-node branch. The SQL wrapper resolves the sole + * standby and validates + * candidatePriority != 0 and both nodes converged BEFORE dispatch (an + * operator-facing ereport(ERROR) on failure, not a table row -- see + * ProceedGroupStateForApiTrigger's own comment on pre/post side + * effects); by the time dispatch runs, activeNode is that standby. + */ + { .pos = 105, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_PERFORM_FAILOVER), + .groupHasExactlyTwoNodes = BOOL_TRUE }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DRAINING), + .comment = "manual failover, 2-node group, primary+standby both converged -> " + "standby prepare_promotion, primary draining" }, + + /* + * perform_failover(), >2-node group -- node_active_protocol.c's + * perform_failover(), >2-node branch. No standby is named at this + * point at all -- there's nothing else for activeNode to be here but + * the primary itself, the one node this specific call is fundamentally + * about. The + * candidatePriority trick and the ProceedGroupState(firstStandbyNode) + * continuation are POST side effects, hand-written in perform_failover() + * itself after this row's own DRAINING assignment has committed -- not + * modeled as a further declarative row or extraAction, since they're + * about biasing an election already left to the heartbeat-driven + * MS-failover cluster rows, not a goal-state assignment of their own. + */ + { .pos = 107, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_PERFORM_FAILOVER), + .groupHasMoreThanTwoNodes = BOOL_TRUE }, + .activeNode = { .isInPrimaryState = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_DRAINING), + .comment = "manual failover, >2-node group -> primary drains, election proceeds " + "via the heartbeat-driven MS-failover cluster rows" }, + + /* + * start_maintenance(), primary, 2-node group -- node_active_protocol.c's + * start_maintenance(), primary branch, 2-node case. The WARNING about + * blocking writes, + * the candidatesCount<1 guard, and the already-in-maintenance + * idempotency check all stay hand-written pre-dispatch, exactly where + * they already are. The firstStandbyNode -> prepare_promotion + * assignment is a POST side effect, hand-written in start_maintenance() + * itself: unlike the >2-node row below, it has no ordering dependency on + * this row's own assignment (neither reads the other's freshly-committed + * state), so it doesn't need extraAction to sequence correctly. + */ + { .pos = 109, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), + .groupHasExactlyTwoNodes = BOOL_TRUE }, + .activeNode = { .isInPrimaryState = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_MAINTENANCE), + .comment = "start_maintenance, primary, 2-node group -> prepare_maintenance " + "(standby separately assigned prepare_promotion)" }, + + /* + * start_maintenance(), primary, >2-node group -- node_active_protocol.c's + * start_maintenance(), primary branch, >2-node case. The ProceedGroupState( + * firstStandbyNode) continuation is a POST side effect, hand-written in + * start_maintenance() itself, called after this row's own + * prepare_maintenance assignment has committed (it re-fetches fresh + * state, so ordering matters here, unlike the 2-node row above). + */ + { .pos = 111, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), + .groupHasMoreThanTwoNodes = BOOL_TRUE }, + .activeNode = { .isInPrimaryState = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_MAINTENANCE), + .comment = "start_maintenance, primary, >2-node group -> prepare_maintenance, " + "election proceeds via the heartbeat-driven MS-failover cluster rows" }, + + /* + * start_maintenance(), secondary, last healthy sync standby -- + * node_active_protocol.c's start_maintenance(), secondary branch. + * lastHealthySyncStandbyGoingToMaintenance is computed by + * BuildApiTriggerNodeActiveContext (see its own comment) only for this + * apiFunction, mirroring start_maintenance()'s own number_sync_standbys==0 + * && secondaryNodesCount==1 && IsHealthySyncStandby(currentNode) check + * verbatim. Must come before the ordinary-secondary row below: both match + * the same activeNode/ primaryNode state pattern, and only the more + * specific condition should win. + */ + { .pos = 113, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), + .lastHealthySyncStandbyGoingToMaintenance = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_REPORTED, + .reportedStates = STATES( + REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP) } + }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_MAINTENANCE), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "start_maintenance, secondary, last healthy sync standby -> " + "wait_maintenance, primary wait_primary (disables sync rep)" }, + + /* + * start_maintenance(), secondary, ordinary case -- node_active_protocol.c's + * start_maintenance(), secondary branch's own final case. + */ + { .pos = 115, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE) }, + .activeNode = { .statePattern = { .kind = NODE_STATE_REPORTED, + .reportedStates = STATES( + REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP) } + }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), + .comment = "start_maintenance, secondary, ordinary case -> maintenance" }, + + /* + * stop_maintenance(), no primary at all. node_active_protocol.c's + * stop_maintenance() handles totalNodesCount==1 (skip dispatch, direct + * ProceedGroupState(currentNode)) and primaryNode==NULL&&totalNodesCount + * ==2 (ereport(ERROR)) as hand-written pre-dispatch branches -- by the + * time this row's own dispatch call runs, primaryNode==NULL only + * happens with totalNodesCount>2, so this row's plain .primaryNode.exists + * = BOOL_FALSE condition is enough; the four outcomes below (this row and + * the next three) are otherwise entirely disambiguated by their own + * declarative conditions, dispatched through a single unconditional + * ProceedGroupStateForApiTrigger() call in stop_maintenance() itself. + */ + { .pos = 117, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, + .primaryNode = { .exists = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "stop_maintenance, no primary -> report_lsn" }, + + /* + * stop_maintenance(), primary fully demoted -> report_lsn, regardless of + * node count: isDemotedPrimary alone is the whole condition. + */ + { .pos = 119, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, + .primaryNode = { .isDemotedPrimary = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "stop_maintenance, primary demoted -> report_lsn" }, + + /* + * stop_maintenance(), failover in progress -> report_lsn. This row's own + * .comment intentionally logs "catchingup" while actually assigning + * REPORT_LSN -- a real message/behavior mismatch, not a modeling error + * in this table. + */ + { .pos = 121, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE), + .failoverInProgress = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "stop_maintenance, failover in progress -> report_lsn (source's own " + "log message says \"catchingup\" here, but the actual call assigns " + "REPORT_LSN -- see this row's own comment above)" }, + + /* + * stop_maintenance(), ordinary case -- catchall: reached only once + * primaryNode exists, isn't demoted, and no failover is in progress. + */ + { .pos = 123, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "stop_maintenance, ordinary case -> catchingup" }, + + /* + * set_node_candidate_priority(), node_active_protocol.c's + * set_node_candidate_priority(). activeNode IS the primary here (mirrors + * BuildForPrimaryNodeNodeActiveContext's own "primaryNode IS activeNode" + * convention): the row's only real target is the primary's own + * apply_settings transition. The "primary is already apply_settings" + * ereport(ERROR) and the "no primary yet, just proceed" no-op are + * hand-written pre-dispatch checks in set_node_candidate_priority() + * itself -- this row is only reached once the wrapper has confirmed a + * primary exists and isn't already apply_settings. + */ + { .pos = 125, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER( + API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY) }, + .activeNode = { .statePattern = { .kind = NODE_STATE_NOT_STABLE, + .reportedStates = STATES( + REPLICATION_STATE_APPLY_SETTINGS) } + }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), + .comment = "set_node_candidate_priority, primary not already apply_settings -> " + "apply_settings" }, + + /* + * set_node_replication_quorum(), node_active_protocol.c's + * set_node_replication_quorum(). Same shape as + * set_node_candidate_priority above. + */ + { .pos = 127, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER( + API_FUNCTION_SET_NODE_REPLICATION_QUORUM) }, + .activeNode = { .statePattern = { .kind = NODE_STATE_NOT_STABLE, + .reportedStates = STATES( + REPLICATION_STATE_APPLY_SETTINGS) } + }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), + .comment = "set_node_replication_quorum, primary not already apply_settings -> " + "apply_settings" }, + + /* + * set_formation_number_sync_standbys(), formation_metadata.c's + * set_formation_number_sync_standbys(). The "primary not in + * primary/wait_primary state" ereport(ERROR) stays + * hand-written pre-dispatch, exactly where it already is -- kept as this + * row's own condition too (belt and suspenders: dump_fsm() should show + * what's actually valid, and a future drift between the two fails loudly + * via this row's own no-match ERROR rather than silently). + */ + { .pos = 129, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, + .conditions = { .apiTrigger = API_TRIGGER( + API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS) }, + .activeNode = { .statePattern = { .kind = NODE_STATE_ASSIGNED, + .assignedStates = STATES( + REPLICATION_STATE_PRIMARY, + REPLICATION_STATE_WAIT_PRIMARY) } + }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), + .comment = "set_formation_number_sync_standbys, primary in primary/wait_primary " + "-> apply_settings" }, + + /* converged to dropped -> remove the node from the catalog entirely */ + { .pos = 201, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DROPPED) }, + .extraAction = ActionRemoveDroppedNode, + .comment = "converged to dropped -> remove the node from the catalog" }, + + /* + * goal already dropped (mid-drop, row above hasn't converged yet) -> no-op + */ + { .pos = 203, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, + .activeNode = { .statePattern = FSM_DROPPED_GOAL }, + .comment = "goal already dropped -> no-op" }, + + /* converged to maintenance -> no-op, frozen until stop_maintenance() */ + { .pos = 205, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_MAINTENANCE) }, + .comment = "converged to maintenance -> no-op, frozen until stop_maintenance()" }, + + /* demote_timeout self-fence re-target (issue #1025) */ + { .pos = 207, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, + .activeNode = { .statePattern = FSM_REPORTED_DEMOTE_TIMEOUT, + .unreachableFromDemoteTimeout = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "reported demote_timeout, assigned goal can't reach it -> demoted" }, + + /* + * alone in group, still preparing for primary maintenance -- no-op, + * same shape as pos 205's own "converged to maintenance" row. This row + * exists purely to keep pos 209's own reportedIsPrepareMaintenance + * exclusion (see its comment on NodeMatchesPattern) from turning into a + * *worse* bug than the one it fixes: join_secondary is safely tolerated + * when left alone because IsParticipatingInPromotion (node_metadata.c) + * already recognizes it, but prepare_maintenance isn't recognized by + * that function, by IsBeingPromoted, or by IsInPrimaryState + * (CanTakeWritesInState(prepare_maintenance) is false) -- so without an + * explicit match here, ProceedGroupStateFromContext's own "primaryNode + * == NULL && !IsFailoverInProgress(...)" guard would ereport(ERROR) on + * every single subsequent heartbeat from this node, instead of safely + * leaving it stuck. Frozen here (no assignment) until an operator + * intervenes -- the same "wait for an operator" outcome pos 211's own + * candidatePriority=0 case already accepts for other source states. + * Only fires when alone: in an ordinary (non-alone) group, the + * candidate standby being promoted in parallel already satisfies + * IsFailoverInProgress for the whole group, so this row must not + * intercept that already-working path. + */ + { .pos = 208, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, + .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, + .comment = "alone in group, still preparing for maintenance -> no-op" }, + + /* + * alone in group, candidate-eligible -- excludes WAIT_STANDBY_STATE too + * (reportedIsWaitStandby's own comment on NodeMatchesPattern): a node + * stuck there never actually started streaming, so there is no safe way + * for the keeper to reach SINGLE from it without risking a system_ + * identifier conflict with its own prior registration. Gated via the + * reported-only reportedIsWaitStandby field rather than folding + * WAIT_STANDBY into FSM_NOT_STABLE_SINGLE's own NOT_STABLE-kind pattern: + * pos 101's own remove_node() fan-out can rewrite this node's goalState + * to report_lsn before this row's own next evaluation, which would + * defeat a goalState-dependent (NOT_STABLE) exclusion the same way pos + * 210's own isInPrimaryState condition was defeated -- confirmed live. + * + * Also excludes JOIN_SECONDARY_STATE (reportedIsJoinSecondary's own + * comment on NodeMatchesPattern): unlike every other source state this + * row matches, a node reporting join_secondary has already had Postgres + * cleanly checkpointed and stopped as part of switching its replication + * target to a newly-elected primary. Its on-disk data is only a + * trustworthy copy of its own last moment as the OLD primary, frozen + * before that new primary ever took a single write -- resuming it + * straight to SINGLE if it ends up alone risks silently discarding + * whatever the new primary committed in the meantime, a real + * split-brain/data-loss risk this row must not create. Same reported- + * only reasoning as reportedIsWaitStandby above for why this is gated on + * reportedState alone rather than folded into FSM_NOT_STABLE_SINGLE. + * + * Also excludes PREPARE_MAINTENANCE_STATE (reportedIsPrepareMaintenance's + * own comment on NodeMatchesPattern): the same split-brain risk as + * join_secondary, reached one step earlier -- start_maintenance() + * assigns this node PREPARE_MAINTENANCE_STATE and its chosen standby + * PREPARE_PROMOTION_STATE in the very same call, and pos 343 lets that + * standby advance all the way to primary the moment this node's own + * reportedState merely converges to prepare_maintenance, with no + * requirement that this node's row ever be removed first. A different, + * already fully-promoted primary can be live and taking writes while + * this node still sits in prepare_maintenance indefinitely. + */ + { .pos = 209, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, + .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, + .candidateEligible = BOOL_TRUE, + .reportedIsWaitStandby = BOOL_FALSE, + .reportedIsJoinSecondary = BOOL_FALSE, + .reportedIsPrepareMaintenance = BOOL_FALSE }, + .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SINGLE), + .comment = "alone in group, candidate-eligible -> single" }, + + /* + * alone in group, not candidate-eligible, but already serving as primary + * -- candidatePriority=0 exists to steer a FUTURE election away from + * this node when an alternative exists; once every other node is gone, + * there is no alternative left to prefer, and demoting the cluster's + * only remaining writable node to report_lsn would strand it there + * indefinitely (no candidate will ever appear to promote instead) for + * no operational benefit -- the least-surprising behavior is to let it + * keep serving as SINGLE, exactly as a candidate-eligible lone primary + * already does via pos 209. Checked before pos 211 below (a strict + * subset of its own condition -- candidateEligible=FALSE, plus + * "already primary-role") so first-match-wins routes this specific case + * here instead. + * + * Gated on FSM_REPORTED_PRIMARY_ROLE_STATES (reportedState alone), not + * isInPrimaryState() (which also requires goalState to already agree): + * a live pgaftest run (keeper_fsm_gap_211_primary_priority_zero.pgaf) + * caught a real self-undermining oscillation with the isInPrimaryState + * version -- this row's own extraAction reassigns goalState to SINGLE, + * which makes isInPrimaryState() evaluate false on the very next + * dispatch (goalState=single no longer agrees with reportedState, which + * is still primary, and single isn't in isInPrimaryState()'s own + * {PRIMARY,APPLY_SETTINGS} second disjunct either) -- so this row + * stopped matching one dispatch after firing, and pos 211 (which has no + * such requirement) fired right behind it and overwrote the assignment + * back to REPORT_LSN. reportedState is untouched by this row's own + * action, so matching on it alone stays stable across dispatches until + * the keeper itself actually converges to single. + */ + { .pos = 210, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, + .activeNode = { .statePattern = FSM_REPORTED_PRIMARY_ROLE_STATES, + .candidateEligible = BOOL_FALSE }, + .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SINGLE), + .comment = "alone in group, already primary despite candidatePriority " + "zero -> single" }, + + /* + * alone in group, not candidate-eligible, and was never already primary + * (a lone surviving standby) -- candidatePriority=0 is honored here: + * wait for an operator or a new peer, don't self-promote. + * + * reportedCanTakeWrites=FALSE explicitly excludes the 4 primary-role + * states pos 210 above already owns (a strict subset of this row's own + * candidateEligible=FALSE + groupHasExactlyOneNode=TRUE condition, so + * pos 210 always intercepts them first regardless of this exclusion -- + * first-match-wins alone already makes this row unreachable for those + * states in practice). The exclusion is added anyway because + * dump_fsm_edges()'s own shadow-detector (EdgeIsShadowedByEarlierRule) + * only recognizes an earlier row as shadowing when it's *fully* + * unconditional for the state -- pos 210 additionally requires + * candidateEligible=FALSE and groupHasExactlyOneNode=TRUE, so it doesn't + * qualify even though those conditions are identical to this row's own. + * Without this exclusion, dump_fsm_edges() (and hence + * keeper_fsm_edges.sql's Step 2a) kept listing "primary/wait_primary/ + * join_primary/apply_settings -> report_lsn" as a live keeper-coverage + * gap even after pos 210 made it unreachable at runtime -- a real + * runtime fix that the diagnostic didn't reflect. Making this row's own + * pattern match its comment ("was never already primary") directly is + * simpler and safer than generalizing the shadow-detector to prove + * conditional (not just unconditional) shadowing between arbitrary row + * pairs. + * + * reportedIsWaitStandby=FALSE also excludes WAIT_STANDBY_STATE (see its + * own comment on NodeMatchesPattern): a node stuck there never actually + * started streaming, so there's no safe way for the keeper to reach + * REPORT_LSN from it either, for the same system_identifier-conflict + * reason pos 209 documents. Reported-only for the same reason as pos + * 209's own use of this field: pos 101's remove_node() fan-out can + * rewrite this node's goalState to report_lsn before this row's own + * next evaluation, which would defeat a goalState-dependent exclusion. + */ + { .pos = 211, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, + .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, + .candidateEligible = BOOL_FALSE, + .reportedCanTakeWrites = BOOL_FALSE, + .reportedIsWaitStandby = BOOL_FALSE }, + .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "alone in group, candidatePriority zero -> report_lsn" }, + + /* + * --- sectionPath[1] == MONITOR_FSM_SECTION_FROM_CONTEXT, under + * SectionReportingNode: the rest of ProceedGroupStateFromContext()'s own + * sequential if-chain -- everything from the timeline-fork check, right + * after the IsInPrimaryState(activeNode) early return, onward. Reached + * only when activeNode is NOT currently primary-role. + */ + + /* + * converged secondary, reportedTLI not an ancestor of the group's reference + * timeline + */ + { .pos = 301, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), + .isComparableToReferenceTli = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = + "converged secondary, reportedTLI not an ancestor of reference -> catchingup" }, + + /* + * replication stall (#997): primary healthy, no standby past + * replication_stall_timeout + */ + { .pos = 303, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .primaryNode = { .isInPrimaryState = BOOL_TRUE, + .isHealthy = BOOL_TRUE }, + .conditions = { .replicationStallExceeded = BOOL_TRUE }, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = + "primary healthy, no standby past replication_stall_timeout -> wait_primary" }, + + /* + * nodesCount>2, primary unhealthy -- draining/maintenance/nothing decided + * inside the action, then the MS-failover cascade unconditionally; must be + * a single row, see ActionRunMultiStandbyFailoverCascade's comment for why. + */ + { .pos = 305, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .primaryNode = { .isUnhealthy = BOOL_TRUE }, + .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE }, + .extraAction = ActionRunMultiStandbyFailoverCascade, + .comment = + "nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade" }, + + /* report_lsn, primary converged wait/join_primary, healthy */ + { .pos = 307, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY, + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .comment = + "report_lsn, primary converged wait/join_primary, healthy -> secondary" }, + + /* report_lsn, primary converged primary, healthy */ + { .pos = 309, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY), + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .comment = "report_lsn, primary converged primary, healthy -> secondary" }, + + /* fast_forward done -> prepare_promotion */ + { .pos = 311, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_FAST_FORWARD) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), + .comment = "fast_forward done -> prepare_promotion" }, + + /* + * continue an already-started MS failover -- a direct `return` in the real + * source, and no later row in this table matches activeNode in + * REPORT_LSN/FAST_FORWARD, so it doesn't matter here whether the + * extraAction's bool stops dispatch or lets it keep scanning. + */ + { .pos = 313, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_REPORT_LSN_OR_FAST_FORWARD }, + .extraAction = ActionRunPlainMSFailoverCascade, + .comment = "report_lsn or fast_forward, continuing an already-started failover -> " + "MS-failover cascade" }, + + /* wait_standby, primary converged wait/join_primary */ + { .pos = 315, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY) }, + .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "wait_standby, primary converged wait/join_primary -> catchingup" }, + + /* wait_standby (quorum member), primary converged primary */ + { .pos = 317, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .replicationQuorum = BOOL_TRUE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), + .comment = "wait_standby (quorum member), primary converged primary -> " + "catchingup + apply_settings" }, + + /* wait_standby (not a quorum member), primary converged primary */ + { .pos = 319, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .replicationQuorum = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = + "wait_standby (not a quorum member), primary converged primary -> catchingup" }, + + /* caught up, same TLI as primary, within sync threshold */ + { .pos = 321, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_CATCHINGUP), + .isHealthy = BOOL_TRUE }, + .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN }, + .conditions = { .activeAndPrimaryTliMatch = BOOL_TRUE, + .walWithinSyncThreshold = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .comment = "caught up, same TLI as primary, within sync threshold -> secondary" }, + + /* + * primary fails, already converged wait_primary (no draining edge, issue + * #1168) + */ + { .pos = 323, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), + .isHealthy = BOOL_TRUE, + .candidateEligible = BOOL_TRUE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY), + .isUnhealthy = BOOL_TRUE }, + .conditions = { .walWithinPromoteThreshold = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), + .comment = "primary fails, already converged wait_primary (issue #1168) -> " + "secondary -> prepare_promotion only (1 of 2)" }, + + /* + * primary fails, not already wait_primary. activeNode (secondary) and + * primaryNode are necessarily two distinct nodes in the same group, so + * groupHasExactlyOneNode = BOOL_FALSE is always true whenever this row + * can match at all -- spelled out explicitly (rather than left implicit) + * because dump_fsm_edges() reads it to prove primaryNode can never be + * genuinely reporting SINGLE here (see + * MonitorFSMTransitionExcludesSingleNode's own comment): SINGLE means + * "alone in my own group," which can't coexist with this row's own + * requirement that a second, distinctly-matched node (activeNode) also + * exists in that same group. + */ + { .pos = 325, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), + .isHealthy = BOOL_TRUE, + .candidateEligible = BOOL_TRUE }, + .primaryNode = { .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, + .isInPrimaryState = BOOL_TRUE, + .isUnhealthy = BOOL_TRUE }, + .conditions = { .walWithinPromoteThreshold = BOOL_TRUE, + .groupHasExactlyOneNode = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DRAINING), + .comment = + "primary fails, not already wait_primary -> secondary -> prepare_promotion, " + "primary -> draining (2 of 2)" }, + + /* wait_maintenance, primary converged wait_primary */ + { .pos = 327, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), + .comment = "wait_maintenance, primary converged wait_primary -> maintenance" }, + + /* wait_maintenance, primary's goal no longer wait_primary */ + { .pos = 329, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, + .primaryNode = { .statePattern = FSM_NOT_ASSIGNED_WAIT_PRIMARY }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), + .comment = + "wait_maintenance, primary's goal no longer wait_primary -> maintenance" }, + + /* prepare_promotion, primary converged prepare_maintenance */ + { .pos = 331, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), + .comment = + "prepare_promotion, primary converged prepare_maintenance -> stop_replication" }, + + /* + * Citus worker, primary present. Unlike pos 325, this row's primaryNode + * carries no .isInPrimaryState requirement, so its SINGLE gap can't be + * ruled out the same way: a primary can converge to SINGLE, then a + * second node register (bumping the primary's own *goal* to + * WAIT_PRIMARY as part of that registration) -- and if the primary dies + * or partitions in that exact window, before ever reporting the new + * goal, its row sits with reportedState=SINGLE/goalState=WAIT_PRIMARY + * indefinitely. GetPrimaryOrDemotedNodeInGroupFromList()'s own phase 1 + * checks only goalState, so it would still resolve this stale node as + * primaryNode -- a genuinely reachable case, not a false positive (see + * PrimaryNodeReportedStateCanBeResolved's own comment for the one + * exclusion -- DROPPED -- that IS sound here, on different grounds). + */ + { .pos = 333, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), + .isCitusWorkerGroup = BOOL_TRUE }, + .primaryNode = { .exists = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = + "Citus worker prepare_promotion, primary present -> wait_primary + demoted" }, + + /* Citus worker, primary removed */ + { .pos = 335, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), + .isCitusWorkerGroup = BOOL_TRUE }, + .primaryNode = { .exists = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "Citus worker prepare_promotion, primary removed -> wait_primary" }, + + /* + * prepare_promotion, primary present, already converged wait_primary (issue + * #1168) + */ + { .pos = 337, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + .primaryNode = { .exists = BOOL_TRUE, + .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY), + .isInMaintenance = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), + .comment = + "prepare_promotion, primary already converged wait_primary (issue #1168) -> " + "stop_replication only (1 of 2)" }, + + /* + * prepare_promotion, primary present, not in maintenance, not already + * wait_primary. Same "no isInPrimaryState convergence requirement, so + * SINGLE stays a genuinely reachable gap" reasoning as pos 333 -- see + * its own comment. + */ + { .pos = 339, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + .primaryNode = { .exists = BOOL_TRUE, + .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, + .isInMaintenance = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTE_TIMEOUT), + .comment = "prepare_promotion, primary present, not in maintenance -> " + "stop_replication + demote_timeout (2 of 2)" }, + + /* prepare_promotion, primary removed */ + { .pos = 341, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + .primaryNode = { .exists = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "prepare_promotion, primary removed -> wait_primary" }, + + /* stop_replication, primary converged prepare_maintenance */ + { .pos = 343, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), + .comment = "stop_replication, primary converged prepare_maintenance -> " + "wait_primary + maintenance" }, + + /* stop_replication, primary converged demote_timeout (3-way OR, 1 of 3) */ + { .pos = 345, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTE_TIMEOUT) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "stop_replication, primary converged demote_timeout -> " + "wait_primary + demoted (1 of 3)" }, + + /* + * stop_replication, primary's drain time expired (3-way OR, 2 of 3). + * Same "no isInPrimaryState convergence requirement, so SINGLE stays a + * genuinely reachable gap" reasoning as pos 333 -- see its own comment. + * dump_fsm_edges() also has no filter for .drainTimeExpired itself + * (unlike isInPrimaryState/reportedCanTakeWrites/etc.), so this row's + * primaryNode candidate universe is the same unnarrowed ANY set. + */ + { .pos = 347, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .primaryNode = { .drainTimeExpired = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "stop_replication, primary's drain time expired -> " + "wait_primary + demoted (2 of 3)" }, + + /* + * stop_replication, primary's goal is wait_primary but presumed dead + * (3-way OR, 3 of 3). primaryIsWaitPrimaryPresumedDead is a .conditions + * field, not a NodeStatusPattern on .primaryNode -- dump_fsm_edges()'s + * state enumeration doesn't read .conditions at all when resolving + * primaryNode's candidate states (only .groupHasExactlyOneNode/ + * .groupHasMoreThanTwoNodes ever feed into it, via singleExcluded), so + * this row's primaryNode candidate universe is the full unnarrowed ANY + * set, same as pos 333/347/351 -- see pos 333's own comment for why + * SINGLE in particular stays a genuinely reachable gap here too. + */ + { .pos = 349, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .conditions = { .primaryIsWaitPrimaryPresumedDead = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "stop_replication, primary's goal wait_primary but presumed dead -> " + "wait_primary + demoted (3 of 3)" }, + + /* + * Citus worker, primary present. Same "no isInPrimaryState convergence + * requirement, so SINGLE stays a genuinely reachable gap" reasoning as + * pos 333 -- see its own comment. + */ + { .pos = 351, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), + .isCitusWorkerGroup = BOOL_TRUE }, + .primaryNode = { .exists = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = + "Citus worker stop_replication, primary present -> wait_primary + demoted" }, + + /* Citus worker, primary removed */ + { .pos = 353, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), + .isCitusWorkerGroup = BOOL_TRUE }, + .primaryNode = { .exists = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "Citus worker stop_replication, primary removed -> wait_primary" }, + + /* demoted, primary reported wait/join_primary with goal primary */ + { .pos = 355, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, + .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY, + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = + "demoted, primary reported wait/join_primary with goal primary -> catchingup" }, + + /* demoted, primary converged wait/join_primary/primary, healthy */ + { .pos = 357, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, + .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN, + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = + "demoted, primary converged wait/join_primary/primary, healthy -> catchingup" }, + + /* + * join_secondary, primary reported wait_primary with goal wait/primary -- + * cascades into a nested pass on primaryNode + */ + { .pos = 359, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, + .primaryNode = { .statePattern = FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .extraAction = ActionRunPrimaryNodeTransition, + .comment = + "join_secondary, primary reported wait_primary with goal wait/primary -> " + "secondary" }, + + /* join_secondary, primary converged primary */ + { .pos = 361, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .comment = "join_secondary, primary converged primary -> secondary" }, + + /* + * --- SectionMSFailover (sectionPath { REPORTING_NODE, MS_FAILOVER }): + * the MS-failover / candidate-selection cluster's own declarative rows + * (pos 363 onward, see TryMSFailoverDeclarativeRow's own comment below). + * BuildCandidateList/SelectFailoverCandidateNode/PromoteSelectedNode + * themselves stay hand-written C, called from + * ProceedGroupStateForMSFailover -- these rows only cover the + * assignments that are expressible as plain per-node facts, gated by + * the exact same hand-written condition that already decides whether to + * make them (so a mismatch here can only widen the row's own no-op + * fallback, never change real behavior). Appended after the ordinary + * REPORTING_NODE rows rather than renumbered into them, so nothing else + * shifts. + */ + + /* + * MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, + * retry + */ + { .pos = 363, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET + }, + .conditions = { .activeNodeAllWalSourcesUnhealthy = BOOL_TRUE, + .guardDataLossEnabled = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES( + REPLICATION_STATE_REPORT_LSN), + .assignedStates = STATES( + REPLICATION_STATE_FAST_FORWARD) } + }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = + "MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, " + "guard_data_loss=true -> report_lsn (retry once a source recovers)" }, + + /* + * MS-failover: candidate ready to stream WAL -> follower joins as secondary + */ + { .pos = 365, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN + }, + .conditions = { .candidatePromotionInProgress = BOOL_TRUE }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .candidateNode = { .isReadyToStreamWAL = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_JOIN_SECONDARY), + .comment = + "MS-failover: activeNode in report_lsn, failover candidate ready to stream " + "WAL -> join_secondary" }, /* - * If the active node just reached the DROPPED state, proceed to remove it - * from the pgautofailover.node table. + * MS-failover: BuildCandidateList's own fan-out loop ("Nodes in + * SECONDARY or CATCHINGUP states are candidates due to report their + * LSN..."). Every AssignGoalState call in that loop is dispatched + * through TryFanOutReportLsnRow (see its own comment) against exactly + * these four rows -- one per distinct fromState shape the loop's own + * if-branches check, since no single NodeStatePattern covers all three + * disjuncts (SECONDARY/CATCHINGUP transitioning, MAINTENANCE->CATCHINGUP, + * DRAINING/DEMOTED stable or DEMOTED->CATCHINGUP transitioning) at once. + * The loop's own skip conditions (old/new primary unless draining or + * demoted, unhealthy-and-not-reporting) are hand-written, evaluated + * ahead of these rows -- by the time one of these four is reached, the + * loop has already established the node is a legitimate fan-out target; + * these rows only need to name which state it's coming from, not + * re-derive that eligibility. + * + * Each carries .conditions.inMSFailoverCluster = BOOL_TRUE: unlike pos + * 363/365, none of these four has an activeNode-state pattern narrow + * enough on its own to stay clear of ProceedGroupStateFromContext's own + * ordinary top-level lookup, which scans under SectionReportingNode -- + * a prefix this cluster's own sectionPath is also under -- and would + * otherwise match them against any ordinary, non-MS-failover heartbeat + * whose reported/goal states happened to line up -- see + * inMSFailoverCluster's own comment on NodeActiveContext. */ - if (IsCurrentState(activeNode, REPLICATION_STATE_DROPPED)) - { - char message[BUFSIZE] = { 0 }; + { .pos = 367, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES( + REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP), + .assignedStates = STATES( + REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP) } + }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = + "MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn " + "(1 of 4)" }, + + { .pos = 369, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES( + REPLICATION_STATE_MAINTENANCE), + .assignedStates = STATES( + REPLICATION_STATE_CATCHINGUP) } + }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = + "MS-failover fan-out: rejoining from maintenance -> report_lsn (2 of 4)" }, + + { .pos = 371, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_STABLE, + .reportedStates = STATES( + REPLICATION_STATE_DRAINING, + REPLICATION_STATE_DEMOTED) } + }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "MS-failover fan-out: old primary converged draining or demoted -> " + "report_lsn (3 of 4)" }, + + { .pos = 373, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES( + REPLICATION_STATE_DEMOTED), + .assignedStates = STATES( + REPLICATION_STATE_CATCHINGUP) } + }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "MS-failover fan-out: old primary demoted, was rejoining a now-failed " + "primary -> report_lsn (4 of 4)" }, - /* time to actually remove the current node */ - RemoveAutoFailoverNode(activeNode); + /* + * MS-failover: PromoteSelectedNode's own two outcomes + * (group_state_machine.c). ResolveAcceptedTimeline and the + * candidatePriority resets stay hand-written side effects, exactly + * where they are; only the final AssignGoalState is dispatched, via + * DispatchMonitorFSMRuleByPos (see its own comment) rather than a + * RuleMatches search -- both rows below share an identical condition + * set (activeNode in report_lsn, no promotion already in flight, the + * pool's most-advanced candidate within the promote threshold), so + * first-match-wins can never distinguish them on its own. The real + * choice -- selectedNode->reportedLSN == candidateList-> + * mostAdvancedReportedLSN, an internal LSN comparison no BoolPattern + * can express -- is made in PromoteSelectedNode itself; both rows exist + * so dump_fsm() shows both reachable outcomes, even though this table's + * own dispatch model can't disambiguate between them on its own. + */ + { .pos = 375, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .candidatePromotionInProgress = BOOL_FALSE, + .mostAdvancedCandidateWithinPromoteThreshold = BOOL_TRUE }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), + .comment = "MS-failover: no promotion in flight, most-advanced candidate within " + "threshold, selected candidate already has all WAL -> prepare_promotion " + "(1 of 2 -- see this row's own comment on why both are listed)" }, + + { .pos = 377, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .candidatePromotionInProgress = BOOL_FALSE, + .mostAdvancedCandidateWithinPromoteThreshold = BOOL_TRUE }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_FAST_FORWARD), + .comment = "MS-failover: no promotion in flight, most-advanced candidate within " + "threshold, selected candidate is lagging -> fast_forward (2 of 2)" }, - LogAndNotifyMessage( - message, BUFSIZE, - "Removing " NODE_FORMAT " from formation \"%s\" and group %d", - NODE_FORMAT_ARGS(activeNode), - activeNode->formationId, - activeNode->groupId); + /* + * The 3 MS-failover counting gates (missingNodesCount/candidateCount/ + * quorumCandidateCount), dispatched from ProceedGroupStateForMSFailover's + * own hand-written `if` blocks via a dedicated + * BuildMSFailoverCandidateGateNodeActiveContext-built nac (see that + * builder's own comment) -- not part of the ordinary fan-out/promotion + * rows above, and never reached by their own SectionMSFailover-wide + * scans thanks to inMSFailoverCandidateGate (see NodeActiveContext's own + * comment on that field). Each pair below shares its own gate's real + * condition, split only on guard_data_loss (true -> decline, false -> + * proceed and log the data-loss risk) -- the decline/continue control + * flow itself stays hand-written C, unchanged; only the message text + * this pair's own extraAction builds is delegated here. + */ + { .pos = 379, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .inMSFailoverCandidateGate = BOOL_TRUE, + .missingNodesCount = AT_LEAST(1), + .guardDataLossEnabled = BOOL_TRUE }, + .extraAction = ActionLogMSFailoverMissingNodesDecline, + .comment = "MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=true " + "-> decline, wait for more reports (1 of 2)" }, + + { .pos = 381, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .inMSFailoverCandidateGate = BOOL_TRUE, + .missingNodesCount = AT_LEAST(1), + .guardDataLossEnabled = BOOL_FALSE }, + .extraAction = ActionLogMSFailoverMissingNodesContinue, + .comment = + "MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=false " + "-> proceed despite possible data loss (2 of 2)" }, - return true; - } + /* + * MS-failover: zero candidates have reported their LSN yet -- a hard, + * silent decline (ProceedGroupStateForMSFailover's own + * candidateCount==0 gate never logs here). Never itself dispatched, + * listed for dump_fsm() completeness only, same as the no_candidate_yet + * row below. + */ + { .pos = 383, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_CANDIDATE_COUNT_GATE + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .inMSFailoverCandidateGate = BOOL_TRUE, + .candidateCount = EXACTLY(0) }, + .comment = + "MS-failover: zero candidates have reported their LSN yet -> silent decline" }, + + { .pos = 385, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .inMSFailoverCandidateGate = BOOL_TRUE, + .sufficientQuorumCandidates = BOOL_FALSE, + .guardDataLossEnabled = BOOL_TRUE }, + .extraAction = ActionLogMSFailoverQuorumDecline, + .comment = + "MS-failover: not enough quorum candidates reported yet, guard_data_loss=true " + "-> decline (1 of 2)" }, + + { .pos = 387, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .inMSFailoverCandidateGate = BOOL_TRUE, + .sufficientQuorumCandidates = BOOL_FALSE, + .guardDataLossEnabled = BOOL_FALSE }, + .extraAction = ActionLogMSFailoverQuorumContinue, + .comment = + "MS-failover: not enough quorum candidates reported yet, guard_data_loss=false " + "-> proceed with fewer than required (2 of 2)" }, - /* node reports secondary/dropped */ - if (activeNode->goalState == REPLICATION_STATE_DROPPED) - { - return true; - } + /* + * MS-failover: no promotion in flight, either not enough candidates have + * reported yet or the most-advanced one is still too far behind the primary + * to safely promote -- a pure no-op besides BuildCandidateList's own + * fan-out (the four rows above); listed for dump_fsm() completeness, never + * itself dispatched (no assignment to attribute). + * inMSFailoverCluster=BOOL_TRUE despite that: without it this row's bare + * candidatePromotionInProgress=BOOL_FALSE (true by default everywhere) plus + * its missing activeNode state pattern (matches ANY state) would let it + * swallow the ordinary top-level lookup's own "BUG: no MonitorFSM row + * matches" detection for any otherwise-unmatched heartbeat, silently + * no-op'ing instead of erroring. + */ + { .pos = 389, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_NO_CANDIDATE_YET + }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .candidatePromotionInProgress = BOOL_FALSE }, + .comment = "MS-failover: no promotion in flight, not enough (or not safe enough) " + "candidates yet -> no-op besides the fan-out above" }, /* - * A node in "maintenance" state can only get out of maintenance through an - * explicit call to stop_maintenance(), the FSM will not assign a new state - * to a node that is currently in maintenance. + * ActionRunMultiStandbyFailoverCascade's own two outcomes (pos 305's + * extraAction). Both rows below share pos 305's own + * gating conditions (primaryNode.isUnhealthy, groupHasMoreThanTwoNodes) + * verbatim -- not a new marker field -- which is what keeps them safe from + * the ordinary top-level lookup: that lookup can only ever reach these two + * rows by first reaching pos 305 itself (earlier in the array, with no + * activeNode-state restriction of its own), which unconditionally + * dispatches through here via its own extraAction before the ordinary scan + * could ever resume past it in the same call. atLeastOneHealthyCandidate is + * computed once by BuildFromContextNodeActiveContext (same + * AutoFailoverOtherNodesListInState + CountHealthyCandidates computation, + * same isUnhealthy/groupNodeCount>2 gate) and reused here rather than + * recomputed. ResolveAcceptedTimeline-style side effects don't apply to + * either row (there are none here); only the plain AssignGoalState calls + * these rows dispatch, each falling back to a hand-written condition on + * no match. */ - if (IsCurrentState(activeNode, REPLICATION_STATE_MAINTENANCE)) - { - return true; - } + { .pos = 391, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE + }, + .primaryNode = { .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, + .isInPrimaryState = BOOL_TRUE, + .isUnhealthy = BOOL_TRUE }, + .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE, + .atLeastOneHealthyCandidate = BOOL_TRUE }, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DRAINING), + .comment = "nodesCount>2, primary unhealthy, in primary role but not yet " + "wait_primary, >=1 healthy candidate -> primary draining" }, + + { .pos = 393, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE + }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE), + .isUnhealthy = BOOL_TRUE }, + .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE }, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), + .comment = "nodesCount>2, primary unhealthy, converged prepare_maintenance -> " + "primary maintenance" }, /* - * A node reporting demote_timeout may have gotten there on its own - * initiative (check_for_network_partitions() in service_keeper.c - * self-fences independently of whatever goal the monitor last assigned -- - * see #1025). If the currently assigned goal isn't one demote_timeout can - * actually reach, the keeper would fatal forever trying to get there. - * Re-target to demoted: always a valid demote_timeout exit - * (DEMOTE_TIMEOUT_STATE -> DEMOTED_STATE, fsm.c:355), and the safe, - * conservative choice -- the node stays fenced from writes until the - * existing "demoted -> catchingup" reintegration path - * (group_state_machine.c:909) or an operator decides otherwise. - * - * Deliberately a plain reportedState check, not IsCurrentState(): the - * whole point is to catch reportedState == demote_timeout while - * goalState is still whatever was assigned before the self-fence -- - * IsCurrentState() requires goalState == reportedState == state, which - * is exactly the case that does NOT need re-targeting (the node is - * already headed somewhere demote_timeout can reach). - */ - if (activeNode->reportedState == REPLICATION_STATE_DEMOTE_TIMEOUT && - activeNode->goalState != REPLICATION_STATE_DEMOTE_TIMEOUT && - activeNode->goalState != REPLICATION_STATE_DEMOTED && - activeNode->goalState != REPLICATION_STATE_PRIMARY && - activeNode->goalState != REPLICATION_STATE_SINGLE) - { - char message[BUFSIZE] = { 0 }; + * --- the PRIMARY_NODE section (sectionPath[0] == + * MONITOR_FSM_SECTION_PRIMARY_NODE, pos 401 onward): the declarative + * replacement for ProceedGroupStateForPrimaryNode()'s own sequential + * if-chain. Here .activeNode maps to the primaryNode parameter, not a + * reporting node -- reached either directly by the top-level driver + * (activeNode already primary-role) or via ActionRunPrimaryNodeTransition's + * nested pass on primaryNode (the join_secondary cascade row above). + */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to demoted: it reports demote_timeout but is assigned %s, " - "which demote_timeout cannot reach.", - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->goalState)); + /* primary alone, another node reached wait_standby */ + { .pos = 401, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SINGLE) }, + .conditions = { .anyOtherNodeWaitingStandby = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "primary alone, another node reached wait_standby -> wait_primary" }, + + /* all nodes async, zero secondaries */ + { .pos = 403, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, + .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, + .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, + .secondaryNodesCountIsZero = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "all nodes async, zero secondaries -> wait_primary " + "(+ unhealthy-secondary fan-out to catchingup)" }, + + /* all nodes async, >=1 secondary */ + { .pos = 405, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, + .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, + .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, + .secondaryNodesCountIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "all nodes async, >=1 secondary -> primary " + "(+ unhealthy-secondary fan-out to catchingup)" }, - AssignGoalState(activeNode, REPLICATION_STATE_DEMOTED, message); + /* + * converged primary/apply_settings (not wait_primary), no quorum + * secondaries, number_sync_standbys=0, no failover in progress (issue #774) + */ + { .pos = 407, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, + .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, + .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, + .failoverInProgress = BOOL_FALSE, + .numberSyncStandbysIsZero = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = + "converged primary/apply_settings, no quorum secondaries, no failover in " + "progress, number_sync_standbys=0 -> wait_primary " + "(+ unhealthy-secondary fan-out to catchingup)" }, + + /* same, but number_sync_standbys>0 -> block writes on primary */ + { .pos = 409, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, + .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, + .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, + .failoverInProgress = BOOL_FALSE, + .numberSyncStandbysIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = + "converged primary/apply_settings, no quorum secondaries, no failover in " + "progress, number_sync_standbys>0 -> primary (block writes) " + "(+ unhealthy-secondary fan-out to catchingup)" }, + + /* wait_primary, >=1 quorum secondary */ + { .pos = 411, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, + .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "wait_primary, >=1 quorum secondary -> primary " + "(+ unhealthy-secondary fan-out to catchingup)" }, + + /* apply_settings, both zero */ + { .pos = 413, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, + .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, + .secondaryQuorumNodesCountIsZero = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "apply_settings, both zero -> wait_primary " + "(+ unhealthy-secondary fan-out to catchingup)" }, + + /* apply_settings, number_sync_standbys != 0 (1 of 2 disjuncts) */ + { .pos = 415, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, + .conditions = { .numberSyncStandbysIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = + "apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) " + "(+ unhealthy-secondary fan-out to catchingup)" }, + + /* apply_settings, sync_standbys=0 but >=1 quorum secondary (2 of 2) */ + { .pos = 417, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, + .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, + .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = + "apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) " + "(+ unhealthy-secondary fan-out to catchingup)" }, - return true; - } + /* + * converged primary/wait_primary/apply_settings, no other condition applies + */ + { .pos = 419, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, + .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = + "converged primary/wait_primary/apply_settings, no other condition applies -> " + "no-op besides the unhealthy-secondary fan-out to catchingup" }, + + /* backwards-compat: join_primary -> primary */ + { .pos = 421, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .comment = "backwards-compat: join_primary -> primary" }, /* - * A node that is alone in its group should be SINGLE. - * - * Exception arises when it used to be other nodes in the group, and the - * only node left has Candidate Priority of zero. In that case the setup is - * clear, it can't allow writes, so it can't be SINGLE. In that case, it - * should be REPORT_LSN, waiting for either a change of settings, or the - * introduction of a new node. + * Terminator, not a real rule -- .pos is deliberately left unset (its + * zero default), which no real row above ever has (every real .pos + * starts at 101). Every loop over MonitorFSM[] in this file stops here + * instead of at a separately hand-maintained size constant (see this + * array's own leading comment and AssertMonitorFSMWellFormed() for why + * that constant was removed): a row inserted anywhere above this one is + * automatically in scope for every one of those loops, and a row + * mistakenly inserted after this one instead is caught by + * AssertMonitorFSMWellFormed()'s own iteration-count sanity check. This + * row must always stay last. */ - if (nodesCount == 1 && - !IsCurrentState(activeNode, REPLICATION_STATE_SINGLE) && - activeNode->candidatePriority > 0) - { - char message[BUFSIZE]; + { .comment = "terminator -- do not add rows after this one" } +}; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to single as there is no other node.", - NODE_FORMAT_ARGS(activeNode)); +/* + * AssertMonitorFSMWellFormed cross-checks each row's own .pos against the + * top-level section its .sectionPath declares, so a row whose pos and + * section have drifted out of sync (e.g. moved into the wrong hundred-block + * without updating its path, or vice versa) is caught here -- loudly, at + * first use -- instead of silently, as a row matching (or failing to match) + * under the wrong prefix. A no-op build (USE_ASSERT_CHECKING off) skips this + * entirely, matching every other structural check in this file (see + * AssignDeclaredGoalState). + * + * .pos is NOT the array index: each section's rows are numbered starting + * at *01 within its own hundred-block (101/201/301/401, matching how many + * top-level MONITOR_FSM_SECTION_* values exist -- *01 rather than *00 so + * the count within the block reads as ordinary 1-based), stepping by 2 + * within the section rather than by 1 -- so a new row can be inserted + * between two existing ones (e.g. 202 between 201 and 203) without + * renumbering anything else in the file. What's checked here is weaker + * than "exactly i*2 + sectionStart + 1" as a result: strictly increasing, + * and within the section's own hundred-block -- consistent with rows being + * added over time at whatever free position is nearest where they belong, + * not at a specific computed slot. + * + * Every row's .sectionPath[0] must be one of the four top-level + * values -- this is what makes it safe for MonitorFSMSectionGetEnum/ + * MonitorFSMSectionGetName (see their own comments) to only ever be called + * with .sectionPath[0], never a deeper path element. + */ +static void +AssertMonitorFSMWellFormed(void) +{ +#ifdef USE_ASSERT_CHECKING - /* other node may have been removed */ - AssignGoalState(activeNode, REPLICATION_STATE_SINGLE, message); + int previousPos = 0; + bool foundResumeAnchor = false; - return true; - } - else if (nodesCount == 1 && - !IsCurrentState(activeNode, REPLICATION_STATE_SINGLE) && - activeNode->candidatePriority == 0) + /* + * MonitorFSM[] ends with a terminator row (.pos left at its zero + * default) rather than a separately maintained count: a terminator + * can't go stale the way a hand-maintained size constant would, since + * it's part of the array's own literal initializer, so any row added + * before it is automatically in scope for every loop below -- no + * separate constant to remember to update when a row is added. The + * iteration cap here is just a sanity net against the terminator itself + * ever being removed or a new row accidentally inserted after it, which + * would otherwise turn every one of these loops into an unbounded scan + * past the end of the array. + */ + for (int i = 0; MonitorFSM[i].pos != 0; i++) { - char message[BUFSIZE]; + Assert(i < 1000); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn as there is no other node" - " and candidate priority is %d.", - NODE_FORMAT_ARGS(activeNode), - activeNode->candidatePriority); + int pos = MonitorFSM[i].pos; + MonitorFSMSection top = MonitorFSM[i].sectionPath[0]; - /* other node may have been removed */ - AssignGoalState(activeNode, REPLICATION_STATE_REPORT_LSN, message); + Assert(pos > previousPos); + previousPos = pos; - return true; - } + if (pos >= 100 && pos < 200) + { + Assert(top == MONITOR_FSM_SECTION_API_TRIGGERED); + } + else if (pos >= 200 && pos < 300) + { + Assert(top == MONITOR_FSM_SECTION_EARLY_CHECKS); + } + else if (pos >= 300 && pos < 400) + { + Assert(top == MONITOR_FSM_SECTION_REPORTING_NODE); + } + else if (pos >= 400 && pos < 500) + { + Assert(top == MONITOR_FSM_SECTION_PRIMARY_NODE); + } + else + { + Assert(false); + } - /* - * We separate out the FSM for the primary server, because that one needs - * to loop over every other node to take decisions. That induces some - * complexity that is best managed in a specialized function. - */ - if (IsInPrimaryState(activeNode)) - { - return ProceedGroupStateForPrimaryNode(ctx, activeNode); + if (pos == MonitorFSM_MultiStandbyCascadeResumeAfterPos) + { + foundResumeAnchor = true; + Assert(SectionPathIsUnderPrefix(MonitorFSM[i].sectionPath, + SectionReportingNode)); + } } - /* - * Derive primaryNode from ctx->groupNodeList (already fetched under the - * lock NodeActive() holds for the whole call) instead of running a - * second, independent AutoFailoverNodeGroup() query -- see - * GetPrimaryOrDemotedNodeInGroupFromList()'s comment for why this - * matters even though every writer now shares the same lock. - */ - AutoFailoverNode *primaryNode = - GetPrimaryOrDemotedNodeInGroupFromList(ctx->groupNodeList); + Assert(foundResumeAnchor); +#endif +} - /* - * We want to have a primaryNode around for most operations, but also need - * to support the case that the primaryNode has been dropped manually by a - * call to remove_node(). So we have two main cases to think about here: - * - * - we have two nodes, one of them has been removed, we catch that earlier - * in this function and assign the remaining one with the SINGLE state, - * - * - we have more than two nodes in total, and the primary has just been - * removed (maybe it was still marked unhealthy and the operator knows it - * won't ever come back so called remove_node() already): in that case in - * remove_node() we set all the other nodes to REPORT_LSN (unless they - * are in MAINTENANCE), and we should be able to make progress with the - * failover without a primary around. - * - * In all other cases we require a primaryNode to be identified. - */ - if (primaryNode == NULL && !IsFailoverInProgress(ctx->groupNodeList)) + +/* + * MonitorFSMSectionGetName returns the (enum) name of a MonitorFSMSection -- + * the SQL-facing spelling used by pgautofailover.fsm_section, dump_fsm(), + * and the rule_section column on pgautofailover.event. Mirrors + * ReplicationStateGetName's role for ReplicationState exactly. + */ +const char * +MonitorFSMSectionGetName(MonitorFSMSection section) +{ + switch (section) { - ereport(ERROR, - (errmsg("ProceedGroupState couldn't find the primary node " - "in formation \"%s\", group %d", - formationId, groupId), - errdetail("activeNode is " NODE_FORMAT - " in state %s", - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->goalState)))); + case MONITOR_FSM_SECTION_API_TRIGGERED: + { + return "api_triggered"; + } + + case MONITOR_FSM_SECTION_EARLY_CHECKS: + { + return "early_checks"; + } + + case MONITOR_FSM_SECTION_REPORTING_NODE: + { + return "reporting_node"; + } + + case MONITOR_FSM_SECTION_PRIMARY_NODE: + { + return "primary_node"; + } + + default: + { + ereport(ERROR, + (errmsg("bug: unknown MonitorFSMSection (%d)", section))); + } } +} - /* - * Detect a genuine timeline fork on a healthy secondary as soon as its - * newly reported timeline is visible, rather than waiting for an - * incidental health-check cycle or an explicit maintenance toggle to - * eventually drive it through catchingup (see #683 and the timeline - * fork detection design). Reuses the exact same ancestry filter the - * report_lsn election path already applies (below, and in - * ProceedGroupStateForMSFailover) -- this only changes when a - * diverged secondary gets pushed to catchingup, not how that's - * decided or how the resync itself recovers it. - * - * Scoped to activeNode currently being SECONDARY_STATE: that's the - * same state the "unhealthy secondary" transition just below already - * uses for the identical CATCHINGUP goal assignment, so this is - * additive to an existing, tested transition rather than a new kind - * of one. A node with reportedTLI == 0 hasn't reported a timeline yet - * (e.g. still in wait_standby) and has nothing to check. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_SECONDARY) && - activeNode->reportedTLI > 0) + +/* + * MonitorApiFunctionGetName returns the (enum) name of a MonitorApiFunction -- + * the dump_fsm()/pgautofailover.fsm section column's own spelling (appended + * after the section name, see MonitorFSMTransitionSectionText) for which + * operator-triggered SQL entry point a MONITOR_FSM_SECTION_API_TRIGGERED + * row's .conditions.apiTrigger requires. API_FUNCTION_NONE has no row that + * matches on it specifically (it's the ordinary heartbeat path's implicit + * value, never written as an explicit API_TRIGGER() in any row), but is + * still given a name here rather than treated as an error, matching + * MonitorFSMSectionGetName's own "every enumerator has a name" discipline. + */ +static const char * +MonitorApiFunctionGetName(MonitorApiFunction apiFunction) +{ + switch (apiFunction) { - int referenceTli = 0; - List *comparableNodeList = - FilterNodesByTimelineAncestry(ctx->groupNodeList, formationId, - groupId, &referenceTli); + case API_FUNCTION_NONE: + { + return "node_active"; + } - bool activeNodeIsComparable = false; - ListCell *cell = NULL; + case API_FUNCTION_REMOVE_NODE: + { + return "remove_node"; + } - foreach(cell, comparableNodeList) + case API_FUNCTION_PERFORM_FAILOVER: { - AutoFailoverNode *node = (AutoFailoverNode *) lfirst(cell); + return "perform_failover"; + } - if (node->nodeId == activeNode->nodeId) - { - activeNodeIsComparable = true; - break; - } + case API_FUNCTION_START_MAINTENANCE: + { + return "start_maintenance"; } - if (referenceTli > 0 && !activeNodeIsComparable) + case API_FUNCTION_STOP_MAINTENANCE: { - char message[BUFSIZE] = { 0 }; + return "stop_maintenance"; + } - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup: its reported timeline %d does not appear " - "to be an ancestor of the group's reference timeline %d -- " - "forcing a resync so the ancestry check can run and, if " - "needed, rewind it onto the correct lineage.", - NODE_FORMAT_ARGS(activeNode), - activeNode->reportedTLI, - referenceTli); + case API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY: + { + return "set_node_candidate_priority"; + } + + case API_FUNCTION_SET_NODE_REPLICATION_QUORUM: + { + return "set_node_replication_quorum"; + } - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); + case API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS: + { + return "set_formation_number_sync_standbys"; + } - return true; + default: + { + ereport(ERROR, + (errmsg("bug: unknown MonitorApiFunction (%d)", apiFunction))); } } +} - /* - * Replication stall detection (issue #997 — 3-DC split-brain). - * - * When the primary is healthy (monitor can reach it) but no standby has - * appeared in pg_stat_replication for longer than - * pgautofailover.replication_stall_timeout, we assign wait_primary. - * That clears synchronous_standby_names so COMMIT no longer hangs. - * - * replication_stall_since is set/cleared by ReportAutoFailoverNodeState() - * each time the keeper calls node_active(). - */ - if (IsInPrimaryState(primaryNode) && - NodeIsHealthy(primaryNode, ctx) && - primaryNode->replicationStallSince != 0 && - TimestampDifferenceExceeds(primaryNode->replicationStallSince, - ctx->now, - ctx->replicationStallTimeoutMs)) - { - char message[BUFSIZE] = { 0 }; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary: no standby has been connected in " - "pg_stat_replication for more than %dms " - "(pgautofailover.replication_stall_timeout).", - NODE_FORMAT_ARGS(primaryNode), - ctx->replicationStallTimeoutMs); +/* + * MonitorFSMTransitionSectionText renders a row's section as the + * dump_fsm()/pgautofailover.fsm section column: plain "reporting_node"-style + * text for most rows, but "api_triggered: remove_node"-style text (section + * name, ": ", API function name) whenever the row's own + * .conditions.apiTrigger requires a specific operator-triggered entry point + * (API_TRIGGER_SPECIFIC -- true for every row in + * MONITOR_FSM_SECTION_API_TRIGGERED, and only those rows) -- one column + * instead of a separate, mostly-NULL api_function column, since the two + * facts are never independently meaningful: a row either has both a section + * and no specific trigger, or both a section and exactly one trigger. + */ +static Datum +MonitorFSMTransitionSectionText(const MonitorFSMTransition *rule) +{ + const char *sectionName = MonitorFSMSectionGetName(rule->sectionPath[0]); - AssignGoalState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY, message); + if (rule->conditions.apiTrigger.kind == API_TRIGGER_SPECIFIC) + { + StringInfoData buf; - return true; + initStringInfo(&buf); + appendStringInfo(&buf, "%s: %s", sectionName, + MonitorApiFunctionGetName(rule->conditions.apiTrigger.function)); + + return CStringGetTextDatum(buf.data); } - /* Multiple Standby failover is handled in its own function. */ - if (nodesCount > 2 && NodeIsUnhealthy(primaryNode, ctx)) + return CStringGetTextDatum(sectionName); +} + + +/* + * MonitorFSMLeafSectionName returns the (enum) name of any MonitorFSMSection + * value, including the fine-grained leaves appended after PRIMARY_NODE -- + * unlike MonitorFSMSectionGetName (SQL-facing, and only ever called with one + * of the 4 top-level values, see that function's own comment), this + * is used purely to render a row's own full section_path text below, one + * path element at a time. + */ +static const char * +MonitorFSMLeafSectionName(MonitorFSMSection section) +{ + switch (section) { - /* - * The WAIT_PRIMARY state encodes the fact that we know there is no - * failover candidate, so there's no point in orchestrating a failover, - * even though the primary node is currently not available. - * - * To be in the WAIT_PRIMARY means that the other nodes are all either - * unhealty or with candidate priority set to zero. - * - * Otherwise stop replication from the primary and proceed with - * candidate election for primary replacement, whenever we have at - * least one candidates for failover. - */ - List *candidateNodesList = - AutoFailoverOtherNodesListInState(primaryNode, - REPLICATION_STATE_SECONDARY); + case MONITOR_FSM_SECTION_API_TRIGGERED: + case MONITOR_FSM_SECTION_EARLY_CHECKS: + case MONITOR_FSM_SECTION_REPORTING_NODE: + case MONITOR_FSM_SECTION_PRIMARY_NODE: + { + return MonitorFSMSectionGetName(section); + } - int candidatesCount = CountHealthyCandidates(candidateNodesList); + case MONITOR_FSM_SECTION_FROM_CONTEXT: + { + return "from_context"; + } - if (IsInPrimaryState(primaryNode) && - !IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) && - candidatesCount >= 1) + case MONITOR_FSM_SECTION_MS_FAILOVER: { - char message[BUFSIZE] = { 0 }; + return "ms_failover"; + } - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to draining after it became unhealthy.", - NODE_FORMAT_ARGS(primaryNode)); + case MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET: + { + return "retry_reset"; + } - AssignGoalState(primaryNode, REPLICATION_STATE_DRAINING, message); + case MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN: + { + return "candidate_join"; } - /* - * In a multiple standby system we can assign maintenance as soon as - * prepare_maintenance has been reached, at the same time than an - * election is triggered. This also allows the operator to disable - * maintenance on the old-primary and have it join the election. - */ - else if (IsCurrentState(primaryNode, REPLICATION_STATE_PREPARE_MAINTENANCE)) + case MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT: { - char message[BUFSIZE] = { 0 }; + return "candidate_fanout"; + } - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance after it converged to prepare_maintenance.", - NODE_FORMAT_ARGS(primaryNode)); + case MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME: + { + return "promotion_outcome"; + } - AssignGoalState(primaryNode, REPLICATION_STATE_MAINTENANCE, message); + case MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE: + { + return "missing_nodes_gate"; } - /* - * ProceedGroupStateForMSFailover chooses the failover candidate when - * there's more than one standby node around, by applying the - * candidatePriority and comparing the reportedLSN. The function also - * orchestrate fetching the missing WAL from the failover candidate if - * that's needed. - * - * When ProceedGroupStateForMSFailover returns true, it means it was - * successfull in driving the failover to the next step, and we should - * stop here. When it return false, it did nothing, and so we want to - * apply the common orchestration code for a failover. - */ - if (ProceedGroupStateForMSFailover(ctx, primaryNode)) + case MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_CANDIDATE_COUNT_GATE: { - return true; + return "candidate_count_gate"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE: + { + return "quorum_candidate_gate"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_NO_CANDIDATE_YET: + { + return "no_candidate_yet"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE: + { + return "draining_or_maintenance"; + } + + default: + { + ereport(ERROR, + (errmsg("bug: unknown MonitorFSMSection (%d)", section))); } } +} - /* - * when report_lsn and the promotion has been done already: - * report_lsn -> secondary - * - * - * Let the main primary loop account for allSecondariesAreHealthy and only - * then decide to assign PRIMARY to the primaryNode. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_REPORT_LSN) && - (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_JOIN_PRIMARY)) && - NodeIsHealthy(primaryNode, ctx)) + +/* + * MonitorFSMTransitionSectionPathText renders a row's full .sectionPath as a + * dotted string (e.g. "reporting_node.ms_failover.retry_reset") for + * dump_fsm()/pgautofailover.fsm's own section_path column -- the only place + * this project ever produces an ltree-shaped string, and only as plain text: + * the cast to real ltree happens in the pgautofailover.fsm view definition + * (pgautofailover.sql), via Postgres's own ordinary type-casting machinery, + * not from this C code. + */ +static Datum +MonitorFSMTransitionSectionPathText(const MonitorFSMTransition *rule) +{ + StringInfoData buf; + + initStringInfo(&buf); + + for (int i = 0; i < MONITOR_FSM_SECTION_PATH_MAX_DEPTH; i++) { - char message[BUFSIZE] = { 0 }; + if (rule->sectionPath[i] == MONITOR_FSM_SECTION_NONE) + { + break; + } + + if (buf.len > 0) + { + appendStringInfoString(&buf, "."); + } + + appendStringInfoString(&buf, MonitorFSMLeafSectionName(rule->sectionPath[i])); + } + + return CStringGetTextDatum(buf.data); +} + + +/* + * MonitorFSMSectionTypeOid returns the OID of the pgautofailover.fsm_section + * type. Mirrors ReplicationStateTypeOid exactly, see its own comment for why + * the String/Value split exists. + */ +Oid +MonitorFSMSectionTypeOid(void) +{ +#if (PG_VERSION_NUM >= 150000) + String *schemaName = makeString(AUTO_FAILOVER_SCHEMA_NAME); + String *typeName = makeString(FSM_SECTION_TYPE_NAME); +#else + Value *schemaName = makeString(AUTO_FAILOVER_SCHEMA_NAME); + Value *typeName = makeString(FSM_SECTION_TYPE_NAME); +#endif + + List *enumTypeNameList = list_make2(schemaName, typeName); + TypeName *enumTypeName = makeTypeNameFromNameList(enumTypeNameList); + Oid enumTypeOid = typenameTypeId(NULL, enumTypeName); + + return enumTypeOid; +} - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to secondary after " NODE_FORMAT - " converged to %s and has been marked healthy.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryNode->reportedState)); - AssignGoalState(activeNode, REPLICATION_STATE_SECONDARY, message); +/* + * MonitorFSMSectionGetEnum returns the SQL enum OID for a given + * MonitorFSMSection value. Mirrors ReplicationStateGetEnum exactly. + */ +Oid +MonitorFSMSectionGetEnum(MonitorFSMSection section) +{ + const char *enumName = MonitorFSMSectionGetName(section); + Oid enumTypeOid = MonitorFSMSectionTypeOid(); - return true; + HeapTuple enumTuple = SearchSysCache2(ENUMTYPOIDNAME, + ObjectIdGetDatum(enumTypeOid), + CStringGetDatum(enumName)); + if (!HeapTupleIsValid(enumTuple)) + { + ereport(ERROR, (errmsg("invalid value for enum: %d", section))); } - /* - * when report_lsn and the promotion has been done already: - * report_lsn -> secondary - * - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_REPORT_LSN) && - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY) && - NodeIsHealthy(primaryNode, ctx)) - { - char message[BUFSIZE]; + Oid sectionOid = HeapTupleGetOid(enumTuple); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to secondary after " NODE_FORMAT - " got selected as the failover candidate.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); + ReleaseSysCache(enumTuple); + + return sectionOid; +} - AssignGoalState(activeNode, REPLICATION_STATE_SECONDARY, message); - return true; +/* + * EnumGetMonitorFSMSection returns the internal value of a fsm_section enum. + * Mirrors EnumGetReplicationState exactly. + */ +MonitorFSMSection +EnumGetMonitorFSMSection(Oid monitorFSMSectionOid) +{ + HeapTuple enumTuple = SearchSysCache1(ENUMOID, + ObjectIdGetDatum(monitorFSMSectionOid)); + if (!HeapTupleIsValid(enumTuple)) + { + ereport(ERROR, (errmsg("invalid input value for enum: %u", + monitorFSMSectionOid))); } - /* - * When the candidate is done fast forwarding the locally missing WAL bits, - * it can be promoted. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_FAST_FORWARD)) + Form_pg_enum enumForm = (Form_pg_enum) GETSTRUCT(enumTuple); + char *enumName = NameStr(enumForm->enumlabel); + MonitorFSMSection section; + + if (strncmp(enumName, "api_triggered", NAMEDATALEN) == 0) { - char message[BUFSIZE] = { 0 }; + section = MONITOR_FSM_SECTION_API_TRIGGERED; + } + else if (strncmp(enumName, "early_checks", NAMEDATALEN) == 0) + { + section = MONITOR_FSM_SECTION_EARLY_CHECKS; + } + else if (strncmp(enumName, "reporting_node", NAMEDATALEN) == 0) + { + section = MONITOR_FSM_SECTION_REPORTING_NODE; + } + else if (strncmp(enumName, "primary_node", NAMEDATALEN) == 0) + { + section = MONITOR_FSM_SECTION_PRIMARY_NODE; + } + else + { + ReleaseSysCache(enumTuple); + ereport(ERROR, (errmsg("bug: unknown fsm_section enum label \"%s\"", enumName))); + } - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to prepare_promotion", - NODE_FORMAT_ARGS(activeNode)); + ReleaseSysCache(enumTuple); - AssignGoalState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION, message); + return section; +} - return true; - } - /* - * There are other cases when we want to continue an already started - * failover. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_REPORT_LSN) || - IsCurrentState(activeNode, REPLICATION_STATE_FAST_FORWARD)) +/* + * NodeStatePatternReportedStatesText renders a NodeStatePattern's own + * reportedStates set as a human- and script-readable comma-separated list + * of pgautofailover.replication_state labels -- the "current (reported) + * state" a row requires of the node this pattern is attached to, for a + * keeper-side cross-check to compare against KeeperFSMTransition.current + * (see fsm.h: current/assigned/pgKind is exactly the (fromState, toState) + * shape this exists to expose the "from" half of). + * + * Only NODE_STATE_STABLE/REPORTED/TRANSITIONING pin down a genuine "current + * state must be one of these" set -- returns NULL for NODE_STATE_ANY (no + * constraint at all) and for NODE_STATE_NOT_STABLE/NOT_ASSIGNED (an + * exclusion, not a "from" set: rendering reportedStates there would read as + * the opposite of what the row actually requires). + */ +static Datum +NodeStatePatternReportedStatesText(const NodeStatePattern *pattern, bool *isNull) +{ + if (pattern->kind != NODE_STATE_STABLE && + pattern->kind != NODE_STATE_REPORTED && + pattern->kind != NODE_STATE_TRANSITIONING) { - return ProceedGroupStateForMSFailover(ctx, primaryNode); + *isNull = true; + return (Datum) 0; } - /* - * when primary node is ready for replication: - * wait_standby -> catchingup - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_WAIT_STANDBY) && - (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_JOIN_PRIMARY))) - { - char message[BUFSIZE]; + StringInfoData buf; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after " NODE_FORMAT - " converged to %s.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryNode->reportedState)); + initStringInfo(&buf); - /* start replication */ - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); + for (int i = 0; i < pattern->reportedStates.count; i++) + { + if (i > 0) + { + appendStringInfoString(&buf, ", "); + } - return true; + appendStringInfoString(&buf, + ReplicationStateGetName( + pattern->reportedStates.states[i])); } - /* - * when primary node is ready for replication: - * wait_standby -> catchingup - * primary -> apply_settings - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_WAIT_STANDBY) && - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY) && - activeNode->replicationQuorum) - { - char message[BUFSIZE]; + *isNull = false; + return CStringGetTextDatum(buf.data); +} - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup and " NODE_FORMAT - " to %s to edit synchronous_standby_names.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryNode->reportedState)); - /* start replication */ - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); +/* + * Appends "name=true"/"name=false" to buf (with a ", " separator when buf isn't + * empty), skipping BOOL_ANY entirely -- BOOL_ANY means this row doesn't care + * about the fact at all, so it's noise, not a guard. Shared by + * NodeStatusPatternConditionsText and NodeActiveContextPatternConditionsText + * below, one line per BoolPattern field rather than a loop: there's no runtime + * array of (name, value) pairs to iterate, since the field names only exist at + * compile time. + */ +#define APPEND_BOOL_CONDITION(buf, name, boolPattern) \ + do { \ + if ((boolPattern) != BOOL_ANY) \ + { \ + if ((buf)->len > 0) \ + { \ + appendStringInfoString((buf), ", "); \ + } \ + appendStringInfo((buf), "%s=%s", (name), \ + (boolPattern) == BOOL_TRUE ? "true" : "false"); \ + } \ + } while (0) - /* edit synchronous_standby_names to add the new standby now */ - AssignGoalState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS, message); +/* + * Same idea as APPEND_BOOL_CONDITION, for an IntPattern field: skips + * INT_PATTERN_ANY, otherwise renders "name=n"/"name>=n"/"name<=n" matching the + * EXACTLY/AT_LEAST/AT_MOST macro that built the pattern. + */ +#define APPEND_INT_CONDITION(buf, name, intPattern) \ + do { \ + if ((intPattern).kind != INT_PATTERN_ANY) \ + { \ + const char *op = (intPattern).kind == INT_PATTERN_AT_LEAST ? ">=" : \ + (intPattern).kind == INT_PATTERN_AT_MOST ? "<=" : "="; \ + if ((buf)->len > 0) \ + { \ + appendStringInfoString((buf), ", "); \ + } \ + appendStringInfo((buf), "%s%s%d", (name), op, (intPattern).value); \ + } \ + } while (0) - return true; +/* + * AppendNodeStateGoalCondition appends a role's own goal-state precondition + * to buf, when its statePattern is NODE_STATE_ASSIGNED/NOT_ASSIGNED: "the + * node's EXISTING goal (before this dispatch runs) must/must not already be + * one of these" -- e.g. "goal=dropped" (pos 203, FSM_DROPPED_GOAL: goal + * already DROPPED, reported state irrelevant) or "goal!=wait_primary" (the + * wait_maintenance/primary's-goal-no-longer-wait_primary row). + * + * This is a genuine match condition, same as any BoolPattern field, but it + * lives on .statePattern (a NodeStatePattern), not .conditions (a + * NodeStatusPattern's BoolPattern fields) -- without this, a row like pos + * 203 renders with every single column empty (no current_state, since + * NodeStatePatternReportedStatesText only renders STABLE/REPORTED/ + * TRANSITIONING; no conditions, since ASSIGNED/NOT_ASSIGNED isn't a + * BoolPattern; no assigned_state, since the row is a no-op), making it look + * unconditional when it very much isn't. Folded into the same + * *_conditions column as the BoolPattern fields (not the *_current_state + * column, which is reserved for a strict "current reported state" fromState + * a keeper-side cross-check can compare against KeeperFSMTransition.current) + * since both are "things that must be true about this node" from a reader's + * point of view. + */ +static void +AppendNodeStateGoalCondition(StringInfoData *buf, const NodeStatePattern *pattern) +{ + if (pattern->kind != NODE_STATE_ASSIGNED && pattern->kind != NODE_STATE_NOT_ASSIGNED) + { + return; } - /* - * when primary node is ready for replication: - * wait_standby -> catchingup - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_WAIT_STANDBY) && - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY) && - !activeNode->replicationQuorum) + if (buf->len > 0) { - char message[BUFSIZE]; + appendStringInfoString(buf, ", "); + } - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup.", - NODE_FORMAT_ARGS(activeNode)); + appendStringInfoString(buf, pattern->kind == NODE_STATE_ASSIGNED ? "goal=" : + "goal!="); - /* start replication */ - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); + for (int i = 0; i < pattern->assignedStates.count; i++) + { + if (i > 0) + { + appendStringInfoString(buf, "|"); + } - return true; + appendStringInfoString(buf, ReplicationStateGetName( + pattern->assignedStates.states[i])); } +} - /* - * when secondary caught up: - * catchingup -> secondary - * + wait_primary -> primary - * - * When we have multiple standby nodes and one of them is joining, or - * re-joining after maintenance, we have to edit the replication setting - * synchronous_standby_names on the primary. The transition from another - * state to PRIMARY includes that edit. If the primary already is in the - * primary state, we assign APPLY_SETTINGS to it to make sure its - * repication settings are updated now. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_CATCHINGUP) && - (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_JOIN_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY)) && - NodeIsHealthy(activeNode, ctx) && - activeNode->reportedTLI == primaryNode->reportedTLI && - WalDifferenceWithin(activeNode, primaryNode, EnableSyncXlogThreshold)) + +/* + * NodeStatusPatternConditionsText renders every non-BOOL_ANY BoolPattern + * field of a NodeStatusPattern (activeNode/primaryNode/candidateNode's own + * per-node guards -- isHealthy foremost among them, per the reason this + * column exists at all) as a compact "name=value, name=value" list, plus + * this role's own goal-state precondition if it has one (see + * AppendNodeStateGoalCondition's own comment). NULL when the row places no + * per-node guard on this role at all -- the common case for rows that only + * match on reported state. + */ +static Datum +NodeStatusPatternConditionsText(const NodeStatusPattern *pattern, bool *isNull) +{ + StringInfoData buf; + + initStringInfo(&buf); + + AppendNodeStateGoalCondition(&buf, &pattern->statePattern); + + APPEND_BOOL_CONDITION(&buf, "exists", pattern->exists); + APPEND_BOOL_CONDITION(&buf, "isHealthy", pattern->isHealthy); + APPEND_BOOL_CONDITION(&buf, "isUnhealthy", pattern->isUnhealthy); + APPEND_BOOL_CONDITION(&buf, "candidateEligible", pattern->candidateEligible); + APPEND_BOOL_CONDITION(&buf, "isInPrimaryState", pattern->isInPrimaryState); + APPEND_BOOL_CONDITION(&buf, "isInMaintenance", pattern->isInMaintenance); + APPEND_BOOL_CONDITION(&buf, "isDemotedPrimary", pattern->isDemotedPrimary); + APPEND_BOOL_CONDITION(&buf, "canTakeWrites", pattern->canTakeWrites); + APPEND_BOOL_CONDITION(&buf, "reportedCanTakeWrites", pattern->reportedCanTakeWrites); + APPEND_BOOL_CONDITION(&buf, "reportedIsWaitStandby", pattern->reportedIsWaitStandby); + APPEND_BOOL_CONDITION(&buf, "reportedIsJoinSecondary", + pattern->reportedIsJoinSecondary); + APPEND_BOOL_CONDITION(&buf, "reportedIsPrepareMaintenance", + pattern->reportedIsPrepareMaintenance); + APPEND_BOOL_CONDITION(&buf, "isReadyToStreamWAL", pattern->isReadyToStreamWAL); + APPEND_BOOL_CONDITION(&buf, "drainTimeExpired", pattern->drainTimeExpired); + APPEND_BOOL_CONDITION(&buf, "isCitusWorkerGroup", pattern->isCitusWorkerGroup); + APPEND_BOOL_CONDITION(&buf, "replicationQuorum", pattern->replicationQuorum); + APPEND_BOOL_CONDITION(&buf, "isComparableToReferenceTli", + pattern->isComparableToReferenceTli); + APPEND_BOOL_CONDITION(&buf, "unreachableFromDemoteTimeout", + pattern->unreachableFromDemoteTimeout); + + if (buf.len == 0) { - char message[BUFSIZE] = { 0 }; + *isNull = true; + return (Datum) 0; + } - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to secondary after it caught up.", - NODE_FORMAT_ARGS(activeNode)); + *isNull = false; + return CStringGetTextDatum(buf.data); +} - /* node is ready for promotion */ - AssignGoalState(activeNode, REPLICATION_STATE_SECONDARY, message); - return true; +/* + * NodeActiveContextPatternConditionsText renders every non-BOOL_ANY + * BoolPattern field of a row's .conditions (NodeActiveContextPattern -- + * group-level guards: counts, WAL thresholds, guard_data_loss, the + * MS-failover cluster's own facts) the same way. .apiTrigger is + * deliberately skipped: which operator function triggers a row is already + * implied by its section (api_triggered) and named in its own comment, so + * repeating it here would be noise, not a guard a reader doesn't already + * have. + */ +static Datum +NodeActiveContextPatternConditionsText(const NodeActiveContextPattern *cond, bool *isNull) +{ + StringInfoData buf; + + initStringInfo(&buf); + + APPEND_BOOL_CONDITION(&buf, "groupHasExactlyOneNode", cond->groupHasExactlyOneNode); + APPEND_BOOL_CONDITION(&buf, "groupHasExactlyTwoNodes", cond->groupHasExactlyTwoNodes); + APPEND_BOOL_CONDITION(&buf, "groupHasMoreThanTwoNodes", + cond->groupHasMoreThanTwoNodes); + APPEND_BOOL_CONDITION(&buf, "anyOtherNodeWaitingStandby", + cond->anyOtherNodeWaitingStandby); + APPEND_BOOL_CONDITION(&buf, "numberSyncStandbysIsZero", + cond->numberSyncStandbysIsZero); + APPEND_BOOL_CONDITION(&buf, "replicationQuorumCountIsZero", + cond->replicationQuorumCountIsZero); + APPEND_BOOL_CONDITION(&buf, "secondaryNodesCountIsZero", + cond->secondaryNodesCountIsZero); + APPEND_BOOL_CONDITION(&buf, "secondaryQuorumNodesCountIsZero", + cond->secondaryQuorumNodesCountIsZero); + APPEND_BOOL_CONDITION(&buf, "atLeastOneHealthyCandidate", + cond->atLeastOneHealthyCandidate); + APPEND_BOOL_CONDITION(&buf, "walWithinPromoteThreshold", + cond->walWithinPromoteThreshold); + APPEND_BOOL_CONDITION(&buf, "walWithinSyncThreshold", cond->walWithinSyncThreshold); + APPEND_BOOL_CONDITION(&buf, "activeAndPrimaryTliMatch", + cond->activeAndPrimaryTliMatch); + APPEND_BOOL_CONDITION(&buf, "primaryIsWaitPrimaryPresumedDead", + cond->primaryIsWaitPrimaryPresumedDead); + APPEND_BOOL_CONDITION(&buf, "failoverInProgress", cond->failoverInProgress); + APPEND_BOOL_CONDITION(&buf, "replicationStallExceeded", + cond->replicationStallExceeded); + APPEND_BOOL_CONDITION(&buf, "lastHealthySyncStandbyGoingToMaintenance", + cond->lastHealthySyncStandbyGoingToMaintenance); + APPEND_BOOL_CONDITION(&buf, "activeNodeAllWalSourcesUnhealthy", + cond->activeNodeAllWalSourcesUnhealthy); + APPEND_BOOL_CONDITION(&buf, "candidatePromotionInProgress", + cond->candidatePromotionInProgress); + APPEND_BOOL_CONDITION(&buf, "mostAdvancedCandidateWithinPromoteThreshold", + cond->mostAdvancedCandidateWithinPromoteThreshold); + APPEND_BOOL_CONDITION(&buf, "guardDataLossEnabled", cond->guardDataLossEnabled); + APPEND_BOOL_CONDITION(&buf, "inMSFailoverCluster", cond->inMSFailoverCluster); + APPEND_BOOL_CONDITION(&buf, "inMSFailoverCandidateGate", + cond->inMSFailoverCandidateGate); + + APPEND_INT_CONDITION(&buf, "missingNodesCount", cond->missingNodesCount); + APPEND_INT_CONDITION(&buf, "candidateCount", cond->candidateCount); + APPEND_INT_CONDITION(&buf, "quorumCandidateCount", cond->quorumCandidateCount); + APPEND_BOOL_CONDITION(&buf, "sufficientQuorumCandidates", + cond->sufficientQuorumCandidates); + + if (buf.len == 0) + { + *isNull = true; + return (Datum) 0; } - /* - * when primary fails: - * secondary -> prepare_promotion - * + primary -> draining - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_SECONDARY) && - IsInPrimaryState(primaryNode) && - NodeIsUnhealthy(primaryNode, ctx) && NodeIsHealthy(activeNode, ctx) && - activeNode->candidatePriority > 0 && - WalDifferenceWithin(activeNode, primaryNode, PromoteXlogThreshold)) - { - char message[BUFSIZE]; + *isNull = false; + return CStringGetTextDatum(buf.data); +} - /* - * A primary already converged to wait_primary has no "draining" to - * go through: it never had a synchronous standby to begin with, so - * there's nothing live to gracefully drain, and KeeperFSM[] has no - * wait_primary -> draining edge anyway (issue #1168). Leave its - * goal untouched here; the prepare_promotion/stop_replication rules - * below apply the same drainTimeoutMs safety margin via report - * staleness instead of a goal-state timestamp, then commit the one - * real, reachable wait_primary -> demoted transition once it has - * genuinely expired. - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY)) - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to prepare_promotion after " NODE_FORMAT - " (at wait_primary) became unhealthy.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - } - else - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to draining and " NODE_FORMAT - " to prepare_promotion " - "after " NODE_FORMAT - " became unhealthy.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - /* shut down the primary */ - AssignGoalState(primaryNode, REPLICATION_STATE_DRAINING, message); - } +PG_FUNCTION_INFO_V1(dump_fsm); - /* keep reading until no more records are available */ - AssignGoalState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION, message); +/* + * dump_fsm exposes MonitorFSM[] to SQL, one row per rule, in table order + * (first-match-wins order -- the same order RuleMatches() itself scans in). + * This is the cross-check surface pgautofailover.check_fsm_reachability() + * (below) is built on: enough to correlate a pgautofailover.event row's + * rule_pos/rule_section (see notifications.h) back to the exact rule that + * produced it, and enough for an operator or a keeper-side reachability + * check to see every transition the monitor's table can produce, without + * reading the C source. The active_node_current_state/ + * other_node_current_state/candidate_node_current_state columns (rendered via + * NodeStatePatternReportedStatesText, see its own + * comment) plus the assigned-state columns already here give a keeper- + * cross-check tool the full (fromState, toState) shape it needs to compare + * against KeeperFSMTransition. The *_conditions columns (rendered via + * NodeStatusPatternConditionsText/NodeActiveContextPatternConditionsText, see + * their own comments) additionally surface every non-default BoolPattern guard + * on the row -- health foremost among them, but also maintenance, candidate + * eligibility, WAL thresholds, guard_data_loss, and the rest -- so a reader (or + * a future automated check) doesn't have to open the C source to see what else + * has to be true for a given row to fire. The section column (rendered via + * MonitorFSMTransitionSectionText) also names + * which operator-triggered SQL entry point a MONITOR_FSM_SECTION_API_TRIGGERED + * row requires, e.g. "api_triggered: remove_node" -- just the plain section + * name for every other row, since only that section's rows carry a specific + * .conditions.apiTrigger. + */ +Datum +dump_fsm(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - return true; + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that " + "cannot accept a set"))); } - /* - * when secondary is put to maintenance and there's no standby left - * wait_maintenance -> maintenance - * wait_primary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_WAIT_MAINTENANCE) && - IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY)) + if (!(rsinfo->allowedModes & SFRM_Materialize)) { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance after " NODE_FORMAT - " converged to wait_primary.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not " + "allowed in this context"))); + } - /* secondary reached maintenance */ - AssignGoalState(activeNode, REPLICATION_STATE_MAINTENANCE, message); + TupleDesc tupdesc; - return true; + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + { + ereport(ERROR, + (errmsg("function returning record called in context " + "that cannot accept type record"))); } - /* - * when secondary is in wait_maintenance state and goal state of primary is - * not wait_primary anymore, e.g. another node joined and made it primary - * again or it got demoted. Then we don't need to wait anymore and we can - * transition directly to maintenance. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_WAIT_MAINTENANCE) && - primaryNode->goalState != REPLICATION_STATE_WAIT_PRIMARY) - { - char message[BUFSIZE]; + MemoryContext perQueryContext = rsinfo->econtext->ecxt_per_query_memory; + MemoryContext oldContext = MemoryContextSwitchTo(perQueryContext); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance after " NODE_FORMAT - " got assigned %s as goal state.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryNode->goalState)); + Tuplestorestate *tupstore = tuplestore_begin_heap(true, false, work_mem); - /* secondary reached maintenance */ - AssignGoalState(activeNode, REPLICATION_STATE_MAINTENANCE, message); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; - return true; - } + MemoryContextSwitchTo(oldContext); - /* - * when primary is put to maintenance - * prepare_promotion -> stop_replication - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION) && - IsCurrentState(primaryNode, REPLICATION_STATE_PREPARE_MAINTENANCE)) + for (int i = 0; MonitorFSM[i].pos != 0; i++) { - char message[BUFSIZE]; + const MonitorFSMTransition *rule = &MonitorFSM[i]; + Datum values[14]; + bool isNull[14] = { false }; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to stop_replication after " NODE_FORMAT - " converged to prepare_maintenance.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); + values[0] = Int32GetDatum(rule->pos); + values[1] = MonitorFSMTransitionSectionText(rule); - /* promote the secondary */ - AssignGoalState(activeNode, REPLICATION_STATE_STOP_REPLICATION, message); + if (rule->comment != NULL) + { + values[2] = CStringGetTextDatum(rule->comment); + } + else + { + isNull[2] = true; + } - return true; - } + values[3] = NodeStatePatternReportedStatesText(&rule->activeNode.statePattern, + &isNull[3]); + values[4] = NodeStatePatternReportedStatesText(&rule->primaryNode.statePattern, + &isNull[4]); + values[5] = NodeStatePatternReportedStatesText(&rule->candidateNode.statePattern, + &isNull[5]); - /* - * when a worker blocked writes: - * prepare_promotion -> wait_primary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION) && - primaryNode && - IsCitusFormation(ctx->formation) && activeNode->groupId > 0) - { - char message[BUFSIZE]; + values[6] = NodeStatusPatternConditionsText(&rule->activeNode, &isNull[6]); + values[7] = NodeStatusPatternConditionsText(&rule->primaryNode, &isNull[7]); + values[8] = NodeStatusPatternConditionsText(&rule->candidateNode, &isNull[8]); + values[9] = NodeActiveContextPatternConditionsText(&rule->conditions, &isNull[9]); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary and " NODE_FORMAT - " to demoted after the coordinator metadata was updated.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); + if (rule->activeNodeAssignedState.kind == GOAL_STATE_SET) + { + values[10] = ObjectIdGetDatum( + ReplicationStateGetEnum(rule->activeNodeAssignedState.state)); + } + else + { + isNull[10] = true; + } - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); + if (rule->otherNodeAssignedState.kind == GOAL_STATE_SET) + { + values[11] = ObjectIdGetDatum( + ReplicationStateGetEnum(rule->otherNodeAssignedState.state)); + } + else + { + isNull[11] = true; + } - /* done draining, node is presumed dead */ - AssignGoalState(primaryNode, REPLICATION_STATE_DEMOTED, message); + values[12] = BoolGetDatum(rule->extraAction != NULL); + values[13] = MonitorFSMTransitionSectionPathText(rule); - return true; + tuplestore_putvalues(tupstore, tupdesc, values, isNull); } - /* - * when a worker blocked writes and the primary has been removed: - * prepare_promotion -> wait_primary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION) && - primaryNode == NULL && - IsCitusFormation(ctx->formation) && activeNode->groupId > 0) - { - char message[BUFSIZE]; + return (Datum) 0; +} - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary after the coordinator metadata was updated.", - NODE_FORMAT_ARGS(activeNode)); - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); +/* + * AllReplicationStates is the full universe of "real" (non-sentinel) + * ReplicationState values -- every one a node can genuinely report or be + * assigned, in the order replication_state.h declares them. Deliberately + * excludes REPLICATION_STATE_UNKNOWN: a meta/sentinel value no MonitorFSM[] + * row ever reports or assigns, and no KeeperFSM[] edge ever targets either. + * Used only to resolve NodeStatePatternResolveFromStates' wildcard/negation + * kinds (ANY, NOT_STABLE, ASSIGNED, NOT_ASSIGNED) into concrete states. + */ +static const ReplicationState AllReplicationStates[] = { + REPLICATION_STATE_INITIAL, + REPLICATION_STATE_SINGLE, + REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_PRIMARY, + REPLICATION_STATE_DRAINING, + REPLICATION_STATE_DEMOTE_TIMEOUT, + REPLICATION_STATE_DEMOTED, + REPLICATION_STATE_CATCHINGUP, + REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_PREPARE_PROMOTION, + REPLICATION_STATE_STOP_REPLICATION, + REPLICATION_STATE_WAIT_STANDBY, + REPLICATION_STATE_MAINTENANCE, + REPLICATION_STATE_JOIN_PRIMARY, + REPLICATION_STATE_APPLY_SETTINGS, + REPLICATION_STATE_PREPARE_MAINTENANCE, + REPLICATION_STATE_WAIT_MAINTENANCE, + REPLICATION_STATE_REPORT_LSN, + REPLICATION_STATE_FAST_FORWARD, + REPLICATION_STATE_JOIN_SECONDARY, + REPLICATION_STATE_DROPPED +}; + +#define ALL_REPLICATION_STATES_COUNT \ + ((int) (sizeof(AllReplicationStates) / sizeof(AllReplicationStates[0]))) - return true; - } - /* - * when node is seeing no more writes: - * prepare_promotion -> stop_replication - * - * refrain from prepare_maintenance -> demote_timeout on the primary, which - * might happen here when secondary has reached prepare_promotion before - * primary has reached prepare_maintenance. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION) && - primaryNode && - !IsInMaintenance(primaryNode)) - { - char message[BUFSIZE]; +/* + * NodeStatePatternResolveFromStates resolves a NodeStatePattern's own "from" + * (reported-state) constraint into a concrete, palloc'd array of + * ReplicationState values -- the genuine edge-source set dump_fsm_edges() + * needs, as opposed to NodeStatePatternReportedStatesText's rendering (which + * only handles STABLE/REPORTED/TRANSITIONING and returns NULL for + * everything else, since that one exists for human display, not edge + * enumeration). + * + * ASSIGNED/NOT_ASSIGNED only ever constrain the node's *goal*, never what it + * currently reports (see each's own comment where declared above), so both + * resolve to the full state universe here, same as ANY. NOT_STABLE resolves + * to the complement of its own reportedStates within that universe. *outCount + * is set to the returned array's length; the caller does not need to free it + * (called only from dump_fsm_edges(), itself already running inside a + * per-query memory context the executor resets on its own). + */ +static ReplicationState * +NodeStatePatternResolveFromStates(const NodeStatePattern *pattern, int *outCount) +{ + ReplicationState *out; - /* - * wait_primary has no reachable demote_timeout edge either (issue - * #1168); leave its goal alone here and let the completion rule - * below apply the drainTimeoutMs safety margin via report - * staleness instead. - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY)) + switch (pattern->kind) + { + case NODE_STATE_STABLE: + case NODE_STATE_REPORTED: + case NODE_STATE_TRANSITIONING: { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to stop_replication after it converged to " - "prepare_promotion.", - NODE_FORMAT_ARGS(activeNode)); + out = (ReplicationState *) + palloc(pattern->reportedStates.count * sizeof(ReplicationState)); + + for (int i = 0; i < pattern->reportedStates.count; i++) + { + out[i] = pattern->reportedStates.states[i]; + } + + *outCount = pattern->reportedStates.count; + return out; } - else + + case NODE_STATE_NOT_STABLE: { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to demote_timeout and " NODE_FORMAT - " to stop_replication after " NODE_FORMAT - " converged to prepare_promotion.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(activeNode)); + out = (ReplicationState *) + palloc(ALL_REPLICATION_STATES_COUNT * sizeof(ReplicationState)); + *outCount = 0; - /* wait for possibly-alive primary to kill itself */ - AssignGoalState(primaryNode, REPLICATION_STATE_DEMOTE_TIMEOUT, message); + for (int i = 0; i < ALL_REPLICATION_STATES_COUNT; i++) + { + if (!MatchStateSet(AllReplicationStates[i], pattern->reportedStates)) + { + out[(*outCount)++] = AllReplicationStates[i]; + } + } + + return out; } - /* perform promotion to stop replication */ - AssignGoalState(activeNode, REPLICATION_STATE_STOP_REPLICATION, message); + case NODE_STATE_ANY: + case NODE_STATE_ASSIGNED: + case NODE_STATE_NOT_ASSIGNED: + default: + { + out = (ReplicationState *) + palloc(ALL_REPLICATION_STATES_COUNT * sizeof(ReplicationState)); - return true; + for (int i = 0; i < ALL_REPLICATION_STATES_COUNT; i++) + { + out[i] = AllReplicationStates[i]; + } + + *outCount = ALL_REPLICATION_STATES_COUNT; + return out; + } } +} - /* - * when primary node has been removed and we are promoting one standby - * prepare_promotion -> stop_replication - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION) && - primaryNode == NULL) + +/* + * NodeStatePatternIncludesState is a single-state membership query built on + * top of NodeStatePatternResolveFromStates -- used by the shadowing check + * below, which needs to ask "does this OTHER row's own pattern also match + * this one concrete state" without caring about the rest of that row's + * resolved set. + */ +static bool +NodeStatePatternIncludesState(const NodeStatePattern *pattern, ReplicationState state) +{ + int count; + ReplicationState *states = NodeStatePatternResolveFromStates(pattern, &count); + + for (int i = 0; i < count; i++) { - char message[BUFSIZE] = { 0 }; + if (states[i] == state) + { + return true; + } + } - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary after " NODE_FORMAT - " converged to prepare_promotion.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(activeNode)); + return false; +} - /* perform promotion to stop replication */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); +/* + * StateCanSatisfyIsInPrimaryState answers, for a hypothetical reportedState + * of state, whether SOME goalState exists making IsInPrimaryState() + * (node_metadata.c) evaluate to required -- i.e. whether state is a genuinely + * possible edge SOURCE for a row whose own .isInPrimaryState field demands + * required, as opposed to merely a state dump_fsm_edges()'s own + * NodeStatePatternResolveFromStates() would enumerate without knowing + * anything about IsInPrimaryState() at all (that function only ever reads + * .statePattern; every other NodeStatusPattern field, isInPrimaryState + * included, is invisible to it by construction, see its own comment). + * + * IsInPrimaryState(node) is exactly: + * (goal == reported && CanTakeWritesInState(goal)) + * || ((goal in {APPLY_SETTINGS, PRIMARY}) && (reported in {PRIMARY, APPLY_SETTINGS})) + * + * For required == true: choosing goal == state makes the first disjunct + * CanTakeWritesInState(state) -- always achievable when true. The second + * disjunct additionally admits state == PRIMARY or APPLY_SETTINGS via an + * appropriate goal choice even when CanTakeWritesInState(state) doesn't + * already cover it (it does, today -- CanTakeWritesInState already includes + * both -- but this is spelled out explicitly since the two conditions are + * independently maintained code and could diverge later). + * + * For required == false: some goal can always be chosen making both + * disjuncts fail (goal different from state, and not the specific + * APPLY_SETTINGS/PRIMARY pairing) -- IsInPrimaryState is never forced true + * for every possible goal by reportedState alone, so "false" is always + * satisfiable regardless of state. + * + * Confirmed real-world impact: pos 303's own .primaryNode.statePattern is + * NODE_STATE_ANY (no restriction at all) alongside .isInPrimaryState = + * BOOL_TRUE -- so dump_fsm_edges() would otherwise enumerate all 21 states + * as candidate primaryNode "current_state" sources for it, most of which + * (catchingup, secondary, dropped, maintenance, ...) IsInPrimaryState() + * could never actually accept regardless of goalState. This function + * narrows that down to the 5 states IsInPrimaryState can ever admit: + * single, primary, wait_primary, join_primary, apply_settings (from + * CanTakeWritesInState's own set). + * + * singleExcluded narrows that 5-state set by one more, conditionally: a rule + * whose own .conditions already prove the group has more than one node + * (groupHasExactlyOneNode = BOOL_FALSE, or the stronger + * groupHasMoreThanTwoNodes = BOOL_TRUE which implies it -- see + * MonitorFSMTransitionExcludesSingleNode) can never have this same node + * simultaneously reporting SINGLE, since SINGLE means "alone in my own + * group" and the row's own precondition already requires a second, + * distinctly-matched node (activeNode/primaryNode's counterpart role) to + * exist in that same group. This is a genuine, provable exclusion (unlike + * the sibling fields discussed below) precisely because it's derived from + * .conditions rather than guessed: pos 325 is the confirmed real-world + * case (its own .conditions carries groupHasExactlyOneNode = BOOL_FALSE for + * exactly this reason). + * + * Deliberately narrow in scope beyond that: only .isInPrimaryState (plus the + * one group-cardinality-derived refinement above) is modeled this way. + * Several sibling fields (isInMaintenance, canTakeWrites, drainTimeExpired, + * unreachableFromDemoteTimeout) are ALSO state-dependent in the same sense + * and could in principle be filtered the same way, but each needs its own + * careful satisfiability proof first -- isInMaintenance in particular looks + * deceptively similar to isInPrimaryState but is NOT safe to treat the same + * way without one: EdgeIsShadowedByEarlierRule's own comment documents a + * concrete case (pos 369) where a reportedState commonly assumed + * incompatible with a goal-dependent condition turned out to be reachable + * anyway, once a real, separate write path (stop_maintenance()'s + * api_triggered dispatch) was accounted for. Extending this file naively to + * every state-dependent field without doing that same due diligence for + * each risks reintroducing exactly that class of bug. + */ +static bool +StateCanSatisfyIsInPrimaryState(ReplicationState state, bool required, + bool singleExcluded) +{ + if (!required) + { return true; } - /* - * when primary node is going to maintenance - * stop_replication -> wait_primary - * prepare_maintenance -> maintenance - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_STOP_REPLICATION) && - IsCurrentState(primaryNode, REPLICATION_STATE_PREPARE_MAINTENANCE)) + if (singleExcluded && state == REPLICATION_STATE_SINGLE) { - char message[BUFSIZE]; + return false; + } + + return CanTakeWritesInState(state) || + state == REPLICATION_STATE_PRIMARY || + state == REPLICATION_STATE_APPLY_SETTINGS; +} - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary and " NODE_FORMAT - " to maintenance.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); +/* + * MonitorFSMTransitionExcludesSingleNode returns true when a rule's own + * .conditions already guarantee the group has more than one node -- either + * directly (groupHasExactlyOneNode = BOOL_FALSE) or via the stronger + * groupHasMoreThanTwoNodes = BOOL_TRUE, which implies it (more than two + * nodes can't be exactly one). Feeds StateCanSatisfyIsInPrimaryState's own + * singleExcluded parameter; see its comment for why this is a sound, + * provable narrowing rather than a guess. + */ +static bool +MonitorFSMTransitionExcludesSingleNode(const NodeActiveContextPattern *cond) +{ + return cond->groupHasExactlyOneNode == BOOL_FALSE || + cond->groupHasMoreThanTwoNodes == BOOL_TRUE; +} - /* old primary node is now ready for maintenance operations */ - AssignGoalState(primaryNode, REPLICATION_STATE_MAINTENANCE, message); +/* + * NodeStatusPatternSurvivesIsInPrimaryState filters a candidate edge-source + * state against pattern's own .isInPrimaryState field (BOOL_ANY -- the vast + * majority of rows -- always survives; see StateCanSatisfyIsInPrimaryState's + * own comment for BOOL_TRUE/BOOL_FALSE, and for singleExcluded). + */ +static bool +NodeStatusPatternSurvivesIsInPrimaryState(const NodeStatusPattern *pattern, + ReplicationState state, + bool singleExcluded) +{ + if (pattern->isInPrimaryState == BOOL_ANY) + { return true; } - /* - * when drain time expires or primary reports it's drained: - * draining -> demoted - * - * NodeIsWaitPrimaryPresumedDead covers the wait_primary equivalent - * (issue #1168): the same drainTimeoutMs safety margin, applied via - * report staleness instead of a demote_timeout goal-state timestamp - * since that state is never reachable from wait_primary. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_STOP_REPLICATION) && - (IsCurrentState(primaryNode, REPLICATION_STATE_DEMOTE_TIMEOUT) || - NodeIsDrainTimeExpired(primaryNode, ctx) || - NodeIsWaitPrimaryPresumedDead(primaryNode, activeNode, ctx))) - { - char message[BUFSIZE]; + return StateCanSatisfyIsInPrimaryState(state, pattern->isInPrimaryState == BOOL_TRUE, + singleExcluded); +} - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary and " NODE_FORMAT - " to demoted after the primary was presumed dead.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); +/* + * PrimaryNodeReportedStateCanBeResolved excludes 3 states from the + * primaryNode/otherNode candidate-state loop only (never activeNode's own + * loop, where all 3 are perfectly ordinary, real current_states -- see pos + * 201/315-319/359-361's own rows). Every row dump_fsm_edges() reaches + * through that second loop (i.e. survives both the api_triggered skip at + * the top of the function and the otherNodesFn skip guarding the loop + * itself) is a reporting_node-section row whose primaryNode ultimately + * comes from the single call to GetPrimaryOrDemotedNodeInGroupFromList() at + * the top of ProceedGroupStateFromContext() -- threaded through unchanged + * into every nested dispatch (BuildFromContextNodeActiveContext's own + * primaryNode/otherNode, and, via ActionRunMultiStandbyFailoverCascade/ + * ActionRunPlainMSFailoverCascade passing nac->primaryNode.node straight + * through, the MS-failover cluster's own use of the same role). + * + * REPLICATION_STATE_DROPPED: that resolver can never return a node + * reporting it, on two independent, purely structural grounds -- + * + * - Its own two-phase logic excludes it outright: phase 1 requires + * CanTakeWritesInState(goalState) (DROPPED is not one of the 5 writable + * states), and phase 2's fallback target set (StateBelongsToPrimary() + * plus an explicit REPLICATION_STATE_DEMOTED check) doesn't include + * DROPPED either. + * + * - It's structurally unreachable in the first place: a node's own + * reportedState only ever becomes DROPPED once its own goalState has + * already been set to DROPPED (the API-triggered remove_node row is the + * only place that ever assigns that goal), and pos 201 (early_checks) + * removes the row from the catalog entirely, atomically, in that exact + * same node_active() call the moment its own reportedState converges to + * DROPPED -- so a DROPPED-reporting node never persists long enough for + * a later, different node's own node_active() call to see it sitting in + * ctx->groupNodeList at all. + * + * REPLICATION_STATE_WAIT_STANDBY and REPLICATION_STATE_JOIN_SECONDARY: a + * different, table-content argument, but resting on the same design intent + * pos 209's own exclusion of both already documents (a node reporting + * either has either not yet started streaming, or already stopped Postgres + * as the OLD primary mid-handoff -- promoting either straight to a writable + * role is exactly the split-brain/data-loss risk pos 209's own comment + * describes). Every row + * matching activeNode against WAIT_STANDBY (pos 315/317/319) or + * JOIN_SECONDARY (pos 359/361) assigns only CATCHINGUP/SECONDARY, never one + * of the 5 writable states -- so no row anywhere ever gives + * GetPrimaryOrDemotedNodeInGroupFromList()'s phase 1 a way to select a node + * reporting either, and neither is in phase 2's own target set. Unlike + * DROPPED's exclusion, this rests on the current table's own contents, not + * a structural invariant -- if a future row is ever added assigning a + * writable goal from either state (which would itself need to defend + * against the same split-brain risk pos 209 already flags), this exclusion + * needs revisiting alongside it. + * + * Unlike singleExcluded, none of this depends on any row's own .conditions + * -- it applies unconditionally to every row reaching this loop, not just + * ones that happen to declare it. + */ +static bool +PrimaryNodeReportedStateCanBeResolved(ReplicationState state) +{ + return state != REPLICATION_STATE_DROPPED && + state != REPLICATION_STATE_WAIT_STANDBY && + state != REPLICATION_STATE_JOIN_SECONDARY; +} - /* done draining, node is presumed dead */ - AssignGoalState(primaryNode, REPLICATION_STATE_DEMOTED, message); +/* + * NodeStatusPatternSurvivesReportedCanTakeWrites filters a candidate + * edge-source state against pattern's own .reportedCanTakeWrites field + * (BOOL_ANY -- the vast majority of rows -- always survives). Unlike + * isInPrimaryState, this needs no separate "does some goal exist making this + * true" satisfiability proof: reportedCanTakeWrites is CanTakeWritesInState + * applied to reportedState alone (see NodeMatchesPattern's own comment), so + * for a hypothetical reportedState of state, whether the pattern's demand is + * satisfiable is just CanTakeWritesInState(state) itself -- no goalState + * involved at all. + */ +static bool +NodeStatusPatternSurvivesReportedCanTakeWrites(const NodeStatusPattern *pattern, + ReplicationState state) +{ + if (pattern->reportedCanTakeWrites == BOOL_ANY) + { return true; } - /* - * when a worker blocked writes: - * stop_replication -> wait_primary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_STOP_REPLICATION) && - primaryNode && - IsCitusFormation(ctx->formation) && activeNode->groupId > 0) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary and " NODE_FORMAT - " to demoted after the coordinator metadata was updated.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); + bool required = (pattern->reportedCanTakeWrites == BOOL_TRUE); - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); + return CanTakeWritesInState(state) == required; +} - /* done draining, node is presumed dead */ - AssignGoalState(primaryNode, REPLICATION_STATE_DEMOTED, message); +/* + * NodeStatusPatternSurvivesReportedIsWaitStandby filters a candidate + * edge-source state against pattern's own .reportedIsWaitStandby field, the + * same shape as NodeStatusPatternSurvivesReportedCanTakeWrites just above -- + * a plain equality on reportedState alone, no goalState-dependent + * satisfiability proof needed. + */ +static bool +NodeStatusPatternSurvivesReportedIsWaitStandby(const NodeStatusPattern *pattern, + ReplicationState state) +{ + if (pattern->reportedIsWaitStandby == BOOL_ANY) + { return true; } - /* - * when a worker blocked writes, and the primary has been dropped: - * stop_replication -> wait_primary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_STOP_REPLICATION) && - primaryNode == NULL && - IsCitusFormation(ctx->formation) && activeNode->groupId > 0) - { - char message[BUFSIZE]; + bool required = (pattern->reportedIsWaitStandby == BOOL_TRUE); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary after the coordinator metadata was updated.", - NODE_FORMAT_ARGS(activeNode)); + return (state == REPLICATION_STATE_WAIT_STANDBY) == required; +} - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); +/* + * NodeStatusPatternSurvivesReportedIsJoinSecondary filters a candidate + * edge-source state against pattern's own .reportedIsJoinSecondary field, + * the same shape as NodeStatusPatternSurvivesReportedIsWaitStandby just + * above -- a plain equality on reportedState alone, no goalState-dependent + * satisfiability proof needed. + */ +static bool +NodeStatusPatternSurvivesReportedIsJoinSecondary(const NodeStatusPattern *pattern, + ReplicationState state) +{ + if (pattern->reportedIsJoinSecondary == BOOL_ANY) + { return true; } - /* - * when a new primary is ready: - * demoted -> catchingup - * - * We accept to move from demoted to catching up as soon as the primary - * node is has reported either wait_primary or join_primary, and even when - * it's already transitioning to primary, thanks to another standby - * concurrently making progress. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_DEMOTED) && - NodeIsHealthy(primaryNode, ctx) && - ((primaryNode->reportedState == REPLICATION_STATE_WAIT_PRIMARY || - primaryNode->reportedState == REPLICATION_STATE_JOIN_PRIMARY) && - primaryNode->goalState == REPLICATION_STATE_PRIMARY)) - { - char message[BUFSIZE]; + bool required = (pattern->reportedIsJoinSecondary == BOOL_TRUE); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after it converged to demotion and " NODE_FORMAT - " converged to primary.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); + return (state == REPLICATION_STATE_JOIN_SECONDARY) == required; +} - /* it's safe to rejoin as a secondary */ - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); +/* + * NodeStatusPatternSurvivesReportedIsPrepareMaintenance filters a candidate + * edge-source state against pattern's own .reportedIsPrepareMaintenance + * field, the same shape as NodeStatusPatternSurvivesReportedIsJoinSecondary + * just above -- a plain equality on reportedState alone, no goalState- + * dependent satisfiability proof needed. + */ +static bool +NodeStatusPatternSurvivesReportedIsPrepareMaintenance(const NodeStatusPattern *pattern, + ReplicationState state) +{ + if (pattern->reportedIsPrepareMaintenance == BOOL_ANY) + { return true; } - /* - * when a new primary is ready: - * demoted -> catchingup - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_DEMOTED) && - NodeIsHealthy(primaryNode, ctx) && - (IsCurrentState(primaryNode, REPLICATION_STATE_JOIN_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY))) - { - char message[BUFSIZE]; + bool required = (pattern->reportedIsPrepareMaintenance == BOOL_TRUE); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after it converged to demotion and " NODE_FORMAT - " converged to %s.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryNode->reportedState)); + return (state == REPLICATION_STATE_PREPARE_MAINTENANCE) == required; +} - /* it's safe to rejoin as a secondary */ - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); - return true; +/* + * NodeStatePatternKindIsReportedStateOnly: true for the pattern kinds whose + * match genuinely depends only on reportedState (ignoring, for STABLE, its + * own additional "reported == goal" requirement -- see this function's own + * comment for why that specific simplification is deliberate). ASSIGNED, + * NOT_ASSIGNED, and TRANSITIONING all have a real, separate dependency on + * goalState (NodeStateMatchesPattern's own switch: ASSIGNED/NOT_ASSIGNED + * check goalState exclusively, TRANSITIONING checks both), which + * NodeStatePatternResolveFromStates papers over for edge-SOURCE purposes by + * resolving them to either the literal reportedStates list (TRANSITIONING, + * silently dropping its own assignedStates half) or the full state universe + * (ASSIGNED/NOT_ASSIGNED, since goalState alone decides those, independently + * of reportedState) -- both correct over-approximations for "what could this + * row's reported-state source legitimately be", but wrong for this function's + * different question, "does this row match unconditionally whenever reported + * state equals state". A row like pos 203 ("goalState == DROPPED, + * reportedState irrelevant", ASSIGNED kind) would otherwise look like it + * resolves to (and therefore unconditionally matches) every one of the 21 + * states, when it actually still requires a completely separate, real fact + * (goalState == DROPPED) that has nothing to do with reportedState at all. + * Treating ASSIGNED/NOT_ASSIGNED rows as eligible for unconditional-match + * shadowing here would wrongly suppress several of pos 209's genuinely + * reachable fanned-out states (wait_standby, prepare_maintenance, + * wait_maintenance, fast_forward, join_secondary), none of which have + * anything to do with the node's goal being DROPPED. + * + * STABLE is kept eligible despite its own "reported == goal" wrinkle: for + * every row actually written using it (a bare FSM_STATE(x), no other + * condition), the codebase's own edge-source resolution + * (NodeStatePatternResolveFromStates) already treats STABLE identically to + * REPORTED (see this file's own comment on pos 205/pos 209), which is what + * makes real shadowing detection work at all -- requiring the stricter, + * fully rigorous "and goal really does equal reported in every possible + * calling context" would need modeling whether some OTHER row's + * otherNodeAssignedState could have changed this exact node's own goalState + * moments earlier, out of scope for a static, per-table check like this one. + */ +static bool +NodeStatePatternKindIsReportedStateOnly(NodeStatePatternKind kind) +{ + switch (kind) + { + case NODE_STATE_ANY: + case NODE_STATE_STABLE: + case NODE_STATE_REPORTED: + case NODE_STATE_NOT_STABLE: + { + return true; + } + + case NODE_STATE_ASSIGNED: + case NODE_STATE_NOT_ASSIGNED: + case NODE_STATE_TRANSITIONING: + default: + { + return false; + } } +} - /* - * when a new primary is ready: - * join_secondary -> secondary - * - * As there's no action to implement on the new selected primary for that - * step, we can make progress as soon as we want to. - * - * The primary could be in one of those states: - * - wait_primary/wait_primary - * - wait_primary/primary - * - * This transition also happens when a former primary node has been - * demoted, and a multiple standbys has taken effect, we have a new primary - * being promoted, and several standby nodes following the new primary. - * - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_JOIN_SECONDARY) && - primaryNode->reportedState == REPLICATION_STATE_WAIT_PRIMARY && - (primaryNode->goalState == REPLICATION_STATE_WAIT_PRIMARY || - primaryNode->goalState == REPLICATION_STATE_PRIMARY)) - { - char message[BUFSIZE] = { 0 }; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to secondary after " NODE_FORMAT - " converged to wait_primary.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); +/* + * NodeStatusPatternOtherFieldsAreAny/NodeStatusPatternIsFullyAny/ + * NodeActiveContextPatternIsAny mirror RuleMatches()'s own field list, + * field-for-field, checking each is at its BOOL_ANY/INT_PATTERN_ANY/ + * API_TRIGGER_NODE_ACTIVE "don't care" default -- if MonitorFSMTransition's + * pattern structs ever gain a new field, RuleMatches() needs to grow a new + * conjunct for it, and these three functions need the matching ANY-check + * added right alongside, or the shadowing detection below silently starts + * ignoring that new field (treating a row as unconditional when it no + * longer is). + */ +static bool +NodeStatusPatternOtherFieldsAreAny(const NodeStatusPattern *pattern) +{ + return pattern->exists == BOOL_ANY && + pattern->isHealthy == BOOL_ANY && + pattern->isUnhealthy == BOOL_ANY && + pattern->candidateEligible == BOOL_ANY && + pattern->isInPrimaryState == BOOL_ANY && + pattern->isInMaintenance == BOOL_ANY && + pattern->isDemotedPrimary == BOOL_ANY && + pattern->canTakeWrites == BOOL_ANY && + pattern->reportedCanTakeWrites == BOOL_ANY && + pattern->reportedIsWaitStandby == BOOL_ANY && + pattern->reportedIsJoinSecondary == BOOL_ANY && + pattern->reportedIsPrepareMaintenance == BOOL_ANY && + pattern->isReadyToStreamWAL == BOOL_ANY && + pattern->drainTimeExpired == BOOL_ANY && + pattern->isCitusWorkerGroup == BOOL_ANY && + pattern->replicationQuorum == BOOL_ANY && + pattern->isComparableToReferenceTli == BOOL_ANY && + pattern->unreachableFromDemoteTimeout == BOOL_ANY; +} - /* it's safe to rejoin as a secondary */ - AssignGoalState(activeNode, REPLICATION_STATE_SECONDARY, message); - /* compute next step for the primary depending on node settings */ - return ProceedGroupStateForPrimaryNode(ctx, primaryNode); - } +static bool +NodeStatusPatternIsFullyAny(const NodeStatusPattern *pattern) +{ + return pattern->statePattern.kind == NODE_STATE_ANY && + NodeStatusPatternOtherFieldsAreAny(pattern); +} - /* - * when a new secondary re-appears after a failover or at a "random" time - * in the FSM cycle, and the wait_primary or join_primary node has already - * made progress to primary. - * - * join_secondary -> secondary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_JOIN_SECONDARY) && - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY)) - { - char message[BUFSIZE]; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to secondary after " NODE_FORMAT - " converged to primary.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); +static bool +NodeActiveContextPatternIsAny(const NodeActiveContextPattern *cond) +{ + return cond->apiTrigger.kind == API_TRIGGER_NODE_ACTIVE && + + cond->groupHasExactlyOneNode == BOOL_ANY && + cond->groupHasExactlyTwoNodes == BOOL_ANY && + cond->groupHasMoreThanTwoNodes == BOOL_ANY && + cond->anyOtherNodeWaitingStandby == BOOL_ANY && + cond->numberSyncStandbysIsZero == BOOL_ANY && + cond->replicationQuorumCountIsZero == BOOL_ANY && + cond->secondaryNodesCountIsZero == BOOL_ANY && + cond->secondaryQuorumNodesCountIsZero == BOOL_ANY && + cond->atLeastOneHealthyCandidate == BOOL_ANY && + cond->walWithinPromoteThreshold == BOOL_ANY && + cond->walWithinSyncThreshold == BOOL_ANY && + cond->activeAndPrimaryTliMatch == BOOL_ANY && + cond->primaryIsWaitPrimaryPresumedDead == BOOL_ANY && + cond->failoverInProgress == BOOL_ANY && + cond->replicationStallExceeded == BOOL_ANY && + cond->lastHealthySyncStandbyGoingToMaintenance == BOOL_ANY && + cond->activeNodeAllWalSourcesUnhealthy == BOOL_ANY && + cond->candidatePromotionInProgress == BOOL_ANY && + cond->mostAdvancedCandidateWithinPromoteThreshold == BOOL_ANY && + cond->guardDataLossEnabled == BOOL_ANY && + cond->inMSFailoverCluster == BOOL_ANY && + cond->inMSFailoverCandidateGate == BOOL_ANY && + + cond->candidateCount.kind == INT_PATTERN_ANY && + cond->quorumCandidateCount.kind == INT_PATTERN_ANY && + cond->missingNodesCount.kind == INT_PATTERN_ANY && + cond->sufficientQuorumCandidates == BOOL_ANY; +} - /* it's safe to rejoin as a secondary */ - AssignGoalState(activeNode, REPLICATION_STATE_SECONDARY, message); - return true; - } +/* + * RuleUnconditionallyMatchesActiveNodeState/RuleUnconditionallyMatchesPrimary + * NodeState: true iff rule's own RuleMatches() would return true for EVERY + * possible NodeActiveContext whose activeNode (resp. primaryNode) reports + * state -- i.e. rule's activeNode.statePattern (resp. primaryNode. + * statePattern) accepts state, and nothing else about the row narrows it any + * further: every other NodeStatus role is entirely unconstrained, the role + * being tested has no OTHER constraint beyond its own state, and every + * .conditions field is at its own "don't care" default. A row like this is + * a pure, unconditional catch-all for that one reported state -- exactly pos + * 205's "converged to maintenance -> no-op" row (see dump_fsm_edges()'s own + * comment on the confirmed pos 209/maintenance case this was built to catch). + * + * Deliberately does NOT require rule->otherNodesFn == NULL: otherNodesFn only + * changes who a matched row's otherNodeAssignedState is assigned to, not + * whether the row matches in the first place, so it has no bearing on + * whether this row shadows another one. + */ +static bool +RuleUnconditionallyMatchesActiveNodeState(const MonitorFSMTransition *rule, + ReplicationState state) +{ + return NodeStatePatternKindIsReportedStateOnly(rule->activeNode.statePattern.kind) && + NodeStatePatternIncludesState(&rule->activeNode.statePattern, state) && + NodeStatusPatternOtherFieldsAreAny(&rule->activeNode) && + NodeStatusPatternIsFullyAny(&rule->primaryNode) && + NodeStatusPatternIsFullyAny(&rule->otherNode) && + NodeStatusPatternIsFullyAny(&rule->candidateNode) && + NodeActiveContextPatternIsAny(&rule->conditions); +} - return false; + +static bool +RuleUnconditionallyMatchesPrimaryNodeState(const MonitorFSMTransition *rule, + ReplicationState state) +{ + return NodeStatePatternKindIsReportedStateOnly(rule->primaryNode.statePattern.kind) && + NodeStatePatternIncludesState(&rule->primaryNode.statePattern, state) && + NodeStatusPatternOtherFieldsAreAny(&rule->primaryNode) && + NodeStatusPatternIsFullyAny(&rule->activeNode) && + NodeStatusPatternIsFullyAny(&rule->otherNode) && + NodeStatusPatternIsFullyAny(&rule->candidateNode) && + NodeActiveContextPatternIsAny(&rule->conditions); } /* - * Group State Machine when a primary node contacts the monitor. + * EdgeIsShadowedByEarlierRule scans MonitorFSM[0 .. beforeIndex) for a row + * that would unconditionally intercept state before dispatch ever reaches + * beforeIndex -- i.e. a real, always-invoked scan of this section (the + * default one, starting from array position 0, which every top-level + * section has independently of any narrower resume-point scan some specific + * caller might also use) would never actually reach beforeIndex for a node + * reporting state, making an edge reported for it purely an artifact of this + * function resolving each row independently (see dump_fsm_edges()'s own + * comment). + * + * Deliberately bounded to rows sharing beforeIndex's own top-level section, + * not the whole array: every section IS reachable via its own independent + * top-level scan (FindAndDispatchMonitorFSMRuleUnderPath's callers), so + * shadowing within one section is straightforward to prove -- the section's + * own default scan, starting from array position 0, is a real call that + * genuinely exists and always runs in that order. + * + * A cross-section version of this check was tried and REJECTED, not merely + * left as a TODO: ProceedGroupStateFromContext() does always try + * SectionEarlyChecks first, before SectionPrimaryNode/SectionReportingNode + * (see its own comment), which looks like it should let an unconditional + * early_checks row (pos 205's "converged to maintenance -> no-op") shadow a + * same-state edge in a later section too. It doesn't, in general: pos 205's + * own STABLE-kind pattern requires reportedState == goalState == maintenance, + * and that equality is NOT guaranteed just because reportedState == + * maintenance -- stop_maintenance() on a multi-node group dispatches through + * the *separate* MONITOR_FSM_SECTION_API_TRIGGERED path (ProceedGroupStateFor + * ApiTrigger, not ProceedGroupStateFromContext at all) and assigns a new goal + * directly, independently of the target node's own next heartbeat -- so a + * node can genuinely present reportedState == maintenance with goalState + * already advanced past it. A first attempt at this cross-section extension + * treated pos 205 as shadowing pos 369 ("MS-failover fan-out: rejoining from + * maintenance -> report_lsn", a TRANSITIONING-kind row requiring exactly + * reportedState == maintenance AND goalState == catchingup) this way, and + * would have wrongly deleted a real, reachable edge -- caught before + * committing by checking the field-level trace by hand, not by any test. + * Soundly generalizing this would require modeling every place a node's own + * goalState can be written independently of its own next reportedState + * update (every apiTrigger row, not just stop_maintenance), which is a much + * larger undertaking than this function's own scope; same-section-only + * stays the safe, committed behavior. This under-approximates real + * shadowing (some cross-section cases go undetected), never over-approximates + * (no risk of wrongly hiding a genuinely reachable edge). */ static bool -ProceedGroupStateForPrimaryNode(GroupStateContext *ctx, - AutoFailoverNode *primaryNode) +EdgeIsShadowedByEarlierRule(int beforeIndex, ReplicationState state, bool primaryNodeSide, + MonitorFSMSection topLevelSection) { - List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); - int otherNodesCount = list_length(otherNodesGroupList); - - /* - * when a first "other" node wants to become standby: - * single -> wait_primary - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_SINGLE)) + for (int j = 0; j < beforeIndex; j++) { - ListCell *nodeCell = NULL; + const MonitorFSMTransition *earlier = &MonitorFSM[j]; - foreach(nodeCell, otherNodesGroupList) + if (earlier->sectionPath[0] != topLevelSection) { - AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); - - if (IsCurrentState(otherNode, REPLICATION_STATE_WAIT_STANDBY)) - { - char message[BUFSIZE]; + continue; + } - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary after " NODE_FORMAT - " joined.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(otherNode)); - - /* prepare replication slot and pg_hba.conf */ - AssignGoalState(primaryNode, - REPLICATION_STATE_WAIT_PRIMARY, - message); - - return true; - } + if (primaryNodeSide + ? RuleUnconditionallyMatchesPrimaryNodeState(earlier, state) + : RuleUnconditionallyMatchesActiveNodeState(earlier, state)) + { + return true; } } - /* - * when secondary unhealthy: - * secondary ➜ catchingup - * primary ➜ wait_primary - * - * We only swith the primary to wait_primary when there's no healthy - * secondary anymore. In other cases, there's by definition at least one - * candidate for failover. - * - * Also we might lose a standby node while already in WAIT_PRIMARY, when - * all the left standby nodes are assigned a candidatePriority of zero. - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS)) + return false; +} + + +PG_FUNCTION_INFO_V1(dump_fsm_edges); + +/* + * dump_fsm_edges exposes MonitorFSM[] as a flat set of concrete + * (pos, current_state, assigned_state) edges -- one row per actual + * (reportedState, assignedState) pair a row can produce, fully resolved (never + * NULL, unlike dump_fsm()'s own pattern-summary columns, which stay unresolved + * for human display). This is the keeper-cross-check surface + * pgautofailover.check_fsm_reachability(jsonb) needs: it anti-joins this + * against a keeper's own + * KeeperFSM[] edges (see KeeperFSMToJSON(), src/bin/pg_autoctl/fsm.c) to find + * any monitor transition with no matching keeper edge -- exactly the bug class + * issue #774 was. + * + * Two potential edges per row, resolved independently: activeNode's own + * (statePattern -> activeNodeAssignedState) when the row actually assigns one, + * and primaryNode's own (statePattern -> otherNodeAssignedState) when it does. + * candidateNode never has an assignment slot of its own (see + * MonitorFSMTransition's own comment), so it never contributes an edge. pgKind + * is deliberately not modeled here: every Citus-specific KeeperFSM[] edge + * already has a NODE_KIND_ANY counterpart with the same (current, assigned) shape + * (fsm_mermaid.c's own comment establishes this as an invariant this codebase + * already relies on elsewhere), so a flat, pgKind-blind edge set on both sides + * is already correct. + * + * Two categories of edges are deliberately never emitted, both confirmed by + * running pg_autoctl inspect fsm check for real and tracing every one of the + * mismatches it reported back to its actual root cause rather than assuming: + * + * - Reflexive (current == assigned) edges. keeper_fsm_reach_assigned_state() + * (src/bin/pg_autoctl/fsm.c) returns true the moment current_role == + * assigned_role, before ever consulting KeeperFSM[] -- confirmed by reading + * that function directly. A self-loop can therefore never have (or need) a + * matching keeper edge; including one here would always report a false gap. + * This alone explained every mismatch on pos 363, 403, 405, 409 during that + * live run. + * + * - MONITOR_FSM_SECTION_API_TRIGGERED rows entirely. Every one of these is + * reached from an operator-facing SQL wrapper (remove_node, perform_failover, + * start_maintenance, ...) that resolves activeNode to a specific, + * already-validated role (almost always the primary) via hand-written C + * *before* dispatch ever runs -- ProceedGroupStateForApiTrigger's own comment, + * and several of these rows' own comments ("there's nothing else for activeNode + * to be here but the primary itself", "activeNode IS the primary here"), + * document this explicitly. The row's own NodeStatePattern for these (typically + * ANY, or a BoolPattern condition like isInPrimaryState with no accompanying + * state-set restriction) is consequently far broader than what's actually + * reachable: it was never meant to double as a full reachability precondition, + * because that precondition already lives in hand-written C outside this table, + * per this design's own "pre/post side effects stay hand-written, not modeled + * as a row" principle. Expanding it here (as this function otherwise correctly + * does for the ordinary heartbeat-driven sections) manufactures the exact same + * kind of false gap for every one of these ten rows, all confirmed by that same + * live run. + * + * A third category is filtered the same way, for a different reason: a + * row's own edges can't be resolved entirely independently, without + * considering any OTHER row, because that would report an edge for a + * current_state that an EARLIER row (lower array index, matching + * unconditionally -- e.g. a bare no-op like pos 205's "converged to + * maintenance -> no-op, frozen until stop_maintenance()") actually + * intercepts first in real first-match-wins dispatch, making that edge + * practically unreachable. A live pgaftest run + * (tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf's own + * investigation) confirmed exactly this: pos 209's "maintenance" edge looks + * like a real gap in isolation, but pos 205 comes first in array order and + * intercepts every node_active() call from a node in MAINTENANCE_STATE + * regardless of group size, so pos 209 can never actually fire for that + * state at all. + * + * EdgeIsShadowedByEarlierRule (below) detects exactly this: for each + * candidate edge, it scans every earlier row sharing the same top-level + * section (every section is its own independent top-level scan, so a row + * outside it is never reachable in the same dispatch pass regardless of + * array order -- see FindAndDispatchMonitorFSMRuleUnderPath's callers) for + * one that would match unconditionally whenever the same NodeStatus role + * reports that same state, regardless of anything else in the dispatch + * context. This is a sound, deliberately conservative check: it only + * suppresses an edge when an earlier row is PROVABLY unconditional for that + * state (every other pattern field at its own "don't care" default -- see + * RuleUnconditionallyMatchesActiveNodeState/...PrimaryNodeState's own + * comment), never merely "plausibly likely to match first" -- a row with + * even one real extra condition is left alone, since whether it actually + * fires first still depends on runtime facts this function can't know + * statically. Every mismatch that live pgaftest run found beyond pos 209's + * (early_checks 211, reporting_node 303/325/333/339/347/349/351) survived + * this check and remains a real, reachable gap -- pos 211's specifically was + * independently confirmed live in that same investigation (a lone + * priority-zero primary really does get assigned report_lsn from PRIMARY_ + * STATE with no shadowing row in front of it, and the keeper really has no + * transition for it). + */ +Datum +dump_fsm_edges(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) { - /* - * We count our nodes in different ways, because of special cases we - * want to be able to address. We want to distinguish nodes that are in - * the replication quorum, nodes that are secondary, and nodes that are - * secondary but do not participate in the quorum. - * - * - replicationQuorumCount is the count of nodes with - * replicationQuorum true, whether or not those nodes are currently - * in the SECONDARY state. - * - * - secondaryNodesCount is the count of nodes that are currently in - * the SECONDARY state. - * - * - secondaryQuorumNodesCount is the count of nodes that are both - * setup to participate in the replication quorum and also currently - * in the SECONDARY state. - */ - int replicationQuorumCount = otherNodesCount; - int secondaryNodesCount = otherNodesCount; - int secondaryQuorumNodesCount = otherNodesCount; + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that " + "cannot accept a set"))); + } - ListCell *nodeCell = NULL; + if (!(rsinfo->allowedModes & SFRM_Materialize)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not " + "allowed in this context"))); + } - foreach(nodeCell, otherNodesGroupList) - { - AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); + TupleDesc tupdesc; - /* - * We force secondary nodes to catching-up even if the node is on - * its way to being a secondary... unless it is currently in the - * reportLSN or join_secondary state, because in those states - * Postgres is stopped, waiting for the new primary to be - * available. - */ - if (otherNode->goalState == REPLICATION_STATE_SECONDARY && - otherNode->reportedState != REPLICATION_STATE_REPORT_LSN && - otherNode->reportedState != REPLICATION_STATE_JOIN_SECONDARY && - NodeIsUnhealthy(otherNode, ctx)) - { - char message[BUFSIZE]; + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + { + ereport(ERROR, + (errmsg("function returning record called in context " + "that cannot accept type record"))); + } - --secondaryNodesCount; - --secondaryQuorumNodesCount; + MemoryContext perQueryContext = rsinfo->econtext->ecxt_per_query_memory; + MemoryContext oldContext = MemoryContextSwitchTo(perQueryContext); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after it became unhealthy.", - NODE_FORMAT_ARGS(otherNode)); + Tuplestorestate *tupstore = tuplestore_begin_heap(true, false, work_mem); - /* other node is behind, no longer eligible for promotion */ - AssignGoalState(otherNode, - REPLICATION_STATE_CATCHINGUP, message); - } - else if (!IsCurrentState(otherNode, REPLICATION_STATE_SECONDARY)) - { - --secondaryNodesCount; - --secondaryQuorumNodesCount; - } + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; - /* at this point we are left with nodes in SECONDARY state */ - else if (IsCurrentState(otherNode, REPLICATION_STATE_SECONDARY) && - !otherNode->replicationQuorum) - { - --secondaryQuorumNodesCount; - } + MemoryContextSwitchTo(oldContext); - /* now separately count nodes setup with replication quorum */ - if (!otherNode->replicationQuorum) - { - --replicationQuorumCount; - } - } + for (int i = 0; MonitorFSM[i].pos != 0; i++) + { + const MonitorFSMTransition *rule = &MonitorFSM[i]; - /* - * Special case first: when given a setup where all the nodes are async - * (replicationQuorumCount == 0) we allow the "primary" state in almost - * all cases, knowing that synchronous_standby_names is still going to - * be computed as ''. - * - * That said, if we don't have a single node in the SECONDARY state, we - * still want to switch to WAIT_PRIMARY to show that something - * unexpected is happening. - */ - if (replicationQuorumCount == 0) + if (rule->sectionPath[0] == MONITOR_FSM_SECTION_API_TRIGGERED) { - Assert(ctx->formation->number_sync_standbys == 0); + continue; + } + + bool singleExcluded = MonitorFSMTransitionExcludesSingleNode(&rule->conditions); - ReplicationState primaryGoalState = - secondaryNodesCount == 0 - ? REPLICATION_STATE_WAIT_PRIMARY - : REPLICATION_STATE_PRIMARY; + if (rule->activeNodeAssignedState.kind == GOAL_STATE_SET) + { + int count; + ReplicationState *states = + NodeStatePatternResolveFromStates(&rule->activeNode.statePattern, &count); - if (primaryNode->goalState != primaryGoalState) + for (int j = 0; j < count; j++) { - char message[BUFSIZE] = { 0 }; + Datum values[3]; + bool isNull[3] = { false }; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to %s because none of the secondary nodes" - " are healthy at the moment.", - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryGoalState)); + if (states[j] == rule->activeNodeAssignedState.state) + { + continue; + } + + if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->activeNode, + states[j], + singleExcluded)) + { + continue; + } - AssignGoalState(primaryNode, primaryGoalState, message); + if (!NodeStatusPatternSurvivesReportedCanTakeWrites(&rule->activeNode, + states[j])) + { + continue; + } - return true; - } + if (!NodeStatusPatternSurvivesReportedIsWaitStandby(&rule->activeNode, + states[j])) + { + continue; + } - /* when all nodes are async, we're done here */ - return true; + if (!NodeStatusPatternSurvivesReportedIsJoinSecondary(&rule->activeNode, + states[j])) + { + continue; + } + + if (!NodeStatusPatternSurvivesReportedIsPrepareMaintenance( + &rule->activeNode, states[j])) + { + continue; + } + + if (EdgeIsShadowedByEarlierRule(i, states[j], false, + rule->sectionPath[0])) + { + continue; + } + + values[0] = Int32GetDatum(rule->pos); + values[1] = ObjectIdGetDatum(ReplicationStateGetEnum(states[j])); + values[2] = ObjectIdGetDatum( + ReplicationStateGetEnum(rule->activeNodeAssignedState.state)); + + tuplestore_putvalues(tupstore, tupdesc, values, isNull); + } } /* - * Disable synchronous replication to maintain availability. - * - * Note that we implement here a trade-off between availability (of - * writes) against durability of the written data. In the case when - * there's a single standby in the group, pg_auto_failover choice is to - * maintain availability of the service, including writes. - * - * In the case when the user has setup a replication quorum of 1 or - * more, then pg_auto_failover does not get in the way. You get what - * you ask for, which is a strong guarantee on durability. - * - * To have number_sync_standbys == 1, you need to have at least 2 - * standby servers. To get to a point where writes are not possible - * anymore, there needs to be a point in time where 2 of the 2 standby - * nodes are unavailable. In that case, pg_auto_failover does not - * change the configured trade-offs. Writes are blocked until one of - * the two defective standby nodes is available again. + * otherNodesFn rows are skipped here: their "other node" target's + * own current-state precondition isn't a NodeStatePattern at all + * (it's whatever filtering the resolver function itself does, e.g. + * OtherNodeIsDueForCatchingUp's own health/state checks) -- reading + * .primaryNode.statePattern for such a row would resolve its + * NODE_STATE_ANY default to all 21 states, fabricating 21 bogus + * edges no keeper FSM could ever have. Same "not every row's edges + * are representable this way" precedent as the api_triggered + * section's own exclusion above. */ - if (!IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) && - secondaryQuorumNodesCount == 0 && - !IsFailoverInProgress(ctx->groupNodeList)) + if (rule->otherNodeAssignedState.kind == GOAL_STATE_SET && + rule->otherNodesFn == NULL) { - /* - * Do not second-guess an already-started failover: a candidate - * that just converged to prepare_promotion (or later stages) has - * left SECONDARY, dropping secondaryQuorumNodesCount to zero even - * though it *is* the failover candidate. Without this guard the - * primary can be reassigned wait_primary right as its own - * candidate is converging, landing it on wait_primary while a - * later rule still expects to find it in draining and assigns - * demote_timeout -- an assignment with no FSM edge from - * wait_primary (issue #774). - * - * Allow wait_primary when number_sync_standbys = 0, otherwise - * block writes on the primary. - */ - ReplicationState primaryGoalState = - ctx->formation->number_sync_standbys == 0 - ? REPLICATION_STATE_WAIT_PRIMARY - : REPLICATION_STATE_PRIMARY; + int count; + ReplicationState *states = + NodeStatePatternResolveFromStates(&rule->primaryNode.statePattern, + &count); - if (primaryNode->goalState != primaryGoalState) + for (int j = 0; j < count; j++) { - char message[BUFSIZE] = { 0 }; + Datum values[3]; + bool isNull[3] = { false }; + + if (states[j] == rule->otherNodeAssignedState.state) + { + continue; + } + + if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->primaryNode, + states[j], + singleExcluded)) + { + continue; + } + + if (!PrimaryNodeReportedStateCanBeResolved(states[j])) + { + continue; + } - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to %s because none of the standby nodes in the quorum" - " are healthy at the moment.", - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryGoalState)); + if (!NodeStatusPatternSurvivesReportedCanTakeWrites(&rule->primaryNode, + states[j])) + { + continue; + } - AssignGoalState(primaryNode, primaryGoalState, message); + if (!NodeStatusPatternSurvivesReportedIsWaitStandby(&rule->primaryNode, + states[j])) + { + continue; + } - return true; - } - } + if (!NodeStatusPatternSurvivesReportedIsJoinSecondary(&rule->primaryNode, + states[j])) + { + continue; + } - /* - * when a node is wait_primary and has at least one healthy candidate - * secondary - * wait_primary ➜ primary - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) && - secondaryQuorumNodesCount > 0) - { - char message[BUFSIZE] = { 0 }; + if (!NodeStatusPatternSurvivesReportedIsPrepareMaintenance( + &rule->primaryNode, states[j])) + { + continue; + } - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to primary now that we have %d healthy " - " secondary nodes in the quorum.", - NODE_FORMAT_ARGS(primaryNode), - secondaryQuorumNodesCount); + if (EdgeIsShadowedByEarlierRule(i, states[j], true, rule->sectionPath[0])) + { + continue; + } - AssignGoalState(primaryNode, REPLICATION_STATE_PRIMARY, message); + values[0] = Int32GetDatum(rule->pos); + values[1] = ObjectIdGetDatum(ReplicationStateGetEnum(states[j])); + values[2] = ObjectIdGetDatum( + ReplicationStateGetEnum(rule->otherNodeAssignedState.state)); - return true; + tuplestore_putvalues(tupstore, tupdesc, values, isNull); + } } + } - /* - * when a node has changed its replication settings: - * apply_settings ➜ wait_primary - * apply_settings ➜ primary - * - * Even when we don't currently have healthy standby nodes to failover - * to, if the number_sync_standbys is greater than zero that means the - * user wants to block writes on the primary, and we do that by - * switching to the primary state after having applied replication - * settings. Think - * - * $ pg_autoctl set formation number-sync-standbys 1 - * - * during an incident to stop the amount of potential data loss. - * - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS)) - { - char message[BUFSIZE] = { 0 }; + return (Datum) 0; +} - ReplicationState primaryGoalState = - ctx->formation->number_sync_standbys == 0 && - secondaryQuorumNodesCount == 0 - ? REPLICATION_STATE_WAIT_PRIMARY - : REPLICATION_STATE_PRIMARY; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to %s after it applied replication properties change.", - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryGoalState)); +/* + * ProceedGroupStateFromContext is the core FSM logic, operating entirely on + * the pre-built GroupStateContext. It does not touch the database for reads; + * writes (AssignGoalState, NotifyStateChange) still go to the DB. + * + * This separation lets test code inject a synthetic context and exercise the + * FSM without a live database connection. + * + * Single-shot, two straight-line lookups at most, not a loop: the cascades + * that need more than one row's worth of assignment in a single call (the + * MS-failover cascade, the join_secondary -> nested primary pass) get there + * via a bounded, named nested search inside their own extraAction (see + * ActionRunMultiStandbyFailoverCascade and ActionRunPrimaryNodeTransition), + * not by this driver looping. + * + * Two lookups, not one straight to SectionPrimaryNode when activeNode is + * already primary-role: the six early-check rows (SectionEarlyChecks) must + * always be tried first regardless -- a primary that just lost its only + * standby must still reach SINGLE via those checks, not get redirected to + * the primary-role section first. Jumping straight past them whenever + * activeNode is already primary-role would skip that case entirely; + * confirmed by the drop_node regression test, which failed exactly this way + * the first time this table's ordering got this wrong. + */ +bool +ProceedGroupStateFromContext(GroupStateContext *ctx) +{ + AutoFailoverNode *activeNode = ctx->activeNode; + char *formationId = ctx->formationId; + int groupId = ctx->groupId; - AssignGoalState(primaryNode, primaryGoalState, message); + /* + * The six early checks run unconditionally, before the IsInPrimaryState + * redirect below -- regardless of whether activeNode currently is the + * primary. primaryNode isn't resolved yet at this point -- and none of + * these six rows reference it -- so NULL is passed and is safe. + */ + NodeActiveContext earlyNac; - return true; - } + BuildFromContextNodeActiveContext(ctx, NULL, &earlyNac); + if (FindAndDispatchMonitorFSMRuleUnderPath(ctx, &earlyNac, SectionEarlyChecks, 0)) + { return true; } /* - * We don't use the join_primary state any more, though for backwards - * compatibility if a node reports JOIN_PRIMARY well then we assign PRIMARY - * to the node. After all it might be that an operator upgrades while a - * node is in JOIN_PRIMARY and we certainly want to be able to handle the - * situation. + * We separate out the FSM for the primary server, because that one needs + * to loop over every other node to take decisions. That induces some + * complexity that is best managed with its own NodeActiveContext, built + * with primaryNode substituted for activeNode's role (see + * BuildForPrimaryNodeNodeActiveContext). + * + * This early return can't become an ordinary row: it's exactly the + * branch point the whole table design has to preserve as an *entry* + * decision, not a matched condition. */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_JOIN_PRIMARY)) + if (IsInPrimaryState(activeNode)) { - char message[BUFSIZE] = { 0 }; + NodeActiveContext primaryNac; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT " to primary", - NODE_FORMAT_ARGS(primaryNode)); + BuildForPrimaryNodeNodeActiveContext(ctx, activeNode, &primaryNac); - AssignGoalState(primaryNode, REPLICATION_STATE_PRIMARY, message); + return FindAndDispatchMonitorFSMRuleUnderPath(ctx, &primaryNac, + SectionPrimaryNode, 0); + } - return true; + /* + * Derive primaryNode from ctx->groupNodeList (already fetched under the + * lock NodeActive() holds for the whole call) instead of running a + * second, independent AutoFailoverNodeGroup() query -- see + * GetPrimaryOrDemotedNodeInGroupFromList()'s comment for why this + * matters even though every writer now shares the same lock. + */ + AutoFailoverNode *primaryNode = + GetPrimaryOrDemotedNodeInGroupFromList(ctx->groupNodeList); + + /* + * We want to have a primaryNode around for most operations, but also need + * to support the case that the primaryNode has been dropped manually by a + * call to remove_node(). So we have two main cases to think about here: + * + * - we have two nodes, one of them has been removed, we catch that earlier + * in this function and assign the remaining one with the SINGLE state, + * + * - we have more than two nodes in total, and the primary has just been + * removed (maybe it was still marked unhealthy and the operator knows it + * won't ever come back so called remove_node() already): in that case in + * remove_node() we set all the other nodes to REPORT_LSN (unless they + * are in MAINTENANCE), and we should be able to make progress with the + * failover without a primary around. + * + * In all other cases we require a primaryNode to be identified. + */ + if (primaryNode == NULL && !IsFailoverInProgress(ctx->groupNodeList)) + { + ereport(ERROR, + (errmsg("ProceedGroupState couldn't find the primary node " + "in formation \"%s\", group %d", + formationId, groupId), + errdetail("activeNode is " NODE_FORMAT + " in state %s", + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->goalState)))); } - return false; + NodeActiveContext nac; + + BuildFromContextNodeActiveContext(ctx, primaryNode, &nac); + + return FindAndDispatchMonitorFSMRuleUnderPath(ctx, &nac, SectionReportingNode, 0); } @@ -1536,6 +5651,249 @@ WalSourceNodesAreAllUnhealthy(GroupStateContext *ctx, } +/* + * BuildMSFailoverNodeActiveContext computes the facts the MS-failover + * cluster's own declarative rows (SectionMSFailover, pos 363 onward) need. + * candidateNode is NULL at exactly one call site (TryFanOutReportLsnRow, wrapping + * BuildCandidateList's own fan-out loop, which hasn't selected a candidate yet) + * -- everywhere else (both call sites TryMSFailoverDeclarativeRow wraps) it's + * non-NULL, called from within ProceedGroupStateForMSFailover's own + * "nodeBeingPromoted != NULL" branch. Either way candidatePromotionInProgress + * is exactly (candidateNode != NULL), and the activeNodeAllWalSourcesUnhealthy + * computation below is skipped whenever candidateNode is NULL, so passing NULL + * never risks matching it against the wrong node. + */ +static void +BuildMSFailoverNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *activeNode, + AutoFailoverNode *candidateNode, NodeActiveContext *nac) +{ + memset(nac, 0, sizeof(NodeActiveContext)); + + BuildNodeStatus(ctx, activeNode, &nac->activeNode); + BuildNodeStatus(ctx, candidateNode, &nac->candidateNode); + + nac->inMSFailoverCluster = true; + nac->guardDataLossEnabled = GuardDataLoss; + nac->candidatePromotionInProgress = (candidateNode != NULL); + + if (candidateNode != NULL && candidateNode->nodeId == activeNode->nodeId) + { + nac->activeNodeAllWalSourcesUnhealthy = + WalSourceNodesAreAllUnhealthy(ctx, ctx->groupNodeList, activeNode); + } +} + + +/* + * BuildMSFailoverCandidateGateNodeActiveContext computes the facts the 3 + * MS-failover counting gates (missingNodesCount/candidateCount/ + * quorumCandidateCount, see ProceedGroupStateForMSFailover's own gate + * checks) match on, from BuildCandidateList's own CandidateList output. + * Called once, right after BuildCandidateList itself, and reused across all + * 3 gate checks -- the counts don't change between them within the same + * node_active() call. candidatePromotionInProgress is unconditionally false + * here: by the time ProceedGroupStateForMSFailover reaches these gates, its + * own "nodeBeingPromoted != NULL" branch has already returned, so no + * candidate is currently being promoted. + */ +static void +BuildMSFailoverCandidateGateNodeActiveContext(GroupStateContext *ctx, + AutoFailoverNode *primaryNode, + CandidateList *candidateList, + NodeActiveContext *nac) +{ + memset(nac, 0, sizeof(NodeActiveContext)); + + BuildNodeStatus(ctx, ctx->activeNode, &nac->activeNode); + BuildNodeStatus(ctx, primaryNode, &nac->primaryNode); + nac->otherNode = nac->primaryNode; /* see NodeActiveContext's own comment on .otherNode */ + + nac->inMSFailoverCluster = true; + nac->inMSFailoverCandidateGate = true; + nac->guardDataLossEnabled = GuardDataLoss; + nac->candidatePromotionInProgress = false; + + nac->candidateCount = candidateList->candidateCount; + nac->quorumCandidateCount = candidateList->quorumCandidateCount; + nac->missingNodesCount = candidateList->missingNodesCount; + + nac->sufficientQuorumCandidates = + candidateList->quorumCandidateCount >= (ctx->formation->number_sync_standbys + 1); +} + + +/* + * ActionLogMSFailoverMissingNodesDecline/Continue and ActionLogMSFailover + * QuorumDecline/Continue build the LogAndNotifyMessage text for + * ProceedGroupStateForMSFailover's own missingNodesCount/quorumCandidateCount + * gates, run as the extraAction of the declarative rows that match these same + * conditions (see the missing_nodes_gate/quorum_candidate_gate rows in + * MonitorFSM[]) -- those rows are the single source of truth for the + * message text. Neither gate assigns a goal state either way, so none of + * these four actions do either -- the control-flow decision itself (decline + * vs. continue) is the hand-written `if (GuardDataLoss)` in + * ProceedGroupStateForMSFailover; only the message text is delegated here. + */ +static void +ActionLogMSFailoverMissingNodesDecline(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + AutoFailoverNode *activeNode = nac->activeNode.node; + + LogAndNotifyMessage( + message, BUFSIZE, + "Failover still in progress after %d nodes reported their LSN " + "and we are waiting for %d nodes to report, " + "activeNode is " NODE_FORMAT + " and reported state \"%s\"", + nac->candidateCount, + nac->missingNodesCount, + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->reportedState)); +} + + +static void +ActionLogMSFailoverMissingNodesContinue(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + AutoFailoverNode *activeNode = nac->activeNode.node; + + LogAndNotifyMessage( + message, BUFSIZE, + "Proceeding with failover despite %d unreported quorum node(s): " + "pgautofailover.guard_data_loss is false. " + "Committed transactions on missing node(s) may be lost. " + "activeNode is " NODE_FORMAT " and reported state \"%s\"", + nac->missingNodesCount, + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->reportedState)); +} + + +static void +ActionLogMSFailoverQuorumDecline(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + AutoFailoverNode *activeNode = nac->activeNode.node; + int minCandidates = ctx->formation->number_sync_standbys + 1; + + LogAndNotifyMessage( + message, BUFSIZE, + "Failover still in progress with %d candidates that participate " + "in the quorum having reported their LSN: %d nodes are required " + "in the quorum to satisfy number_sync_standbys=%d in " + "formation \"%s\", activeNode is " NODE_FORMAT + " and reported state \"%s\"", + nac->quorumCandidateCount, + minCandidates, + ctx->formation->number_sync_standbys, + ctx->formation->formationId, + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->reportedState)); +} + + +static void +ActionLogMSFailoverQuorumContinue(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + AutoFailoverNode *activeNode = nac->activeNode.node; + int minCandidates = ctx->formation->number_sync_standbys + 1; + + LogAndNotifyMessage( + message, BUFSIZE, + "Proceeding with failover with only %d quorum candidate(s) despite " + "number_sync_standbys=%d requiring %d: " + "pgautofailover.guard_data_loss is false. " + "The new primary may start in wait_primary state with fewer " + "sync standbys than required. " + "activeNode is " NODE_FORMAT " and reported state \"%s\"", + nac->quorumCandidateCount, + ctx->formation->number_sync_standbys, + minCandidates, + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->reportedState)); +} + + +/* + * TryMSFailoverDeclarativeRow attempts the MS-failover cluster's own two + * declarative rows (pos 363/365) for activeNode/candidateNode, returning + * whether one matched and was dispatched. Callers only invoke this from + * exactly the hand-written C condition the matched row's own conditions + * mirror (see each call site's own comment) -- this is not a new source of + * behavior, only a new, attributed path to the same AssignGoalState call + * the hand-written code already made unconditionally at that point. A + * false return (row's own conditions didn't line up with what the caller's + * condition already established -- should not happen, but this file's own + * "no dead ends in a hot path" principle applies here too) falls back to + * the caller's own pre-existing plain AssignGoalState call, never to an + * ereport(ERROR): unlike the operator-triggered API_TRIGGERED section, + * this is reached from the ordinary node_active() heartbeat path, where a + * hard error is not an acceptable failure mode. + */ +static bool +TryMSFailoverDeclarativeRow(GroupStateContext *ctx, AutoFailoverNode *activeNode, + AutoFailoverNode *candidateNode) +{ + NodeActiveContext msNac; + + BuildMSFailoverNodeActiveContext(ctx, activeNode, candidateNode, &msNac); + + return FindAndDispatchMonitorFSMRuleUnderPath(ctx, &msNac, SectionMSFailover, 0); +} + + +/* + * TryFanOutReportLsnRow attempts to dispatch BuildCandidateList's own + * fan-out (pos 367/369/371/373) for a single node that the hand-written + * loop has already determined is a legitimate report_lsn target. A thin + * wrapper over TryMSFailoverDeclarativeRow with candidateNode=NULL, which + * correctly leaves candidatePromotionInProgress false and skips the + * activeNodeAllWalSourcesUnhealthy computation (see + * BuildMSFailoverNodeActiveContext's own comment) -- so it can never + * accidentally match pos 363/365/375/377/379, all of which require + * activeNode to already be in report_lsn, which these fan-out nodes never + * are yet. Falls back to the caller's own pre-existing AssignGoalState call + * on no match, exactly like TryMSFailoverDeclarativeRow itself. + */ +static bool +TryFanOutReportLsnRow(GroupStateContext *ctx, AutoFailoverNode *node) +{ + return TryMSFailoverDeclarativeRow(ctx, node, NULL); +} + + +/* + * DispatchMonitorFSMRuleByPos dispatches the single MonitorFSM[] row whose + * .pos equals the given value, unconditionally -- no RuleMatches check. + * Used exactly once, by PromoteSelectedNode (see pos 375/377's own + * comment), for the one pair of rows in the whole table first-match-wins + * can never disambiguate on its own: the caller has already made the real + * choice (an internal LSN comparison no BoolPattern can express) before + * calling this. Returns false if no row has that .pos -- should never + * happen for a literal, hand-maintained pos value, but the caller still + * falls back to its own pre-existing AssignGoalState call rather than + * ereport(ERROR), matching TryMSFailoverDeclarativeRow's own "no dead ends + * in a hot path" principle. + */ +static bool +DispatchMonitorFSMRuleByPos(GroupStateContext *ctx, NodeActiveContext *nac, int pos) +{ + for (int i = 0; MonitorFSM[i].pos != 0; i++) + { + if (MonitorFSM[i].pos == pos) + { + DispatchMonitorFSMRule(ctx, nac, &MonitorFSM[i]); + return true; + } + } + + return false; +} + + /* * ProceedGroupStateForMSFailover implements Group State Machine transition to * orchestrate a failover when we have more than one standby. @@ -1604,19 +5962,31 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, { if (GuardDataLoss) { - LogAndNotifyMessage( - message, BUFSIZE, - "Failover candidate " NODE_FORMAT - " is stuck in fast_forward: all WAL source nodes are " - "unhealthy and pgautofailover.guard_data_loss is true. " - "Resetting candidate to report_lsn to retry when a " - "source recovers. Use pg_autoctl perform failover " - "--allow-data-loss to promote with available WAL.", - NODE_FORMAT_ARGS(activeNode)); - - AssignGoalState(activeNode, - REPLICATION_STATE_REPORT_LSN, - message); + /* + * Dispatched through MonitorFSM[]'s own MS-failover + * declarative row (pos 363) when possible, for + * dump_fsm() visibility and rule_pos attribution -- + * falling back to the plain AssignGoalState call below + * only if that row's own conditions somehow didn't + * line up with the ones just checked above (should + * never happen; see TryMSFailoverDeclarativeRow's own + * comment). + */ + if (!TryMSFailoverDeclarativeRow(ctx, activeNode, activeNode)) + { + /* + * Can't happen: the if-condition just above already + * establishes exactly what pos 363's own conditions + * require (same activeNodeAllWalSourcesUnhealthy/ + * guardDataLossEnabled facts, see + * BuildMSFailoverNodeActiveContext). + */ + ereport(ERROR, + (errmsg("BUG: pos 363 didn't match " NODE_FORMAT + " although its own conditions should " + "always hold here", + NODE_FORMAT_ARGS(activeNode)))); + } return true; } @@ -1632,7 +6002,7 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, } } - return ProceedWithMSFailover(activeNode, nodeBeingPromoted); + return ProceedWithMSFailover(ctx, activeNode, nodeBeingPromoted); } LogAndNotifyMessage( @@ -1660,7 +6030,7 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, elog(LOG, "Found candidate " NODE_FORMAT, NODE_FORMAT_ARGS(nodeBeingPromoted)); - return ProceedWithMSFailover(activeNode, nodeBeingPromoted); + return ProceedWithMSFailover(ctx, activeNode, nodeBeingPromoted); } } @@ -1701,6 +6071,21 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, BuildCandidateList(ctx, comparableNodesGroupList, &candidateList); + /* + * gateNac carries the 3 counting gates' own facts (missingNodesCount/ + * candidateCount/quorumCandidateCount/sufficientQuorumCandidates) for + * MonitorFSM[]'s own declarative rows under reporting_node.ms_failover. + * promotion_outcome.*_gate -- see BuildMSFailoverCandidateGateNodeActive + * Context's own comment. Each gate below makes its own decline-vs- + * continue decision in plain C; only the message text each branch logs + * is delegated to the matching row's own extraAction, so the table + * stays the single source of truth for what gets logged and why. + */ + NodeActiveContext gateNac; + + BuildMSFailoverCandidateGateNodeActiveContext(ctx, primaryNode, &candidateList, + &gateNac); + /* * Time to select a candidate? * @@ -1714,33 +6099,14 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, */ if (candidateList.missingNodesCount > 0) { - char message[BUFSIZE] = { 0 }; + (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, &gateNac, + SectionMSFailoverMissingNodesGate, + 0); if (GuardDataLoss) { - LogAndNotifyMessage( - message, BUFSIZE, - "Failover still in progress after %d nodes reported their LSN " - "and we are waiting for %d nodes to report, " - "activeNode is " NODE_FORMAT - " and reported state \"%s\"", - candidateList.candidateCount, - candidateList.missingNodesCount, - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->reportedState)); - return false; } - - LogAndNotifyMessage( - message, BUFSIZE, - "Proceeding with failover despite %d unreported quorum node(s): " - "pgautofailover.guard_data_loss is false. " - "Committed transactions on missing node(s) may be lost. " - "activeNode is " NODE_FORMAT " and reported state \"%s\"", - candidateList.missingNodesCount, - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->reportedState)); } /* @@ -1756,7 +6122,11 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, */ int minCandidates = ctx->formation->number_sync_standbys + 1; - /* no candidates is a hard pass */ + /* + * no candidates is a hard pass, with no log message -- see MonitorFSM[]'s + * own candidate_count_gate row for this same fact, matched declaratively + * but never itself dispatched, purely for dump_fsm() completeness. + */ if (candidateList.candidateCount == 0) { return false; @@ -1765,40 +6135,14 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, /* not enough candidates to promote and then accept writes, pass */ if (candidateList.quorumCandidateCount < minCandidates) { - char message[BUFSIZE] = { 0 }; + (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, &gateNac, + SectionMSFailoverQuorumCandidateGate, + 0); if (GuardDataLoss) { - LogAndNotifyMessage( - message, BUFSIZE, - "Failover still in progress with %d candidates that participate " - "in the quorum having reported their LSN: %d nodes are required " - "in the quorum to satisfy number_sync_standbys=%d in " - "formation \"%s\", activeNode is " NODE_FORMAT - " and reported state \"%s\"", - candidateList.quorumCandidateCount, - minCandidates, - ctx->formation->number_sync_standbys, - ctx->formation->formationId, - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->reportedState)); - return false; } - - LogAndNotifyMessage( - message, BUFSIZE, - "Proceeding with failover with only %d quorum candidate(s) despite " - "number_sync_standbys=%d requiring %d: " - "pgautofailover.guard_data_loss is false. " - "The new primary may start in wait_primary state with fewer " - "sync standbys than required. " - "activeNode is " NODE_FORMAT " and reported state \"%s\"", - candidateList.quorumCandidateCount, - ctx->formation->number_sync_standbys, - minCandidates, - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->reportedState)); } /* enough candidates to promote and then accept writes, let's do it! */ @@ -1867,7 +6211,7 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, return false; } - return PromoteSelectedNode(selectedNode, + return PromoteSelectedNode(ctx, selectedNode, primaryNode, &candidateList); } @@ -2016,17 +6360,21 @@ BuildCandidateList(GroupStateContext *ctx, List *nodesGroupList, (node->reportedState == REPLICATION_STATE_DEMOTED && node->goalState == REPLICATION_STATE_CATCHINGUP)))) { - char message[BUFSIZE] = { 0 }; - ++(candidateList->missingNodesCount); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn to find the failover candidate", - NODE_FORMAT_ARGS(node)); - - AssignGoalState(node, REPLICATION_STATE_REPORT_LSN, message); + if (!TryFanOutReportLsnRow(ctx, node)) + { + /* + * Can't happen: the if-condition just above already + * establishes exactly what pos 367-373's own conditions + * require. + */ + ereport(ERROR, + (errmsg("BUG: no MS-failover fan-out row matched " + NODE_FORMAT " although its own conditions " + "should always hold here", + NODE_FORMAT_ARGS(node)))); + } continue; } @@ -2051,7 +6399,7 @@ BuildCandidateList(GroupStateContext *ctx, List *nodesGroupList, * WAL it's still missing from another standby node. */ static bool -ProceedWithMSFailover(AutoFailoverNode *activeNode, +ProceedWithMSFailover(GroupStateContext *ctx, AutoFailoverNode *activeNode, AutoFailoverNode *candidateNode) { Assert(candidateNode != NULL); @@ -2064,17 +6412,25 @@ ProceedWithMSFailover(AutoFailoverNode *activeNode, if (IsCurrentState(activeNode, REPLICATION_STATE_REPORT_LSN) && CandidateNodeIsReadyToStreamWAL(candidateNode)) { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to join_secondary after " NODE_FORMAT - " got selected as the failover candidate.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(candidateNode)); - - AssignGoalState(activeNode, REPLICATION_STATE_JOIN_SECONDARY, message); + /* + * Dispatched through MonitorFSM[]'s own MS-failover declarative row + * (pos 365) when possible -- see TryMSFailoverDeclarativeRow's own + * comment; falls back to the plain AssignGoalState call below only + * if that row's own conditions somehow didn't line up with the + * ones just checked above. + */ + if (!TryMSFailoverDeclarativeRow(ctx, activeNode, candidateNode)) + { + /* + * Can't happen: the if-condition just above already establishes + * exactly what pos 365's own conditions require. + */ + ereport(ERROR, + (errmsg("BUG: pos 365 didn't match " NODE_FORMAT + " although its own conditions should always " + "hold here", + NODE_FORMAT_ARGS(activeNode)))); + } return true; } @@ -2263,7 +6619,8 @@ SelectFailoverCandidateNode(GroupStateContext *ctx, * PromoteSelectedNode assigns goal state to the selected node to failover to. */ static bool -PromoteSelectedNode(AutoFailoverNode *selectedNode, +PromoteSelectedNode(GroupStateContext *ctx, + AutoFailoverNode *selectedNode, AutoFailoverNode *primaryNode, CandidateList *candidateList) { @@ -2371,6 +6728,10 @@ PromoteSelectedNode(AutoFailoverNode *selectedNode, if (selectedNode->reportedLSN == candidateList->mostAdvancedReportedLSN) { char message[BUFSIZE] = { 0 }; + NodeActiveContext promotionNac; + + memset(&promotionNac, 0, sizeof(NodeActiveContext)); + BuildNodeStatus(ctx, selectedNode, &promotionNac.activeNode); if (primaryNode) { @@ -2393,9 +6754,16 @@ PromoteSelectedNode(AutoFailoverNode *selectedNode, candidateList->candidateCount); } - AssignGoalState(selectedNode, - REPLICATION_STATE_PREPARE_PROMOTION, - message); + if (!DispatchMonitorFSMRuleByPos(ctx, &promotionNac, 375)) + { + /* + * can't happen: pos 375 is a fixed, always-present row (see + * AssertMonitorFSMWellFormed) -- DispatchMonitorFSMRuleByPos only + * fails to find a pos that doesn't exist in the table at all. + */ + ereport(ERROR, + (errmsg("BUG: MonitorFSM[] has no row with pos = 375"))); + } /* leave the other nodes in ReportLSN state for now */ return true; @@ -2403,6 +6771,10 @@ PromoteSelectedNode(AutoFailoverNode *selectedNode, else { char message[BUFSIZE] = { 0 }; + NodeActiveContext promotionNac; + + memset(&promotionNac, 0, sizeof(NodeActiveContext)); + BuildNodeStatus(ctx, selectedNode, &promotionNac.activeNode); if (primaryNode) { @@ -2425,8 +6797,16 @@ PromoteSelectedNode(AutoFailoverNode *selectedNode, candidateList->candidateCount); } - AssignGoalState(selectedNode, - REPLICATION_STATE_FAST_FORWARD, message); + if (!DispatchMonitorFSMRuleByPos(ctx, &promotionNac, 377)) + { + /* + * can't happen: pos 377 is a fixed, always-present row (see + * AssertMonitorFSMWellFormed) -- DispatchMonitorFSMRuleByPos only + * fails to find a pos that doesn't exist in the table at all. + */ + ereport(ERROR, + (errmsg("BUG: MonitorFSM[] has no row with pos = 377"))); + } return true; } diff --git a/src/monitor/group_state_machine.h b/src/monitor/group_state_machine.h index 17effdd17..c40276f95 100644 --- a/src/monitor/group_state_machine.h +++ b/src/monitor/group_state_machine.h @@ -19,6 +19,99 @@ #include "formation_metadata.h" #include "node_metadata.h" +/* + * MonitorFSMSection identifies which node of the section hierarchy a row's + * .sectionPath (see MonitorFSMTransition/MonitorFSMSectionPath in + * group_state_machine.c) belongs to at some depth -- a row's own path is a + * small array of these, e.g. { MONITOR_FSM_SECTION_REPORTING_NODE, + * MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET }, + * matched via prefix/ancestor containment (SectionPathIsUnderPrefix) instead + * of the hand-maintained array-index-range constants this replaces -- see + * that mechanism's own comment for why (the design doc's own "Open items" + * flagged the old index constants as needing "to stay in sync with the + * table by hand as rows are added, removed, or reordered"). + * + * Only the first four values below (API_TRIGGERED/EARLY_CHECKS/ + * REPORTING_NODE/PRIMARY_NODE, unchanged in name and meaning from before this + * mechanism existed) are ever legal as a row's .sectionPath[0] -- every row's + * path always starts with exactly one of these four, enforced by + * AssertMonitorFSMWellFormed(). Declared here (not just in the .c file) so + * this top-level tag can be exposed to SQL as pgautofailover.fsm_section + * (mapped by NAME, not ordinal -- see MonitorFSMSectionGetEnum/ + * EnumGetMonitorFSMSection -- so inserting MONITOR_FSM_SECTION_NONE ahead of + * them, or appending new fine-grained values after PRIMARY_NODE, changes no + * SQL-visible behavior at all), the same way ReplicationState is exposed as + * pgautofailover.replication_state (see replication_state.h/.c for that + * pattern, mirrored below). + * + * API_TRIGGERED comes first (pos 101-1xx) even though it's the newest + * section, chronologically: an operator-triggered call (perform_failover, + * remove_node, start/stop_maintenance, set_node_candidate_priority, + * set_node_replication_quorum, set_formation_number_sync_standbys) is a + * distinct, self-contained entry point, not a continuation of the + * heartbeat-driven EARLY_CHECKS/REPORTING_NODE/PRIMARY_NODE chain -- placing + * it in its own leading hundred-block keeps that same "each section is one + * contiguous pos range, matching one real entry point" property the other + * three sections already have, rather than wedging operator rows into gaps + * within the heartbeat sections. + * + * Values from MONITOR_FSM_SECTION_FROM_CONTEXT onward are fine-grained leaf + * labels, only ever used at path depth 1+ (reporting_node.from_context, + * reporting_node.ms_failover, and its own sub-leaves) -- purely for + * dump_fsm()'s own section_path column readability today; no caller needs + * to bound a search this narrowly (see each leaf's own row comment in + * MonitorFSM[]). + */ +typedef enum MonitorFSMSection +{ + MONITOR_FSM_SECTION_NONE = 0, /* terminator / unused trailing path slot */ + MONITOR_FSM_SECTION_API_TRIGGERED, + MONITOR_FSM_SECTION_EARLY_CHECKS, + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_PRIMARY_NODE, + + MONITOR_FSM_SECTION_FROM_CONTEXT, /* reporting_node.from_context */ + MONITOR_FSM_SECTION_MS_FAILOVER, /* reporting_node.ms_failover */ + MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_CANDIDATE_COUNT_GATE, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_NO_CANDIDATE_YET, + MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE, +} MonitorFSMSection; + +/* public function declarations, mirroring replication_state.h's pattern */ +extern Oid MonitorFSMSectionTypeOid(void); +extern MonitorFSMSection EnumGetMonitorFSMSection(Oid monitorFSMSectionOid); +extern Oid MonitorFSMSectionGetEnum(MonitorFSMSection section); +extern const char * MonitorFSMSectionGetName(MonitorFSMSection section); + +/* + * MonitorApiFunction identifies which operator-triggered SQL entry point + * produced a ProceedGroupStateForApiTrigger() dispatch call -- see that + * function's own comment in group_state_machine.c, and the + * MONITOR_FSM_SECTION_API_TRIGGERED rows in MonitorFSM[] that match against + * it via .conditions.apiTrigger. API_FUNCTION_NONE is never passed to + * ProceedGroupStateForApiTrigger() itself -- it's the ordinary node_active() + * heartbeat path's own implicit value (see NodeActiveContext's apiFunction + * field), kept ordinal 0 so every row written before this mechanism existed + * keeps meaning exactly what it always meant, with zero changes. + */ +typedef enum MonitorApiFunction +{ + API_FUNCTION_NONE = 0, + API_FUNCTION_REMOVE_NODE, + API_FUNCTION_PERFORM_FAILOVER, + API_FUNCTION_START_MAINTENANCE, + API_FUNCTION_STOP_MAINTENANCE, + API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY, + API_FUNCTION_SET_NODE_REPLICATION_QUORUM, + API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS, +} MonitorApiFunction; + /* * AutoFailoverNodeState describes the current state of a node in a group. */ @@ -67,6 +160,9 @@ extern bool BuildGroupStateContext(GroupStateContext *ctx, AutoFailoverNode *activeNode); extern bool ProceedGroupState(AutoFailoverNode *activeNode); extern bool ProceedGroupStateFromContext(GroupStateContext *ctx); +extern bool ProceedGroupStateForApiTrigger(MonitorApiFunction apiFunction, + AutoFailoverNode *activeNode, + AutoFailoverNode *primaryNode); /* GUCs */ extern int EnableSyncXlogThreshold; diff --git a/src/monitor/health_check_metadata.c b/src/monitor/health_check_metadata.c index fed9483e2..b1c4b63e9 100644 --- a/src/monitor/health_check_metadata.c +++ b/src/monitor/health_check_metadata.c @@ -28,6 +28,7 @@ #include "pgstat.h" #include "utils/builtins.h" #include "utils/memutils.h" +#include "utils/resowner.h" #include "utils/snapmgr.h" @@ -76,13 +77,56 @@ LoadNodeHealthList(void) pgstat_report_activity(STATE_RUNNING, query.data); - spiStatus = SPI_execute(query.data, false, 0); - /* * When we start the monitor during an upgrade (from 1.3 to 1.4), the * background worker might be reading the 1.3 pgautofailover catalogs - * still, where the "nodehost" column does not exist. + * still, where the "nodehost" column does not exist. The same + * happens for the brief window a regress/isolation test spends with + * the extension deliberately left at a non-2.3 (or dropped) schema + * (dummy_update.sql, upgrade.sql): the query below is a genuine + * parse-time ereport(ERROR) in either case, not a soft SPI-status + * failure SPI_execute() itself can report -- it always throws, + * aborting the enclosing transaction, so it has to be caught with a + * subtransaction here rather than checked via spiStatus after the + * fact (the "if (spiStatus != SPI_OK_SELECT)" shape this comment + * used to describe never actually ran for this case; it only + * covered a SPI-level failure with no matching real code path). */ + MemoryContext spiErrContext = CurrentMemoryContext; + ResourceOwner spiErrOwner = CurrentResourceOwner; + + BeginInternalSubTransaction(NULL); + + PG_TRY(); + { + spiStatus = SPI_execute(query.data, false, 0); + + ReleaseCurrentSubTransaction(); + MemoryContextSwitchTo(spiErrContext); + CurrentResourceOwner = spiErrOwner; + } + PG_CATCH(); + { + ErrorData *edata; + + MemoryContextSwitchTo(spiErrContext); + edata = CopyErrorData(); + FlushErrorState(); + + RollbackAndReleaseCurrentSubTransaction(); + MemoryContextSwitchTo(spiErrContext); + CurrentResourceOwner = spiErrOwner; + + ereport(WARNING, + (errmsg("health check skipping this round: %s", edata->message))); + + FreeErrorData(edata); + + EndSPITransaction(); + return NIL; + } + PG_END_TRY(); + if (spiStatus != SPI_OK_SELECT) { EndSPITransaction(); diff --git a/src/monitor/keeper_fsm_edges.json b/src/monitor/keeper_fsm_edges.json new file mode 100644 index 000000000..20ab42ac9 --- /dev/null +++ b/src/monitor/keeper_fsm_edges.json @@ -0,0 +1,526 @@ +[ + { + "current": "init", + "assigned": "single" + }, + { + "current": "init", + "assigned": "single" + }, + { + "current": "init", + "assigned": "single" + }, + { + "current": "dropped", + "assigned": "single" + }, + { + "current": "dropped", + "assigned": "single" + }, + { + "current": "dropped", + "assigned": "report_lsn" + }, + { + "current": "primary", + "assigned": "single" + }, + { + "current": "wait_primary", + "assigned": "single" + }, + { + "current": "join_primary", + "assigned": "single" + }, + { + "current": "primary", + "assigned": "draining" + }, + { + "current": "draining", + "assigned": "demoted" + }, + { + "current": "primary", + "assigned": "demoted" + }, + { + "current": "primary", + "assigned": "demote_timeout" + }, + { + "current": "join_primary", + "assigned": "draining" + }, + { + "current": "join_primary", + "assigned": "demoted" + }, + { + "current": "join_primary", + "assigned": "demote_timeout" + }, + { + "current": "apply_settings", + "assigned": "draining" + }, + { + "current": "apply_settings", + "assigned": "demoted" + }, + { + "current": "apply_settings", + "assigned": "demote_timeout" + }, + { + "current": "primary", + "assigned": "prepare_maintenance" + }, + { + "current": "prepare_maintenance", + "assigned": "maintenance" + }, + { + "current": "primary", + "assigned": "maintenance" + }, + { + "current": "draining", + "assigned": "demote_timeout" + }, + { + "current": "demote_timeout", + "assigned": "demoted" + }, + { + "current": "wait_primary", + "assigned": "demoted" + }, + { + "current": "init", + "assigned": "demoted" + }, + { + "current": "single", + "assigned": "demoted" + }, + { + "current": "catchingup", + "assigned": "demoted" + }, + { + "current": "secondary", + "assigned": "demoted" + }, + { + "current": "prepare_promotion", + "assigned": "demoted" + }, + { + "current": "stop_replication", + "assigned": "demoted" + }, + { + "current": "maintenance", + "assigned": "demoted" + }, + { + "current": "prepare_maintenance", + "assigned": "demoted" + }, + { + "current": "wait_maintenance", + "assigned": "demoted" + }, + { + "current": "report_lsn", + "assigned": "demoted" + }, + { + "current": "fast_forward", + "assigned": "demoted" + }, + { + "current": "init", + "assigned": "demote_timeout" + }, + { + "current": "single", + "assigned": "demote_timeout" + }, + { + "current": "demoted", + "assigned": "demote_timeout" + }, + { + "current": "catchingup", + "assigned": "demote_timeout" + }, + { + "current": "secondary", + "assigned": "demote_timeout" + }, + { + "current": "prepare_promotion", + "assigned": "demote_timeout" + }, + { + "current": "stop_replication", + "assigned": "demote_timeout" + }, + { + "current": "maintenance", + "assigned": "demote_timeout" + }, + { + "current": "prepare_maintenance", + "assigned": "demote_timeout" + }, + { + "current": "wait_maintenance", + "assigned": "demote_timeout" + }, + { + "current": "report_lsn", + "assigned": "demote_timeout" + }, + { + "current": "fast_forward", + "assigned": "demote_timeout" + }, + { + "current": "demoted", + "assigned": "single" + }, + { + "current": "demoted", + "assigned": "single" + }, + { + "current": "demote_timeout", + "assigned": "single" + }, + { + "current": "demote_timeout", + "assigned": "single" + }, + { + "current": "draining", + "assigned": "single" + }, + { + "current": "draining", + "assigned": "single" + }, + { + "current": "secondary", + "assigned": "single" + }, + { + "current": "secondary", + "assigned": "single" + }, + { + "current": "secondary", + "assigned": "single" + }, + { + "current": "catchingup", + "assigned": "single" + }, + { + "current": "catchingup", + "assigned": "single" + }, + { + "current": "catchingup", + "assigned": "single" + }, + { + "current": "prepare_promotion", + "assigned": "single" + }, + { + "current": "prepare_promotion", + "assigned": "single" + }, + { + "current": "prepare_promotion", + "assigned": "single" + }, + { + "current": "stop_replication", + "assigned": "single" + }, + { + "current": "stop_replication", + "assigned": "single" + }, + { + "current": "report_lsn", + "assigned": "single" + }, + { + "current": "wait_maintenance", + "assigned": "single" + }, + { + "current": "fast_forward", + "assigned": "single" + }, + { + "current": "single", + "assigned": "wait_primary" + }, + { + "current": "primary", + "assigned": "join_primary" + }, + { + "current": "primary", + "assigned": "wait_primary" + }, + { + "current": "join_primary", + "assigned": "wait_primary" + }, + { + "current": "wait_primary", + "assigned": "join_primary" + }, + { + "current": "wait_primary", + "assigned": "primary" + }, + { + "current": "join_primary", + "assigned": "primary" + }, + { + "current": "demote_timeout", + "assigned": "primary" + }, + { + "current": "wait_standby", + "assigned": "catchingup" + }, + { + "current": "demoted", + "assigned": "catchingup" + }, + { + "current": "secondary", + "assigned": "catchingup" + }, + { + "current": "catchingup", + "assigned": "secondary" + }, + { + "current": "catchingup", + "assigned": "secondary" + }, + { + "current": "secondary", + "assigned": "prepare_promotion" + }, + { + "current": "secondary", + "assigned": "prepare_promotion" + }, + { + "current": "catchingup", + "assigned": "prepare_promotion" + }, + { + "current": "catchingup", + "assigned": "prepare_promotion" + }, + { + "current": "prepare_promotion", + "assigned": "stop_replication" + }, + { + "current": "prepare_promotion", + "assigned": "stop_replication" + }, + { + "current": "stop_replication", + "assigned": "wait_primary" + }, + { + "current": "stop_replication", + "assigned": "wait_primary" + }, + { + "current": "stop_replication", + "assigned": "wait_primary" + }, + { + "current": "prepare_promotion", + "assigned": "wait_primary" + }, + { + "current": "prepare_promotion", + "assigned": "wait_primary" + }, + { + "current": "init", + "assigned": "wait_standby" + }, + { + "current": "dropped", + "assigned": "wait_standby" + }, + { + "current": "secondary", + "assigned": "wait_standby" + }, + { + "current": "secondary", + "assigned": "wait_maintenance" + }, + { + "current": "catchingup", + "assigned": "wait_maintenance" + }, + { + "current": "secondary", + "assigned": "maintenance" + }, + { + "current": "catchingup", + "assigned": "maintenance" + }, + { + "current": "wait_maintenance", + "assigned": "maintenance" + }, + { + "current": "maintenance", + "assigned": "catchingup" + }, + { + "current": "prepare_maintenance", + "assigned": "catchingup" + }, + { + "current": "primary", + "assigned": "apply_settings" + }, + { + "current": "wait_primary", + "assigned": "apply_settings" + }, + { + "current": "apply_settings", + "assigned": "primary" + }, + { + "current": "apply_settings", + "assigned": "single" + }, + { + "current": "apply_settings", + "assigned": "wait_primary" + }, + { + "current": "apply_settings", + "assigned": "join_primary" + }, + { + "current": "secondary", + "assigned": "report_lsn" + }, + { + "current": "catchingup", + "assigned": "report_lsn" + }, + { + "current": "maintenance", + "assigned": "report_lsn" + }, + { + "current": "prepare_maintenance", + "assigned": "report_lsn" + }, + { + "current": "wait_maintenance", + "assigned": "report_lsn" + }, + { + "current": "fast_forward", + "assigned": "report_lsn" + }, + { + "current": "prepare_promotion", + "assigned": "report_lsn" + }, + { + "current": "stop_replication", + "assigned": "report_lsn" + }, + { + "current": "demote_timeout", + "assigned": "report_lsn" + }, + { + "current": "join_secondary", + "assigned": "report_lsn" + }, + { + "current": "report_lsn", + "assigned": "prepare_promotion" + }, + { + "current": "report_lsn", + "assigned": "prepare_promotion" + }, + { + "current": "report_lsn", + "assigned": "fast_forward" + }, + { + "current": "fast_forward", + "assigned": "prepare_promotion" + }, + { + "current": "fast_forward", + "assigned": "prepare_promotion" + }, + { + "current": "report_lsn", + "assigned": "join_secondary" + }, + { + "current": "report_lsn", + "assigned": "secondary" + }, + { + "current": "join_secondary", + "assigned": "secondary" + }, + { + "current": "draining", + "assigned": "report_lsn" + }, + { + "current": "demoted", + "assigned": "report_lsn" + }, + { + "current": "init", + "assigned": "report_lsn" + }, + { + "current": "any", + "assigned": "dropped" + }, + { + "current": "any", + "assigned": "dropped" + } +] diff --git a/src/monitor/metadata.h b/src/monitor/metadata.h index d974d8bff..b564d3c05 100644 --- a/src/monitor/metadata.h +++ b/src/monitor/metadata.h @@ -22,6 +22,7 @@ #define AUTO_FAILOVER_NODE_TABLE "pgautofailover.node" #define AUTO_FAILOVER_EVENT_TABLE "pgautofailover.event" #define REPLICATION_STATE_TYPE_NAME "replication_state" +#define FSM_SECTION_TYPE_NAME "fsm_section" /* diff --git a/src/monitor/node_active_protocol.c b/src/monitor/node_active_protocol.c index 24a0d0163..a1e22db01 100644 --- a/src/monitor/node_active_protocol.c +++ b/src/monitor/node_active_protocol.c @@ -1162,7 +1162,6 @@ remove_node_by_host(PG_FUNCTION_ARGS) static bool RemoveNode(int64 nodeId, bool force) { - ListCell *nodeCell = NULL; char message[BUFSIZE] = { 0 }; /* @@ -1248,50 +1247,15 @@ RemoveNode(int64 nodeId, bool force) return true; } - /* review the FSM for every other node, when removing the primary */ - if (currentNodeIsPrimary) - { - foreach(nodeCell, otherNodesGroupList) - { - AutoFailoverNode *node = (AutoFailoverNode *) lfirst(nodeCell); - - if (node == NULL) - { - /* shouldn't happen */ - ereport(ERROR, (errmsg("BUG: node is NULL"))); - continue; - } - - /* skip nodes that are currently in maintenance */ - if (IsInMaintenance(node)) - { - continue; - } - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn after primary node removal.", - NODE_FORMAT_ARGS(node)); - - SetNodeGoalState(node, REPLICATION_STATE_REPORT_LSN, message); - } - } - /* - * Mark the node as being dropped, so that the pg_autoctl node-active - * process can implement further actions at drop time. + * Dispatch through MonitorFSM[]'s API_TRIGGERED section (see + * ProceedGroupStateForApiTrigger's own comment): assigns dropped to + * currentNode, and, when it canTakeWrites, fans out report_lsn to every + * surviving non-maintenance standby first (both folded into one row -- + * see that row's own comment for why splitting them across two rows, + * as an earlier draft of this table did, would have been a real bug). */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " from formation \"%s\" and group %d to \"dropped\"" - " to implement node removal.", - NODE_FORMAT_ARGS(currentNode), - currentNode->formationId, - currentNode->groupId); - - SetNodeGoalState(currentNode, REPLICATION_STATE_DROPPED, message); + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_REMOVE_NODE, currentNode, NULL); /* * Adjust number-sync-standbys if necessary. @@ -1536,21 +1500,13 @@ perform_failover(PG_FUNCTION_ARGS) "perform a manual failover"))); } - char message[BUFSIZE] = { 0 }; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to draining and " NODE_FORMAT - " to prepare_promotion after a user-initiated failover.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(secondaryNode)); - - SetNodeGoalState(primaryNode, - REPLICATION_STATE_DRAINING, message); - - SetNodeGoalState(secondaryNode, - REPLICATION_STATE_PREPARE_PROMOTION, message); + /* + * Dispatch through MonitorFSM[]'s API_TRIGGERED section (see + * ProceedGroupStateForApiTrigger's own comment): standby -> + * prepare_promotion, primary -> draining. + */ + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_PERFORM_FAILOVER, + secondaryNode, primaryNode); } else { @@ -1558,16 +1514,15 @@ perform_failover(PG_FUNCTION_ARGS) AutoFailoverNode *firstStandbyNode = linitial(standbyNodesGroupList); char message[BUFSIZE] = { 0 }; - /* so we have at least one candidate, let's get started */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - "at LSN %X/%X to draining after a user-initiated failover.", - NODE_FORMAT_ARGS(primaryNode), - (uint32) (primaryNode->reportedLSN >> 32), - (uint32) primaryNode->reportedLSN); - - SetNodeGoalState(primaryNode, REPLICATION_STATE_DRAINING, message); + /* + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary -> + * draining. The candidatePriority trick and the continuation + * dispatch below are POST side effects, hand-written here exactly + * as before -- see that row's own comment for why they don't + * become part of the row itself. + */ + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_PERFORM_FAILOVER, + primaryNode, NULL); /* * When a failover is performed with all the nodes up and running, the @@ -1901,49 +1856,37 @@ start_maintenance(PG_FUNCTION_ARGS) if (totalNodesCount == 2) { /* - * Set the primary to prepare_maintenance now, and if we have a - * single secondary we assign it prepare_promotion, otherwise we - * need to elect a secondary, same as in perform_failover. + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary + * -> prepare_maintenance. The lone standby -> prepare_promotion + * is a POST side effect, hand-written here: it has no ordering + * dependency on this row's own assignment (neither reads the + * other's freshly-committed state), so it doesn't need + * extraAction to sequence correctly. */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to prepare_maintenance " - "after a user-initiated start_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_START_MAINTENANCE, + currentNode, NULL); - SetNodeGoalState(currentNode, - REPLICATION_STATE_PREPARE_MAINTENANCE, message); - - AutoFailoverNode *otherNode = firstStandbyNode; - - /* - * We put the only secondary node straight to prepare_replication. - */ LogAndNotifyMessage( message, BUFSIZE, "Setting goal state of " NODE_FORMAT - " to prepare_maintenance and " NODE_FORMAT " to prepare_promotion " "after a user-initiated start_maintenance call.", - NODE_FORMAT_ARGS(currentNode), - NODE_FORMAT_ARGS(otherNode)); + NODE_FORMAT_ARGS(firstStandbyNode)); - SetNodeGoalState(otherNode, + SetNodeGoalState(firstStandbyNode, REPLICATION_STATE_PREPARE_PROMOTION, message); } else { - /* put the primary directly to maintenance */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance " - "after a user-initiated start_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); - - SetNodeGoalState(currentNode, - REPLICATION_STATE_PREPARE_MAINTENANCE, message); + /* + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary + * -> prepare_maintenance. The continuation dispatch below is a + * POST side effect, hand-written here, called after this row's + * own assignment has committed (it re-fetches fresh state, so + * ordering matters here, unlike the 2-node case above). + */ + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_START_MAINTENANCE, + currentNode, NULL); /* now proceed with the failover, starting with the first standby */ (void) ProceedGroupState(firstStandbyNode); @@ -1970,30 +1913,18 @@ start_maintenance(PG_FUNCTION_ARGS) * state of any standby node yet, we get there when the count is one * (not zero). */ - if (formation->number_sync_standbys == 0 && secondaryNodesCount == 1 && - IsHealthySyncStandby(currentNode)) - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary and " NODE_FORMAT - " to wait_maintenance " - "after a user-initiated start_maintenance call.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(currentNode)); - SetNodeGoalState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY, message); - SetNodeGoalState(currentNode, REPLICATION_STATE_WAIT_MAINTENANCE, message); - } - else - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance " - "after a user-initiated start_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); - SetNodeGoalState(currentNode, REPLICATION_STATE_MAINTENANCE, message); - } + + /* + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: the + * last-healthy-sync-standby row (wait_maintenance + primary + * wait_primary) and the ordinary row (maintenance) are both + * declarative, dual-role rows there -- see + * BuildApiTriggerNodeActiveContext's own comment for how + * lastHealthySyncStandbyGoingToMaintenance mirrors this exact + * condition. + */ + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_START_MAINTENANCE, + currentNode, primaryNode); } else { @@ -2029,8 +1960,6 @@ stop_maintenance(PG_FUNCTION_ARGS) int64 nodeId = PG_GETARG_INT64(0); - char message[BUFSIZE] = { 0 }; - AutoFailoverNode *currentNode = LockNodeGroupAndFetch(nodeId); if (currentNode == NULL) { @@ -2087,69 +2016,17 @@ stop_maintenance(PG_FUNCTION_ARGS) "group %d", currentNode->formationId, currentNode->groupId))); } - else if ((primaryNode == NULL || IsDemotedPrimary(primaryNode)) && - totalNodesCount > 2) - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn after a user-initiated stop_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); - - SetNodeGoalState(currentNode, REPLICATION_STATE_REPORT_LSN, message); - - PG_RETURN_BOOL(true); - } - else if (IsDemotedPrimary(primaryNode)) - { - /* - * The primary is fully demoted (Postgres stopped, e.g. after a - * #1025 self-fence recovery): there's nothing left running to - * stream from, so catchingup would just retry a doomed replication - * connection forever. Join the report_lsn crew instead -- once this - * node reports its LSN, the candidate-scanning code in - * ProceedGroupStateForMSFailover() picks up the demoted primary too - * (it's still IsDemotedPrimary()) and the normal election proceeds. - */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn after a user-initiated stop_maintenance call, " - "as " NODE_FORMAT " is demoted and has nothing to catch up from.", - NODE_FORMAT_ARGS(currentNode), - NODE_FORMAT_ARGS(primaryNode)); - - SetNodeGoalState(currentNode, REPLICATION_STATE_REPORT_LSN, message); - - PG_RETURN_BOOL(true); - } /* - * When a failover is in progress and stop_maintenance() is called (by - * means of pg_autoctl disable maintenance or otherwise), then we should - * join the crew on REPORT_LSN: the last known primary can be presumed - * down. + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: no-primary, + * primary-demoted, failover-in-progress, and the ordinary catchingup + * catchall are all declarative rows there -- see each row's own + * comment. Every one of the four real branches this replaces produces + * the same PG_RETURN_BOOL(true), so they collapse to a single dispatch + * call followed by one shared return. */ - if (IsFailoverInProgress(groupNodesList)) - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after a user-initiated stop_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); - - SetNodeGoalState(currentNode, REPLICATION_STATE_REPORT_LSN, message); - } - else - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after a user-initiated stop_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); - - SetNodeGoalState(currentNode, REPLICATION_STATE_CATCHINGUP, message); - } + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_STOP_MAINTENANCE, + currentNode, primaryNode); PG_RETURN_BOOL(true); } @@ -2265,8 +2142,6 @@ set_node_candidate_priority(PG_FUNCTION_ARGS) } else { - char message[BUFSIZE]; - AutoFailoverNode *primaryNode = GetPrimaryNodeInGroup(currentNode->formationId, currentNode->groupId); @@ -2278,21 +2153,17 @@ set_node_candidate_priority(PG_FUNCTION_ARGS) * * If we don't currently have a primary node anyway, we can just * proceed with the change. + * + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary -> + * apply_settings. Kept hand-written, not routed through the + * table's own no-match ERROR: this exact message text is + * preserved unchanged so it stays test-stable. */ if (primaryNode && !IsCurrentState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS)) { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to apply_settings after updating " NODE_FORMAT - " candidate priority to %d.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(currentNode), - currentNode->candidatePriority); - - SetNodeGoalState(primaryNode, - REPLICATION_STATE_APPLY_SETTINGS, message); + (void) ProceedGroupStateForApiTrigger( + API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY, primaryNode, NULL); } /* if primaryNode is not NULL, then current state is APPLY_SETTINGS */ @@ -2410,8 +2281,6 @@ set_node_replication_quorum(PG_FUNCTION_ARGS) } else { - char message[BUFSIZE]; - AutoFailoverNode *primaryNode = GetPrimaryNodeInGroup(currentNode->formationId, currentNode->groupId); @@ -2423,21 +2292,17 @@ set_node_replication_quorum(PG_FUNCTION_ARGS) * * If we don't currently have a primary node anyway, we can just * proceed with the change. + * + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary -> + * apply_settings. Kept hand-written, not routed through the table's + * own no-match ERROR: this exact message text is preserved + * unchanged so it stays test-stable. */ if (primaryNode && !IsCurrentState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS)) { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to apply_settings after updating " NODE_FORMAT - " replication quorum to %s.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(currentNode), - currentNode->replicationQuorum ? "true" : "false"); - - SetNodeGoalState(primaryNode, - REPLICATION_STATE_APPLY_SETTINGS, message); + (void) ProceedGroupStateForApiTrigger( + API_FUNCTION_SET_NODE_REPLICATION_QUORUM, primaryNode, NULL); } /* if primaryNode is not NULL, then current state is APPLY_SETTINGS */ diff --git a/src/monitor/notifications.c b/src/monitor/notifications.c index 3db58c73c..f295a3c0e 100644 --- a/src/monitor/notifications.c +++ b/src/monitor/notifications.c @@ -28,6 +28,13 @@ #include "utils/json.h" #include "utils/pg_lsn.h" +#include "group_state_machine.h" + + +/* see notifications.h for why these exist and how they're used */ +int CurrentMonitorFSMRulePos = 0; +int CurrentMonitorFSMRuleSection = 0; + /* * LogAndNotifyMessage emits the given message both as a log entry and also as @@ -123,6 +130,17 @@ InsertEvent(AutoFailoverNode *node, char *description) Oid goalStateOid = ReplicationStateGetEnum(node->goalState); Oid reportedStateOid = ReplicationStateGetEnum(node->reportedState); Oid replicationStateTypeOid = ReplicationStateTypeOid(); + Oid fsmSectionTypeOid = MonitorFSMSectionTypeOid(); + + /* + * CurrentMonitorFSMRulePos/Section (see notifications.h) are only set + * while a MonitorFSM[] row's own extraAction/goal-state assignment is + * actually running; 0 means this event was produced by an ordinary + * AssignGoalState call from outside the declarative dispatch table, so + * rule_pos/rule_section stay NULL for it rather than a misleading 0 + * (not a valid .pos value -- every real section starts at 101 or above). + */ + bool haveRule = CurrentMonitorFSMRulePos != 0; Oid argTypes[] = { TEXTOID, /* formationid */ @@ -138,7 +156,9 @@ InsertEvent(AutoFailoverNode *node, char *description) LSNOID, /* reportedLSN */ INT4OID, /* candidate_priority */ BOOLOID, /* replication_quorum */ - TEXTOID /* description */ + TEXTOID, /* description */ + INT4OID, /* rule_pos */ + fsmSectionTypeOid /* rule_section */ }; Datum argValues[] = { @@ -155,7 +175,18 @@ InsertEvent(AutoFailoverNode *node, char *description) LSNGetDatum(node->reportedLSN), /* reportedLSN */ Int32GetDatum(node->candidatePriority), /* candidate_priority */ BoolGetDatum(node->replicationQuorum), /* replication_quorum */ - CStringGetTextDatum(description) /* description */ + CStringGetTextDatum(description), /* description */ + Int32GetDatum(CurrentMonitorFSMRulePos), /* rule_pos, ignored if NULL below */ + haveRule + ? ObjectIdGetDatum(MonitorFSMSectionGetEnum( + (MonitorFSMSection) CurrentMonitorFSMRuleSection)) + : ObjectIdGetDatum(InvalidOid) /* rule_section, ignored if NULL below */ + }; + + char argNulls[] = { + ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', + haveRule ? ' ' : 'n', /* rule_pos */ + haveRule ? ' ' : 'n' /* rule_section */ }; const int argCount = sizeof(argValues) / sizeof(argValues[0]); @@ -165,14 +196,14 @@ InsertEvent(AutoFailoverNode *node, char *description) "INSERT INTO " AUTO_FAILOVER_EVENT_TABLE "(formationid, nodeid, groupid, nodename, nodehost, nodeport," " reportedstate, goalstate, reportedrepstate, reportedtli, reportedlsn," - " candidatepriority, replicationquorum, description) " - "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) " + " candidatepriority, replicationquorum, description, rule_pos, rule_section) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) " "RETURNING eventid"; SPI_connect(); int spiStatus = SPI_execute_with_args(insertQuery, argCount, argTypes, - argValues, NULL, false, 0); + argValues, argNulls, false, 0); if (spiStatus == SPI_OK_INSERT_RETURNING && SPI_processed > 0) { diff --git a/src/monitor/notifications.h b/src/monitor/notifications.h index 00e064211..9bf22a317 100644 --- a/src/monitor/notifications.h +++ b/src/monitor/notifications.h @@ -41,3 +41,32 @@ void LogAndNotifyMessage(char *message, size_t size, const char *fmt, ...) __att int64 NotifyStateChange(AutoFailoverNode *node, char *description); int64 InsertEvent(AutoFailoverNode *node, char *description); + +/* + * Set by group_state_machine.c's declarative dispatch just before invoking a + * matched MonitorFSM[] row's extraAction/goal-state assignment, and restored + * to whatever it was before immediately after (nested dispatch -- the + * MS-failover cascade's and join_secondary's own bounded nested searches -- + * saves/restores rather than clobbers, so the outer row's own subsequent + * assignments are still attributed correctly). Lets InsertEvent() attribute + * the resulting pgautofailover.event row to the rule that produced it + * (rule_pos/rule_section columns), without threading an extra parameter + * through AssignGoalState/SetNodeGoalState/NotifyStateChange and every one + * of their many call sites outside the declarative dispatch table. 0 means + * "no rule attributed": rule_pos/rule_section are left NULL in that case, + * since 0 is not a valid .pos value (every real section starts at 100 or + * above). This is NOT the case for operator-triggered SQL functions or for + * ProceedGroupStateForMSFailover's own hand-written internals -- both DO + * get a real, non-NULL rule_pos: the former dispatch via + * ProceedGroupStateForApiTrigger, which itself calls DispatchMonitorFSMRule; + * the latter (BuildCandidateList, PromoteSelectedNode, etc.) are only ever + * invoked from inside an outer row's own extraAction, so this global is + * already non-zero (attributed to that OUTER row, not one of their own -- + * misleading, since MS-failover's candidate-selection internals were never + * decomposed into declarative rows) by the time they call AssignGoalState + * directly. 0/NULL only really happens for call sites genuinely outside any + * DispatchMonitorFSMRule call at all -- e.g. perform_failover()'s own + * candidate-priority bookkeeping in node_active_protocol.c. + */ +extern int CurrentMonitorFSMRulePos; +extern int CurrentMonitorFSMRuleSection; diff --git a/src/monitor/pgautofailover--2.2--2.3.sql b/src/monitor/pgautofailover--2.2--2.3.sql index 7b858f5a1..871e6b83a 100644 --- a/src/monitor/pgautofailover--2.2--2.3.sql +++ b/src/monitor/pgautofailover--2.2--2.3.sql @@ -457,3 +457,235 @@ $$; grant execute on function pgautofailover.get_most_advanced_standby(text,int,bigint) to autoctl_node; + +-- +-- Expose the monitor's declarative dispatch table (MonitorFSM[] in +-- group_state_machine.c) to SQL: pgautofailover.dump_fsm()/dump_fsm_edges()/ +-- pgautofailover.fsm/check_fsm_reachability(), plus rule_pos/rule_section +-- attribution on pgautofailover.event. See pgautofailover.sql's own +-- comments on each object below for the full rationale -- unchanged here, +-- this is the same DDL, just applied incrementally to an existing 2.2 +-- install instead of as part of a fresh CREATE EXTENSION. +-- + +-- pgautofailover.fsm's section_path column (below) is cast to ltree (a +-- plain SQL cast through Postgres's own type-casting machinery, not a +-- C-level dependency -- see that view's own comment). Note there is +-- deliberately no "CREATE EXTENSION IF NOT EXISTS ltree" in this script: +-- ALTER EXTENSION ... UPDATE checks every "requires" entry in the *target* +-- version's control file is already installed before it ever runs the +-- upgrade script body for any step on the path, so a CREATE EXTENSION +-- placed here would never get a chance to run -- the ALTER EXTENSION +-- statement itself already fails first ("required extension \"ltree\" is +-- not installed"). The actual fix is client-side, in +-- monitor_extension_update() (monitor.c), which now creates "ltree" before +-- issuing the ALTER EXTENSION statement -- mirroring the exact same +-- pre-existing pattern that function already uses for btree_gist. + +-- Mirrors group_state_machine.h's MonitorFSMSection: which of the three +-- real control-flow regions of the monitor's declarative dispatch table +-- (MonitorFSM[] in group_state_machine.c) a rule belongs to. See dump_fsm() +-- and pgautofailover.fsm below, and the rule_section column on +-- pgautofailover.event. +CREATE TYPE pgautofailover.fsm_section + AS ENUM + ( + 'api_triggered', + 'early_checks', + 'reporting_node', + 'primary_node' + ); + +-- Which MonitorFSM[] row this event is attributed to (see +-- CurrentMonitorFSMRulePos in notifications.h for the mechanism) -- see +-- pgautofailover.sql's own comment on the pgautofailover.event table +-- definition for the full rationale of what is and isn't attributed. +ALTER TABLE pgautofailover.event + ADD COLUMN IF NOT EXISTS rule_pos int, + ADD COLUMN IF NOT EXISTS rule_section pgautofailover.fsm_section; + +-- Exposes the monitor's declarative dispatch table (MonitorFSM[] in +-- group_state_machine.c) to SQL, one row per rule, in first-match-wins +-- order -- see dump_fsm()'s own C-side comment for exactly what this does +-- and doesn't cover. pgautofailover.event.rule_pos/rule_section (added +-- above) join back to this view's pos column to show which rule produced +-- a given event. section is plain text here (not pgautofailover.fsm_section): +-- an api_triggered row's section names its specific operator-triggered +-- entry point too, e.g. "api_triggered: remove_node". +CREATE FUNCTION pgautofailover.dump_fsm() +RETURNS TABLE + ( + pos int, + section text, + comment text, + active_node_current_state text, + other_node_current_state text, + candidate_node_current_state text, + active_node_conditions text, + other_node_conditions text, + candidate_node_conditions text, + group_conditions text, + active_node_assigned_state pgautofailover.replication_state, + other_node_assigned_state pgautofailover.replication_state, + has_extra_action bool, + section_path text + ) +LANGUAGE C SECURITY DEFINER +AS 'MODULE_PATHNAME', $$dump_fsm$$; + +grant execute on function pgautofailover.dump_fsm() to autoctl_node; + +-- section_path is cast to ltree here (a plain SQL cast, going through +-- Postgres's own ordinary type-casting machinery) rather than in the C +-- function itself: dump_fsm() only ever builds the dotted text, this view +-- is the sole place pgautofailover takes a dependency on ltree. +CREATE VIEW pgautofailover.fsm AS + SELECT pos, + section, + comment, + active_node_current_state, + other_node_current_state, + candidate_node_current_state, + active_node_conditions, + other_node_conditions, + candidate_node_conditions, + group_conditions, + active_node_assigned_state, + other_node_assigned_state, + has_extra_action, + section_path::ltree AS section_path + FROM pgautofailover.dump_fsm() + ORDER BY pos; + +-- Flat, fully-resolved (pos, current_state, assigned_state) edges derived +-- from MonitorFSM[] -- see dump_fsm_edges()'s own C-side comment. Never +-- queried directly by an operator; check_fsm_reachability() below is built +-- on top of it. +CREATE FUNCTION pgautofailover.dump_fsm_edges() +RETURNS TABLE + ( + pos int, + current_state pgautofailover.replication_state, + assigned_state pgautofailover.replication_state + ) +LANGUAGE C SECURITY DEFINER +AS 'MODULE_PATHNAME', $$dump_fsm_edges$$; + +grant execute on function pgautofailover.dump_fsm_edges() to autoctl_node; + +-- Compares the monitor's own declarative dispatch table against a keeper's +-- KeeperFSM[] edges (serialized to JSON by KeeperFSMToJSON(), +-- src/bin/pg_autoctl/fsm.c, and sent here by "pg_autoctl inspect fsm check") +-- and returns every monitor edge with no matching keeper entry -- an empty +-- result means every transition the monitor can ever assign has somewhere +-- for the keeper to go. keeper_edges is expected to be a jsonb array of +-- {"current": ..., "assigned": ...} objects, one per KeeperFSMTransition +-- row. "current" can be the literal string "any" (KeeperFSMToJSON()'s own +-- sentinel for a row whose real .current is ANY_STATE): matched against +-- every e.current_state without ever casting it to +-- pgautofailover.replication_state, via a CASE (not "k.current = 'any' OR +-- k.current::...= e.current_state", which does not reliably guarantee the +-- cast is skipped once the left side matches -- CASE WHEN/THEN is the only +-- construct Postgres guarantees short-circuits). Any other current value, +-- and "assigned" always, still go through the enum cast unconditionally -- +-- a keeper reporting a state name this enum doesn't recognize still fails +-- loudly, with a real cast error, rather than silently never matching. +CREATE FUNCTION pgautofailover.check_fsm_reachability(keeper_edges jsonb) +RETURNS TABLE + ( + pos int, + current_state pgautofailover.replication_state, + assigned_state pgautofailover.replication_state, + comment text + ) +LANGUAGE sql +AS $$ + SELECT e.pos, e.current_state, e.assigned_state, f.comment + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE NOT EXISTS ( + SELECT 1 + FROM jsonb_to_recordset(keeper_edges) AS k(current text, assigned text) + WHERE k.assigned::pgautofailover.replication_state = e.assigned_state + AND CASE WHEN k.current = 'any' THEN true + ELSE k.current::pgautofailover.replication_state = e.current_state + END) + ORDER BY e.pos; +$$; + +grant execute on function pgautofailover.check_fsm_reachability(jsonb) to autoctl_node; + +-- Re-create last_events()'s three overloads to also select the new +-- rule_pos/rule_section columns added to pgautofailover.event above. +-- Return type is SETOF pgautofailover.event (the whole row type), which +-- already reflects the table's new columns automatically once ALTER TABLE +-- above has run -- only the function bodies' own column lists need +-- updating, via CREATE OR REPLACE (same signature, so no DROP needed). +CREATE OR REPLACE FUNCTION pgautofailover.last_events + ( + count int default 10 + ) +RETURNS SETOF pgautofailover.event LANGUAGE SQL STRICT +AS $$ +with last_events as +( + select eventid, eventtime, formationid, + nodeid, groupid, nodename, nodehost, nodeport, + reportedstate, goalstate, + reportedrepstate, reportedtli, reportedlsn, + candidatepriority, replicationquorum, description, + rule_pos, rule_section + from pgautofailover.event +order by eventid desc + limit count +) +select * from last_events order by eventtime, eventid; +$$; + +CREATE OR REPLACE FUNCTION pgautofailover.last_events + ( + formation_id text default 'default', + count int default 10 + ) +RETURNS SETOF pgautofailover.event LANGUAGE SQL STRICT +AS $$ +with last_events as +( + select eventid, eventtime, formationid, + nodeid, groupid, nodename, nodehost, nodeport, + reportedstate, goalstate, + reportedrepstate, reportedtli, reportedlsn, + candidatepriority, replicationquorum, description, + rule_pos, rule_section + from pgautofailover.event + where formationid = formation_id + order by eventid desc + limit count +) +select * from last_events order by eventtime, eventid; +$$; + +CREATE OR REPLACE FUNCTION pgautofailover.last_events + ( + formation_id text, + group_id int, + count int default 10 + ) +RETURNS SETOF pgautofailover.event LANGUAGE SQL STRICT +AS $$ +with last_events as +( + select eventid, eventtime, formationid, + nodeid, groupid, nodename, nodehost, nodeport, + reportedstate, goalstate, + reportedrepstate, reportedtli, reportedlsn, + candidatepriority, replicationquorum, description, + rule_pos, rule_section + from pgautofailover.event + where formationid = formation_id + and groupid = group_id + order by eventid desc + limit count +) +select * from last_events order by eventtime, eventid; +$$; diff --git a/src/monitor/pgautofailover.control b/src/monitor/pgautofailover.control index 081bc691b..6fcd33149 100644 --- a/src/monitor/pgautofailover.control +++ b/src/monitor/pgautofailover.control @@ -2,4 +2,4 @@ comment = 'pg_auto_failover' default_version = '2.3' module_pathname = '$libdir/pgautofailover' relocatable = false -requires = 'btree_gist' +requires = 'btree_gist, ltree' diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index b7494bf65..8545bf38c 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -176,6 +176,20 @@ CREATE TABLE pgautofailover.node -- we expect few rows and lots of UPDATE, let's benefit from HOT WITH (fillfactor = 25); +-- Mirrors group_state_machine.h's MonitorFSMSection: which of the three +-- real control-flow regions of the monitor's declarative dispatch table +-- (MonitorFSM[] in group_state_machine.c) a rule belongs to. See dump_fsm() +-- and pgautofailover.fsm below, and the rule_section column on +-- pgautofailover.event. +CREATE TYPE pgautofailover.fsm_section + AS ENUM + ( + 'api_triggered', + 'early_checks', + 'reporting_node', + 'primary_node' + ); + CREATE TABLE pgautofailover.event ( eventid bigserial not null, @@ -195,6 +209,32 @@ CREATE TABLE pgautofailover.event replicationquorum bool, description text, + -- Which MonitorFSM[] row this event is attributed to (see + -- CurrentMonitorFSMRulePos in notifications.h for the mechanism): set + -- for the whole duration of DispatchMonitorFSMRule's call to a row, + -- including everything that row's own extraAction runs -- which + -- covers BOTH operator-triggered SQL functions (dispatched via + -- ProceedGroupStateForApiTrigger, itself a DispatchMonitorFSMRule + -- call) and ProceedGroupStateForMSFailover's raw AssignGoalState + -- calls (only ever reached from inside an outer row's extraAction, + -- e.g. pos 305/363 -- see ActionRunMultiStandbyFailoverCascade/ + -- ActionRunPlainMSFailoverCascade). Neither is NULL: both get + -- attributed to that OUTER triggering row, not a row of their own, + -- since MS-failover's candidate-selection internals were never + -- decomposed into declarative rows (see this array's own comment on + -- BuildCandidateList/PromoteSelectedNode) -- this can be misleading + -- (an event from PromoteSelectedNode's LSN-driven promotion decision + -- shows up attributed to pos 305/363's own, quite different, + -- comment). Truly NULL only when the assignment happened from a code + -- path that never runs inside any DispatchMonitorFSMRule call at all + -- (e.g. perform_failover()'s own candidate-priority bookkeeping in + -- node_active_protocol.c, or a handful of other NotifyStateChange + -- call sites outside group_state_machine.c entirely). rule_pos is the + -- row's human-facing position (see dump_fsm()/pgautofailover.fsm), + -- not an array index. + rule_pos int, + rule_section pgautofailover.fsm_section, + PRIMARY KEY (eventid) ); @@ -229,6 +269,117 @@ CREATE TABLE pgautofailover.accepted_timeline PRIMARY KEY (formationid, groupid, decided_at) ); +-- Exposes the monitor's declarative dispatch table (MonitorFSM[] in +-- group_state_machine.c) to SQL, one row per rule, in first-match-wins +-- order -- see dump_fsm()'s own C-side comment for exactly what this does +-- and doesn't cover. pgautofailover.event.rule_pos/rule_section (added +-- above) join back to this view's pos column to show which rule produced +-- a given event. section is plain text here (not pgautofailover.fsm_section): +-- an api_triggered row's section names its specific operator-triggered +-- entry point too, e.g. "api_triggered: remove_node". +CREATE FUNCTION pgautofailover.dump_fsm() +RETURNS TABLE + ( + pos int, + section text, + comment text, + active_node_current_state text, + other_node_current_state text, + candidate_node_current_state text, + active_node_conditions text, + other_node_conditions text, + candidate_node_conditions text, + group_conditions text, + active_node_assigned_state pgautofailover.replication_state, + other_node_assigned_state pgautofailover.replication_state, + has_extra_action bool, + section_path text + ) +LANGUAGE C SECURITY DEFINER +AS 'MODULE_PATHNAME', $$dump_fsm$$; + +grant execute on function pgautofailover.dump_fsm() to autoctl_node; + +-- section_path is cast to ltree here (a plain SQL cast, going through +-- Postgres's own ordinary type-casting machinery) rather than in the C +-- function itself: dump_fsm() only ever builds the dotted text, this view +-- is the sole place pgautofailover takes a dependency on ltree. +CREATE VIEW pgautofailover.fsm AS + SELECT pos, + section, + comment, + active_node_current_state, + other_node_current_state, + candidate_node_current_state, + active_node_conditions, + other_node_conditions, + candidate_node_conditions, + group_conditions, + active_node_assigned_state, + other_node_assigned_state, + has_extra_action, + section_path::ltree AS section_path + FROM pgautofailover.dump_fsm() + ORDER BY pos; + +-- Flat, fully-resolved (pos, current_state, assigned_state) edges derived +-- from MonitorFSM[] -- see dump_fsm_edges()'s own C-side comment. Never +-- queried directly by an operator; check_fsm_reachability() below is built +-- on top of it. +CREATE FUNCTION pgautofailover.dump_fsm_edges() +RETURNS TABLE + ( + pos int, + current_state pgautofailover.replication_state, + assigned_state pgautofailover.replication_state + ) +LANGUAGE C SECURITY DEFINER +AS 'MODULE_PATHNAME', $$dump_fsm_edges$$; + +grant execute on function pgautofailover.dump_fsm_edges() to autoctl_node; + +-- Compares the monitor's own declarative dispatch table against a keeper's +-- KeeperFSM[] edges (serialized to JSON by KeeperFSMToJSON(), +-- src/bin/pg_autoctl/fsm.c, and sent here by "pg_autoctl inspect fsm check") +-- and returns every monitor edge with no matching keeper entry -- an empty +-- result means every transition the monitor can ever assign has somewhere +-- for the keeper to go. keeper_edges is expected to be a jsonb array of +-- {"current": ..., "assigned": ...} objects, one per KeeperFSMTransition +-- row. "current" can be the literal string "any" (KeeperFSMToJSON()'s own +-- sentinel for a row whose real .current is ANY_STATE): matched against +-- every e.current_state without ever casting it to +-- pgautofailover.replication_state, via a CASE (not "k.current = 'any' OR +-- k.current::...= e.current_state", which does not reliably guarantee the +-- cast is skipped once the left side matches -- CASE WHEN/THEN is the only +-- construct Postgres guarantees short-circuits). Any other current value, +-- and "assigned" always, still go through the enum cast unconditionally -- +-- a keeper reporting a state name this enum doesn't recognize still fails +-- loudly, with a real cast error, rather than silently never matching. +CREATE FUNCTION pgautofailover.check_fsm_reachability(keeper_edges jsonb) +RETURNS TABLE + ( + pos int, + current_state pgautofailover.replication_state, + assigned_state pgautofailover.replication_state, + comment text + ) +LANGUAGE sql +AS $$ + SELECT e.pos, e.current_state, e.assigned_state, f.comment + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE NOT EXISTS ( + SELECT 1 + FROM jsonb_to_recordset(keeper_edges) AS k(current text, assigned text) + WHERE k.assigned::pgautofailover.replication_state = e.assigned_state + AND CASE WHEN k.current = 'any' THEN true + ELSE k.current::pgautofailover.replication_state = e.current_state + END) + ORDER BY e.pos; +$$; + +grant execute on function pgautofailover.check_fsm_reachability(jsonb) to autoctl_node; + GRANT SELECT ON ALL TABLES IN SCHEMA pgautofailover TO autoctl_node; CREATE FUNCTION pgautofailover.set_node_system_identifier @@ -560,7 +711,8 @@ with last_events as nodeid, groupid, nodename, nodehost, nodeport, reportedstate, goalstate, reportedrepstate, reportedtli, reportedlsn, - candidatepriority, replicationquorum, description + candidatepriority, replicationquorum, description, + rule_pos, rule_section from pgautofailover.event order by eventid desc limit count @@ -587,7 +739,8 @@ with last_events as nodeid, groupid, nodename, nodehost, nodeport, reportedstate, goalstate, reportedrepstate, reportedtli, reportedlsn, - candidatepriority, replicationquorum, description + candidatepriority, replicationquorum, description, + rule_pos, rule_section from pgautofailover.event where formationid = formation_id order by eventid desc @@ -616,7 +769,8 @@ with last_events as nodeid, groupid, nodename, nodehost, nodeport, reportedstate, goalstate, reportedrepstate, reportedtli, reportedlsn, - candidatepriority, replicationquorum, description + candidatepriority, replicationquorum, description, + rule_pos, rule_section from pgautofailover.event where formationid = formation_id and groupid = group_id diff --git a/src/monitor/regress_schedule b/src/monitor/regress_schedule index c44e50f93..4d2ca7bdb 100644 --- a/src/monitor/regress_schedule +++ b/src/monitor/regress_schedule @@ -31,6 +31,9 @@ # exclusively. test: create_extension +test: fsm +test: check_fsm_reachability +test: keeper_fsm_edges test: monitor test: workers test: node_active_protocol @@ -38,9 +41,11 @@ test: guard_data_loss test: fast_forward test: drop_node test: stale_primary_report +test: candidate_count_gate test: lock_and_fetch_migration test: timeline_fork_detection test: failover_candidate_leaves_secondary +test: cluster_init_failover_rule_attribution test: dummy_update test: drop_extension test: upgrade diff --git a/src/monitor/sql/candidate_count_gate.sql b/src/monitor/sql/candidate_count_gate.sql new file mode 100644 index 000000000..a08ab7703 --- /dev/null +++ b/src/monitor/sql/candidate_count_gate.sql @@ -0,0 +1,202 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Regression test for ProceedGroupStateForMSFailover's candidateCount == 0 +-- gate (reporting_node.ms_failover.promotion_outcome.candidate_count_gate in +-- MonitorFSM[]): the window where the primary has gone unhealthy but NOT A +-- SINGLE standby has yet reported reaching report_lsn. +-- +-- Unlike the other two counting gates -- missingNodesCount (guard_data_loss.sql) +-- and quorumCandidateCount (stale_primary_report.sql), both named explicitly +-- in those files' own header comments -- no existing test exercises this one +-- by name. It has no dedicated log message either (the original hand-written +-- code silently `return`s false here, and the declarative row that now +-- matches this same condition carries no extraAction, matching that exactly) +-- so there is no pgautofailover.event row to check for it; the only +-- observable effect is what does NOT happen: neither standby should reach +-- prepare_promotion/fast_forward this round. +-- +-- guard_data_loss is set to false: with the default (true), the +-- missingNodesCount > 0 gate above this one in ProceedGroupStateForMSFailover +-- would itself decline and return before ever reaching this gate, since both +-- standbys are still SECONDARY/CATCHINGUP (each counted as missing, per +-- BuildCandidateList's own fan-out branch) at the moment this test polls +-- them. +-- +-- startup_grace_period is also lowered to 1, same as guard_data_loss.sql/ +-- fast_forward.sql/stale_primary_report.sql: NodeIsUnhealthy() only honors a +-- BAD health reading once at least this many seconds have passed since the +-- monitor process itself started (PgStartTime), to avoid spurious failovers +-- right after the monitor restarts. The default (10s) is longer than this +-- whole schedule takes to reach this test file when run automated, which +-- would silently make p never register as unhealthy and this test's own +-- ProceedGroupStateForMSFailover call never even fire. + +\x on + +-- ── formation and node registration ───────────────────────────────────────── + +SELECT pgautofailover.create_formation('ccg_test', 'pgsql', 'postgres', true, 1); + +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_p', 5432, + 'postgres', 'ccg_p', 1); + +SELECT nodeid AS np FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p' \gset + +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_s1', 5432, + 'postgres', 'ccg_s1', 1); + +SELECT nodeid AS ns1 FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_s1' \gset + +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_s2', 5432, + 'postgres', 'ccg_s2', 1); + +SELECT nodeid AS ns2 FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_s2' \gset + +-- ── bootstrap: drive the FSM to primary + secondary + secondary ───────────── +-- Same sequence as guard_data_loss.sql's own bootstrap. + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'single'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'wait_standby'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'single', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'wait_standby'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'wait_standby'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +-- p: primary (refresh to pick up second secondary) +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); + +-- Verify bootstrap: p=primary, s1=secondary, s2=secondary. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; + +-- ── manufacture: primary unhealthy, NEITHER standby has reported yet ──────── +-- +-- p goes unhealthy and is demoted to draining/draining (same manufactured +-- shape as guard_data_loss.sql/stale_primary_report.sql). s1/s2 are left +-- exactly as bootstrap left them -- secondary/secondary, neither has been +-- assigned report_lsn yet -- so candidateCount is 0 for both of them: no +-- standby has reached report_lsn, the exact window this gate covers. + +SET pgautofailover.startup_grace_period = 1; +SET pgautofailover.guard_data_loss TO false; + +UPDATE pgautofailover.node + SET health = 0, + healthchecktime = now(), + reporttime = now() - interval '60 seconds' + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p'; + +UPDATE pgautofailover.node + SET goalstate = 'draining', reportedstate = 'draining' + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p'; + +-- Verify the manufactured state before the test call. +SELECT nodename, goalstate, reportedstate, health + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; + +-- ── test: poll exactly one secondary, candidateCount == 0 for both ────────── +-- +-- s1 reports secondary/0-5000 again (nothing new from its own point of +-- view). This drives ProceedGroupState(s1) -> ActionRunMultiStandbyFailover +-- Cascade -> ProceedGroupStateForMSFailover, which runs BuildCandidateList +-- over the WHOLE group (not just s1): both s1 and s2 are still SECONDARY/ +-- CATCHINGUP, so BOTH get fanned out to report_lsn in this same call (the +-- fan-out rows, already covered elsewhere) -- but candidateCount is 0 (no +-- node's reportedState is report_lsn yet), so the candidate_count_gate row +-- matches and the function returns false: no candidate is selected. + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +-- s1 and s2 must both have been fanned out to report_lsn (goalstate), but +-- NEITHER may have reached prepare_promotion/fast_forward: the +-- candidate_count_gate declined before any candidate could be selected. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; + +RESET pgautofailover.guard_data_loss; +RESET pgautofailover.startup_grace_period; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('ccg_test', count => 100); diff --git a/src/monitor/sql/check_fsm_reachability.sql b/src/monitor/sql/check_fsm_reachability.sql new file mode 100644 index 000000000..b4668fa74 --- /dev/null +++ b/src/monitor/sql/check_fsm_reachability.sql @@ -0,0 +1,80 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Exercises the mechanics of pgautofailover.check_fsm_reachability(jsonb) +-- (and the pgautofailover.dump_fsm_edges() it's built on) against small, +-- synthetic keeper-edge inputs -- not the real KeeperFSM[] table, which +-- lives in the pg_autoctl binary, not this database. The real, end-to-end +-- completeness check (does the real KeeperFSM[] cover every real +-- MonitorFSM[] edge) is run separately, live, via +-- "pg_autoctl inspect fsm check" against a real monitor+keeper pair -- this +-- test only confirms the SQL-side comparison mechanism itself behaves +-- correctly: an edge present in the keeper_edges parameter drops out of the +-- mismatch list, an edge absent from it stays in, and an unrecognized state +-- name fails loudly rather than silently never matching. + +-- Every edge dump_fsm_edges() can produce, fully resolved: a fixed, +-- reviewable count for the current MonitorFSM[] -- changes only when a row +-- is added, removed, or edited there, same as fsm.sql's own row count. +SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); + +-- pos 301 ("converged secondary, reportedTLI not an ancestor of reference +-- -> catchingup", single edge) and pos 343 ("stop_replication, primary +-- converged prepare_maintenance -> wait_primary + maintenance", two edges, +-- one per role) are stable, non-reflexive, non-api_triggered rows -- good, +-- deterministic targets to check both single- and dual-edge rows against. +SELECT pos, current_state, assigned_state + FROM pgautofailover.dump_fsm_edges() + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + +-- Two categories of edges dump_fsm_edges() deliberately never produces, see +-- its own comment for why: reflexive (current == assigned) edges, and the +-- whole api_triggered section (every row there resolves activeNode to a +-- specific role via hand-written C before dispatch, so its own +-- NodeStatePattern was never meant to double as a full reachability +-- precondition). pos 403 ("all nodes async, zero secondaries -> +-- wait_primary") has both an ordinary edge (primary/apply_settings -> +-- wait_primary) and a reflexive one (wait_primary -> wait_primary) in its +-- own source pattern -- only the former should appear. pos 105 (api +-- triggered: perform_failover) should produce no edges at all. +SELECT pos, current_state, assigned_state + FROM pgautofailover.dump_fsm_edges() + WHERE pos = 403 + ORDER BY current_state; + +SELECT count(*) AS api_triggered_edge_count + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE f.section LIKE 'api_triggered%'; + +-- An empty keeper_edges: every single edge dump_fsm_edges() produces comes +-- back as a mismatch, so this count must equal total_edge_count above. +SELECT count(*) AS missing_with_empty_keeper_edges + FROM pgautofailover.check_fsm_reachability('[]'::jsonb); + +-- Providing exactly pos 301's own edge, plus one of pos 343's two edges +-- (leaving its other edge, prepare_maintenance->maintenance, still +-- unmatched): pos 301 must disappear entirely from the mismatch list, pos +-- 343 must still appear, but only once. +SELECT pos, current_state, assigned_state + FROM pgautofailover.check_fsm_reachability( + '[{"current":"secondary","assigned":"catchingup"}, + {"current":"stop_replication","assigned":"wait_primary"}]'::jsonb) + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + +-- Providing both of pos 343's edges too: it must now disappear as well. +SELECT pos, current_state, assigned_state + FROM pgautofailover.check_fsm_reachability( + '[{"current":"secondary","assigned":"catchingup"}, + {"current":"stop_replication","assigned":"wait_primary"}, + {"current":"prepare_maintenance","assigned":"maintenance"}]'::jsonb) + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + +-- An unrecognized state name in keeper_edges must fail loudly (a real cast +-- error), not silently never match -- the same "fail loudly on drift" +-- instinct as AssignDeclaredGoalState's own trust-check. +SELECT * FROM pgautofailover.check_fsm_reachability( + '[{"current":"not_a_real_state","assigned":"catchingup"}]'::jsonb); diff --git a/src/monitor/sql/cluster_init_failover_rule_attribution.sql b/src/monitor/sql/cluster_init_failover_rule_attribution.sql new file mode 100644 index 000000000..76d54302c --- /dev/null +++ b/src/monitor/sql/cluster_init_failover_rule_attribution.sql @@ -0,0 +1,126 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- End-to-end demonstration of the rule_pos/rule_section attribution +-- mechanism (notifications.c's CurrentMonitorFSMRulePos/RuleSection, +-- InsertEvent()): registers a two-node formation, drives it through the +-- heartbeat-only bootstrap to primary + secondary, triggers a manual +-- perform_failover(), and then joins pgautofailover.event against +-- pgautofailover.fsm on rule_pos = pos to show, for every state +-- transition the monitor produced, exactly which MonitorFSM[] row was +-- selected and executed -- both for the ordinary heartbeat-driven +-- bootstrap rows (api_triggered = false in this join, since rule_pos is +-- only set for rows actually reached through the declarative dispatch +-- table) and for the operator-triggered perform_failover call itself. + +\x on + +-- ── formation and node registration ───────────────────────────────────────── + +SELECT pgautofailover.create_formation('cifra_test', 'pgsql', 'postgres', true, 0); + +SELECT * + FROM pgautofailover.register_node('cifra_test', 'cifra_p', 5432, + 'postgres', 'cifra_p', 1); + +SELECT nodeid AS np FROM pgautofailover.node + WHERE formationid = 'cifra_test' AND nodename = 'cifra_p' \gset + +SELECT * + FROM pgautofailover.register_node('cifra_test', 'cifra_s', 5432, + 'postgres', 'cifra_s', 1); + +SELECT nodeid AS ns FROM pgautofailover.node + WHERE formationid = 'cifra_test' AND nodename = 'cifra_s' \gset + +-- ── bootstrap: drive the FSM to primary + secondary ───────────────────────── +-- +-- Mirrors drop_node.sql's bootstrap sequence (register -> single -> +-- wait_primary -> [standby: wait_standby -> catchingup -> secondary] -> +-- primary), including its "confirm" round-trips. + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'single'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'wait_standby'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'single', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'wait_standby'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +-- Verify bootstrap: p=primary, s=secondary. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'cifra_test' + ORDER BY nodename; + +-- ── manual failover ────────────────────────────────────────────────────── +-- +-- Two-node group: dispatches through MonitorFSM[]'s API_TRIGGERED section +-- (pos 105, "manual failover, 2-node group, primary+standby both converged +-- -> standby prepare_promotion, primary draining"), attributing both +-- resulting event rows to that one rule. + +SELECT pgautofailover.perform_failover('cifra_test', 0); + +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'cifra_test' + ORDER BY nodename; + +-- ── which rule fired for which event? ─────────────────────────────────────── +-- +-- Every event row this formation produced, joined against the FSM table on +-- rule_pos = pos: rule_pos/rule_section are NULL for the ordinary +-- heartbeat-driven bootstrap transitions above whenever the matched row +-- happens to be identified only by array position in earlier sessions' +-- tests -- here every one of them was reached through the same declarative +-- MonitorFSM[] dispatch table, so each carries its own attribution too. The +-- final two rows (both attributed to pos 105) are the perform_failover() +-- call's own dual assignment (standby -> prepare_promotion, primary -> +-- draining), selected and executed from the API_TRIGGERED section. + +SELECT e.eventid, e.nodename, e.reportedstate, e.goalstate, + e.rule_pos, e.rule_section, f.comment AS rule_comment + FROM pgautofailover.event e + LEFT JOIN pgautofailover.fsm f ON f.pos = e.rule_pos + WHERE e.formationid = 'cifra_test' + ORDER BY e.eventid; diff --git a/src/monitor/sql/drop_node.sql b/src/monitor/sql/drop_node.sql index 79c35754c..edc8195ad 100644 --- a/src/monitor/sql/drop_node.sql +++ b/src/monitor/sql/drop_node.sql @@ -171,3 +171,15 @@ SELECT nodename, goalstate, reportedstate FROM pgautofailover.node WHERE formationid = 'dn_test' ORDER BY nodename; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('dn_test', count => 100); diff --git a/src/monitor/sql/failover_candidate_leaves_secondary.sql b/src/monitor/sql/failover_candidate_leaves_secondary.sql index 8a587e4cd..9287d59f2 100644 --- a/src/monitor/sql/failover_candidate_leaves_secondary.sql +++ b/src/monitor/sql/failover_candidate_leaves_secondary.sql @@ -127,3 +127,15 @@ SELECT nodename, reportedstate, goalstate FROM pgautofailover.node WHERE formationid = 'fclma_test' ORDER BY nodename; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('fclma_test', count => 100); diff --git a/src/monitor/sql/fast_forward.sql b/src/monitor/sql/fast_forward.sql index cc3bda9c9..7c1ad5832 100644 --- a/src/monitor/sql/fast_forward.sql +++ b/src/monitor/sql/fast_forward.sql @@ -278,3 +278,15 @@ SELECT node_name, node_lsn, node_is_primary RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('ff_test', count => 100); diff --git a/src/monitor/sql/fsm.sql b/src/monitor/sql/fsm.sql new file mode 100644 index 000000000..782944dce --- /dev/null +++ b/src/monitor/sql/fsm.sql @@ -0,0 +1,24 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Plain dump of the monitor's declarative dispatch table, via the +-- pgautofailover.fsm view (pgautofailover.dump_fsm() ordered by pos). This +-- is a static, compile-time-fixed table -- unaffected by any node/formation +-- state -- so its expected output changes only when a row is added, +-- removed, or edited in MonitorFSM[] (group_state_machine.c), giving that +-- change an explicit, reviewable regression diff. +-- +-- \x on: with the *_conditions columns added, a plain tabular row is far +-- wider than a terminal (or this file's own diff-ability), and reads far +-- worse than one field-per-line. + +\x on + +SELECT pos, section, section_path, + active_node_current_state, other_node_current_state, candidate_node_current_state, + active_node_conditions, other_node_conditions, candidate_node_conditions, + group_conditions, + active_node_assigned_state, other_node_assigned_state, has_extra_action, + comment + FROM pgautofailover.fsm + ORDER BY pos; diff --git a/src/monitor/sql/guard_data_loss.sql b/src/monitor/sql/guard_data_loss.sql index 09066b844..e75ff0f8a 100644 --- a/src/monitor/sql/guard_data_loss.sql +++ b/src/monitor/sql/guard_data_loss.sql @@ -220,3 +220,15 @@ SELECT nodename, goalstate, reportedstate RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('gdl_test', count => 100); diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql new file mode 100644 index 000000000..276738592 --- /dev/null +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -0,0 +1,116 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Two-step check of the real keeper FSM (KeeperFSM[], src/bin/pg_autoctl/ +-- fsm.c) against the monitor's own MonitorFSM[] table, using the +-- keeper_fsm_edges.json fixture committed alongside this file. That fixture +-- is generated from the real KeeperFSM[] via +-- "pg_autoctl inspect fsm list --json" (KeeperFSMToJSON(), fsm.c) -- see +-- cli_do_fsm_list's own comment for how that command runs with zero setup +-- (no --pgdata, no live cluster) -- and must be regenerated by hand +-- whenever KeeperFSM[] changes; this test only reads the fixture, it never +-- runs pg_autoctl itself. +-- +-- Step 1: load the fixture client-side (psql's own backtick file +-- embedding, not server-side pg_read_file() -- the latter is +-- superuser/pg_read_server_files-gated and resolves relative paths against +-- $PGDATA, not this test's own directory) into a real table, one row per +-- distinct keeper edge, so this test's own expected/keeper_fsm_edges.out +-- shows the whole keeper FSM line by line, human-reviewable, with a diff on +-- every change to KeeperFSM[] -- the same discipline fsm.sql's own dump +-- already gives MonitorFSM[]. +-- +-- current_state is plain text, not pgautofailover.replication_state: a row +-- whose real KeeperFSM[] .current is ANY_STATE (state_matches()'s wildcard) +-- is serialized by KeeperFSMToJSON() as the literal string "any" (see its +-- own comment, fsm.c), which is not a legal enum value by design -- it's a +-- sentinel Step 2a/2b below match structurally, not a real reported state. +-- Every other value is still round-tripped through the enum (CASE ... ELSE +-- ... ::pgautofailover.replication_state ... END) so a typo'd or unrecognized +-- state name in the fixture still fails loudly here, same as +-- check_fsm_reachability()'s own cast does for the live-cluster path. +-- +-- DISTINCT: nothing in KeeperFSM[] guarantees two different rows can't +-- resolve to the exact same (current, assigned) pair, and the JSON has no +-- per-row provenance to tell such duplicates apart by, so they carry no +-- extra information here and would only clutter the reviewable list. +\set keeper_json `cat keeper_fsm_edges.json` + +CREATE TABLE keeper_fsm_edges AS +SELECT DISTINCT + CASE WHEN (edge ->> 'current') = 'any' + THEN 'any' + ELSE ((edge ->> 'current')::pgautofailover.replication_state)::text + END AS current_state, + (edge ->> 'assigned')::pgautofailover.replication_state AS assigned_state + FROM jsonb_array_elements(:'keeper_json'::jsonb) AS edge; + +SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; + +-- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() +-- edge the keeper_fsm_edges table above has no matching row for. A +-- non-empty result here is a real, actionable gap: the monitor can assign a +-- transition the keeper has no KeeperFSM[] row to perform. See +-- dump_fsm_edges()'s own comment (group_state_machine.c) for exactly what +-- it resolves and what it deliberately excludes (e.g. the api_triggered +-- section, resolved via hand-written C rather than a NodeStatePattern). +-- +-- k.current_state = 'any' matches every e.current_state -- a keeper row +-- covering every current state also covers this specific one. +-- +-- GROUPING SETS adds one summary row per rule (pos, assigned_state, comment) +-- -- current_state NULL, n = how many current_states that single +-- MonitorFSM[] rule fans out to -- alongside the ordinary per-current_state +-- detail rows, so a rule using a broad NodeStatePattern (matching many +-- states at once) is immediately visible as one big number instead of +-- having to count its own detail rows by hand. NULLS FIRST puts each rule's +-- summary row right before its own detail rows, as a header. +-- +-- Expected result: empty. Every MonitorFSM[] rule currently has a matching +-- KeeperFSM[] row for every current_state it can assign a transition from. +SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE NOT EXISTS ( + SELECT 1 + FROM keeper_fsm_edges k + WHERE k.assigned_state = e.assigned_state + AND (k.current_state = 'any' OR k.current_state = e.current_state::text) + ) + GROUP BY GROUPING SETS ( + (e.pos, e.assigned_state, f.comment, e.current_state), + (e.pos, e.assigned_state, f.comment) + ) + ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; + +-- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper +-- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the +-- full, fully-resolved edge set the monitor's table can reach, see its own +-- comment). A non-empty result here means the keeper is prepared to +-- transition through a (current, assigned) pair the monitor itself would +-- never assign -- either genuinely dead keeper code, or a real coverage +-- gap on the monitor side, same "investigate before assuming which" caveat +-- as step 2a's own comment. +-- +-- For a k.current_state = 'any' row, "matches" is existential: at least one +-- current_state for which the monitor assigns this same target is enough +-- to say the target is implementable at all, so a gap here means the +-- monitor can NEVER produce this assigned_state from ANY current state -- +-- e.g. "any -> dropped" (KeeperFSM[]'s two ANY_STATE -> DROPPED rows +-- collapse to this single row) is a standing, expected exception: the +-- monitor's own DROPPED assignment (remove_node(), pos 101/103) lives +-- entirely in the api_triggered section, which dump_fsm_edges() +-- deliberately excludes (see its own comment), so it can never appear +-- here. Same "investigate before assuming which" caveat as the rest of +-- this file applies to every other row below. +SELECT k.current_state, k.assigned_state + FROM keeper_fsm_edges k + WHERE NOT EXISTS ( + SELECT 1 + FROM pgautofailover.dump_fsm_edges() e + WHERE e.assigned_state = k.assigned_state + AND (k.current_state = 'any' OR k.current_state = e.current_state::text) + ) + ORDER BY k.current_state, k.assigned_state; + +DROP TABLE keeper_fsm_edges; diff --git a/src/monitor/sql/lock_and_fetch_migration.sql b/src/monitor/sql/lock_and_fetch_migration.sql index d0d8242d5..e9dccb5cc 100644 --- a/src/monitor/sql/lock_and_fetch_migration.sql +++ b/src/monitor/sql/lock_and_fetch_migration.sql @@ -172,3 +172,15 @@ SELECT nodename, goalstate, reportedstate FROM pgautofailover.node WHERE formationid = 'lafm_test' ORDER BY nodename; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('lafm_test', count => 100); diff --git a/src/monitor/sql/monitor.sql b/src/monitor/sql/monitor.sql index b8c0a0685..aa2099916 100644 --- a/src/monitor/sql/monitor.sql +++ b/src/monitor/sql/monitor.sql @@ -79,3 +79,20 @@ select * -- should fail as there's no primary at this point select pgautofailover.perform_failover(); + +-- last_events() (all three overloads) returns SETOF pgautofailover.event, so +-- its own SELECT list must match that composite type's full column set -- +-- including rule_pos/rule_section -- or the call errors at parse time +-- ("Final statement returns too few columns") before ever running. Not +-- exercised anywhere else in this test suite, so a regression here (e.g. a +-- future column added to pgautofailover.event without updating these three +-- function bodies) would otherwise go unnoticed until a live +-- "pg_autoctl show events"/"pg_autoctl watch" call broke in production. +select count(*) >= 0 as last_events_count_ok + from pgautofailover.last_events(10); + +select count(*) >= 0 as last_events_by_formation_count_ok + from pgautofailover.last_events(formation_id => 'default', count => 10); + +select count(*) >= 0 as last_events_by_formation_and_group_count_ok + from pgautofailover.last_events('default', 0, 10); diff --git a/src/monitor/sql/node_active_protocol.sql b/src/monitor/sql/node_active_protocol.sql index 355435fda..3f7fa3c95 100644 --- a/src/monitor/sql/node_active_protocol.sql +++ b/src/monitor/sql/node_active_protocol.sql @@ -440,3 +440,21 @@ SELECT pgautofailover.report_postgres_version(NULL, 170003); -- unknown node_id: silent no-op, not an error SELECT pgautofailover.report_postgres_version(-1, 170003); + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events, for each of the two formations used +-- above. Exercises pgautofailover.last_events() against a real scenario -- +-- its own SELECT list didn't match pgautofailover.event's column set for a +-- long time, breaking it outright, and nothing in this suite ever called +-- it to notice (see monitor.sql's own minimal-repro coverage). Filtering +-- by formationid isolates each summary from the other, and from every +-- other test in this schedule sharing the same event table -- safe +-- regardless of where in this file (or the whole schedule) it runs. +-- eventid/eventtime omitted: eventid is a database-wide sequence shared by +-- every test in this schedule (see regress_schedule's own comment) and +-- eventtime is a live timestamp -- neither is a stable value to pin here. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('fsm_test', count => 100); + +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('killed_test', count => 100); diff --git a/src/monitor/sql/stale_primary_report.sql b/src/monitor/sql/stale_primary_report.sql index 6db9732c4..45183caa6 100644 --- a/src/monitor/sql/stale_primary_report.sql +++ b/src/monitor/sql/stale_primary_report.sql @@ -208,3 +208,15 @@ SELECT nodename, goalstate, reportedstate FROM pgautofailover.node WHERE formationid = 'spr_test' ORDER BY nodename; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('spr_test', count => 100); diff --git a/src/monitor/sql/timeline_fork_detection.sql b/src/monitor/sql/timeline_fork_detection.sql index 67b987789..c605b98de 100644 --- a/src/monitor/sql/timeline_fork_detection.sql +++ b/src/monitor/sql/timeline_fork_detection.sql @@ -317,3 +317,18 @@ SELECT formationid, groupid, accepted_tli, resolved_at IS NOT NULL AS resolved RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events, for each of the two formations used +-- above. Exercises pgautofailover.last_events() against a real scenario -- +-- its own SELECT list didn't match pgautofailover.event's column set for a +-- long time, breaking it outright, and nothing in this suite ever called +-- it to notice (see monitor.sql's own minimal-repro coverage). eventid/ +-- eventtime omitted: eventid is a database-wide sequence shared by every +-- test in this schedule (see regress_schedule's own comment) and +-- eventtime is a live timestamp -- neither is a stable value to pin here. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_unit', count => 100); + +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_election', count => 100); diff --git a/src/monitor/sql/workers.sql b/src/monitor/sql/workers.sql index d251168fc..5d44aa667 100644 --- a/src/monitor/sql/workers.sql +++ b/src/monitor/sql/workers.sql @@ -30,3 +30,15 @@ select * dbname => 'citus', desired_group_id => 1, node_kind => 'worker'); + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +select reportedstate, goalstate, rule_pos, rule_section, description + from pgautofailover.last_events('citus', count => 100); diff --git a/tests/tap/schedule b/tests/tap/schedule index 417560ee0..3b29df16a 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -31,6 +31,20 @@ fast_forward demote_timeout_wait_primary_deadlock wait_primary_draining_deadlock timeline_fork_report_lsn_deadlock +keeper_fsm_gap_209_wait_maintenance +keeper_fsm_gap_211_wait_maintenance +keeper_fsm_gap_209_wait_standby +keeper_fsm_gap_211_wait_standby +keeper_fsm_gap_211_primary_priority_zero +keeper_fsm_gap_new_node_joins_report_lsn_group +keeper_fsm_gap_candidate_fast_forward_left_alone +keeper_fsm_gap_priority_zero_fast_forward_left_alone +keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff +keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion +keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion +keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff +keeper_fsm_gap_stop_replication_report_lsn_priority +keeper_fsm_gap_stop_replication_report_lsn_new_node extension_update tablespaces installcheck diff --git a/tests/tap/schedules/node-fsm-gaps.sch b/tests/tap/schedules/node-fsm-gaps.sch new file mode 100644 index 000000000..2dc601930 --- /dev/null +++ b/tests/tap/schedules/node-fsm-gaps.sch @@ -0,0 +1,23 @@ +# Keeper/monitor FSM edge-gap coverage, split out of node.sch (~14 min): +# node.sch's own comment already measured ~30 min for its original 14 specs +# against the CI step's 20-minute timeout before these were ever added, and +# adding these on top pushed every PG version over the limit (CI run +# 83400372049: all 6 node schedule jobs timed out at 20 minutes, stalling +# around spec #20/28 -- a cumulative time-budget overrun, not a single +# stuck test). Run on PG17 only, same rationale as multi-alternate/ +# multi-misc/multi-async/citus-1/citus-2 in ci.yml: this is FSM logic, not +# version-specific code paths. +keeper_fsm_gap_209_wait_maintenance +keeper_fsm_gap_211_wait_maintenance +keeper_fsm_gap_209_wait_standby +keeper_fsm_gap_211_wait_standby +keeper_fsm_gap_211_primary_priority_zero +keeper_fsm_gap_new_node_joins_report_lsn_group +keeper_fsm_gap_candidate_fast_forward_left_alone +keeper_fsm_gap_priority_zero_fast_forward_left_alone +keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff +keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion +keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion +keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff +keeper_fsm_gap_stop_replication_report_lsn_priority +keeper_fsm_gap_stop_replication_report_lsn_new_node diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index 84ce20484..1c9ed4fef 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -1,6 +1,9 @@ # Node lifecycle, monitor operations, and Debian/tablespace layouts (~30 min). # Merged from former node, monitor, and node-extra schedules to reduce CI job -# count and GitHub Actions runner queue pressure. +# count and GitHub Actions runner queue pressure. The keeper/monitor FSM +# edge-gap specs that used to live here were split out to node-fsm-gaps.sch +# (PG17-only) once this schedule's own combined runtime started timing out +# the CI step on every PG version -- see that file's own header comment. create_standby_with_pgdata launch_deferred_set_metadata fsm_step_report_advance diff --git a/tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf b/tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf new file mode 100644 index 000000000..a2af9b69d --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf @@ -0,0 +1,87 @@ +# Regression spec for MonitorFSM[] pos 209 ("alone in group, +# candidate-eligible -> single", src/monitor/group_state_machine.c). +# +# This spec originally documented a real gap: pos 209's NodeStatePattern is +# willing to assign SINGLE_STATE to a lone remaining node reporting +# WAIT_MAINTENANCE_STATE as its current_state, but KeeperFSM[] +# (src/bin/pg_autoctl/fsm.c) had no WAIT_MAINTENANCE_STATE -> SINGLE_STATE +# row at all -- node2 would get stuck reporting wait_maintenance forever, +# logging "pg_autoctl does not know how to reach state \"single\" from +# \"wait_maintenance\"". +# +# Fixed by adding that row to KeeperFSM[], reusing fsm_promote_standby (the +# same function already reused by every other converged-standby source +# state: SECONDARY/CATCHINGUP/PREP_PROMOTION/STOP_REPLICATION/REPORT_LSN) -- +# entering WAIT_MAINTENANCE_STATE itself runs no transition function (a +# converged standby just marks intent to go to maintenance, Postgres keeps +# running and replicating normally), so promoting it exactly like any other +# actively-replicating standby is safe. +# +# This spec now asserts that fixed behavior: +# +# 1. node1 (primary) is network-disconnected, so it can never acknowledge +# a replication-quorum change. +# 2. node2 (the only secondary, hence the last quorum member) is told to +# enter maintenance. Per service_keeper.c's own comment, a secondary +# that is the last quorum member goes SECONDARY -> WAIT_MAINTENANCE +# first, waiting for the primary to drop it from synchronous_standby_ +# names before proceeding to MAINTENANCE -- since node1 can never see +# or acknowledge that request, node2 is durably stuck at +# WAIT_MAINTENANCE_STATE, not a narrow timing window. +# 3. node1 is dropped directly via pgautofailover.remove_node(), bypassing +# node1 entirely (it doesn't need to cooperate for the monitor's own +# catalog to drop it) -- leaving node2 alone in the group, still +# reporting wait_maintenance. +# 4. The monitor recomputes: groupHasExactlyOneNode is now true, node2's +# own reported current_state is "wait_maintenance", and its candidate +# priority is > 0, so pos 209 fires and the monitor assigns SINGLE. +# 5. node2's keeper now has a matching KeeperFSM[] row and actually +# converges to single. + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_001_disconnect_primary { + network disconnect node1 +} + +step test_002_secondary_stuck_at_wait_maintenance { + # pg_autoctl enable maintenance itself polls for convergence and exits + # non-zero when it can't reach "maintenance" quickly (the primary is + # unreachable, exactly the point of this scenario) -- the real state + # transition still happens regardless of this command's own exit code, + # confirmed by the wait below. + exec-fails node2 pg_autoctl enable maintenance + wait until node2 state is wait_maintenance timeout 60s +} + +step test_003_drop_primary_converges_to_single { + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + wait until node2 assigned-state = single timeout 60s + wait until node2 state is single timeout 60s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { single } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { single } +} diff --git a/tests/tap/specs/keeper_fsm_gap_209_wait_standby.pgaf b/tests/tap/specs/keeper_fsm_gap_209_wait_standby.pgaf new file mode 100644 index 000000000..5a9b2d331 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_209_wait_standby.pgaf @@ -0,0 +1,84 @@ +# Regression spec for MonitorFSM[] pos 209 ("alone in group, +# candidate-eligible -> single", src/monitor/group_state_machine.c), for its +# wait_standby current_state. +# +# This spec originally set out to reproduce the same kind of gap as +# keeper_fsm_gap_209_wait_maintenance.pgaf: pos 209's NodeStatePattern was +# willing to assign SINGLE_STATE to a lone remaining node reporting +# WAIT_STANDBY_STATE, but KeeperFSM[] had no matching row. Reproducing it +# live (network-disconnect node1 before node2 ever streams, then drop node1) +# confirmed the gap is real -- but also surfaced something worse than a +# missing keeper transition: a node stuck at WAIT_STANDBY_STATE has never +# actually started streaming (no pg_basebackup done, see fsm_init_standby, +# the WAIT_STANDBY_STATE -> CATCHINGUP_STATE transition, fsm.c), so there is +# no safe way to reach SINGLE_STATE from it at all. The only local action +# available (fsm_init_primary, the same one used for INIT_STATE/ +# DROPPED_STATE -> SINGLE_STATE) does a *fresh* initdb, generating a new +# system_identifier -- which conflicts with the one this node already +# registered under, tripping the monitor's own same_system_identifier_ +# within_group exclusion constraint (confirmed live: "Failed to transition +# from state \"wait_standby\" to state \"single\""). +# +# Rather than force this through on the keeper side, pos 209 was narrowed +# instead (the reportedIsWaitStandby field, group_state_machine.c) to stop +# assigning SINGLE to a wait_standby node at all. +# +# Note this does NOT mean node2's own goalstate stays "wait_standby" +# forever: pos 101 (remove_node()'s own fan-out, a separate, unconditional +# row -- ".otherNodeAssignedState = GOAL(REPORT_LSN)" for "every surviving +# non-maintenance standby") still reassigns it to report_lsn as a direct, +# synchronous side effect of dropping node1, regardless of pos 209/211. That +# reassignment is harmless here: KeeperFSM[] has no WAIT_STANDBY_STATE -> +# REPORT_LSN_STATE row either, so node2's keeper simply stays parked (its +# own reportedstate never leaves wait_standby) instead of attempting +# anything -- the actual safety property this spec cares about. An earlier +# version of this spec asserted goalstate also stays at wait_standby, which +# is not what actually happens and isn't the real invariant to check. +# +# 1. node2 is created with launch deferred, then started with node1 +# already network-disconnected -- it registers (reaching +# WAIT_STANDBY_STATE) but can never reach node1 to fetch its initial +# pg_basebackup, so it's durably stuck reporting wait_standby, +# unhealthy -- not a narrow timing window. +# 2. node1 is dropped directly via pgautofailover.remove_node(), leaving +# node2 alone in the group, still reporting wait_standby, with default +# (nonzero) candidate-priority. +# 3. The monitor recomputes: groupHasExactlyOneNode is now true, but pos +# 209 no longer matches wait_standby at all, so it never assigns +# SINGLE. node2's reportedstate stays wait_standby -- it never attempts +# (and never risks) an unsafe fresh-init transition. + +cluster { + monitor + formation { + node1 + node2 create and launch deferred + } +} + +setup { + wait until node1 state is single timeout 60s +} + +teardown { + compose down +} + +step test_001_start_node2_isolated_from_node1 { + network disconnect node1 + exec node2 pg_autoctl node start + wait until node2 state is wait_standby timeout 60s +} + +step test_002_drop_node1_leaves_node2_reportedstate_untouched { + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + sleep 10s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { wait_standby } + logs node2 contains "Still waiting for the monitor to drive us to state" +} diff --git a/tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf b/tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf new file mode 100644 index 000000000..99e7e3132 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf @@ -0,0 +1,90 @@ +# Regression spec for MonitorFSM[] pos 210/211 ("alone in group, +# candidatePriority zero", src/monitor/group_state_machine.c). +# +# This spec originally documented a real gap: pos 211 ("alone in group, +# candidatePriority zero -> report_lsn") fired even when the lone remaining +# node was already serving as primary, and KeeperFSM[] (src/bin/pg_autoctl/ +# fsm.c) has no PRIMARY_STATE -> REPORT_LSN_STATE row -- so the node got +# stuck reporting "primary" forever, still accepting writes, while the +# monitor believed it had been demoted. +# +# The fix is on the monitor side, not the keeper side: demoting the +# cluster's only remaining writable node to report_lsn once every other +# node is gone serves no purpose (there is no alternative candidate left to +# promote instead), so pos 210 now assigns SINGLE instead, exactly as a +# candidate-eligible lone primary already gets via pos 209. +# +# This spec now asserts that fixed behavior: +# +# 1. node2 is promoted to primary, then its own candidate-priority is set +# to 0. +# 2. node1 (now the only secondary) is dropped, leaving node2 alone in the +# group -- still reporting PRIMARY_STATE, since nothing prompts it to +# change role on its own. +# 3. The monitor recomputes: groupHasExactlyOneNode is now true, node2 is +# still primary-role (reportedState in {primary, wait_primary, +# join_primary, apply_settings}), and its candidate priority is 0 (not +# eligible) -- pos 210 fires and the monitor assigns SINGLE. +# 4. node2's keeper has an ordinary PRIMARY_STATE -> SINGLE_STATE +# transition (KeeperFSM[]), so it actually converges: reportedstate +# becomes "single", matching the assigned goal. +# +# pos 210's own condition is gated on node2's *reported* state alone (not +# IsInPrimaryState(), which also requires goalState to already agree) -- +# an earlier version of this row used isInPrimaryState=true and, live in +# this exact scenario, self-undermined one dispatch after firing: assigning +# SINGLE changed goalState away from what IsInPrimaryState() requires, so +# the row stopped matching on the very next dispatch and pos 211 (no such +# requirement) fired right behind it, overwriting the assignment back to +# REPORT_LSN. This spec is what caught that oscillation live. + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_001_promote_node2_and_zero_its_priority { + promote node2 + wait until node2 state is primary + and node1 state is secondary + timeout 90s + exec node2 pg_autoctl set node candidate-priority 0 +} + +step test_002_drop_secondary_leaves_lone_priority_zero_primary { + exec node1 pg_autoctl drop node --no-wait + wait until node1 stopped timeout 60s + # The monitor now sees exactly one node (node2) in the group, still + # reporting "primary", with candidate-priority 0 -- pos 210 must fire + # and assign "single" (not pos 211's "report_lsn": that would strand a + # node with no alternative candidate left to hand off to). + wait until node2 assigned-state = single timeout 60s +} + +step test_003_node2_converges_to_single { + # node2 has an ordinary PRIMARY_STATE -> SINGLE_STATE transition + # (KeeperFSM[]), so unlike the original report_lsn assignment, this one + # is actually reachable -- confirm it converges rather than getting + # stuck reporting "primary" forever. + wait until node2 state is single timeout 60s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { single } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { single } +} diff --git a/tests/tap/specs/keeper_fsm_gap_211_wait_maintenance.pgaf b/tests/tap/specs/keeper_fsm_gap_211_wait_maintenance.pgaf new file mode 100644 index 000000000..558d1d404 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_211_wait_maintenance.pgaf @@ -0,0 +1,86 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, +# candidatePriority zero -> report_lsn", src/monitor/group_state_machine.c), +# for its wait_maintenance current_state -- a separate gap from +# keeper_fsm_gap_209_wait_maintenance.pgaf's own scenario (that one is pos +# 209's candidate-eligible sibling, assigning single instead of report_lsn). +# +# This spec originally documented a real gap: pos 211's NodeStatePattern is +# willing to assign REPORT_LSN_STATE to a lone remaining node reporting +# WAIT_MAINTENANCE_STATE as its current_state, but KeeperFSM[] +# (src/bin/pg_autoctl/fsm.c) had no WAIT_MAINTENANCE_STATE -> REPORT_LSN_STATE +# row at all -- node2 would get stuck reporting wait_maintenance forever. +# +# Fixed by adding that row to KeeperFSM[], reusing fsm_report_lsn (the same +# function already reused by SECONDARY/CATCHINGUP/MAINTENANCE/ +# PREPARE_MAINTENANCE_STATE -> REPORT_LSN_STATE) -- entering +# WAIT_MAINTENANCE_STATE itself runs no transition function (a converged +# standby just marks intent to go to maintenance, Postgres keeps running and +# replicating normally), so it's safe to treat exactly like those other +# actively-replicating standby states. +# +# Scenario: identical to keeper_fsm_gap_209_wait_maintenance.pgaf's own, +# except node2's candidate-priority is set to 0 first, so pos 211 (not pos +# 209) is the one that fires once node2 is alone: +# +# 1. node2's candidate-priority is set to 0. +# 2. node1 (primary) is network-disconnected, so it can never acknowledge +# a replication-quorum change. +# 3. node2 (the only secondary, hence the last quorum member) is told to +# enter maintenance, and gets durably stuck at WAIT_MAINTENANCE_STATE +# for the same reason as the pos 209 spec (node1 can never see or +# acknowledge the request). +# 4. node1 is dropped directly via pgautofailover.remove_node(), leaving +# node2 alone in the group, still reporting wait_maintenance, with +# candidate-priority 0 -- pos 211 fires and the monitor assigns +# REPORT_LSN. +# 5. node2's keeper now has a matching KeeperFSM[] row and actually +# converges to report_lsn. + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_001_zero_priority_and_disconnect_primary { + exec node2 pg_autoctl set node candidate-priority 0 + network disconnect node1 +} + +step test_002_secondary_stuck_at_wait_maintenance { + # pg_autoctl enable maintenance itself polls for convergence and exits + # non-zero when it can't reach "maintenance" quickly (the primary is + # unreachable, exactly the point of this scenario) -- the real state + # transition still happens regardless of this command's own exit code, + # confirmed by the wait below. + exec-fails node2 pg_autoctl enable maintenance + wait until node2 state is wait_maintenance timeout 60s +} + +step test_003_drop_primary_converges_to_report_lsn { + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + wait until node2 assigned-state = report_lsn timeout 60s + wait until node2 state is report_lsn timeout 60s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { report_lsn } +} diff --git a/tests/tap/specs/keeper_fsm_gap_211_wait_standby.pgaf b/tests/tap/specs/keeper_fsm_gap_211_wait_standby.pgaf new file mode 100644 index 000000000..005c0b751 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_211_wait_standby.pgaf @@ -0,0 +1,78 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, +# candidatePriority zero -> report_lsn", src/monitor/group_state_machine.c), +# for its wait_standby current_state -- the candidatePriority=0 sibling of +# keeper_fsm_gap_209_wait_standby.pgaf's own scenario. See that spec's header +# for the full story: a node stuck at WAIT_STANDBY_STATE never actually +# started streaming, so there is no safe way to reach REPORT_LSN_STATE (or +# SINGLE_STATE) from it without risking a system_identifier conflict with +# its own prior registration. +# +# pos 211 was narrowed the same way pos 209 was (the reportedIsWaitStandby +# field, group_state_machine.c) -- this spec asserts the same safe, +# conservative behavior for the candidatePriority=0 case. +# +# Note this does NOT mean node2's own goalstate stays "wait_standby" +# forever: pos 101 (remove_node()'s own fan-out, a separate, unconditional +# row -- ".otherNodeAssignedState = GOAL(REPORT_LSN)" for "every surviving +# non-maintenance standby") still reassigns it to report_lsn as a direct, +# synchronous side effect of dropping node1, regardless of pos 209/211 -- +# happens to be the very state pos 211 would itself assign anyway, so +# there's nothing to "correct" it. That reassignment is harmless here: +# KeeperFSM[] has no WAIT_STANDBY_STATE -> REPORT_LSN_STATE row either, so +# node2's keeper simply stays parked (its own reportedstate never leaves +# wait_standby) instead of attempting anything -- the actual safety +# property this spec cares about. +# +# 1. node2 is created with launch deferred, then started with node1 +# already network-disconnected -- it registers (reaching +# WAIT_STANDBY_STATE) but can never reach node1 to fetch its initial +# pg_basebackup, so it's durably stuck reporting wait_standby, +# unhealthy. +# 2. node2's candidate-priority is set to 0 (registration itself must +# complete first for this CLI call to have a monitor connection to +# use, even though the node is otherwise stuck), so pos 211 (not pos +# 209) is the one that would otherwise apply once node2 is alone. +# 3. node1 is dropped directly via pgautofailover.remove_node(), leaving +# node2 alone in the group, still reporting wait_standby, with +# candidate-priority 0. +# 4. The monitor recomputes: groupHasExactlyOneNode is now true, but pos +# 211 no longer matches wait_standby at all, so it never (re)assigns +# REPORT_LSN on its own account. node2's reportedstate stays +# wait_standby -- it never attempts (and never risks) an unsafe +# fresh-init transition. + +cluster { + monitor + formation { + node1 + node2 create and launch deferred + } +} + +setup { + wait until node1 state is single timeout 60s +} + +teardown { + compose down +} + +step test_001_start_node2_isolated_and_zero_priority { + network disconnect node1 + exec node2 pg_autoctl node start + wait until node2 state is wait_standby timeout 60s + exec node2 pg_autoctl set node candidate-priority 0 +} + +step test_002_drop_node1_leaves_node2_reportedstate_untouched { + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + sleep 10s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { wait_standby } + logs node2 contains "Still waiting for the monitor to drive us to state" +} diff --git a/tests/tap/specs/keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf b/tests/tap/specs/keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf new file mode 100644 index 000000000..53782d1e2 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf @@ -0,0 +1,221 @@ +# Regression spec for MonitorFSM[] pos 209 ("alone in group, +# candidate-eligible -> single", src/monitor/group_state_machine.c), for its +# fast_forward current_state. +# +# Until this session, KeeperFSM[] (src/bin/pg_autoctl/fsm.c) had no +# FAST_FORWARD_STATE -> SINGLE_STATE row at all: a node that ends up alone +# in its group while genuinely reporting "fast_forward" (its WAL-source peer +# and the old primary both gone) would get assigned SINGLE by pos 209 (whose +# own NodeStatePattern, FSM_NOT_STABLE_SINGLE, is broad enough to match a +# node reporting fast_forward) but have no way to reach it -- stuck logging +# "pg_autoctl does not know how to reach state \"single\" from +# \"fast_forward\"". +# +# Fixed by adding that row to KeeperFSM[], reusing fsm_promote_standby (the +# same function already reused by every other converged-standby source +# state: SECONDARY/CATCHINGUP/PREP_PROMOTION/STOP_REPLICATION/REPORT_LSN/ +# WAIT_MAINTENANCE) -- fast_forward only means "Postgres is running as a +# caught-up-enough standby", so promoting it directly is exactly as safe as +# promoting any of those other states. +# +# A real fast_forward assignment only happens mid MS-failover, when the +# monitor's selected candidate is behind the most advanced available node +# and must fetch the missing WAL before it can safely promote (pos 377, +# ProceedGroupStateForMSFailover / PromoteSelectedNode). This spec +# reproduces that genuinely, using the same network-disconnect + +# INSERT + CHECKPOINT recipe already established by +# tests/tap/specs/multi_ifdown.pgaf and +# tests/tap/specs/debug_citus_worker_fast_forward.pgaf. +# +# First draft of this spec drove node3 as an ordinary (autonomous) node and +# tried to catch the narrow fast_forward *assigned*-state window by external +# polling before removing node1/node2 -- confirmed live (CI run 30759600065) +# that this is still genuinely racy: with a WAL gap this small (5000 rows), +# node3's own local fetch completes and reports back to the monitor within +# about 2 seconds of the assignment landing, and the monitor's own cascade +# (node2, still alive, keeps re-triggering evaluation of node3's candidate +# status) advances assigned-state on to prepare_promotion before the +# external "wait until" poll can reliably observe fast_forward in between -- +# even though it polls assigned-state, not reported-state. +# +# Fixed for real this time by declaring node3 "suspended" (step mode, +# PG_AUTOCTL_SUSPENDED -- src/bin/pg_autoctl/service_keeper.c): its +# node-active service never ticks on its own, so nothing it reports to the +# monitor ever changes except in direct response to an explicit "fsm step +# node3" command. This removes the race entirely, for a reason confirmed +# directly against FSM_REPORT_LSN_OR_FAST_FORWARD's own definition +# (src/monitor/group_state_machine.c): the monitor's cascade past +# fast_forward requires reportedState == goalState == FAST_FORWARD (a +# NODE_STATE_STABLE match) -- as long as node3's own reportedState column +# never updates (because node3 itself never contacts the monitor), the +# *assignment* of fast_forward can still land purely from node2's own +# regular ticking (which is what selects node3 as the candidate and points +# it at node2 as WAL source in the first place), but the cascade *past* it +# structurally cannot fire, however long node2 keeps re-triggering +# evaluation -- there is no more race window to lose. +# +# 1. 3-node formation (node1 primary, node2 + node3 secondaries). node3 is +# declared suspended -- it must not be the first node in the +# formation (compose_gen only attaches a healthcheck to the first data +# node so that later nodes can depend_on it being healthy, and a +# suspended node can never satisfy that healthcheck on its own, since +# nothing restarts Postgres after the one-shot node-init phase until +# the first explicit "fsm step"). node2's candidate-priority is set to +# 0 -- it stays a synchronous, always-caught-up standby that is never +# itself selected as a failover candidate, existing purely to be +# node3's WAL-fetch source later. node3 keeps its default (nonzero) +# candidate-priority -- pos 209's own candidate-eligible case. +# 2. node3 is network-disconnected, then node1 (still primary) receives +# real write traffic + a CHECKPOINT -- node2 (still connected, still +# synchronous) replicates and catches up to the new LSN; node3, cut +# off, is now genuinely behind. +# 3. node1 is network-disconnected (killing the primary) and node3 is +# reconnected in the same step -- this forces an MS-failover election. +# node2 (candidate-priority 0) is excluded from candidacy entirely +# (GroupListCandidates), so node3 is the only candidate -- but the +# monitor can't yet tell it needs a fetch: that decision depends on +# comparing node3's own current LSN against node2's, and the monitor +# has no fresh LSN for node3 until node3 itself makes contact. So the +# first thing that lands, driven purely by node2's own regular +# ticking, is the ordinary standby fan-out: goalState = REPORT_LSN. +# This can be polled with a comfortable timeout and no race at all -- +# node3 itself (suspended) has made no contact yet, so nothing moves +# it off this goal until the explicit steps below. +# 4. A first explicit "fsm step node3" reports node3's own still-stale +# "secondary" (unreported since before this whole scenario started) +# and, in the same call, performs the local secondary -> report_lsn +# transition -- recording node3's own genuinely-behind LSN, still +# unreported to the monitor. +# 5. A second explicit "fsm step node3" reports "report_lsn" -- with +# node3's real LSN now visible to the monitor for the first time, +# BuildCandidateList can finally compare it against node2's LSN, +# discover node3 is behind, and assign FAST_FORWARD_STATE, pointing +# at node2 as the WAL source. node3 acts on that brand new goal in +# this very same call: a live, physical WAL fetch from node2's still- +# running Postgres instance -- still unreported. node1's and node2's +# rows still exist at both of these steps (node1 merely network- +# disconnected, not yet removed), so pos 209's "alone in group" check +# does not fire on either contact. +# 6. Immediately (the fetch above already completed synchronously, inside +# that one call): node1's and node2's rows are dropped directly via +# pgautofailover.remove_node(..., true) -- "alone in group" is true +# from this point on. node2 is still alive and connected, so +# force=true is required (without it, remove_node() only marks +# goalState=dropped and waits on node2's own cooperative shutdown, +# which both takes an indeterminate amount of time and would kill the +# WAL source's Postgres mid-transfer for anyone still relying on it). +# 7. A third explicit "fsm step node3" now reports "fast_forward" (the +# state the previous step already reached locally) -- with node1 and +# node2 both gone, pos 209's "alone in group, candidate-eligible" now +# matches this exact report and assigns SINGLE directly -- no cascade +# through prepare_promotion is possible, since node2's row is already +# gone. node3's keeper now has a matching KeeperFSM[] row (the fix +# under test) and actually converges to single via that same call. +# 8. A fourth explicit "fsm step node3" has node3 actually report +# "single" back to the monitor. + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 suspended + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s + promote node1 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s +} + +teardown { + compose down +} + +step test_001_set_candidate_priorities { + exec node2 pg_autoctl set node candidate-priority 0 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 60s + sql node1 { CREATE TABLE t1 (a int); } +} + +step test_002_disconnect_node3_and_diverge { + network disconnect node3 + sql node1 { + INSERT INTO t1 SELECT x FROM generate_series(1, 5000) as gs(x); + } + sql node1 { CHECKPOINT; } +} + +step test_003_failover_and_remove_peers_while_still_fetching { + network disconnect node1 + network connect node3 + # node3 is suspended, so this can be polled with a comfortable timeout + # and no race at all: node3 itself has made no contact yet, and won't + # until the explicit "fsm step" calls below -- this assignment lands + # purely from node2's own regular ticking (the ordinary standby fan-out + # to REPORT_LSN once node1 is unhealthy), and then simply stays there. + # Confirmed live: the monitor cannot decide fast_forward vs. a direct + # promotion without first learning node3's own actual LSN, which needs + # a real contact from node3 itself -- so the goal genuinely stops here, + # not at fast_forward, until node3 makes contact. + wait until node3 assigned-state = report_lsn timeout 60s + # This call reports node3's still-stale "secondary" and performs the + # local secondary -> report_lsn transition (recording its own current, + # genuinely-behind LSN) -- still unreported. node1's row still exists + # (just network-disconnected, not yet removed) and node2 is still + # present, so pos 209's "alone in group" check does not fire on this + # contact. + fsm step node3 + # This call reports "report_lsn" (with node3's real LSN, now visible to + # the monitor for the first time) -- only now can BuildCandidateList + # compare it against node2's LSN, discover node3 is behind, and assign + # FAST_FORWARD_STATE pointing at node2. node3 acts on that brand new + # goal in this very same call: fsm_fast_forward, a live WAL fetch from + # node2's still-running Postgres instance -- still unreported. + fsm step node3 + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node2'; + } +} + +step test_004_node3_converges_straight_to_single { + # node3 is still suspended: this call reports "fast_forward" (the state + # the previous step already reached locally, but had not yet reported) + # -- with node1 and node2 both gone, pos 209's "alone in group, + # candidate-eligible" now fires on this exact report and assigns + # single, performing the fast_forward -> single transition under test + # in this very call. + fsm step node3 + # One more call to have node3 actually report "single" back. + fsm step node3 + wait until node3 state = single timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { single } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { single } +} diff --git a/tests/tap/specs/keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf b/tests/tap/specs/keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf new file mode 100644 index 000000000..2af88068a --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf @@ -0,0 +1,96 @@ +# Regression spec confirming a real requirement flagged during the pos +# 209/211 "alone in group" investigation: once a lone node is parked at +# report_lsn with candidate-priority 0 (pos 211's own "wait for an operator +# or a new peer" outcome), is there actually a way for the cluster to +# recover once a new peer DOES show up? +# +# Answer: yes, already fully working, no monitor or keeper change needed -- +# confirmed live. register_node()'s own logic (RegisterNode, +# node_active_protocol.c ~line 662-694) already handles this exact case: if +# there's no primary and no promotion already in flight, it searches for a +# node with candidatePriority == 0 AND IsCurrentState(REPORT_LSN); if found, +# the new node is ALSO assigned initialState = REPORT_LSN (not +# WAIT_STANDBY), which uses fsm_init_from_standby / keeper_get_most_ +# advanced_standby (NOT keeper_get_primary, which would find nothing here -- +# GetPrimaryOrDemotedNodeInGroupFromList requires CanTakeWritesInState or +# StateBelongsToPrimary, neither of which report_lsn satisfies) to +# basebackup from the existing report_lsn node instead of a "real" primary. +# +# Once the new node registers, the group now has TWO report_lsn-reporting +# nodes -- exactly the ordinary MS-failover candidate-selection scenario +# (BuildCandidateList/SelectFailoverCandidateNode/PromoteSelectedNode). +# Since the new node has ordinary (nonzero) candidate-priority while the +# original node is still 0, the new node is selected as the failover +# candidate and promoted -- through fast_forward (if marginally behind the +# original node's exact LSN after basebackup), prepare_promotion, +# wait_primary, primary -- while the ORIGINAL report_lsn node transitions +# report_lsn -> join_secondary -> secondary, following the new node as its +# primary. The cluster ends up fully HA again with zero manual +# intervention beyond starting the new node. +# +# This was discovered by literally trying it live: an earlier draft of this +# spec asserted the new node would become a plain secondary of the +# original node, which is backwards -- it actually becomes the new +# primary, and the original (priority-zero) node becomes its secondary. +# +# 1. node1 (primary) + node2 (secondary) bootstrap normally. +# 2. node2's candidate-priority is set to 0. +# 3. node1 is network-disconnected, node2 told to enter maintenance (last +# quorum member -> wait_maintenance), then node1 is dropped directly -- +# leaving node2 alone, converged at report_lsn (candidate-priority 0). +# (Same mechanism as keeper_fsm_gap_211_wait_maintenance.pgaf.) +# 4. node3 (created with launch deferred) is started only now, once node2 +# is confirmed stuck at report_lsn. It registers, basebackups from +# node2, and the cluster elects it as the new primary. + +cluster { + monitor + formation { + node1 + node2 + node3 create and launch deferred + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_001_zero_priority_and_disconnect_primary { + exec node2 pg_autoctl set node candidate-priority 0 + network disconnect node1 +} + +step test_002_secondary_stuck_at_wait_maintenance { + exec-fails node2 pg_autoctl enable maintenance + wait until node2 state is wait_maintenance timeout 60s +} + +step test_003_drop_primary_converges_to_report_lsn { + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + wait until node2 assigned-state = report_lsn timeout 60s + wait until node2 state is report_lsn timeout 60s +} + +step test_004_node3_joins_and_becomes_primary { + exec node3 pg_autoctl node start + wait until node3 state is primary + and node2 state is secondary + timeout 90s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { primary } + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { secondary } +} diff --git a/tests/tap/specs/keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf b/tests/tap/specs/keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf new file mode 100644 index 000000000..ab62d6693 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf @@ -0,0 +1,131 @@ +# Regression spec for MonitorFSM[] pos 209 ("alone in group, +# candidate-eligible -> single", src/monitor/group_state_machine.c), for its +# prepare_maintenance current_state -- and for the new pos 208 no-op row +# added alongside it. +# +# Unlike keeper_fsm_gap_209_wait_maintenance.pgaf's own secondary-side +# scenario (already fixed by adding a KeeperFSM[] row), prepare_maintenance +# is deliberately NOT wired up the same way: it's the PRIMARY's own +# pre-maintenance state (start_maintenance() only ever assigns it to a node +# that .isInPrimaryState, see pos 109/111), entered via +# fsm_stop_postgres_for_primary_maintenance (fsm.c) -- which cleanly +# checkpoints and STOPS Postgres, in preparation for handing off to a +# standby that was assigned PREPARE_PROMOTION_STATE in the very same +# start_maintenance() call. +# +# Investigating this while looking at pos 209's own remaining Step 2a gaps +# found that "prepare_maintenance -> single" would be a real data-loss bug, +# not a missing convenience: pos 343 (this same file) lets that standby +# advance all the way to WAIT_PRIMARY/PRIMARY the moment the old primary's +# own reportedState merely *converges* to prepare_maintenance, with no +# requirement that the old primary's row ever be removed first. So a node +# can sit in prepare_maintenance indefinitely while a different, +# already-fully-promoted primary is live and taking writes elsewhere -- if +# that new primary later also vanishes, promoting the OLD node straight to +# SINGLE would discard everything the new primary committed in between. +# This is the exact same split-brain risk pos 209's own +# reportedIsJoinSecondary exclusion already guards against for +# join_secondary, just reached one step earlier in the handoff. +# +# Fixed by adding reportedIsPrepareMaintenance = BOOL_FALSE to pos 209 +# (excluding this source state, mirroring reportedIsJoinSecondary) -- NOT by +# adding a KeeperFSM[] row. No keeper change was made at all: there's +# nothing safe to promote it to. +# +# That exclusion alone would have been an improvement over a data-loss bug, +# but a strictly worse *availability* bug on its own: unlike join_secondary +# (already recognized by node_metadata.c's IsParticipatingInPromotion, so a +# lone join_secondary node safely no-ops), a lone node reporting +# prepare_maintenance is recognized by neither that function, IsBeingPromoted, +# nor IsInPrimaryState (CanTakeWritesInState(prepare_maintenance) is false). +# Without an explicit match, ProceedGroupStateFromContext's own "primaryNode +# == NULL && !IsFailoverInProgress(...)" guard would ereport(ERROR) on +# *every single subsequent heartbeat* from this node -- worse than the +# original bug, which merely got the node stuck silently. Fixed by adding +# pos 208, a plain no-op row (same shape as pos 205's own "converged to +# maintenance" row) matching exactly this "alone, reporting +# prepare_maintenance" combination. +# +# This spec proves both fixes live: +# +# 1. node1 (primary) is put into maintenance directly via +# pgautofailover.start_maintenance() (the underlying SQL function, not +# the `pg_autoctl enable maintenance` CLI -- that CLI blocks polling +# for full convergence to "maintenance", which would race against the +# very next step below; calling the SQL function directly gives full +# control over timing, the same lesson learned constructing +# keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf). This assigns node1 +# PREPARE_MAINTENANCE_STATE and node2 (the only standby) +# PREPARE_PROMOTION_STATE in the same call. +# 2. Once node1 is confirmed *assigned* prepare_maintenance (still +# converging, Postgres not yet stopped): node2's row is force-removed +# (it's still alive and connected, so force=true is required) -- +# before node2 could ever advance to wait_primary/primary. This makes +# "alone in group" true well before any handoff race could resolve +# either way. +# 3. node1 finishes converging (Postgres stops, current_state becomes +# prepare_maintenance) with nothing left in its group. Before this +# session's fix, pos 209 would have assigned it SINGLE (a data-loss +# risk); after the fix, pos 209 declines (reportedIsPrepareMaintenance +# exclusion) and pos 208 matches instead (plain no-op). +# 4. Assert node1 stays parked at prepare_maintenance (goalstate never +# becomes single) -- and, to prove pos 208 actually prevents the +# ereport(ERROR) rather than just happening not to fire yet, assert +# node1's own reporttime keeps advancing well after being left alone: +# if pos 208 were missing, every subsequent node_active() call from +# node1 would hit that ereport(ERROR) instead of successfully updating +# its own report time. + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_001_start_maintenance_and_remove_standby_while_still_converging { + sql monitor { + SELECT pgautofailover.start_maintenance(nodeid) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + wait until node1 assigned-state = prepare_maintenance timeout 60s + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node2'; + } +} + +step test_002_node1_converges_and_stays_parked { + wait until node1 state is prepare_maintenance timeout 60s + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node1'; + } + expect { prepare_maintenance } + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node1'; + } + expect { prepare_maintenance } +} + +step test_003_node1_keeps_checking_in_not_stuck_erroring { + sleep 10s + sql monitor { + SELECT (now() - reporttime) < interval '10 seconds' + FROM pgautofailover.node WHERE nodename = 'node1'; + } + expect { t } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node1'; + } + expect { prepare_maintenance } +} diff --git a/tests/tap/specs/keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf new file mode 100644 index 000000000..421982554 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf @@ -0,0 +1,120 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, candidatePriority +# zero -> report_lsn", src/monitor/group_state_machine.c), for its +# prepare_promotion current_state, and for the matching KeeperFSM[] row +# added in fsm.c: +# +# { +# PREP_PROMOTION_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, +# COMMENT_SECONDARY_TO_REPORT_LSN, +# &fsm_report_lsn, +# FSM_PHASE_FAILOVER +# }, +# +# When this fix was first written, live pgaftest reproduction was attempted +# and abandoned: fsm_prepare_standby_for_promotion (the transition INTO +# prepare_promotion) is a no-op, so in ordinary autonomous operation the +# monitor's own cascade advances assigned=prepare_promotion straight through +# to stop_replication within the same heartbeat -- faster than any external +# test script's own "remove the other peer" SQL call could land in between. +# See src/monitor/sql/keeper_fsm_edges.sql's own comment for the full story +# and the static/code-level verification that was used instead at the time. +# +# Step mode (PG_AUTOCTL_STEP_MODE, src/bin/pg_autoctl/step_socket.c) removes +# that race entirely: node3's node-active service never ticks on its own, so +# the spec can drive it to converge locally to prepare_promotion via one +# explicit "fsm step node3" and then simply *not issue the next one* for as +# long as it likes -- there is no cascade to race against, because nothing +# happens without an explicit command. Once node3 is confirmed frozen there +# (still unreported to the monitor), the spec zeroes its own candidate +# priority and force-removes the old primary's row, then a single further +# "fsm step node3" reports prepare_promotion, receives the newly-applicable +# report_lsn assignment from pos 211, and performs the transition under +# test in the very same call. +# +# Cluster shape: plain 2-node (no MS-failover candidate-selection ambiguity +# needed here, unlike the join_secondary sibling spec) -- node3 is the +# formation's only standby, so failing node1 assigns it straight through +# secondary -> prepare_promotion with no report_lsn hop at all (confirmed +# live: querying node3's own assigned-state right after node1 goes away +# already shows "prepare_promotion", never "report_lsn" first). + +cluster { + monitor + formation { + node1 + node3 suspended candidate-priority 50 + } +} + +setup { + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node3 state is secondary + timeout 60s +} + +teardown { + compose down +} + +step test_001_kill_primary_and_converge_locally_to_prepare_promotion { + # A hard kill never gives node1 a chance to self-report (unlike a + # graceful shutdown, which would report prepare_maintenance on its own + # way out and, as a side effect of processing *that* report, cascade a + # fresh goal to node3 too) -- so nothing recalculates node3's own goal + # state until node3 itself makes contact. node3 never ticks on its own + # (suspended), so this explicit "fsm step node3" is what prompts + # that contact -- and, in the very same call, discovers the fresh + # prepare_promotion assignment and performs the transition into it. + # There's no way to observe the assignment without also applying it. + # The sleep gives the monitor's own health-check worker (a periodic + # background process, unrelated to and slower than any of our + # explicit steps) time to actually notice node1 is gone before we + # step node3 -- stepping too early would just be a same-state no-op, + # since the monitor wouldn't yet have anything new to assign. + compose kill node1 + sleep 30s + fsm step node3 + wait until node3 assigned-state = prepare_promotion timeout 30s +} + +step test_002_zero_priority_and_remove_old_primary_row { + # node3's own current_role is already prepare_promotion (from the step + # above), but that has not been reported back to the monitor yet -- the + # step above reported the *old* "secondary" state, before transitioning. + # This has to happen *before* node3's next contact, not after: the very + # next time node3 reports "prepare_promotion" as its current state, the + # monitor's ordinary cascade would advance it straight to + # stop_replication (a dead end -- see the header comment) unless pos + # 211's "alone in group, candidatePriority zero" condition is already + # true by then. So the priority drop and the peer removal both need to + # land while node3 is still sitting here, frozen, unreported. + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } +} + +step test_003_node3_converges_straight_to_report_lsn { + fsm step node3 + # The call above already performed the prepare_promotion -> report_lsn + # transition under test, but keeper_fsm_step() only reports the state + # it had *before* that transition -- one more call is needed for node3 + # to actually report "report_lsn" back to the monitor. + fsm step node3 + wait until node3 state = report_lsn timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} diff --git a/tests/tap/specs/keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf new file mode 100644 index 000000000..63795e277 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf @@ -0,0 +1,206 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, +# candidatePriority zero -> report_lsn", src/monitor/group_state_machine.c), +# for its fast_forward current_state -- the candidatePriority=0 sibling of +# keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf's own scenario. See +# that spec's header for the shared setup (network disconnect + INSERT + +# CHECKPOINT to create genuine LSN divergence, the same recipe already +# established by tests/tap/specs/multi_ifdown.pgaf and +# tests/tap/specs/debug_citus_worker_fast_forward.pgaf), for why node3 is +# declared "suspended" (step mode), and for the two-step +# report_lsn-then-fast_forward mechanics this now relies on (confirmed live, +# both here and in the sibling spec): the monitor can't decide fast_forward +# vs. a direct promotion without first learning node3's own actual LSN, +# which needs a real contact from node3 itself, so the first assignment +# that lands is the ordinary standby fan-out to REPORT_LSN, not +# fast_forward directly. +# +# First draft of this spec (like its sibling) drove node3 as an ordinary +# (autonomous) node and tried to catch the narrow fast_forward +# *assigned*-state window by external polling before removing node1/node2, +# and even switched from `pg_autoctl set node candidate-priority` to a +# direct SQL call for zeroing node3's own priority once that CLI's own +# confirmation loop turned out to be slow enough to lose the race on its +# own. Confirmed live (CI run 30759600065) that even that tightened window +# was still genuinely racy: with a WAL gap this small (5000 rows), node3's +# own local fetch completes and reports back to the monitor within about 2 +# seconds of the assignment landing, and the monitor's own cascade (node2, +# still alive, keeps re-triggering evaluation of node3's candidate status) +# advances assigned-state on to prepare_promotion before the external "wait +# until" poll can reliably observe fast_forward in between. +# +# Fixed for real this time the same way as the sibling spec: node3 is +# declared "suspended" (step mode, PG_AUTOCTL_SUSPENDED -- +# src/bin/pg_autoctl/service_keeper.c), so nothing it reports to the +# monitor ever changes except in direct response to an explicit "fsm step +# node3" command. Confirmed directly against FSM_REPORT_LSN_OR_FAST_FORWARD +# (src/monitor/group_state_machine.c): the monitor's cascade past +# fast_forward requires reportedState == goalState == FAST_FORWARD (a +# NODE_STATE_STABLE match), so as long as node3 itself never contacts the +# monitor, there is no race window left to lose -- and since zeroing +# node3's own candidate-priority is now also driven by an explicit step +# (not a race against the CLI's own timing), the plain +# `pg_autoctl set node candidate-priority` CLI could be used again just as +# well; the direct SQL call is kept here anyway, simplest and just as +# correct. +# +# This spec asserts the candidatePriority=0 sibling row that was added +# alongside pos 209's: +# +# { +# FAST_FORWARD_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, +# COMMENT_SECONDARY_TO_REPORT_LSN, +# &fsm_report_lsn, +# FSM_PHASE_FAILOVER +# }, +# +# reusing fsm_report_lsn exactly like every other converged-standby source +# state that already had a REPORT_LSN_STATE-bound row (SECONDARY/CATCHINGUP/ +# MAINTENANCE/PREPARE_MAINTENANCE/WAIT_MAINTENANCE) -- no separate WAL fetch +# is attempted or needed, fast_forward already means Postgres is running as +# a caught-up-enough standby. +# +# node3 needs its ordinary (nonzero) candidate-priority to actually be +# selected as the failover candidate and assigned fast_forward in the first +# place -- GroupListCandidates excludes candidatePriority == 0 nodes from +# selection entirely, so a node can only reach fast_forward while still +# candidate-eligible. Its priority is only dropped to 0 afterwards, once it +# has already performed the fast_forward transition locally (confirmed via +# an explicit step, not a race). +# +# 1. Identical setup to keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf: +# node2 (candidate-priority 0, WAL source) stays connected and caught +# up; node3 (default nonzero candidate-priority, suspended) is +# disconnected while node1 receives writes, then reconnected as node1 +# is disconnected -- forcing an MS-failover election that selects +# node3 (the only candidate) as the group's failover target. +# 2. This can't yet be resolved to fast_forward without node3's own real +# LSN, so the first thing that lands (driven purely by node2's own +# regular ticking) is the ordinary standby fan-out: goalState = +# REPORT_LSN. Polled with a comfortable timeout and no race at all -- +# node3 itself (suspended) has made no contact yet. +# 3. A first explicit "fsm step node3" reports node3's own still-stale +# "secondary" and, in the same call, performs the local secondary -> +# report_lsn transition -- recording node3's own genuinely-behind LSN, +# still unreported. +# 4. A second explicit "fsm step node3" reports "report_lsn" -- with +# node3's real LSN now visible, BuildCandidateList compares it against +# node2's, discovers node3 is behind, and assigns FAST_FORWARD_STATE, +# pointing at node2. node3 acts on that brand new goal in the very +# same call: a live, physical WAL fetch from node2's still-running +# Postgres instance -- still unreported. node1's and node2's rows +# still exist at both of these steps (node1 merely network- +# disconnected, not yet removed), so pos 211's "alone in group" check +# does not fire on either contact. +# 5. Immediately (the fetch above already completed synchronously, inside +# that one call): node3's own candidate-priority is set to 0, and +# node1's and node2's rows are dropped directly via +# pgautofailover.remove_node(..., true) -- node2 is still alive and +# connected, so force=true is required (its actual Postgres instance +# and the already-completed fetch from node3 are unaffected by +# removing its monitor row). +# 6. A third explicit "fsm step node3" now reports "fast_forward" (the +# state the previous step already reached locally) -- with node1 and +# node2 both gone and candidate-priority 0, pos 211's "alone in group, +# candidatePriority zero" now matches this exact report and assigns +# REPORT_LSN directly -- no cascade through prepare_promotion is +# possible, since node2's row is already gone. node3's keeper now has +# a matching KeeperFSM[] row (the fix under test) and actually +# converges to report_lsn via that same call. +# 7. A fourth explicit "fsm step node3" has node3 actually report +# "report_lsn" back to the monitor. + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 suspended + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s + promote node1 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s +} + +teardown { + compose down +} + +step test_001_set_candidate_priorities { + exec node2 pg_autoctl set node candidate-priority 0 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 60s + sql node1 { CREATE TABLE t1 (a int); } +} + +step test_002_disconnect_node3_and_diverge { + network disconnect node3 + sql node1 { + INSERT INTO t1 SELECT x FROM generate_series(1, 5000) as gs(x); + } + sql node1 { CHECKPOINT; } +} + +step test_003_failover_zero_priority_and_remove_peers_while_still_fetching { + network disconnect node1 + network connect node3 + # node3 is suspended: nothing moves this off REPORT_LSN until the + # explicit steps below (see header for why the monitor lands here + # first, not directly at fast_forward). + wait until node3 assigned-state = report_lsn timeout 60s + # Reports node3's still-stale "secondary" and performs the local + # secondary -> report_lsn transition -- still unreported. + fsm step node3 + # Reports "report_lsn" (with node3's real LSN, now visible for the + # first time) -- the monitor discovers node3 is behind node2 and + # assigns FAST_FORWARD_STATE; node3 acts on it in this same call (a + # live WAL fetch from node2) -- still unreported. node1's and node2's + # rows still exist at this point, so pos 211 does not fire yet. + fsm step node3 + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node2'; + } +} + +step test_004_node3_converges_straight_to_report_lsn { + # Reports "fast_forward" (reached locally by the previous step) -- with + # node1 and node2 both gone and candidate-priority 0, pos 211 now fires + # and assigns report_lsn, performing the fast_forward -> report_lsn + # transition under test in this very call. + fsm step node3 + # One more call to have node3 actually report "report_lsn" back. + fsm step node3 + wait until node3 state = report_lsn timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} diff --git a/tests/tap/specs/keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf new file mode 100644 index 000000000..7107ec062 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf @@ -0,0 +1,145 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, candidatePriority +# zero -> report_lsn", src/monitor/group_state_machine.c), for its +# join_secondary current_state, and for the matching KeeperFSM[] row added in +# fsm.c: +# +# { +# JOIN_SECONDARY_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, +# COMMENT_SECONDARY_TO_REPORT_LSN, +# &fsm_report_lsn, +# FSM_PHASE_FAILOVER +# }, +# +# Unlike its prepare_promotion and demote_timeout siblings, join_secondary +# only exists on the *losing* side of a genuine MS-failover candidate +# election, so this scenario needs three nodes and two candidates: node1 is +# the original primary; node2 and node3 both start as ordinary (nonzero +# candidate-priority) standbys, so both are eligible and get sent through +# report_lsn to have their LSNs compared once node1 is gone. Both are +# declared "suspended" so the spec can pace the whole election by hand -- +# node2 is driven to actually win (report_lsn -> prepare_promotion), and +# node3, the loser, is driven only as far as report_lsn and then frozen +# there while node2 pulls ahead, exactly reproducing the real window in +# which the monitor assigns join_secondary to the node that didn't win. +# +# Every "fsm step" call the setup/steps below make is deterministic and +# non-racing precisely because both node2 and node3 are step-mode: nothing +# advances except in response to an explicit command, so there is no +# autonomous cascade to lose a race against, unlike attempting this same +# scenario against ordinary (autonomous) standbys. +# +# test_003's first call steps node2 from prepare_promotion onward; whether +# that particular attempt (reaching stop_replication) itself succeeds or +# fails is irrelevant to what this spec is testing, and by this point node1 +# (the old primary) has already been force-killed, so it can go either way. +# What matters is the side effect that already landed regardless: node2's +# report of "I am now in prepare_promotion" reached the monitor, which is +# what makes it assign join_secondary to node3 in the first place. + +cluster { + monitor + formation { + node1 + node2 suspended candidate-priority 50 + node3 suspended candidate-priority 50 + } +} + +setup { + wait until node2 state = catchingup timeout 60s + wait until node3 state = catchingup timeout 60s + fsm step node2 + fsm step node3 + fsm step node2 + fsm step node3 + fsm step node2 + fsm step node3 + fsm step node2 + fsm step node3 + wait until node1 state is primary timeout 60s + wait until node2 state is secondary timeout 60s + wait until node3 state is secondary timeout 60s +} + +teardown { + compose down +} + +step test_001_kill_primary_and_converge_both_locally_to_report_lsn { + # A hard kill never gives node1 a chance to self-report (unlike a + # graceful shutdown), so nothing recalculates either standby's own goal + # state until each makes its own contact -- neither node2 nor node3 + # ticks on its own (suspended), so it takes one explicit "fsm step" + # per node here, not a bare wait. Each call both discovers the new + # report_lsn assignment *and* performs the transition into it, in the + # same call -- there's no way to observe the assignment without also + # applying it. The sleep gives the monitor's own health-check worker + # (a periodic background process, unrelated to and slower than any of + # our explicit steps) time to actually notice node1 is gone before we + # step either standby -- stepping too early would just be a + # same-state no-op, since the monitor wouldn't yet have anything new + # to assign. + compose kill node1 + sleep 30s + fsm step node2 + fsm step node3 + wait until node2 assigned-state = report_lsn timeout 30s + wait until node3 assigned-state = report_lsn timeout 30s +} + +step test_002_winner_reports_report_lsn_and_converges_to_prepare_promotion { + # This call reports node2's own report_lsn state to the monitor (which + # is what lets it be selected) and, in the same call, converges node2 + # locally to prepare_promotion -- still unreported at this point. + fsm step node2 +} + +step test_003_winner_reports_prepare_promotion_and_loser_gets_join_secondary { + # This call reports node2's own prepare_promotion state to the monitor + # and then attempts the prepare_promotion -> stop_replication + # transition. Whether that attempt itself succeeds or fails (it can go + # either way depending on exactly how reachable node1 still looks at + # this point) is irrelevant to what this spec is testing -- plain exec + # (not the "fsm step" sugar, which retries on failure) accepts either + # outcome without burning its retry budget. What matters is the report + # of "prepare_promotion" that already landed before that attempt: that + # is what makes the monitor assign join_secondary to node3 next. + exec node2 pg_autoctl manual fsm step --pgdata /var/lib/postgres/pgaf + # node3 still reports "report_lsn" (unreported since test_001); this + # call refreshes that report, and only now -- with node2's own + # prepare_promotion report already landed above -- does the monitor + # assign join_secondary in response. + fsm step node3 +} + +step test_004_zero_priority_and_remove_other_two_nodes { + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node2'; + } +} + +step test_005_node3_converges_straight_to_report_lsn { + fsm step node3 + # The call above already performed the join_secondary -> report_lsn + # transition under test, but keeper_fsm_step() only reports the state + # it had *before* that transition -- one more call is needed for node3 + # to actually report "report_lsn" back to the monitor. + fsm step node3 + wait until node3 state = report_lsn timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} diff --git a/tests/tap/specs/keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf new file mode 100644 index 000000000..b80ebad61 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf @@ -0,0 +1,123 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, candidatePriority +# zero -> report_lsn", src/monitor/group_state_machine.c), for its +# demote_timeout current_state, and for the matching KeeperFSM[] row added in +# fsm.c: +# +# { +# DEMOTE_TIMEOUT_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, +# COMMENT_SECONDARY_TO_REPORT_LSN, +# &fsm_report_lsn, +# FSM_PHASE_FAILOVER +# }, +# +# Unlike the prepare_promotion sibling spec, here the node under test is the +# primary being demoted, not a standby being promoted. node3 (suspended) +# is declared *second*, not first: compose_gen only attaches a healthcheck +# to the first data node (so that later nodes can depend_on it being +# healthy, making the initial election deterministic) -- a suspended +# node can never satisfy that healthcheck on its own (`pg_autoctl status` +# requires Postgres to be running, and nothing restarts Postgres after the +# one-shot node-init phase stops it until the very first "fsm step" call), +# so it must never be the first node in the formation. +# +# This means node3 starts life as an ordinary standby and has to be +# *promoted* into place before it can be demoted -- two separate +# single-candidate MS-failover cycles chained together, both driven the +# same deterministic way the sibling specs use: node3's node-active +# service never ticks on its own, so every step below is an explicit, +# non-racing "fsm step node3" call, and the spec is free to pause for as +# long as it likes between any two of them. + +cluster { + monitor + formation { + node1 + node3 suspended candidate-priority 50 + } +} + +setup { + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node3 state is secondary + timeout 60s +} + +teardown { + compose down +} + +step test_001_promote_node3 { + sql monitor { + SELECT pgautofailover.perform_promotion('default', 'node3'); + } + wait until node3 assigned-state = prepare_promotion timeout 60s +} + +step test_002_step_node3_through_to_primary { + # node3 needs to walk secondary -> prepare_promotion -> stop_replication + # -> wait_primary -> primary, each hop its own explicit "fsm step". + # There's no harm in calling it a few more times than strictly + # necessary -- once a hop is already applied, the next call is simply a + # no-op "ensure current state" -- so blast enough of them to reach + # "primary" regardless of exactly how many hops this happened to take + # this time around, rather than hand-counting an exact number that's + # sensitive to timing. + fsm step node3 + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is secondary timeout 90s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node3 state is primary timeout 60s + # One more step to have node3 actually *report* primary to the monitor + # -- perform_failover() below requires a node already confirmed primary. + fsm step node3 +} + +step test_003_perform_failover_and_wait_demote_timeout_assignment { + sql monitor { + SELECT pgautofailover.perform_failover('default', 0); + } + wait until node3 assigned-state = demote_timeout timeout 60s +} + +step test_004_converge_locally_without_reporting { + # node3's own current_role is now demote_timeout, but this has not been + # reported back to the monitor yet, and won't be until the spec issues + # another explicit "fsm step node3" below. + fsm step node3 +} + +step test_005_zero_priority_and_remove_new_primary_row { + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } +} + +step test_006_node3_converges_straight_to_report_lsn { + fsm step node3 + # The call above already performed the demote_timeout -> report_lsn + # transition under test, but keeper_fsm_step() only reports the state + # it had *before* that transition -- one more call is needed for node3 + # to actually report "report_lsn" back to the monitor. + fsm step node3 + wait until node3 state = report_lsn timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} diff --git a/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf new file mode 100644 index 000000000..d0d2e6266 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf @@ -0,0 +1,153 @@ +# Regression spec for MonitorFSM[] pos 211's stop_replication current_state +# ("alone in group, candidatePriority zero -> report_lsn", +# src/monitor/group_state_machine.c), and for the matching KeeperFSM[] row +# added in fsm.c (see keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf's +# own header for the full row and the shared construction technique -- this +# spec's setup/test_001/test_002/test_003 steps are identical to that one, +# reaching the exact same "node3 parked at report_lsn, candidate-priority +# 0, alone in group" state). +# +# This exercises the *second* documented way out of that parked state: a +# new node registers, and RegisterNode's own existing report_lsn- +# candidate-priority-0 special case (node_active_protocol.c, already +# proven live for a different source state by +# keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf) basebackups it from +# node3 instead of from a "real" primary, then the ordinary MS-failover +# candidate-selection machinery (BuildCandidateList/ +# SelectFailoverCandidateNode/PromoteSelectedNode) picks the new node +# (ordinary nonzero priority) over node3 (priority 0) and promotes it, +# while node3 follows it back in as a secondary -- respecting node3's own +# candidate-priority=0 the whole time, exactly like the priority-raise +# sibling spec respects it by simply never promoting node3 on its own. +# +# node4 is basebacked up directly from node3's own frozen, disconnected +# report_lsn snapshot; by the time node4 finishes and reports in, node3 +# has moved a little further along in real time, so node4 *does* pass +# through a brief fast_forward hop (report_lsn -> fast_forward -> +# prepare_promotion -> wait_primary -> primary) to pick up the small +# remaining gap before promoting -- unlike +# keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf's own sibling +# scenario, where the comment notes fast_forward is conditional on +# exactly this kind of timing. node3 itself, once node4 is selected as +# the winning candidate, goes straight report_lsn -> secondary -- not +# through join_secondary as originally guessed here (join_secondary is +# for a node still mid-handoff from an old primary it hasn't finished +# checkpointing away from; node3 has no such old-primary handoff pending, +# it's simply been sitting parked, so PromoteSelectedNode's own fan-out +# takes the more direct edge). Confirmed live, the same way the sibling +# spec's own header already flags having had to fix its first guess about +# which node ends up primary. + +cluster { + monitor + formation { + node1 + node3 suspended candidate-priority 50 + node4 create and launch deferred + } +} + +setup { + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node3 state is secondary + timeout 60s +} + +teardown { + compose down +} + +step test_001_kill_primary_and_converge_locally_to_stop_replication { + compose kill node1 + sleep 30s + fsm step node3 + wait until node3 assigned-state = prepare_promotion timeout 30s + fsm step node3 + wait until node3 assigned-state = stop_replication timeout 30s +} + +step test_002_zero_priority_and_remove_old_primary_row { + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } +} + +step test_003_node3_converges_to_report_lsn { + fsm step node3 + fsm step node3 + wait until node3 state = report_lsn timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} + +step test_004_node4_joins_and_becomes_primary { + # node3 stays suspended for this whole spec (step mode is a + # startup-time setting, not something a running spec can toggle off + # again once the earlier steps no longer need it), and that turns out + # to matter here in a way it didn't in the earlier steps: + # node_active()'s own HBA refresh (keeper_refresh_other_nodes, + # service_keeper.c's step-mode dispatch calls it before every step, + # same as autopilot's own per-tick loop does) is what grants node4's + # IP the pg_hba.conf rule it needs to even *reach* node3 for its + # initial basebackup -- and node3 never runs that refresh on its own + # while parked, unreported. Confirmed live: without an explicit "fsm + # step node3" right after starting node4, node4 spins forever on "no + # pg_hba.conf entry for replication connection", never even reaching + # report_lsn itself, let alone getting basebacked-up-from and + # promoted. So node3 needs a step immediately, before node4 can make + # any progress at all -- not just later, once node4's own report + # fans out a fresh join_secondary goal to node3 (which still needs + # its own later explicit steps too, same as every earlier step in + # this spec). One immediate step turned out not to be enough either: + # "pg_autoctl node start" returns as soon as the container process is + # up, before node4 has necessarily finished registering with the + # monitor -- stepping node3 too early just finds nothing new to + # refresh yet. node4's own supervisor keeps retrying every few + # seconds regardless, so a couple of staggered steps a few seconds + # apart reliably lands at least one after node4's row exists. + # + # node4 can't reach primary until node3 has actually converged to + # secondary and is counted as a synced standby -- so node3's own + # steps have to happen, and land, *before* waiting on node4's own + # final state, not after (confirmed live: waiting for node4 first + # just times out with node4 stuck at wait_primary forever, since + # nothing ever drives node3's own report_lsn -> secondary hop). + exec node4 pg_autoctl node start + sleep 10s + fsm step node3 + sleep 10s + fsm step node3 + wait until node3 assigned-state = secondary timeout 60s + fsm step node3 + fsm step node3 + wait until node3 state is secondary timeout 30s + wait until node4 state is primary timeout 90s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node4'; + } + expect { primary } + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { secondary } +} + +sequence + test_001_kill_primary_and_converge_locally_to_stop_replication + test_002_zero_priority_and_remove_old_primary_row + test_003_node3_converges_to_report_lsn + test_004_node4_joins_and_becomes_primary diff --git a/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf new file mode 100644 index 000000000..4d9af3f6d --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf @@ -0,0 +1,137 @@ +# Regression spec for MonitorFSM[] pos 211's stop_replication current_state +# ("alone in group, candidatePriority zero -> report_lsn", +# src/monitor/group_state_machine.c), and for the matching KeeperFSM[] row +# added in fsm.c: +# +# { +# STOP_REPLICATION_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, +# COMMENT_SECONDARY_TO_REPORT_LSN, +# &fsm_report_lsn, +# FSM_PHASE_FAILOVER +# }, +# +# This exercises the first of the two documented ways out of that parked +# report_lsn state: raising candidate-priority back above 0. The second +# way out (a new node registers and takes over as primary) is +# keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf's own job. +# +# Same construction technique as +# keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf, one +# step further: that spec freezes node3 at prepare_promotion and lets +# pos 211 assign it report_lsn directly (a no-op hop in, so nothing here +# needs step-mode to hold still for it). Reaching stop_replication instead +# needs one more explicit "fsm step node3" first, since entering +# stop_replication is *not* a no-op -- fsm_stop_replication really does +# call fsm_promote_standby, physically promoting node3's own Postgres onto +# a new timeline -- and step mode is exactly what lets the spec hold node3 +# frozen there, unreported, for as long as it takes to zero its priority +# and remove the old primary's row before node3's next contact. +# +# Cluster shape: plain 2-node, same reasoning as the prepare_promotion +# sibling spec -- node3 is the formation's only standby, so failing node1 +# assigns it straight through secondary -> prepare_promotion -> (this +# spec's own extra step) stop_replication, with no report_lsn hop in +# between. + +cluster { + monitor + formation { + node1 + node3 suspended candidate-priority 50 + } +} + +setup { + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node3 state is secondary + timeout 60s +} + +teardown { + compose down +} + +step test_001_kill_primary_and_converge_locally_to_stop_replication { + # First "fsm step node3": reports "secondary" (unchanged), receives + # the fresh prepare_promotion assignment, and performs that hop in the + # same call (fsm_prepare_standby_for_promotion is a no-op -- Postgres + # is untouched, still an ordinary streaming standby). + # + # Second "fsm step node3": reports "prepare_promotion" (the state the + # first call just reached, not yet reported before now), receives the + # ordinary promotion cascade's next assignment (stop_replication -- + # node1's row still exists at this point, just unhealthy, so this is + # the everyday 2-node failover path, not pos 211 yet), and performs + # *that* hop in the same call -- this is where node3's own Postgres + # actually gets promoted (fsm_stop_replication -> fsm_promote_standby). + compose kill node1 + sleep 30s + fsm step node3 + wait until node3 assigned-state = prepare_promotion timeout 30s + fsm step node3 + wait until node3 assigned-state = stop_replication timeout 30s +} + +step test_002_zero_priority_and_remove_old_primary_row { + # Same timing requirement as the prepare_promotion sibling spec: both + # of these have to land before node3's next contact, while it's still + # frozen at stop_replication, unreported -- otherwise the ordinary + # cascade would carry on past stop_replication instead of pos 211's + # "alone in group, candidatePriority zero" condition ever getting a + # chance to apply. + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } +} + +step test_003_node3_converges_to_report_lsn { + # First call performs the stop_replication -> report_lsn transition + # under test (reports "stop_replication", receives report_lsn from + # pos 211, applies it via the new KeeperFSM[] row -- fsm_report_lsn, + # no live peer contacted). Second call reports the result back. + fsm step node3 + fsm step node3 + wait until node3 state = report_lsn timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} + +step test_004_raise_priority_and_converge_to_single { + # Respects the operator's own candidate-priority=0 declaration the + # whole time up to here: nothing auto-promotes node3 while it stays + # 0 (BuildCandidateList/GroupListCandidates would never select it). + # Only once priority is explicitly raised back above 0 does pos 209 + # ("alone in group, candidate-eligible -> single") start matching, and + # report_lsn -> single is already an ordinary existing KeeperFSM[] + # edge -- no new keeper code needed for this half. + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 50); + } + fsm step node3 + fsm step node3 + wait until node3 state = single timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { single } +} + +sequence + test_001_kill_primary_and_converge_locally_to_stop_replication + test_002_zero_priority_and_remove_old_primary_row + test_003_node3_converges_to_report_lsn + test_004_raise_priority_and_converge_to_single