From 1530e93aeb774b8ee83d034439efad6e8019365e Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 28 Jul 2026 21:15:33 +0200 Subject: [PATCH 01/52] Checkpoint: three-array FSM dispatch with looping continuation Working, fully-tested implementation (14/14 regress, 6/6 isolation, 49/49 + 101/101 pgaftest, all in Docker) but deviates from the design doc at /Users/dim/dev/temp/monitor-fsm-data-driven-refactor-prompt.md in two ways the doc explicitly argues against: - three separate arrays (MonitorFSM_EarlyChecks[], MonitorFSM_FromContext[], MonitorFSM_ForPrimaryNode[]) instead of one MonitorFSM[] with named section-boundary constants. - a looping dispatch site (resume from index+1 when a row's extraAction returns false) instead of the doc's single-shot driver + bounded named jump inside extraAction. This is the continuesDispatch mechanism the doc's own design phase tried and reverted -- and it reproduced exactly the double-invocation failure mode the doc predicted, fixed here by merging three sibling rows into one instead of adopting the doc's named-boundary jump. Keeping this commit as a known-good reference point before reworking towards the doc's actual single-array/single-shot-dispatch design. --- src/monitor/group_state_machine.c | 2527 +++++++++++++++-------------- 1 file changed, 1286 insertions(+), 1241 deletions(-) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 85ed020a2..44f9d194c 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -79,6 +79,577 @@ static bool WalDifferenceWithin(AutoFailoverNode *secondaryNode, AutoFailoverNode *primaryNode, int64 delta); +/* + * --------------------------------------------------------------------- + * Declarative dispatch for ProceedGroupStateFromContext() and + * ProceedGroupStateForPrimaryNode(): each function's own sequential + * if-chain is replaced by a table of MonitorFSMTransition rows, matched + * first-match-wins by RuleMatches(). ProceedGroupStateForMSFailover() and + * everything it calls (BuildCandidateList, SelectFailoverCandidateNode, + * PromoteSelectedNode, ProceedWithMSFailover, WalSourceNodesAreAllUnhealthy) + * stays hand-written C exactly as before, reached from the table via + * extraAction -- the candidate-selection algorithm (priority sort, LSN + * comparison, WAL-fetch orchestration) doesn't reduce to declarative + * conditions any more cleanly than it did before this change. + * --------------------------------------------------------------------- + */ + +typedef enum BoolPattern +{ + BOOL_ANY = 0, + BOOL_FALSE, + BOOL_TRUE +} BoolPattern; + +static bool +MatchBoolPattern(bool actual, BoolPattern pattern) +{ + switch (pattern) + { + case BOOL_FALSE: + { + return !actual; + } + + case BOOL_TRUE: + { + return actual; + } + + case BOOL_ANY: + default: + { + return true; + } + } +} + + +typedef enum NodeStatePatternKind +{ + NODE_STATE_ANY = 0, + NODE_STATE_STABLE, + NODE_STATE_NOT_STABLE, + NODE_STATE_REPORTED, + NODE_STATE_ASSIGNED, + NODE_STATE_NOT_ASSIGNED, + NODE_STATE_TRANSITIONING +} NodeStatePatternKind; + +/* Fixed-size (not pointer-based): a pointer initialized from the address of + * a compound literal nested inside another compound literal is not a + * constant expression by the C standard -- clang accepts it as an extension + * in file-scope initializers, but gcc (used by the project's Debian/CI + * build) rejects it with "initializer element is not constant". A fixed-size + * value member sidesteps the whole address-of-compound-literal question: + * no STATES() call in this file passes more than 3 states. + */ +typedef struct ReplicationStateSet +{ + ReplicationState states[4]; + int count; +} ReplicationStateSet; + +#define STATES_NARG_(_1, _2, _3, _4, N, ...) N +#define STATES_NARG(...) STATES_NARG_(__VA_ARGS__, 4, 3, 2, 1) + +/* Builds a { states[4], count } pair inline -- no NONE-terminated array to + * remember to close, no separate _STATES[] declaration to name. Deliberately + * NOT wrapped in a "(ReplicationStateSet) { ... }" compound-literal cast: + * every use is as a designated-initializer value nested inside another + * static aggregate (a MonitorFSMTransition table row, or one of the named + * FSM_* pattern constants below), and a bare brace-list is a plain + * sub-object initializer there -- fully portable ISO C. A compound-literal + * cast would make it a distinct object in its own right, and gcc (unlike + * clang) rejects using one of those, even by value, to initialize part of + * another object with static storage duration ("initializer element is not + * constant") -- see the note on ReplicationStateSet above for the same + * problem one level down. + */ +#define STATES(...) \ + { \ + .states = { __VA_ARGS__ }, \ + .count = STATES_NARG(__VA_ARGS__) \ + } + +typedef struct NodeStatePattern +{ + NodeStatePatternKind kind; + ReplicationStateSet reportedStates; + ReplicationStateSet assignedStates; +} NodeStatePattern; + +static bool +MatchStateSet(ReplicationState actual, ReplicationStateSet declared) +{ + for (int i = 0; i < declared.count; i++) + { + if (declared.states[i] == actual) + { + return true; + } + } + return false; +} + + +/* Same reasoning as STATES() above: no compound-literal cast, since every + * use is nested inside another static aggregate's designated initializer + * (e.g. ".statePattern = FSM_STATE(x)" inside a MonitorFSMTransition row). */ +#define FSM_STATE(x) \ + { .kind = NODE_STATE_STABLE, .reportedStates = STATES(x) } + +/* group_state_machine.c:504-523/1059-1106 -- three IsCurrentState(primaryNode, X) ORed */ +static const NodeStatePattern FSM_PRIMARY_OR_WAIT_OR_JOIN = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_JOIN_PRIMARY, + REPLICATION_STATE_PRIMARY), +}; + +/* WAIT_PRIMARY/JOIN_PRIMARY only, not PRIMARY -- a distinct, narrower set from the one above */ +static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_JOIN_PRIMARY), +}; + +/* the "primary role" states inside ProceedGroupStateForPrimaryNode -- a different + * three-element set from FSM_PRIMARY_OR_WAIT_OR_JOIN above (no JOIN_PRIMARY, has APPLY_SETTINGS) */ +static const NodeStatePattern FSM_PRIMARY_ROLE_STATES = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_PRIMARY, + REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_APPLY_SETTINGS), +}; + +/* same "primary role" scope, minus WAIT_PRIMARY -- a narrower enumerated STABLE set */ +static const NodeStatePattern FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_PRIMARY, + REPLICATION_STATE_APPLY_SETTINGS), +}; + +/* reported WAIT_PRIMARY, goal in {WAIT_PRIMARY, PRIMARY} -- join_secondary's cascade row */ +static const NodeStatePattern FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY = { + .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY), + .assignedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, REPLICATION_STATE_PRIMARY), +}; + +/* reported in {WAIT_PRIMARY,JOIN_PRIMARY}, goal PRIMARY -- demoted->catchingup, first disjunct */ +static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY = { + .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, REPLICATION_STATE_JOIN_PRIMARY), + .assignedStates = STATES(REPLICATION_STATE_PRIMARY), +}; + +/* goalState != WAIT_PRIMARY, reportedState irrelevant -- wait_maintenance's second row */ +static const NodeStatePattern FSM_NOT_ASSIGNED_WAIT_PRIMARY = { + .kind = NODE_STATE_NOT_ASSIGNED, + .assignedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY), +}; + +/* !IsCurrentState(node, WAIT_PRIMARY): not converged to wait_primary, for any reason */ +static const NodeStatePattern FSM_NOT_STABLE_WAIT_PRIMARY = { + .kind = NODE_STATE_NOT_STABLE, + .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY), +}; + +/* !IsCurrentState(node, SINGLE) */ +static const NodeStatePattern FSM_NOT_STABLE_SINGLE = { + .kind = NODE_STATE_NOT_STABLE, + .reportedStates = STATES(REPLICATION_STATE_SINGLE), +}; + +/* goalState == DROPPED, reportedState irrelevant */ +static const NodeStatePattern FSM_DROPPED_GOAL = { + .kind = NODE_STATE_ASSIGNED, + .assignedStates = STATES(REPLICATION_STATE_DROPPED), +}; + +/* reportedState == DEMOTE_TIMEOUT, goalState irrelevant. NOT FSM_STATE(DEMOTE_TIMEOUT), which + * would also require goalState == DEMOTE_TIMEOUT -- the opposite of what this self-fence guard + * needs: it must catch a node whose goalState is still whatever was assigned before the + * self-fence fired (see the real comment on this guard, preserved below). */ +static const NodeStatePattern FSM_REPORTED_DEMOTE_TIMEOUT = { + .kind = NODE_STATE_REPORTED, + .reportedStates = STATES(REPLICATION_STATE_DEMOTE_TIMEOUT), +}; + +/* reportedState in {REPORT_LSN, FAST_FORWARD}, converged -- "continue an already started + * failover" guard, the direct (non-cascading) entry into ProceedGroupStateForMSFailover */ +static const NodeStatePattern FSM_REPORT_LSN_OR_FAST_FORWARD = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_REPORT_LSN, REPLICATION_STATE_FAST_FORWARD), +}; + + +static bool +NodeStateMatchesPattern(const AutoFailoverNode *node, const NodeStatePattern *pattern) +{ + if (node == NULL) + { + /* No node this round (no primary). Only NODE_STATE_ANY can still match -- every other + * kind needs a real reportedState/goalState to compare, which a nonexistent node simply + * doesn't have. Existence itself is checked separately via NodeStatusPattern.exists. */ + return pattern->kind == NODE_STATE_ANY; + } + + ReplicationState reported = node->reportedState; + ReplicationState goal = node->goalState; + + switch (pattern->kind) + { + case NODE_STATE_ANY: + { + return true; + } + + case NODE_STATE_STABLE: + { + return (reported == goal) && MatchStateSet(reported, pattern->reportedStates); + } + + case NODE_STATE_NOT_STABLE: + { + return !((reported == goal) && MatchStateSet(reported, pattern->reportedStates)); + } + + case NODE_STATE_REPORTED: + { + return MatchStateSet(reported, pattern->reportedStates); + } + + case NODE_STATE_ASSIGNED: + { + return MatchStateSet(goal, pattern->assignedStates); + } + + case NODE_STATE_NOT_ASSIGNED: + { + return !MatchStateSet(goal, pattern->assignedStates); + } + + case NODE_STATE_TRANSITIONING: + { + return MatchStateSet(reported, pattern->reportedStates) && + MatchStateSet(goal, pattern->assignedStates); + } + + default: + { + return false; + } + } +} + + +/* + * NodeStatus: every per-node fact this table's conditions need, computed once + * per role (activeNode, primaryNode) at the top of dispatch by BuildNodeStatus(). + */ +typedef struct NodeStatus +{ + AutoFailoverNode *node; + GroupStateContext *ctx; + bool isHealthy; + bool isUnhealthy; + bool candidateEligible; + bool isCitusWorkerGroup; + bool replicationQuorum; + bool isComparableToReferenceTli; +} NodeStatus; + +typedef struct NodeStatusPattern +{ + BoolPattern exists; + NodeStatePattern statePattern; + BoolPattern isHealthy; + BoolPattern isUnhealthy; + BoolPattern candidateEligible; + BoolPattern isInPrimaryState; + BoolPattern isInMaintenance; + BoolPattern drainTimeExpired; + BoolPattern isCitusWorkerGroup; + BoolPattern replicationQuorum; + BoolPattern isComparableToReferenceTli; + BoolPattern unreachableFromDemoteTimeout; +} NodeStatusPattern; + +static void +BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *status) +{ + memset(status, 0, sizeof(NodeStatus)); + status->node = node; + status->ctx = ctx; + + if (node == NULL) + { + /* NodeIsUnhealthy(NULL, ctx) returns true -- a nonexistent node being "unhealthy" is + * exactly the semantics the original if-chain relies on. */ + status->isUnhealthy = true; + return; + } + + status->isHealthy = NodeIsHealthy(node, ctx); + status->isUnhealthy = NodeIsUnhealthy(node, ctx); + status->candidateEligible = node->candidatePriority > 0; + status->isCitusWorkerGroup = IsCitusFormation(ctx->formation) && node->groupId > 0; + status->replicationQuorum = node->replicationQuorum; +} + + +/* + * isInPrimaryState/isInMaintenance/drainTimeExpired/unreachableFromDemoteTimeout + * are deliberately NOT cached in NodeStatus and computed live here from + * status->node instead: unlike isHealthy/isUnhealthy/candidateEligible + * (health/priority facts that can't change mid-dispatch), these four are + * pure functions of node->goalState/reportedState, and a matched row's + * extraAction can reassign the very node being matched (e.g. DRAINING on + * primaryNode) in the SAME node_active() call, before dispatch continues to + * a later row -- exactly like NodeStateMatchesPattern below, which reads + * status->node's fields live for the same reason. Caching these as + * snapshot booleans (an earlier version of this code did) left later rows + * in the same call matching against a stale "still in primary state" fact + * even after primaryNode had just been moved to DRAINING -- confirmed by + * concurrent_health_check_and_report, which requires the "secondary -> + * prepare_promotion" row to correctly stop matching once primaryNode is no + * longer IsInPrimaryState(). + */ +static bool +NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) +{ + bool unreachableFromDemoteTimeout = + status->node != NULL && + status->node->goalState != REPLICATION_STATE_DEMOTE_TIMEOUT && + status->node->goalState != REPLICATION_STATE_DEMOTED && + status->node->goalState != REPLICATION_STATE_PRIMARY && + status->node->goalState != REPLICATION_STATE_SINGLE; + + return MatchBoolPattern(status->node != NULL, pattern->exists) && + NodeStateMatchesPattern(status->node, &pattern->statePattern) && + MatchBoolPattern(status->isHealthy, pattern->isHealthy) && + MatchBoolPattern(status->isUnhealthy, pattern->isUnhealthy) && + MatchBoolPattern(status->candidateEligible, pattern->candidateEligible) && + MatchBoolPattern(IsInPrimaryState(status->node), pattern->isInPrimaryState) && + MatchBoolPattern(IsInMaintenance(status->node), pattern->isInMaintenance) && + MatchBoolPattern(NodeIsDrainTimeExpired(status->node, status->ctx), + pattern->drainTimeExpired) && + MatchBoolPattern(status->isCitusWorkerGroup, pattern->isCitusWorkerGroup) && + MatchBoolPattern(status->replicationQuorum, pattern->replicationQuorum) && + MatchBoolPattern(status->isComparableToReferenceTli, + pattern->isComparableToReferenceTli) && + MatchBoolPattern(unreachableFromDemoteTimeout, + pattern->unreachableFromDemoteTimeout); +} + + +/* + * NodeActiveContext: group-level facts, computed once per dispatch call + * alongside the two NodeStatus roles above. + */ +typedef struct NodeActiveContext +{ + NodeStatus activeNode; + NodeStatus primaryNode; + + bool groupHasExactlyOneNode; + bool groupHasMoreThanTwoNodes; + bool anyOtherNodeWaitingStandby; + + bool numberSyncStandbysIsZero; + bool replicationQuorumCountIsZero; + bool secondaryNodesCountIsZero; + bool secondaryQuorumNodesCountIsZero; + bool atLeastOneHealthyCandidate; + + bool walWithinPromoteThreshold; + bool walWithinSyncThreshold; + bool activeAndPrimaryTliMatch; + + bool primaryIsWaitPrimaryPresumedDead; + bool failoverInProgress; + bool replicationStallExceeded; +} NodeActiveContext; + +typedef struct NodeActiveContextPattern +{ + BoolPattern groupHasExactlyOneNode; + BoolPattern groupHasMoreThanTwoNodes; + BoolPattern anyOtherNodeWaitingStandby; + + BoolPattern numberSyncStandbysIsZero; + BoolPattern replicationQuorumCountIsZero; + BoolPattern secondaryNodesCountIsZero; + BoolPattern secondaryQuorumNodesCountIsZero; + BoolPattern atLeastOneHealthyCandidate; + + BoolPattern walWithinPromoteThreshold; + BoolPattern walWithinSyncThreshold; + BoolPattern activeAndPrimaryTliMatch; + + BoolPattern primaryIsWaitPrimaryPresumedDead; + BoolPattern failoverInProgress; + BoolPattern replicationStallExceeded; +} NodeActiveContextPattern; + + +typedef enum GoalStateAssignmentKind +{ + GOAL_STATE_NONE = 0, + GOAL_STATE_SET +} GoalStateAssignmentKind; + +typedef struct GoalStateAssignment +{ + GoalStateAssignmentKind kind; + ReplicationState state; +} GoalStateAssignment; + +/* No compound-literal cast -- see STATES()'s comment: every use nests inside + * another static aggregate's designated initializer. */ +#define GOAL(x) { .kind = GOAL_STATE_SET, .state = (x) } + +/* + * Returns true when this row's match should stand and dispatch should stop + * here (the overwhelming majority of extraActions -- a pure side effect + * alongside a genuine, unconditional transition). Returns false when the + * row's own facts didn't actually lead anywhere useful this round (the + * MS-failover cascade declining because e.g. not all candidates have + * reported yet) and the SAME call should keep searching later rows for + * activeNode's own transition -- exactly what the real source's fallthrough + * (no `return` after the DRAINING/MAINTENANCE assignment, see + * ActionRunMultiStandbyFailoverCascade below) does. Assignments on the matched row + * itself still apply either way; this only controls whether dispatch + * continues afterward. + */ +typedef bool (*MonitorExtraActionFunction) (GroupStateContext *ctx, + NodeActiveContext *nac, + char *message); + +typedef struct MonitorFSMTransition +{ + NodeStatusPattern activeNode; + NodeStatusPattern primaryNode; + NodeActiveContextPattern conditions; + + GoalStateAssignment activeNodeAssignedState; + GoalStateAssignment otherNodeAssignedState; /* target: nac->primaryNode.node */ + + MonitorExtraActionFunction extraAction; + + const char *comment; +} MonitorFSMTransition; + + +static bool +RuleMatches(const NodeActiveContext *nac, const MonitorFSMTransition *rule) +{ + const NodeActiveContextPattern *cond = &rule->conditions; + + return NodeMatchesPattern(&nac->activeNode, &rule->activeNode) && + NodeMatchesPattern(&nac->primaryNode, &rule->primaryNode) && + + MatchBoolPattern(nac->groupHasExactlyOneNode, cond->groupHasExactlyOneNode) && + MatchBoolPattern(nac->groupHasMoreThanTwoNodes, cond->groupHasMoreThanTwoNodes) && + MatchBoolPattern(nac->anyOtherNodeWaitingStandby, cond->anyOtherNodeWaitingStandby) && + MatchBoolPattern(nac->numberSyncStandbysIsZero, cond->numberSyncStandbysIsZero) && + MatchBoolPattern(nac->replicationQuorumCountIsZero, + cond->replicationQuorumCountIsZero) && + MatchBoolPattern(nac->secondaryNodesCountIsZero, cond->secondaryNodesCountIsZero) && + MatchBoolPattern(nac->secondaryQuorumNodesCountIsZero, + cond->secondaryQuorumNodesCountIsZero) && + MatchBoolPattern(nac->atLeastOneHealthyCandidate, cond->atLeastOneHealthyCandidate) && + MatchBoolPattern(nac->walWithinPromoteThreshold, cond->walWithinPromoteThreshold) && + MatchBoolPattern(nac->walWithinSyncThreshold, cond->walWithinSyncThreshold) && + MatchBoolPattern(nac->activeAndPrimaryTliMatch, cond->activeAndPrimaryTliMatch) && + MatchBoolPattern(nac->primaryIsWaitPrimaryPresumedDead, + cond->primaryIsWaitPrimaryPresumedDead) && + MatchBoolPattern(nac->failoverInProgress, cond->failoverInProgress) && + MatchBoolPattern(nac->replicationStallExceeded, cond->replicationStallExceeded); +} + + +static int +FindMatchingMonitorFSMRuleIndexFrom(const MonitorFSMTransition table[], int tableSize, + int startIndex, const NodeActiveContext *nac) +{ + for (int i = startIndex; i < tableSize; i++) + { + if (RuleMatches(nac, &table[i])) + { + return i; + } + } + return -1; +} + + +static int +FindMatchingMonitorFSMRuleIndex(const MonitorFSMTransition table[], int tableSize, + const NodeActiveContext *nac) +{ + return FindMatchingMonitorFSMRuleIndexFrom(table, tableSize, 0, nac); +} + + +/* + * AssignDeclaredGoalState asserts that a rule only ever assigns a state it + * actually declared: drift between a row's own assignment slots and what it + * assigns fails loudly (an Assert) instead of silently rotting. + */ +static void +AssignDeclaredGoalState(const MonitorFSMTransition *rule, AutoFailoverNode *node, + ReplicationState state, char *message) +{ +#ifdef USE_ASSERT_CHECKING + bool declared = + (rule->activeNodeAssignedState.kind == GOAL_STATE_SET && + rule->activeNodeAssignedState.state == state) || + (rule->otherNodeAssignedState.kind == GOAL_STATE_SET && + rule->otherNodeAssignedState.state == state); + + Assert(declared); +#endif + + AssignGoalState(node, state, message); +} + + +static bool +DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, + const MonitorFSMTransition *rule) +{ + char message[BUFSIZE] = { 0 }; + bool stopDispatch = true; + + if (rule->comment != NULL) + { + snprintf(message, BUFSIZE, "%s", rule->comment); + } + + if (rule->extraAction != NULL) + { + stopDispatch = rule->extraAction(ctx, nac, message); + } + + if (rule->activeNodeAssignedState.kind == GOAL_STATE_SET) + { + AssignDeclaredGoalState(rule, nac->activeNode.node, + rule->activeNodeAssignedState.state, message); + } + + if (rule->otherNodeAssignedState.kind == GOAL_STATE_SET) + { + AssignDeclaredGoalState(rule, nac->primaryNode.node, + rule->otherNodeAssignedState.state, message); + } + + return stopDispatch; +} + + /* GUC variables */ int EnableSyncXlogThreshold = DEFAULT_XLOG_SEG_SIZE; int PromoteXlogThreshold = DEFAULT_XLOG_SEG_SIZE; @@ -138,1038 +709,812 @@ ProceedGroupState(AutoFailoverNode *activeNode) /* - * ProceedGroupStateFromContext is the core FSM logic, operating entirely on - * the pre-built GroupStateContext. It does not touch the database for reads; - * writes (AssignGoalState, NotifyStateChange) still go to the DB. - * - * This separation lets test code inject a synthetic context and exercise the - * FSM without a live database connection. + * OtherNodeIsDueForCatchingUp is shared between the count computation in + * BuildForPrimaryNodeNodeActiveContext() and the fan-out assignment in + * ActionCatchupUnhealthySecondaries() below, so the two can never drift + * apart on which nodes they mean by "unhealthy secondary" -- both need to + * agree, since the counts drive which row matches and the fan-out is that + * row's own side effect. */ -bool -ProceedGroupStateFromContext(GroupStateContext *ctx) +static bool +OtherNodeIsDueForCatchingUp(GroupStateContext *ctx, AutoFailoverNode *otherNode) { - AutoFailoverNode *activeNode = ctx->activeNode; - char *formationId = ctx->formationId; - int groupId = ctx->groupId; - int nodesCount = ctx->groupNodeCount; + return otherNode->goalState == REPLICATION_STATE_SECONDARY && + otherNode->reportedState != REPLICATION_STATE_REPORT_LSN && + otherNode->reportedState != REPLICATION_STATE_JOIN_SECONDARY && + NodeIsUnhealthy(otherNode, ctx); +} - /* - * If the active node just reached the DROPPED state, proceed to remove it - * from the pgautofailover.node table. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_DROPPED)) - { - char message[BUFSIZE] = { 0 }; - /* time to actually remove the current node */ - RemoveAutoFailoverNode(activeNode); +static bool +ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *message) +{ + RemoveAutoFailoverNode(ctx->activeNode); - LogAndNotifyMessage( - message, BUFSIZE, - "Removing " NODE_FORMAT " from formation \"%s\" and group %d", - NODE_FORMAT_ARGS(activeNode), - activeNode->formationId, - activeNode->groupId); + return true; +} - return true; - } - /* node reports secondary/dropped */ - if (activeNode->goalState == REPLICATION_STATE_DROPPED) +/* + * ActionRunMultiStandbyFailoverCascade implements the whole + * nodesCount>2-unhealthy-primary block as a single extraAction: the DRAINING/ + * MAINTENANCE/nothing if/else-if decision, followed unconditionally by + * ProceedGroupStateForMSFailover(). The real source never `return`s after + * assigning DRAINING/MAINTENANCE to the primary -- it always falls through to + * try ProceedGroupStateForMSFailover next, in the SAME outer if-block, and if + * THAT declines (returns false), falls through further still to the rest of + * ProceedGroupStateFromContext's own if-chain (the report_lsn/prepare_ + * promotion/stop_replication/... rows, for this SAME activeNode). + * + * This has to be ONE row/action, not three separate declarative rows sharing + * this action (as an earlier version of this file had it): once dispatch + * continues past a declined row, it keeps scanning forward and a later, + * broader row matching the same outer "nodesCount>2, primary unhealthy" + * condition (the catch-all "neither DRAINING nor MAINTENANCE applies" case) + * would match too and re-invoke ProceedGroupStateForMSFailover a *second* + * time in the same node_active() call -- something the original single-pass + * if/else-if structure never does. Confirmed by concurrent_second_primary_ + * death_report and concurrent_health_check_and_report, which got stuck (the + * former) or produced a spurious second cascade invocation changing the + * outcome (the latter) until this was folded into a single row/action pair. + */ +static bool +ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + AutoFailoverNode *primaryNode = nac->primaryNode.node; + + List *candidateNodesList = + AutoFailoverOtherNodesListInState(primaryNode, REPLICATION_STATE_SECONDARY); + int candidatesCount = CountHealthyCandidates(candidateNodesList); + + if (IsInPrimaryState(primaryNode) && + !IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) && + candidatesCount >= 1) { - return true; - } + char drainingMessage[BUFSIZE] = { 0 }; - /* - * A node in "maintenance" state can only get out of maintenance through an - * explicit call to stop_maintenance(), the FSM will not assign a new state - * to a node that is currently in maintenance. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_MAINTENANCE)) + snprintf(drainingMessage, BUFSIZE, + "Setting goal state of " NODE_FORMAT + " to draining after it became unhealthy.", + NODE_FORMAT_ARGS(primaryNode)); + + AssignGoalState(primaryNode, REPLICATION_STATE_DRAINING, drainingMessage); + } + else if (IsCurrentState(primaryNode, REPLICATION_STATE_PREPARE_MAINTENANCE)) { - return true; + char maintenanceMessage[BUFSIZE] = { 0 }; + + snprintf(maintenanceMessage, BUFSIZE, + "Setting goal state of " NODE_FORMAT + " to maintenance after it converged to prepare_maintenance.", + NODE_FORMAT_ARGS(primaryNode)); + + AssignGoalState(primaryNode, REPLICATION_STATE_MAINTENANCE, maintenanceMessage); } - /* - * A node reporting demote_timeout may have gotten there on its own - * initiative (check_for_network_partitions() in service_keeper.c - * self-fences independently of whatever goal the monitor last assigned -- - * see #1025). If the currently assigned goal isn't one demote_timeout can - * actually reach, the keeper would fatal forever trying to get there. - * Re-target to demoted: always a valid demote_timeout exit - * (DEMOTE_TIMEOUT_STATE -> DEMOTED_STATE, fsm.c:355), and the safe, - * conservative choice -- the node stays fenced from writes until the - * existing "demoted -> catchingup" reintegration path - * (group_state_machine.c:909) or an operator decides otherwise. - * - * Deliberately a plain reportedState check, not IsCurrentState(): the - * whole point is to catch reportedState == demote_timeout while - * goalState is still whatever was assigned before the self-fence -- - * IsCurrentState() requires goalState == reportedState == state, which - * is exactly the case that does NOT need re-targeting (the node is - * already headed somewhere demote_timeout can reach). - */ - if (activeNode->reportedState == REPLICATION_STATE_DEMOTE_TIMEOUT && - activeNode->goalState != REPLICATION_STATE_DEMOTE_TIMEOUT && - activeNode->goalState != REPLICATION_STATE_DEMOTED && - activeNode->goalState != REPLICATION_STATE_PRIMARY && - activeNode->goalState != REPLICATION_STATE_SINGLE) - { - char message[BUFSIZE] = { 0 }; + return ProceedGroupStateForMSFailover(ctx, primaryNode); +} - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to demoted: it reports demote_timeout but is assigned %s, " - "which demote_timeout cannot reach.", - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->goalState)); - AssignGoalState(activeNode, REPLICATION_STATE_DEMOTED, message); +/* + * ActionRunPlainMSFailoverCascade is the "continue an already-started + * failover" call site: activeNode itself is REPORT_LSN or FAST_FORWARD, and + * the real source just `return`s ProceedGroupStateForMSFailover()'s result + * directly, with no DRAINING/MAINTENANCE decision attached. + */ +static bool +ActionRunPlainMSFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, char *message) +{ + return ProceedGroupStateForMSFailover(ctx, nac->primaryNode.node); +} - return true; - } - /* - * A node that is alone in its group should be SINGLE. - * - * Exception arises when it used to be other nodes in the group, and the - * only node left has Candidate Priority of zero. In that case the setup is - * clear, it can't allow writes, so it can't be SINGLE. In that case, it - * should be REPORT_LSN, waiting for either a change of settings, or the - * introduction of a new node. - */ - if (nodesCount == 1 && - !IsCurrentState(activeNode, REPLICATION_STATE_SINGLE) && - activeNode->candidatePriority > 0) - { - char message[BUFSIZE]; +static bool +ActionRunPrimaryNodeTransition(GroupStateContext *ctx, NodeActiveContext *nac, char *message) +{ + (void) ProceedGroupStateForPrimaryNode(ctx, nac->primaryNode.node); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to single as there is no other node.", - NODE_FORMAT_ARGS(activeNode)); + return true; +} - /* other node may have been removed */ - AssignGoalState(activeNode, REPLICATION_STATE_SINGLE, message); - return true; - } - else if (nodesCount == 1 && - !IsCurrentState(activeNode, REPLICATION_STATE_SINGLE) && - activeNode->candidatePriority == 0) +static bool +ActionCatchupUnhealthySecondaries(GroupStateContext *ctx, NodeActiveContext *nac, char *message) +{ + AutoFailoverNode *primaryNode = nac->activeNode.node; + List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); + ListCell *nodeCell = NULL; + + foreach(nodeCell, otherNodesGroupList) { - char message[BUFSIZE]; + AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn as there is no other node" - " and candidate priority is %d.", - NODE_FORMAT_ARGS(activeNode), - activeNode->candidatePriority); + if (OtherNodeIsDueForCatchingUp(ctx, otherNode)) + { + char otherMessage[BUFSIZE] = { 0 }; - /* other node may have been removed */ - AssignGoalState(activeNode, REPLICATION_STATE_REPORT_LSN, message); + snprintf(otherMessage, BUFSIZE, + "Setting goal state of " NODE_FORMAT + " to catchingup after it became unhealthy.", + NODE_FORMAT_ARGS(otherNode)); - return true; + AssignGoalState(otherNode, REPLICATION_STATE_CATCHINGUP, otherMessage); + } } - /* - * We separate out the FSM for the primary server, because that one needs - * to loop over every other node to take decisions. That induces some - * complexity that is best managed in a specialized function. - */ - if (IsInPrimaryState(activeNode)) - { - return ProceedGroupStateForPrimaryNode(ctx, activeNode); - } + return true; +} - /* - * Derive primaryNode from ctx->groupNodeList (already fetched under the - * lock NodeActive() holds for the whole call) instead of running a - * second, independent AutoFailoverNodeGroup() query -- see - * GetPrimaryOrDemotedNodeInGroupFromList()'s comment for why this - * matters even though every writer now shares the same lock. - */ - AutoFailoverNode *primaryNode = - GetPrimaryOrDemotedNodeInGroupFromList(ctx->groupNodeList); - /* - * We want to have a primaryNode around for most operations, but also need - * to support the case that the primaryNode has been dropped manually by a - * call to remove_node(). So we have two main cases to think about here: - * - * - we have two nodes, one of them has been removed, we catch that earlier - * in this function and assign the remaining one with the SINGLE state, - * - * - we have more than two nodes in total, and the primary has just been - * removed (maybe it was still marked unhealthy and the operator knows it - * won't ever come back so called remove_node() already): in that case in - * remove_node() we set all the other nodes to REPORT_LSN (unless they - * are in MAINTENANCE), and we should be able to make progress with the - * failover without a primary around. - * - * In all other cases we require a primaryNode to be identified. - */ - if (primaryNode == NULL && !IsFailoverInProgress(ctx->groupNodeList)) - { - ereport(ERROR, - (errmsg("ProceedGroupState couldn't find the primary node " - "in formation \"%s\", group %d", - formationId, groupId), - errdetail("activeNode is " NODE_FORMAT - " in state %s", - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->goalState)))); - } +/* + * BuildFromContextNodeActiveContext computes every fact MonitorFSM_FromContext + * needs, mirroring exactly what the original ProceedGroupStateFromContext() + * if-chain read inline. primaryNode may be NULL (failover already in + * progress, primary removed). + */ +static void +BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *primaryNode, + NodeActiveContext *nac) +{ + AutoFailoverNode *activeNode = ctx->activeNode; - /* - * Detect a genuine timeline fork on a healthy secondary as soon as its - * newly reported timeline is visible, rather than waiting for an - * incidental health-check cycle or an explicit maintenance toggle to - * eventually drive it through catchingup (see #683 and the timeline - * fork detection design). Reuses the exact same ancestry filter the - * report_lsn election path already applies (below, and in - * ProceedGroupStateForMSFailover) -- this only changes when a - * diverged secondary gets pushed to catchingup, not how that's - * decided or how the resync itself recovers it. - * - * Scoped to activeNode currently being SECONDARY_STATE: that's the - * same state the "unhealthy secondary" transition just below already - * uses for the identical CATCHINGUP goal assignment, so this is - * additive to an existing, tested transition rather than a new kind - * of one. A node with reportedTLI == 0 hasn't reported a timeline yet - * (e.g. still in wait_standby) and has nothing to check. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_SECONDARY) && - activeNode->reportedTLI > 0) + memset(nac, 0, sizeof(NodeActiveContext)); + + BuildNodeStatus(ctx, activeNode, &nac->activeNode); + BuildNodeStatus(ctx, primaryNode, &nac->primaryNode); + + /* isComparableToReferenceTli defaults to true (row :328 doesn't fire) -- a node that hasn't + * reported a timeline yet (reportedTLI == 0) has nothing to check, same as the original. */ + nac->activeNode.isComparableToReferenceTli = true; + if (activeNode->reportedTLI > 0) { int referenceTli = 0; List *comparableNodeList = - FilterNodesByTimelineAncestry(ctx->groupNodeList, formationId, - groupId, &referenceTli); + FilterNodesByTimelineAncestry(ctx->groupNodeList, ctx->formationId, + ctx->groupId, &referenceTli); - bool activeNodeIsComparable = false; - ListCell *cell = NULL; - - foreach(cell, comparableNodeList) + if (referenceTli > 0) { - AutoFailoverNode *node = (AutoFailoverNode *) lfirst(cell); + bool comparable = false; + ListCell *cell = NULL; - if (node->nodeId == activeNode->nodeId) + foreach(cell, comparableNodeList) { - activeNodeIsComparable = true; - break; + AutoFailoverNode *node = (AutoFailoverNode *) lfirst(cell); + + if (node->nodeId == activeNode->nodeId) + { + comparable = true; + break; + } } + + nac->activeNode.isComparableToReferenceTli = comparable; } + } - if (referenceTli > 0 && !activeNodeIsComparable) - { - char message[BUFSIZE] = { 0 }; + nac->groupHasExactlyOneNode = (ctx->groupNodeCount == 1); + nac->groupHasMoreThanTwoNodes = (ctx->groupNodeCount > 2); + nac->failoverInProgress = IsFailoverInProgress(ctx->groupNodeList); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup: its reported timeline %d does not appear " - "to be an ancestor of the group's reference timeline %d -- " - "forcing a resync so the ancestry check can run and, if " - "needed, rewind it onto the correct lineage.", - NODE_FORMAT_ARGS(activeNode), - activeNode->reportedTLI, - referenceTli); + nac->activeAndPrimaryTliMatch = + primaryNode != NULL && activeNode->reportedTLI == primaryNode->reportedTLI; - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); + nac->walWithinPromoteThreshold = + WalDifferenceWithin(activeNode, primaryNode, PromoteXlogThreshold); + nac->walWithinSyncThreshold = + WalDifferenceWithin(activeNode, primaryNode, EnableSyncXlogThreshold); - return true; - } - } + nac->primaryIsWaitPrimaryPresumedDead = + NodeIsWaitPrimaryPresumedDead(primaryNode, activeNode, ctx); - /* - * Replication stall detection (issue #997 — 3-DC split-brain). - * - * When the primary is healthy (monitor can reach it) but no standby has - * appeared in pg_stat_replication for longer than - * pgautofailover.replication_stall_timeout, we assign wait_primary. - * That clears synchronous_standby_names so COMMIT no longer hangs. - * - * replication_stall_since is set/cleared by ReportAutoFailoverNodeState() - * each time the keeper calls node_active(). - */ - if (IsInPrimaryState(primaryNode) && - NodeIsHealthy(primaryNode, ctx) && + nac->replicationStallExceeded = + primaryNode != NULL && primaryNode->replicationStallSince != 0 && TimestampDifferenceExceeds(primaryNode->replicationStallSince, - ctx->now, - ctx->replicationStallTimeoutMs)) + ctx->now, ctx->replicationStallTimeoutMs); + + if (ctx->groupNodeCount > 2 && nac->primaryNode.isUnhealthy) { - char message[BUFSIZE] = { 0 }; + List *candidateNodesList = + AutoFailoverOtherNodesListInState(primaryNode, REPLICATION_STATE_SECONDARY); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary: no standby has been connected in " - "pg_stat_replication for more than %dms " - "(pgautofailover.replication_stall_timeout).", - NODE_FORMAT_ARGS(primaryNode), - ctx->replicationStallTimeoutMs); + nac->atLeastOneHealthyCandidate = CountHealthyCandidates(candidateNodesList) >= 1; + } +} - AssignGoalState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY, message); - return true; - } +/* + * BuildForPrimaryNodeNodeActiveContext computes every fact + * MonitorFSM_ForPrimaryNode needs, mirroring the counting loop that used to + * be inline at the top of ProceedGroupStateForPrimaryNode() (the same loop + * OtherNodeIsDueForCatchingUp's condition drives the fan-out assignment + * for, in ActionCatchupUnhealthySecondaries above). + */ +static void +BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *primaryNode, + NodeActiveContext *nac) +{ + memset(nac, 0, sizeof(NodeActiveContext)); - /* Multiple Standby failover is handled in its own function. */ - if (nodesCount > 2 && NodeIsUnhealthy(primaryNode, ctx)) - { - /* - * The WAIT_PRIMARY state encodes the fact that we know there is no - * failover candidate, so there's no point in orchestrating a failover, - * even though the primary node is currently not available. - * - * To be in the WAIT_PRIMARY means that the other nodes are all either - * unhealty or with candidate priority set to zero. - * - * Otherwise stop replication from the primary and proceed with - * candidate election for primary replacement, whenever we have at - * least one candidates for failover. - */ - List *candidateNodesList = - AutoFailoverOtherNodesListInState(primaryNode, - REPLICATION_STATE_SECONDARY); + BuildNodeStatus(ctx, primaryNode, &nac->activeNode); + /* .primaryNode role is unused by every MonitorFSM_ForPrimaryNode row -- primaryNode IS + * activeNode here, so every condition is expressed against .activeNode directly. */ - int candidatesCount = CountHealthyCandidates(candidateNodesList); + List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); + int otherNodesCount = list_length(otherNodesGroupList); - if (IsInPrimaryState(primaryNode) && - !IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) && - candidatesCount >= 1) - { - char message[BUFSIZE] = { 0 }; + int replicationQuorumCount = otherNodesCount; + int secondaryNodesCount = otherNodesCount; + int secondaryQuorumNodesCount = otherNodesCount; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to draining after it became unhealthy.", - NODE_FORMAT_ARGS(primaryNode)); + ListCell *nodeCell = NULL; - AssignGoalState(primaryNode, REPLICATION_STATE_DRAINING, message); - } + foreach(nodeCell, otherNodesGroupList) + { + AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); - /* - * In a multiple standby system we can assign maintenance as soon as - * prepare_maintenance has been reached, at the same time than an - * election is triggered. This also allows the operator to disable - * maintenance on the old-primary and have it join the election. - */ - else if (IsCurrentState(primaryNode, REPLICATION_STATE_PREPARE_MAINTENANCE)) + if (OtherNodeIsDueForCatchingUp(ctx, otherNode)) { - char message[BUFSIZE] = { 0 }; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance after it converged to prepare_maintenance.", - NODE_FORMAT_ARGS(primaryNode)); + --secondaryNodesCount; + --secondaryQuorumNodesCount; + } + else if (!IsCurrentState(otherNode, REPLICATION_STATE_SECONDARY)) + { + --secondaryNodesCount; + --secondaryQuorumNodesCount; + } + else if (IsCurrentState(otherNode, REPLICATION_STATE_SECONDARY) && + !otherNode->replicationQuorum) + { + --secondaryQuorumNodesCount; + } - AssignGoalState(primaryNode, REPLICATION_STATE_MAINTENANCE, message); + if (!otherNode->replicationQuorum) + { + --replicationQuorumCount; } - /* - * ProceedGroupStateForMSFailover chooses the failover candidate when - * there's more than one standby node around, by applying the - * candidatePriority and comparing the reportedLSN. The function also - * orchestrate fetching the missing WAL from the failover candidate if - * that's needed. - * - * When ProceedGroupStateForMSFailover returns true, it means it was - * successfull in driving the failover to the next step, and we should - * stop here. When it return false, it did nothing, and so we want to - * apply the common orchestration code for a failover. - */ - if (ProceedGroupStateForMSFailover(ctx, primaryNode)) + if (IsCurrentState(otherNode, REPLICATION_STATE_WAIT_STANDBY)) { - return true; + nac->anyOtherNodeWaitingStandby = true; } } - /* - * when report_lsn and the promotion has been done already: - * report_lsn -> secondary - * - * - * Let the main primary loop account for allSecondariesAreHealthy and only - * then decide to assign PRIMARY to the primaryNode. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_REPORT_LSN) && - (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_JOIN_PRIMARY)) && - NodeIsHealthy(primaryNode, ctx)) - { - char message[BUFSIZE] = { 0 }; + nac->replicationQuorumCountIsZero = (replicationQuorumCount == 0); + nac->secondaryNodesCountIsZero = (secondaryNodesCount == 0); + nac->secondaryQuorumNodesCountIsZero = (secondaryQuorumNodesCount == 0); + nac->numberSyncStandbysIsZero = (ctx->formation->number_sync_standbys == 0); + nac->failoverInProgress = IsFailoverInProgress(ctx->groupNodeList); +} - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to secondary after " NODE_FORMAT - " converged to %s and has been marked healthy.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryNode->reportedState)); - AssignGoalState(activeNode, REPLICATION_STATE_SECONDARY, message); +/* + * MonitorFSM_EarlyChecks: the six checks the real if-chain runs BEFORE the + * IsInPrimaryState(activeNode) early return (group_state_machine.c:284) -- + * DROPPED, goal==DROPPED, MAINTENANCE, the demote_timeout self-fence, and + * both "alone in group" rows. These fire regardless of whether activeNode is + * currently the primary (a primary that just lost its only standby must + * still reach SINGLE here, before ever redirecting to + * ProceedGroupStateForPrimaryNode) -- confirmed by the drop_node regression + * test, which failed the first time this table put the primary-state + * redirect ahead of these six checks instead of after them. None of these + * six rows reference .primaryNode at all, so they can be matched against a + * NodeActiveContext built with primaryNode == NULL, before primaryNode is + * even resolved. + */ +static const MonitorFSMTransition MonitorFSM_EarlyChecks[] = { + /* converged to dropped -> remove the node from the catalog entirely */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DROPPED) }, + .extraAction = ActionRemoveDroppedNode, + .comment = "converged to dropped -> remove the node from the catalog" }, + + /* goal already dropped (mid-drop, row above hasn't converged yet) -> no-op */ + { .activeNode = { .statePattern = FSM_DROPPED_GOAL }, + .comment = "goal already dropped -> no-op" }, + + /* converged to maintenance -> no-op, frozen until stop_maintenance() */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_MAINTENANCE) }, + .comment = "converged to maintenance -> no-op, frozen until stop_maintenance()" }, + + /* demote_timeout self-fence re-target (issue #1025) */ + { .activeNode = { .statePattern = FSM_REPORTED_DEMOTE_TIMEOUT, + .unreachableFromDemoteTimeout = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "reported demote_timeout, assigned goal can't reach it -> demoted" }, + + /* alone in group, candidate-eligible */ + { .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, + .candidateEligible = BOOL_TRUE }, + .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SINGLE), + .comment = "alone in group, candidate-eligible -> single" }, + + /* alone in group, not candidate-eligible */ + { .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, + .candidateEligible = BOOL_FALSE }, + .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "alone in group, candidatePriority zero -> report_lsn" }, +}; + +#define MonitorFSM_EarlyChecks_SIZE \ + (sizeof(MonitorFSM_EarlyChecks) / sizeof(MonitorFSMTransition)) + - return true; - } +/* + * MonitorFSM_FromContext: the declarative replacement for the rest of + * ProceedGroupStateFromContext()'s own sequential if-chain -- everything + * from the timeline-fork check (group_state_machine.c:328, right after the + * IsInPrimaryState(activeNode) early return) onward. Rows are kept in the + * exact order the original if-chain checked them in: first-match-wins over + * this array is a straight extraction, not a behaviour change. + */ +static const MonitorFSMTransition MonitorFSM_FromContext[] = { + /* converged secondary, reportedTLI not an ancestor of the group's reference timeline */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), + .isComparableToReferenceTli = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "converged secondary, reportedTLI not an ancestor of reference -> catchingup" }, + + /* replication stall (#997): primary healthy, no standby past replication_stall_timeout */ + { .primaryNode = { .isInPrimaryState = BOOL_TRUE, + .isHealthy = BOOL_TRUE }, + .conditions = { .replicationStallExceeded = BOOL_TRUE }, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "primary healthy, no standby past replication_stall_timeout -> wait_primary" }, + + /* nodesCount>2, primary unhealthy -- draining/maintenance/nothing decided inside the + * action, then the MS-failover cascade unconditionally; must be a single row, see + * ActionRunMultiStandbyFailoverCascade's comment for why. */ + { .primaryNode = { .isUnhealthy = BOOL_TRUE }, + .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE }, + .extraAction = ActionRunMultiStandbyFailoverCascade, + .comment = "nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade" }, + + /* report_lsn, primary converged wait/join_primary, healthy */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY, + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .comment = "report_lsn, primary converged wait/join_primary, healthy -> secondary" }, + + /* report_lsn, primary converged primary, healthy */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY), + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .comment = "report_lsn, primary converged primary, healthy -> secondary" }, + + /* fast_forward done -> prepare_promotion */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_FAST_FORWARD) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), + .comment = "fast_forward done -> prepare_promotion" }, + + /* continue an already-started MS failover -- a direct `return` in the real source, and no + * later row in this table matches activeNode in REPORT_LSN/FAST_FORWARD, so it doesn't + * matter here whether the extraAction's bool stops dispatch or lets it keep scanning. */ + { .activeNode = { .statePattern = FSM_REPORT_LSN_OR_FAST_FORWARD }, + .extraAction = ActionRunPlainMSFailoverCascade, + .comment = "report_lsn or fast_forward, continuing an already-started failover -> " + "MS-failover cascade" }, + + /* wait_standby, primary converged wait/join_primary */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY) }, + .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "wait_standby, primary converged wait/join_primary -> catchingup" }, + + /* wait_standby (quorum member), primary converged primary */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .replicationQuorum = BOOL_TRUE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), + .comment = "wait_standby (quorum member), primary converged primary -> " + "catchingup + apply_settings" }, + + /* wait_standby (not a quorum member), primary converged primary */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .replicationQuorum = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "wait_standby (not a quorum member), primary converged primary -> catchingup" }, + + /* caught up, same TLI as primary, within sync threshold */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_CATCHINGUP), + .isHealthy = BOOL_TRUE }, + .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN }, + .conditions = { .activeAndPrimaryTliMatch = BOOL_TRUE, + .walWithinSyncThreshold = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .comment = "caught up, same TLI as primary, within sync threshold -> secondary" }, + + /* primary fails, already converged wait_primary (no draining edge, issue #1168) */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), + .isHealthy = BOOL_TRUE, + .candidateEligible = BOOL_TRUE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY), + .isUnhealthy = BOOL_TRUE }, + .conditions = { .walWithinPromoteThreshold = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), + .comment = "primary fails, already converged wait_primary (issue #1168) -> " + "secondary -> prepare_promotion only (1 of 2)" }, + + /* primary fails, not already wait_primary */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), + .isHealthy = BOOL_TRUE, + .candidateEligible = BOOL_TRUE }, + .primaryNode = { .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, + .isInPrimaryState = BOOL_TRUE, + .isUnhealthy = BOOL_TRUE }, + .conditions = { .walWithinPromoteThreshold = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DRAINING), + .comment = "primary fails, not already wait_primary -> secondary -> prepare_promotion, " + "primary -> draining (2 of 2)" }, + + /* wait_maintenance, primary converged wait_primary */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), + .comment = "wait_maintenance, primary converged wait_primary -> maintenance" }, + + /* wait_maintenance, primary's goal no longer wait_primary */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, + .primaryNode = { .statePattern = FSM_NOT_ASSIGNED_WAIT_PRIMARY }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), + .comment = "wait_maintenance, primary's goal no longer wait_primary -> maintenance" }, + + /* prepare_promotion, primary converged prepare_maintenance */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), + .comment = "prepare_promotion, primary converged prepare_maintenance -> stop_replication" }, + + /* Citus worker, primary present */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), + .isCitusWorkerGroup = BOOL_TRUE }, + .primaryNode = { .exists = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "Citus worker prepare_promotion, primary present -> wait_primary + demoted" }, + + /* Citus worker, primary removed */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), + .isCitusWorkerGroup = BOOL_TRUE }, + .primaryNode = { .exists = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "Citus worker prepare_promotion, primary removed -> wait_primary" }, + + /* prepare_promotion, primary present, already converged wait_primary (issue #1168) */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + .primaryNode = { .exists = BOOL_TRUE, + .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY), + .isInMaintenance = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), + .comment = "prepare_promotion, primary already converged wait_primary (issue #1168) -> " + "stop_replication only (1 of 2)" }, + + /* prepare_promotion, primary present, not in maintenance, not already wait_primary */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + .primaryNode = { .exists = BOOL_TRUE, + .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, + .isInMaintenance = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTE_TIMEOUT), + .comment = "prepare_promotion, primary present, not in maintenance -> " + "stop_replication + demote_timeout (2 of 2)" }, + + /* prepare_promotion, primary removed */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + .primaryNode = { .exists = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "prepare_promotion, primary removed -> wait_primary" }, + + /* stop_replication, primary converged prepare_maintenance */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), + .comment = "stop_replication, primary converged prepare_maintenance -> " + "wait_primary + maintenance" }, + + /* stop_replication, primary converged demote_timeout (3-way OR, 1 of 3) */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTE_TIMEOUT) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "stop_replication, primary converged demote_timeout -> " + "wait_primary + demoted (1 of 3)" }, + + /* stop_replication, primary's drain time expired (3-way OR, 2 of 3) */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .primaryNode = { .drainTimeExpired = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "stop_replication, primary's drain time expired -> " + "wait_primary + demoted (2 of 3)" }, + + /* stop_replication, primary's goal is wait_primary but presumed dead (3-way OR, 3 of 3) */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .conditions = { .primaryIsWaitPrimaryPresumedDead = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "stop_replication, primary's goal wait_primary but presumed dead -> " + "wait_primary + demoted (3 of 3)" }, + + /* Citus worker, primary present */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), + .isCitusWorkerGroup = BOOL_TRUE }, + .primaryNode = { .exists = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "Citus worker stop_replication, primary present -> wait_primary + demoted" }, + + /* Citus worker, primary removed */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), + .isCitusWorkerGroup = BOOL_TRUE }, + .primaryNode = { .exists = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "Citus worker stop_replication, primary removed -> wait_primary" }, + + /* demoted, primary reported wait/join_primary with goal primary */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, + .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY, + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "demoted, primary reported wait/join_primary with goal primary -> catchingup" }, + + /* demoted, primary converged wait/join_primary/primary, healthy */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, + .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN, + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "demoted, primary converged wait/join_primary/primary, healthy -> catchingup" }, + + /* join_secondary, primary reported wait_primary with goal wait/primary -- cascades into a + * nested pass on primaryNode */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, + .primaryNode = { .statePattern = FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .extraAction = ActionRunPrimaryNodeTransition, + .comment = "join_secondary, primary reported wait_primary with goal wait/primary -> " + "secondary" }, + + /* join_secondary, primary converged primary */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .comment = "join_secondary, primary converged primary -> secondary" }, +}; + +#define MonitorFSM_FromContext_SIZE \ + (sizeof(MonitorFSM_FromContext) / sizeof(MonitorFSMTransition)) - /* - * when report_lsn and the promotion has been done already: - * report_lsn -> secondary - * - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_REPORT_LSN) && - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY) && - NodeIsHealthy(primaryNode, ctx)) - { - char message[BUFSIZE]; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to secondary after " NODE_FORMAT - " got selected as the failover candidate.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); +/* + * MonitorFSM_ForPrimaryNode: the declarative replacement for + * ProceedGroupStateForPrimaryNode()'s own sequential if-chain. Here + * .activeNode maps to the primaryNode parameter, not a reporting node -- + * see ProceedGroupStateForPrimaryNode() below. + */ +static const MonitorFSMTransition MonitorFSM_ForPrimaryNode[] = { + /* primary alone, another node reached wait_standby */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SINGLE) }, + .conditions = { .anyOtherNodeWaitingStandby = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "primary alone, another node reached wait_standby -> wait_primary" }, + + /* all nodes async, zero secondaries */ + { .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, + .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, + .secondaryNodesCountIsZero = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "all nodes async, zero secondaries -> wait_primary" }, + + /* all nodes async, >=1 secondary */ + { .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, + .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, + .secondaryNodesCountIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "all nodes async, >=1 secondary -> primary" }, + + /* converged primary/apply_settings (not wait_primary), no quorum secondaries, + * number_sync_standbys=0, no failover in progress (issue #774) */ + { .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, + .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, + .failoverInProgress = BOOL_FALSE, + .numberSyncStandbysIsZero = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "converged primary/apply_settings, no quorum secondaries, no failover in " + "progress, number_sync_standbys=0 -> wait_primary" }, + + /* same, but number_sync_standbys>0 -> block writes on primary */ + { .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, + .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, + .failoverInProgress = BOOL_FALSE, + .numberSyncStandbysIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "converged primary/apply_settings, no quorum secondaries, no failover in " + "progress, number_sync_standbys>0 -> primary (block writes)" }, + + /* wait_primary, >=1 quorum secondary */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, + .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "wait_primary, >=1 quorum secondary -> primary" }, + + /* apply_settings, both zero */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, + .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, + .secondaryQuorumNodesCountIsZero = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "apply_settings, both zero -> wait_primary" }, + + /* apply_settings, number_sync_standbys != 0 (1 of 2 disjuncts) */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, + .conditions = { .numberSyncStandbysIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts)" }, + + /* apply_settings, sync_standbys=0 but >=1 quorum secondary (2 of 2) */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, + .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, + .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2)" }, + + /* converged primary/wait_primary/apply_settings, no other condition applies */ + { .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "converged primary/wait_primary/apply_settings, no other condition applies -> " + "no-op besides the unhealthy-secondary fan-out" }, + + /* backwards-compat: join_primary -> primary */ + { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .comment = "backwards-compat: join_primary -> primary" }, +}; + +#define MonitorFSM_ForPrimaryNode_SIZE \ + (sizeof(MonitorFSM_ForPrimaryNode) / sizeof(MonitorFSMTransition)) - AssignGoalState(activeNode, REPLICATION_STATE_SECONDARY, message); - return true; - } +/* + * ProceedGroupStateFromContext is the core FSM logic, operating entirely on + * the pre-built GroupStateContext. It does not touch the database for reads; + * writes (AssignGoalState, NotifyStateChange) still go to the DB. + * + * This separation lets test code inject a synthetic context and exercise the + * FSM without a live database connection. + */ +bool +ProceedGroupStateFromContext(GroupStateContext *ctx) +{ + AutoFailoverNode *activeNode = ctx->activeNode; + char *formationId = ctx->formationId; + int groupId = ctx->groupId; /* - * When the candidate is done fast forwarding the locally missing WAL bits, - * it can be promoted. + * MonitorFSM_EarlyChecks first, before the IsInPrimaryState redirect + * below -- these six checks run unconditionally in the real source, + * regardless of whether activeNode currently is the primary (a primary + * that just lost its only standby must still reach SINGLE here, not get + * redirected to ProceedGroupStateForPrimaryNode first). primaryNode + * isn't resolved yet at this point -- and none of these six rows + * reference it -- so NULL is passed and is safe. */ - if (IsCurrentState(activeNode, REPLICATION_STATE_FAST_FORWARD)) - { - char message[BUFSIZE] = { 0 }; + NodeActiveContext earlyNac; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to prepare_promotion", - NODE_FORMAT_ARGS(activeNode)); + BuildFromContextNodeActiveContext(ctx, NULL, &earlyNac); - AssignGoalState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION, message); + int earlyIndex = FindMatchingMonitorFSMRuleIndex(MonitorFSM_EarlyChecks, + MonitorFSM_EarlyChecks_SIZE, &earlyNac); - return true; + if (earlyIndex >= 0) + { + return DispatchMonitorFSMRule(ctx, &earlyNac, &MonitorFSM_EarlyChecks[earlyIndex]); } /* - * There are other cases when we want to continue an already started - * failover. + * We separate out the FSM for the primary server, because that one needs + * to loop over every other node to take decisions. That induces some + * complexity that is best managed in a specialized function. + * + * This early return can't become a MonitorFSM_FromContext row: it's + * exactly the branch point the whole table design has to preserve as an + * *entry* decision, not a matched condition -- see the note above + * MonitorFSM_FromContext. */ - if (IsCurrentState(activeNode, REPLICATION_STATE_REPORT_LSN) || - IsCurrentState(activeNode, REPLICATION_STATE_FAST_FORWARD)) + if (IsInPrimaryState(activeNode)) { - return ProceedGroupStateForMSFailover(ctx, primaryNode); + return ProceedGroupStateForPrimaryNode(ctx, activeNode); } /* - * when primary node is ready for replication: - * wait_standby -> catchingup + * Derive primaryNode from ctx->groupNodeList (already fetched under the + * lock NodeActive() holds for the whole call) instead of running a + * second, independent AutoFailoverNodeGroup() query -- see + * GetPrimaryOrDemotedNodeInGroupFromList()'s comment for why this + * matters even though every writer now shares the same lock. */ - if (IsCurrentState(activeNode, REPLICATION_STATE_WAIT_STANDBY) && - (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_JOIN_PRIMARY))) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after " NODE_FORMAT - " converged to %s.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryNode->reportedState)); - - /* start replication */ - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); - - return true; - } + AutoFailoverNode *primaryNode = + GetPrimaryOrDemotedNodeInGroupFromList(ctx->groupNodeList); /* - * when primary node is ready for replication: - * wait_standby -> catchingup - * primary -> apply_settings + * We want to have a primaryNode around for most operations, but also need + * to support the case that the primaryNode has been dropped manually by a + * call to remove_node(). So we have two main cases to think about here: + * + * - we have two nodes, one of them has been removed, we catch that earlier + * in this function and assign the remaining one with the SINGLE state, + * + * - we have more than two nodes in total, and the primary has just been + * removed (maybe it was still marked unhealthy and the operator knows it + * won't ever come back so called remove_node() already): in that case in + * remove_node() we set all the other nodes to REPORT_LSN (unless they + * are in MAINTENANCE), and we should be able to make progress with the + * failover without a primary around. + * + * In all other cases we require a primaryNode to be identified. */ - if (IsCurrentState(activeNode, REPLICATION_STATE_WAIT_STANDBY) && - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY) && - activeNode->replicationQuorum) + if (primaryNode == NULL && !IsFailoverInProgress(ctx->groupNodeList)) { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup and " NODE_FORMAT - " to %s to edit synchronous_standby_names.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryNode->reportedState)); - - /* start replication */ - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); + ereport(ERROR, + (errmsg("ProceedGroupState couldn't find the primary node " + "in formation \"%s\", group %d", + formationId, groupId), + errdetail("activeNode is " NODE_FORMAT + " in state %s", + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->goalState)))); + } - /* edit synchronous_standby_names to add the new standby now */ - AssignGoalState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS, message); + NodeActiveContext nac; - return true; - } + BuildFromContextNodeActiveContext(ctx, primaryNode, &nac); /* - * when primary node is ready for replication: - * wait_standby -> catchingup + * A matched row's extraAction may return false ("continue") to mirror + * the real source's fallthrough after the MS-failover cascade: when + * ProceedGroupStateForMSFailover() (via ActionRunMultiStandbyFailoverCascade) + * declines to act, the original if-chain keeps evaluating the rest of + * its conditions in the very same call instead of stopping. Re-scanning + * from startIndex on the same nac is safe here: RuleMatches() reads node + * state through the live AutoFailoverNode pointers stored in nac, so any + * goal state just assigned by this same dispatch (e.g. DRAINING on + * primaryNode) is already visible to the next match attempt. */ - if (IsCurrentState(activeNode, REPLICATION_STATE_WAIT_STANDBY) && - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY) && - !activeNode->replicationQuorum) + int startIndex = 0; + + while (true) { - char message[BUFSIZE]; + int index = FindMatchingMonitorFSMRuleIndexFrom(MonitorFSM_FromContext, + MonitorFSM_FromContext_SIZE, + startIndex, &nac); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup.", - NODE_FORMAT_ARGS(activeNode)); + if (index < 0) + { + return false; + } - /* start replication */ - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); + if (DispatchMonitorFSMRule(ctx, &nac, &MonitorFSM_FromContext[index])) + { + return true; + } - return true; - } - - /* - * when secondary caught up: - * catchingup -> secondary - * + wait_primary -> primary - * - * When we have multiple standby nodes and one of them is joining, or - * re-joining after maintenance, we have to edit the replication setting - * synchronous_standby_names on the primary. The transition from another - * state to PRIMARY includes that edit. If the primary already is in the - * primary state, we assign APPLY_SETTINGS to it to make sure its - * repication settings are updated now. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_CATCHINGUP) && - (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_JOIN_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY)) && - NodeIsHealthy(activeNode, ctx) && - activeNode->reportedTLI == primaryNode->reportedTLI && - WalDifferenceWithin(activeNode, primaryNode, EnableSyncXlogThreshold)) - { - char message[BUFSIZE] = { 0 }; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to secondary after it caught up.", - NODE_FORMAT_ARGS(activeNode)); - - /* node is ready for promotion */ - AssignGoalState(activeNode, REPLICATION_STATE_SECONDARY, message); - - return true; - } - - /* - * when primary fails: - * secondary -> prepare_promotion - * + primary -> draining - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_SECONDARY) && - IsInPrimaryState(primaryNode) && - NodeIsUnhealthy(primaryNode, ctx) && NodeIsHealthy(activeNode, ctx) && - activeNode->candidatePriority > 0 && - WalDifferenceWithin(activeNode, primaryNode, PromoteXlogThreshold)) - { - char message[BUFSIZE]; - - /* - * A primary already converged to wait_primary has no "draining" to - * go through: it never had a synchronous standby to begin with, so - * there's nothing live to gracefully drain, and KeeperFSM[] has no - * wait_primary -> draining edge anyway (issue #1168). Leave its - * goal untouched here; the prepare_promotion/stop_replication rules - * below apply the same drainTimeoutMs safety margin via report - * staleness instead of a goal-state timestamp, then commit the one - * real, reachable wait_primary -> demoted transition once it has - * genuinely expired. - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY)) - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to prepare_promotion after " NODE_FORMAT - " (at wait_primary) became unhealthy.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - } - else - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to draining and " NODE_FORMAT - " to prepare_promotion " - "after " NODE_FORMAT - " became unhealthy.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - - /* shut down the primary */ - AssignGoalState(primaryNode, REPLICATION_STATE_DRAINING, message); - } - - /* keep reading until no more records are available */ - AssignGoalState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION, message); - - return true; - } - - /* - * when secondary is put to maintenance and there's no standby left - * wait_maintenance -> maintenance - * wait_primary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_WAIT_MAINTENANCE) && - IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY)) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance after " NODE_FORMAT - " converged to wait_primary.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - - /* secondary reached maintenance */ - AssignGoalState(activeNode, REPLICATION_STATE_MAINTENANCE, message); - - return true; - } - - /* - * when secondary is in wait_maintenance state and goal state of primary is - * not wait_primary anymore, e.g. another node joined and made it primary - * again or it got demoted. Then we don't need to wait anymore and we can - * transition directly to maintenance. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_WAIT_MAINTENANCE) && - primaryNode->goalState != REPLICATION_STATE_WAIT_PRIMARY) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance after " NODE_FORMAT - " got assigned %s as goal state.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryNode->goalState)); - - /* secondary reached maintenance */ - AssignGoalState(activeNode, REPLICATION_STATE_MAINTENANCE, message); - - return true; - } - - /* - * when primary is put to maintenance - * prepare_promotion -> stop_replication - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION) && - IsCurrentState(primaryNode, REPLICATION_STATE_PREPARE_MAINTENANCE)) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to stop_replication after " NODE_FORMAT - " converged to prepare_maintenance.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - - /* promote the secondary */ - AssignGoalState(activeNode, REPLICATION_STATE_STOP_REPLICATION, message); - - return true; - } - - /* - * when a worker blocked writes: - * prepare_promotion -> wait_primary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION) && - primaryNode && - IsCitusFormation(ctx->formation) && activeNode->groupId > 0) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary and " NODE_FORMAT - " to demoted after the coordinator metadata was updated.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); - - /* done draining, node is presumed dead */ - AssignGoalState(primaryNode, REPLICATION_STATE_DEMOTED, message); - - return true; - } - - /* - * when a worker blocked writes and the primary has been removed: - * prepare_promotion -> wait_primary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION) && - primaryNode == NULL && - IsCitusFormation(ctx->formation) && activeNode->groupId > 0) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary after the coordinator metadata was updated.", - NODE_FORMAT_ARGS(activeNode)); - - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); - - return true; - } - - /* - * when node is seeing no more writes: - * prepare_promotion -> stop_replication - * - * refrain from prepare_maintenance -> demote_timeout on the primary, which - * might happen here when secondary has reached prepare_promotion before - * primary has reached prepare_maintenance. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION) && - primaryNode && - !IsInMaintenance(primaryNode)) - { - char message[BUFSIZE]; - - /* - * wait_primary has no reachable demote_timeout edge either (issue - * #1168); leave its goal alone here and let the completion rule - * below apply the drainTimeoutMs safety margin via report - * staleness instead. - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY)) - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to stop_replication after it converged to " - "prepare_promotion.", - NODE_FORMAT_ARGS(activeNode)); - } - else - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to demote_timeout and " NODE_FORMAT - " to stop_replication after " NODE_FORMAT - " converged to prepare_promotion.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(activeNode)); - - /* wait for possibly-alive primary to kill itself */ - AssignGoalState(primaryNode, REPLICATION_STATE_DEMOTE_TIMEOUT, message); - } - - /* perform promotion to stop replication */ - AssignGoalState(activeNode, REPLICATION_STATE_STOP_REPLICATION, message); - - return true; - } - - /* - * when primary node has been removed and we are promoting one standby - * prepare_promotion -> stop_replication - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_PREPARE_PROMOTION) && - primaryNode == NULL) - { - char message[BUFSIZE] = { 0 }; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary after " NODE_FORMAT - " converged to prepare_promotion.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(activeNode)); - - /* perform promotion to stop replication */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); - - return true; - } - - /* - * when primary node is going to maintenance - * stop_replication -> wait_primary - * prepare_maintenance -> maintenance - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_STOP_REPLICATION) && - IsCurrentState(primaryNode, REPLICATION_STATE_PREPARE_MAINTENANCE)) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary and " NODE_FORMAT - " to maintenance.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); - - /* old primary node is now ready for maintenance operations */ - AssignGoalState(primaryNode, REPLICATION_STATE_MAINTENANCE, message); - - return true; - } - - /* - * when drain time expires or primary reports it's drained: - * draining -> demoted - * - * NodeIsWaitPrimaryPresumedDead covers the wait_primary equivalent - * (issue #1168): the same drainTimeoutMs safety margin, applied via - * report staleness instead of a demote_timeout goal-state timestamp - * since that state is never reachable from wait_primary. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_STOP_REPLICATION) && - (IsCurrentState(primaryNode, REPLICATION_STATE_DEMOTE_TIMEOUT) || - NodeIsDrainTimeExpired(primaryNode, ctx) || - NodeIsWaitPrimaryPresumedDead(primaryNode, activeNode, ctx))) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary and " NODE_FORMAT - " to demoted after the primary was presumed dead.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); - - /* done draining, node is presumed dead */ - AssignGoalState(primaryNode, REPLICATION_STATE_DEMOTED, message); - - return true; - } - - /* - * when a worker blocked writes: - * stop_replication -> wait_primary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_STOP_REPLICATION) && - primaryNode && - IsCitusFormation(ctx->formation) && activeNode->groupId > 0) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary and " NODE_FORMAT - " to demoted after the coordinator metadata was updated.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); - - /* done draining, node is presumed dead */ - AssignGoalState(primaryNode, REPLICATION_STATE_DEMOTED, message); - - return true; - } - - /* - * when a worker blocked writes, and the primary has been dropped: - * stop_replication -> wait_primary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_STOP_REPLICATION) && - primaryNode == NULL && - IsCitusFormation(ctx->formation) && activeNode->groupId > 0) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary after the coordinator metadata was updated.", - NODE_FORMAT_ARGS(activeNode)); - - /* node is now taking writes */ - AssignGoalState(activeNode, REPLICATION_STATE_WAIT_PRIMARY, message); - - return true; - } - - /* - * when a new primary is ready: - * demoted -> catchingup - * - * We accept to move from demoted to catching up as soon as the primary - * node is has reported either wait_primary or join_primary, and even when - * it's already transitioning to primary, thanks to another standby - * concurrently making progress. - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_DEMOTED) && - NodeIsHealthy(primaryNode, ctx) && - ((primaryNode->reportedState == REPLICATION_STATE_WAIT_PRIMARY || - primaryNode->reportedState == REPLICATION_STATE_JOIN_PRIMARY) && - primaryNode->goalState == REPLICATION_STATE_PRIMARY)) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after it converged to demotion and " NODE_FORMAT - " converged to primary.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - - /* it's safe to rejoin as a secondary */ - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); - - return true; - } - - /* - * when a new primary is ready: - * demoted -> catchingup - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_DEMOTED) && - NodeIsHealthy(primaryNode, ctx) && - (IsCurrentState(primaryNode, REPLICATION_STATE_JOIN_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY))) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after it converged to demotion and " NODE_FORMAT - " converged to %s.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryNode->reportedState)); - - /* it's safe to rejoin as a secondary */ - AssignGoalState(activeNode, REPLICATION_STATE_CATCHINGUP, message); - - return true; + startIndex = index + 1; } - - /* - * when a new primary is ready: - * join_secondary -> secondary - * - * As there's no action to implement on the new selected primary for that - * step, we can make progress as soon as we want to. - * - * The primary could be in one of those states: - * - wait_primary/wait_primary - * - wait_primary/primary - * - * This transition also happens when a former primary node has been - * demoted, and a multiple standbys has taken effect, we have a new primary - * being promoted, and several standby nodes following the new primary. - * - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_JOIN_SECONDARY) && - primaryNode->reportedState == REPLICATION_STATE_WAIT_PRIMARY && - (primaryNode->goalState == REPLICATION_STATE_WAIT_PRIMARY || - primaryNode->goalState == REPLICATION_STATE_PRIMARY)) - { - char message[BUFSIZE] = { 0 }; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to secondary after " NODE_FORMAT - " converged to wait_primary.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - - /* it's safe to rejoin as a secondary */ - AssignGoalState(activeNode, REPLICATION_STATE_SECONDARY, message); - - /* compute next step for the primary depending on node settings */ - return ProceedGroupStateForPrimaryNode(ctx, primaryNode); - } - - /* - * when a new secondary re-appears after a failover or at a "random" time - * in the FSM cycle, and the wait_primary or join_primary node has already - * made progress to primary. - * - * join_secondary -> secondary - */ - if (IsCurrentState(activeNode, REPLICATION_STATE_JOIN_SECONDARY) && - IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY)) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to secondary after " NODE_FORMAT - " converged to primary.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(primaryNode)); - - /* it's safe to rejoin as a secondary */ - AssignGoalState(activeNode, REPLICATION_STATE_SECONDARY, message); - - return true; - } - - return false; } @@ -1180,319 +1525,19 @@ static bool ProceedGroupStateForPrimaryNode(GroupStateContext *ctx, AutoFailoverNode *primaryNode) { - List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); - int otherNodesCount = list_length(otherNodesGroupList); - - /* - * when a first "other" node wants to become standby: - * single -> wait_primary - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_SINGLE)) - { - ListCell *nodeCell = NULL; - - foreach(nodeCell, otherNodesGroupList) - { - AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); - - if (IsCurrentState(otherNode, REPLICATION_STATE_WAIT_STANDBY)) - { - char message[BUFSIZE]; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary after " NODE_FORMAT - " joined.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(otherNode)); - - /* prepare replication slot and pg_hba.conf */ - AssignGoalState(primaryNode, - REPLICATION_STATE_WAIT_PRIMARY, - message); - - return true; - } - } - } - - /* - * when secondary unhealthy: - * secondary ➜ catchingup - * primary ➜ wait_primary - * - * We only swith the primary to wait_primary when there's no healthy - * secondary anymore. In other cases, there's by definition at least one - * candidate for failover. - * - * Also we might lose a standby node while already in WAIT_PRIMARY, when - * all the left standby nodes are assigned a candidatePriority of zero. - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) || - IsCurrentState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS)) - { - /* - * We count our nodes in different ways, because of special cases we - * want to be able to address. We want to distinguish nodes that are in - * the replication quorum, nodes that are secondary, and nodes that are - * secondary but do not participate in the quorum. - * - * - replicationQuorumCount is the count of nodes with - * replicationQuorum true, whether or not those nodes are currently - * in the SECONDARY state. - * - * - secondaryNodesCount is the count of nodes that are currently in - * the SECONDARY state. - * - * - secondaryQuorumNodesCount is the count of nodes that are both - * setup to participate in the replication quorum and also currently - * in the SECONDARY state. - */ - int replicationQuorumCount = otherNodesCount; - int secondaryNodesCount = otherNodesCount; - int secondaryQuorumNodesCount = otherNodesCount; - - ListCell *nodeCell = NULL; - - foreach(nodeCell, otherNodesGroupList) - { - AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); - - /* - * We force secondary nodes to catching-up even if the node is on - * its way to being a secondary... unless it is currently in the - * reportLSN or join_secondary state, because in those states - * Postgres is stopped, waiting for the new primary to be - * available. - */ - if (otherNode->goalState == REPLICATION_STATE_SECONDARY && - otherNode->reportedState != REPLICATION_STATE_REPORT_LSN && - otherNode->reportedState != REPLICATION_STATE_JOIN_SECONDARY && - NodeIsUnhealthy(otherNode, ctx)) - { - char message[BUFSIZE]; - - --secondaryNodesCount; - --secondaryQuorumNodesCount; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after it became unhealthy.", - NODE_FORMAT_ARGS(otherNode)); - - /* other node is behind, no longer eligible for promotion */ - AssignGoalState(otherNode, - REPLICATION_STATE_CATCHINGUP, message); - } - else if (!IsCurrentState(otherNode, REPLICATION_STATE_SECONDARY)) - { - --secondaryNodesCount; - --secondaryQuorumNodesCount; - } - - /* at this point we are left with nodes in SECONDARY state */ - else if (IsCurrentState(otherNode, REPLICATION_STATE_SECONDARY) && - !otherNode->replicationQuorum) - { - --secondaryQuorumNodesCount; - } - - /* now separately count nodes setup with replication quorum */ - if (!otherNode->replicationQuorum) - { - --replicationQuorumCount; - } - } - - /* - * Special case first: when given a setup where all the nodes are async - * (replicationQuorumCount == 0) we allow the "primary" state in almost - * all cases, knowing that synchronous_standby_names is still going to - * be computed as ''. - * - * That said, if we don't have a single node in the SECONDARY state, we - * still want to switch to WAIT_PRIMARY to show that something - * unexpected is happening. - */ - if (replicationQuorumCount == 0) - { - Assert(ctx->formation->number_sync_standbys == 0); - - ReplicationState primaryGoalState = - secondaryNodesCount == 0 - ? REPLICATION_STATE_WAIT_PRIMARY - : REPLICATION_STATE_PRIMARY; + NodeActiveContext nac; - if (primaryNode->goalState != primaryGoalState) - { - char message[BUFSIZE] = { 0 }; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to %s because none of the secondary nodes" - " are healthy at the moment.", - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryGoalState)); - - AssignGoalState(primaryNode, primaryGoalState, message); - - return true; - } - - /* when all nodes are async, we're done here */ - return true; - } - - /* - * Disable synchronous replication to maintain availability. - * - * Note that we implement here a trade-off between availability (of - * writes) against durability of the written data. In the case when - * there's a single standby in the group, pg_auto_failover choice is to - * maintain availability of the service, including writes. - * - * In the case when the user has setup a replication quorum of 1 or - * more, then pg_auto_failover does not get in the way. You get what - * you ask for, which is a strong guarantee on durability. - * - * To have number_sync_standbys == 1, you need to have at least 2 - * standby servers. To get to a point where writes are not possible - * anymore, there needs to be a point in time where 2 of the 2 standby - * nodes are unavailable. In that case, pg_auto_failover does not - * change the configured trade-offs. Writes are blocked until one of - * the two defective standby nodes is available again. - */ - if (!IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) && - secondaryQuorumNodesCount == 0 && - !IsFailoverInProgress(ctx->groupNodeList)) - { - /* - * Do not second-guess an already-started failover: a candidate - * that just converged to prepare_promotion (or later stages) has - * left SECONDARY, dropping secondaryQuorumNodesCount to zero even - * though it *is* the failover candidate. Without this guard the - * primary can be reassigned wait_primary right as its own - * candidate is converging, landing it on wait_primary while a - * later rule still expects to find it in draining and assigns - * demote_timeout -- an assignment with no FSM edge from - * wait_primary (issue #774). - * - * Allow wait_primary when number_sync_standbys = 0, otherwise - * block writes on the primary. - */ - ReplicationState primaryGoalState = - ctx->formation->number_sync_standbys == 0 - ? REPLICATION_STATE_WAIT_PRIMARY - : REPLICATION_STATE_PRIMARY; - - if (primaryNode->goalState != primaryGoalState) - { - char message[BUFSIZE] = { 0 }; + BuildForPrimaryNodeNodeActiveContext(ctx, primaryNode, &nac); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to %s because none of the standby nodes in the quorum" - " are healthy at the moment.", - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryGoalState)); + int index = FindMatchingMonitorFSMRuleIndex(MonitorFSM_ForPrimaryNode, + MonitorFSM_ForPrimaryNode_SIZE, &nac); - AssignGoalState(primaryNode, primaryGoalState, message); - - return true; - } - } - - /* - * when a node is wait_primary and has at least one healthy candidate - * secondary - * wait_primary ➜ primary - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) && - secondaryQuorumNodesCount > 0) - { - char message[BUFSIZE] = { 0 }; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to primary now that we have %d healthy " - " secondary nodes in the quorum.", - NODE_FORMAT_ARGS(primaryNode), - secondaryQuorumNodesCount); - - AssignGoalState(primaryNode, REPLICATION_STATE_PRIMARY, message); - - return true; - } - - /* - * when a node has changed its replication settings: - * apply_settings ➜ wait_primary - * apply_settings ➜ primary - * - * Even when we don't currently have healthy standby nodes to failover - * to, if the number_sync_standbys is greater than zero that means the - * user wants to block writes on the primary, and we do that by - * switching to the primary state after having applied replication - * settings. Think - * - * $ pg_autoctl set formation number-sync-standbys 1 - * - * during an incident to stop the amount of potential data loss. - * - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS)) - { - char message[BUFSIZE] = { 0 }; - - ReplicationState primaryGoalState = - ctx->formation->number_sync_standbys == 0 && - secondaryQuorumNodesCount == 0 - ? REPLICATION_STATE_WAIT_PRIMARY - : REPLICATION_STATE_PRIMARY; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to %s after it applied replication properties change.", - NODE_FORMAT_ARGS(primaryNode), - ReplicationStateGetName(primaryGoalState)); - - AssignGoalState(primaryNode, primaryGoalState, message); - - return true; - } - - return true; - } - - /* - * We don't use the join_primary state any more, though for backwards - * compatibility if a node reports JOIN_PRIMARY well then we assign PRIMARY - * to the node. After all it might be that an operator upgrades while a - * node is in JOIN_PRIMARY and we certainly want to be able to handle the - * situation. - */ - if (IsCurrentState(primaryNode, REPLICATION_STATE_JOIN_PRIMARY)) + if (index < 0) { - char message[BUFSIZE] = { 0 }; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT " to primary", - NODE_FORMAT_ARGS(primaryNode)); - - AssignGoalState(primaryNode, REPLICATION_STATE_PRIMARY, message); - - return true; + return false; } - return false; + return DispatchMonitorFSMRule(ctx, &nac, &MonitorFSM_ForPrimaryNode[index]); } From ec4516f1b674e8094672d54626121a2865f68275 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 28 Jul 2026 21:32:35 +0200 Subject: [PATCH 02/52] Align FSM dispatch with the design doc: one array, single-shot dispatch Reworks the previous checkpoint (9c9c9b9) to match /Users/dim/dev/temp/monitor-fsm-data-driven-refactor-prompt.md's actual design, which that commit deviated from in two ways the doc explicitly argues against: - Merges MonitorFSM_EarlyChecks[]/FromContext[]/ForPrimaryNode[] into one MonitorFSM[] array (48 rows), with named boundary constants (MonitorFSM_FromContextStart=6, MonitorFSM_MSFailoverClusterStart=9, MonitorFSM_PrimaryNodeSectionStart=37) instead of three separate arrays standing in for section boundaries. - Replaces the looping dispatch site (extraAction returning bool, resuming from index+1 on decline -- functionally the continuesDispatch mechanism the doc's own design phase tried and reverted) with the doc's single-shot driver: ProceedGroupStateFromContext now makes at most a small, fixed number of straight-line lookups and dispatches at most one row per call. MonitorExtraActionFunction is void again; ActionRunMultiStandbyFailoverCascade and ActionRunPrimaryNodeTransition each do one bounded, explicitly-named nested search+dispatch instead of signaling the driver to keep scanning. One deliberate deviation from the doc's own top-level driver snippet, kept and documented in ProceedGroupStateFromContext's comment: the six early-check rows are always tried first via their own lookup, rather than being skipped via 'startIndex = isInPrimaryState ? PrimaryNode SectionStart : 0'. The literal doc snippet would skip them whenever activeNode is already primary-role, silently reintroducing the drop_node regression (a primary that just lost its only standby must still reach SINGLE via those checks before any primary-role redirect). ProceedGroupStateForPrimaryNode is removed as a separate function -- folded into the top-level driver's primary-role branch and ActionRunPrimaryNodeTransition, both now querying MonitorFSM[] directly via the shared FindAndDispatchMonitorFSMRule helper. Verified: 14/14 regress + 6/6 isolation, both locally and in the Docker-based installcheck (pgaf-base:bookworm, ephemeral pg_virtualenv cluster) -- same as the three-array checkpoint, confirming this is a structural realignment with the doc, not a behavior change. --- src/monitor/group_state_machine.c | 379 +++++++++++++++++------------- 1 file changed, 218 insertions(+), 161 deletions(-) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 44f9d194c..743d21a8b 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -54,8 +54,6 @@ typedef struct CandidateList /* private function forward declarations */ -static bool ProceedGroupStateForPrimaryNode(GroupStateContext *ctx, - AutoFailoverNode *primaryNode); static bool ProceedGroupStateForMSFailover(GroupStateContext *ctx, AutoFailoverNode *primaryNode); static bool ProceedWithMSFailover(AutoFailoverNode *activeNode, @@ -81,9 +79,10 @@ static bool WalDifferenceWithin(AutoFailoverNode *secondaryNode, /* * --------------------------------------------------------------------- - * Declarative dispatch for ProceedGroupStateFromContext() and - * ProceedGroupStateForPrimaryNode(): each function's own sequential - * if-chain is replaced by a table of MonitorFSMTransition rows, matched + * Declarative dispatch for ProceedGroupStateFromContext(): its own + * sequential if-chain, and the sequential if-chain that used to be a + * separate ProceedGroupStateForPrimaryNode() function, are both replaced by + * one table of MonitorFSMTransition rows (MonitorFSM[] below), matched * first-match-wins by RuleMatches(). ProceedGroupStateForMSFailover() and * everything it calls (BuildCandidateList, SelectFailoverCandidateNode, * PromoteSelectedNode, ProceedWithMSFailover, WalSourceNodesAreAllUnhealthy) @@ -512,19 +511,23 @@ typedef struct GoalStateAssignment #define GOAL(x) { .kind = GOAL_STATE_SET, .state = (x) } /* - * Returns true when this row's match should stand and dispatch should stop - * here (the overwhelming majority of extraActions -- a pure side effect - * alongside a genuine, unconditional transition). Returns false when the - * row's own facts didn't actually lead anywhere useful this round (the - * MS-failover cascade declining because e.g. not all candidates have - * reported yet) and the SAME call should keep searching later rows for - * activeNode's own transition -- exactly what the real source's fallthrough - * (no `return` after the DRAINING/MAINTENANCE assignment, see - * ActionRunMultiStandbyFailoverCascade below) does. Assignments on the matched row - * itself still apply either way; this only controls whether dispatch - * continues afterward. + * A row's extraAction runs before its own activeNodeAssignedState/ + * otherNodeAssignedState are applied (matching the original if-chain's + * order). When a row's real-source counterpart falls through to more of the + * function after its own assignment (the MS-failover cascade, the + * join_secondary -> nested primary pass), the action itself performs one + * bounded, explicitly-named nested search+dispatch over MonitorFSM[] -- + * see ActionRunMultiStandbyFailoverCascade and ActionRunPrimaryNodeTransition + * below -- rather than signaling the top-level driver to keep scanning. + * There is deliberately no generic "continue dispatch" flag here: an earlier + * version of this file had extraAction return bool for exactly that purpose, + * and it reproduced a real bug (a sibling row in the same "family" matching + * a second time after the intended row declined -- see + * ActionRunMultiStandbyFailoverCascade's comment) that a bounded, named jump + * cannot have, because it can only ever land on one specific row family, not + * wander into whichever row happens to be next. */ -typedef bool (*MonitorExtraActionFunction) (GroupStateContext *ctx, +typedef void (*MonitorExtraActionFunction) (GroupStateContext *ctx, NodeActiveContext *nac, char *message); @@ -542,6 +545,64 @@ typedef struct MonitorFSMTransition const char *comment; } MonitorFSMTransition; +/* + * MonitorFSM[] is one array, not several: see its own definition far below + * for why ("One array, not three" in the design doc this table implements). + * Forward-declared here so the extraActions defined above it (which each + * perform one bounded, named nested search over it) can reference it by + * name; the boundary constants below are forward-declared the same way, for + * the same reason -- both actions and the top-level driver need them. + * + * These three indices partition MonitorFSM[] into the sections the real + * if-chain's control flow actually has: + * + * [0, MonitorFSM_FromContextStart) the six checks + * ProceedGroupStateFromContext() + * runs before its one real branch + * point (IsInPrimaryState(activeNode)), + * regardless of which way that + * branch goes. + * [MonitorFSM_FromContextStart, the rest of + * MonitorFSM_PrimaryNodeSectionStart) ProceedGroupStateFromContext(), + * reached only when activeNode is + * NOT currently primary-role. + * [MonitorFSM_MSFailoverClusterStart, the sub-range of the row above + * MonitorFSM_PrimaryNodeSectionStart) that ActionRunMultiStandby + * FailoverCascade resumes into + * when ProceedGroupStateForMSFailover() + * declines, mirroring the real + * source's fallthrough to + * "whatever is textually next". + * [MonitorFSM_PrimaryNodeSectionStart, ProceedGroupStateForPrimaryNode()'s + * MonitorFSM_SIZE) own rows, reached either directly + * by the top-level driver (activeNode + * already primary-role) or via + * ActionRunPrimaryNodeTransition's + * nested pass on primaryNode + * (join_secondary's cascade row). + * + * Kept as plain hardcoded integers, exactly as the design doc's own + * placeholders are -- recomputed by hand whenever a row is added, removed, + * or moved across a boundary. A wrong value here fails loudly and + * immediately (either a compile-time out-of-bounds slice that scans zero + * rows and never matches, or a row from the wrong section matching + * unexpectedly) rather than silently: the regress/isolation suite this + * table is checked against covers every one of these boundaries already. + */ +static const MonitorFSMTransition MonitorFSM[]; + +#define MonitorFSM_FromContextStart 6 +#define MonitorFSM_MSFailoverClusterStart 9 +#define MonitorFSM_PrimaryNodeSectionStart 37 +#define MonitorFSM_SIZE 48 + +/* Forward-declared for the same reason as MonitorFSM[] above: used by + * extraActions (ActionRunPrimaryNodeTransition) defined before its real + * definition further down. */ +static void BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, + AutoFailoverNode *primaryNode, + NodeActiveContext *nac); + static bool RuleMatches(const NodeActiveContext *nac, const MonitorFSMTransition *rule) @@ -586,14 +647,6 @@ FindMatchingMonitorFSMRuleIndexFrom(const MonitorFSMTransition table[], int tabl } -static int -FindMatchingMonitorFSMRuleIndex(const MonitorFSMTransition table[], int tableSize, - const NodeActiveContext *nac) -{ - return FindMatchingMonitorFSMRuleIndexFrom(table, tableSize, 0, nac); -} - - /* * AssignDeclaredGoalState asserts that a rule only ever assigns a state it * actually declared: drift between a row's own assignment slots and what it @@ -617,12 +670,11 @@ AssignDeclaredGoalState(const MonitorFSMTransition *rule, AutoFailoverNode *node } -static bool +static void DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, const MonitorFSMTransition *rule) { char message[BUFSIZE] = { 0 }; - bool stopDispatch = true; if (rule->comment != NULL) { @@ -631,7 +683,7 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, if (rule->extraAction != NULL) { - stopDispatch = rule->extraAction(ctx, nac, message); + rule->extraAction(ctx, nac, message); } if (rule->activeNodeAssignedState.kind == GOAL_STATE_SET) @@ -645,8 +697,33 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, AssignDeclaredGoalState(rule, nac->primaryNode.node, rule->otherNodeAssignedState.state, message); } +} + + +/* + * FindAndDispatchMonitorFSMRule bounds a search over MonitorFSM[] to + * [startIndex, endIndex) and dispatches the first match, if any -- the one + * building block every call site in this file needs (the top-level driver's + * three straight-line lookups, and the two extraActions that perform their + * own single bounded nested search: ActionRunMultiStandbyFailoverCascade and + * ActionRunPrimaryNodeTransition below). Returns whether a row matched, so + * callers that need to distinguish "matched and handled" from "nothing in + * this range applied" can. + */ +static bool +FindAndDispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, + int startIndex, int endIndex) +{ + int index = FindMatchingMonitorFSMRuleIndexFrom(MonitorFSM, endIndex, startIndex, nac); + + if (index < 0) + { + return false; + } - return stopDispatch; + DispatchMonitorFSMRule(ctx, nac, &MonitorFSM[index]); + + return true; } @@ -726,12 +803,10 @@ OtherNodeIsDueForCatchingUp(GroupStateContext *ctx, AutoFailoverNode *otherNode) } -static bool +static void ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *message) { RemoveAutoFailoverNode(ctx->activeNode); - - return true; } @@ -757,8 +832,16 @@ ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *me * death_report and concurrent_health_check_and_report, which got stuck (the * former) or produced a spurious second cascade invocation changing the * outcome (the latter) until this was folded into a single row/action pair. + * + * When ProceedGroupStateForMSFailover() declines, the fallthrough to "the + * rest of ProceedGroupStateFromContext" is a single bounded nested search + * from MonitorFSM_MSFailoverClusterStart, not a flag back to the top-level + * driver: FindAndDispatchMonitorFSMRule's own internal loop already finds + * whichever row is the correct next match, however many rows down that is, + * in one call -- no repeated re-dispatch needed to walk past intervening + * non-matches. */ -static bool +static void ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, char *message) { @@ -793,7 +876,11 @@ ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext * AssignGoalState(primaryNode, REPLICATION_STATE_MAINTENANCE, maintenanceMessage); } - return ProceedGroupStateForMSFailover(ctx, primaryNode); + if (!ProceedGroupStateForMSFailover(ctx, primaryNode)) + { + (void) FindAndDispatchMonitorFSMRule(ctx, nac, MonitorFSM_MSFailoverClusterStart, + MonitorFSM_PrimaryNodeSectionStart); + } } @@ -801,25 +888,37 @@ ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext * * ActionRunPlainMSFailoverCascade is the "continue an already-started * failover" call site: activeNode itself is REPORT_LSN or FAST_FORWARD, and * the real source just `return`s ProceedGroupStateForMSFailover()'s result - * directly, with no DRAINING/MAINTENANCE decision attached. + * directly, with no DRAINING/MAINTENANCE decision attached and no further + * fallthrough either way -- so its return value is simply discarded here. */ -static bool +static void ActionRunPlainMSFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, char *message) { - return ProceedGroupStateForMSFailover(ctx, nac->primaryNode.node); + (void) ProceedGroupStateForMSFailover(ctx, nac->primaryNode.node); } -static bool +/* + * ActionRunPrimaryNodeTransition is the join_secondary cascade: the real + * source's tail call into ProceedGroupStateForPrimaryNode(ctx, primaryNode) + * substitutes primaryNode for activeNode and re-enters that function's own + * dispatch from scratch -- modeled here as one bounded nested search over + * MonitorFSM[]'s ForPrimaryNode section, built with primaryNode playing the + * activeNode role (see BuildForPrimaryNodeNodeActiveContext). + */ +static void ActionRunPrimaryNodeTransition(GroupStateContext *ctx, NodeActiveContext *nac, char *message) { - (void) ProceedGroupStateForPrimaryNode(ctx, nac->primaryNode.node); + NodeActiveContext primaryNac; - return true; + BuildForPrimaryNodeNodeActiveContext(ctx, nac->primaryNode.node, &primaryNac); + + (void) FindAndDispatchMonitorFSMRule(ctx, &primaryNac, MonitorFSM_PrimaryNodeSectionStart, + MonitorFSM_SIZE); } -static bool +static void ActionCatchupUnhealthySecondaries(GroupStateContext *ctx, NodeActiveContext *nac, char *message) { AutoFailoverNode *primaryNode = nac->activeNode.node; @@ -842,8 +941,6 @@ ActionCatchupUnhealthySecondaries(GroupStateContext *ctx, NodeActiveContext *nac AssignGoalState(otherNode, REPLICATION_STATE_CATCHINGUP, otherMessage); } } - - return true; } @@ -926,10 +1023,11 @@ BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *prim /* - * BuildForPrimaryNodeNodeActiveContext computes every fact - * MonitorFSM_ForPrimaryNode needs, mirroring the counting loop that used to - * be inline at the top of ProceedGroupStateForPrimaryNode() (the same loop - * OtherNodeIsDueForCatchingUp's condition drives the fan-out assignment + * BuildForPrimaryNodeNodeActiveContext computes every fact the + * ForPrimaryNode section of MonitorFSM[] (from MonitorFSM_PrimaryNodeSectionStart + * onward) needs, mirroring the counting loop that used to be inline at the + * top of the old, now-folded-in ProceedGroupStateForPrimaryNode() (the same + * loop OtherNodeIsDueForCatchingUp's condition drives the fan-out assignment * for, in ActionCatchupUnhealthySecondaries above). */ static void @@ -939,8 +1037,9 @@ BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *p memset(nac, 0, sizeof(NodeActiveContext)); BuildNodeStatus(ctx, primaryNode, &nac->activeNode); - /* .primaryNode role is unused by every MonitorFSM_ForPrimaryNode row -- primaryNode IS - * activeNode here, so every condition is expressed against .activeNode directly. */ + /* .primaryNode role is unused by every row in MonitorFSM[]'s ForPrimaryNode section -- + * primaryNode IS activeNode here, so every condition is expressed against .activeNode + * directly. */ List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); int otherNodesCount = list_length(otherNodesGroupList); @@ -991,20 +1090,28 @@ BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *p /* - * MonitorFSM_EarlyChecks: the six checks the real if-chain runs BEFORE the - * IsInPrimaryState(activeNode) early return (group_state_machine.c:284) -- - * DROPPED, goal==DROPPED, MAINTENANCE, the demote_timeout self-fence, and - * both "alone in group" rows. These fire regardless of whether activeNode is - * currently the primary (a primary that just lost its only standby must - * still reach SINGLE here, before ever redirecting to - * ProceedGroupStateForPrimaryNode) -- confirmed by the drop_node regression - * test, which failed the first time this table put the primary-state - * redirect ahead of these six checks instead of after them. None of these - * six rows reference .primaryNode at all, so they can be matched against a - * NodeActiveContext built with primaryNode == NULL, before primaryNode is - * even resolved. + * MonitorFSM[]: one array, not several -- see the boundary-constant comment + * above the MonitorFSMTransition typedef for the section layout and why it's + * a single ordered list rather than one array per real C function. Rows are + * kept in the exact order the original if-chain(s) checked them in: + * first-match-wins over this array is a straight extraction, not a + * behaviour change, exactly as it was over the three separate arrays this + * replaces. + * + * --- [0, MonitorFSM_FromContextStart): the six checks the real if-chain + * runs BEFORE the IsInPrimaryState(activeNode) early return + * (group_state_machine.c:284) -- DROPPED, goal==DROPPED, MAINTENANCE, the + * demote_timeout self-fence, and both "alone in group" rows. These fire + * regardless of whether activeNode is currently the primary (a primary that + * just lost its only standby must still reach SINGLE here, before ever + * redirecting into the ProceedGroupStateForPrimaryNode section) -- confirmed + * by the drop_node regression test, which failed the first time this table + * put the primary-state redirect ahead of these six checks instead of after + * them. None of these six rows reference .primaryNode at all, so they can be + * matched against a NodeActiveContext built with primaryNode == NULL, before + * primaryNode is even resolved. */ -static const MonitorFSMTransition MonitorFSM_EarlyChecks[] = { +static const MonitorFSMTransition MonitorFSM[] = { /* converged to dropped -> remove the node from the catalog entirely */ { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DROPPED) }, .extraAction = ActionRemoveDroppedNode, @@ -1037,21 +1144,13 @@ static const MonitorFSMTransition MonitorFSM_EarlyChecks[] = { .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), .comment = "alone in group, candidatePriority zero -> report_lsn" }, -}; - -#define MonitorFSM_EarlyChecks_SIZE \ - (sizeof(MonitorFSM_EarlyChecks) / sizeof(MonitorFSMTransition)) + /* --- [MonitorFSM_FromContextStart, MonitorFSM_PrimaryNodeSectionStart): the rest of + * ProceedGroupStateFromContext()'s own sequential if-chain -- everything from the + * timeline-fork check (group_state_machine.c:328, right after the + * IsInPrimaryState(activeNode) early return) onward. Reached only when activeNode is + * NOT currently primary-role. */ -/* - * MonitorFSM_FromContext: the declarative replacement for the rest of - * ProceedGroupStateFromContext()'s own sequential if-chain -- everything - * from the timeline-fork check (group_state_machine.c:328, right after the - * IsInPrimaryState(activeNode) early return) onward. Rows are kept in the - * exact order the original if-chain checked them in: first-match-wins over - * this array is a straight extraction, not a behaviour change. - */ -static const MonitorFSMTransition MonitorFSM_FromContext[] = { /* converged secondary, reportedTLI not an ancestor of the group's reference timeline */ { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isComparableToReferenceTli = BOOL_FALSE }, @@ -1288,19 +1387,13 @@ static const MonitorFSMTransition MonitorFSM_FromContext[] = { .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), .comment = "join_secondary, primary converged primary -> secondary" }, -}; - -#define MonitorFSM_FromContext_SIZE \ - (sizeof(MonitorFSM_FromContext) / sizeof(MonitorFSMTransition)) + /* --- [MonitorFSM_PrimaryNodeSectionStart, MonitorFSM_SIZE): the declarative replacement + * for ProceedGroupStateForPrimaryNode()'s own sequential if-chain. Here .activeNode maps + * to the primaryNode parameter, not a reporting node -- reached either directly by the + * top-level driver (activeNode already primary-role) or via ActionRunPrimaryNodeTransition's + * nested pass on primaryNode (the join_secondary cascade row above). */ -/* - * MonitorFSM_ForPrimaryNode: the declarative replacement for - * ProceedGroupStateForPrimaryNode()'s own sequential if-chain. Here - * .activeNode maps to the primaryNode parameter, not a reporting node -- - * see ProceedGroupStateForPrimaryNode() below. - */ -static const MonitorFSMTransition MonitorFSM_ForPrimaryNode[] = { /* primary alone, another node reached wait_standby */ { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SINGLE) }, .conditions = { .anyOtherNodeWaitingStandby = BOOL_TRUE }, @@ -1386,9 +1479,6 @@ static const MonitorFSMTransition MonitorFSM_ForPrimaryNode[] = { .comment = "backwards-compat: join_primary -> primary" }, }; -#define MonitorFSM_ForPrimaryNode_SIZE \ - (sizeof(MonitorFSM_ForPrimaryNode) / sizeof(MonitorFSMTransition)) - /* * ProceedGroupStateFromContext is the core FSM logic, operating entirely on @@ -1397,6 +1487,24 @@ static const MonitorFSMTransition MonitorFSM_ForPrimaryNode[] = { * * This separation lets test code inject a synthetic context and exercise the * FSM without a live database connection. + * + * Single-shot, three straight-line lookups at most -- matching the design + * doc's own top-level driver, not a loop: the cascades that need more than + * one row's worth of assignment in a single call (the MS-failover cascade, + * the join_secondary -> nested primary pass) get there via a bounded, named + * nested search inside their own extraAction (see ActionRunMultiStandby + * FailoverCascade and ActionRunPrimaryNodeTransition), not by this driver + * looping. + * + * Two lookups, not the design doc's one ("startIndex = isInPrimaryState ? + * MonitorFSM_PrimaryNodeSectionStart : 0"): the six early-check rows must + * always be tried first, regardless of whether activeNode is already + * primary-role -- a primary that just lost its only standby must still + * reach SINGLE via those checks, not get redirected to the + * ProceedGroupStateForPrimaryNode section first. Jumping straight past them + * whenever activeNode is already primary-role would skip that case + * entirely; confirmed by the drop_node regression test, which failed + * exactly this way the first time this table's ordering got this wrong. */ bool ProceedGroupStateFromContext(GroupStateContext *ctx) @@ -1406,39 +1514,40 @@ ProceedGroupStateFromContext(GroupStateContext *ctx) int groupId = ctx->groupId; /* - * MonitorFSM_EarlyChecks first, before the IsInPrimaryState redirect - * below -- these six checks run unconditionally in the real source, - * regardless of whether activeNode currently is the primary (a primary - * that just lost its only standby must still reach SINGLE here, not get - * redirected to ProceedGroupStateForPrimaryNode first). primaryNode - * isn't resolved yet at this point -- and none of these six rows - * reference it -- so NULL is passed and is safe. + * The six early checks run unconditionally, before the IsInPrimaryState + * redirect below -- regardless of whether activeNode currently is the + * primary. primaryNode isn't resolved yet at this point -- and none of + * these six rows reference it -- so NULL is passed and is safe. */ NodeActiveContext earlyNac; BuildFromContextNodeActiveContext(ctx, NULL, &earlyNac); - int earlyIndex = FindMatchingMonitorFSMRuleIndex(MonitorFSM_EarlyChecks, - MonitorFSM_EarlyChecks_SIZE, &earlyNac); - - if (earlyIndex >= 0) + if (FindAndDispatchMonitorFSMRule(ctx, &earlyNac, 0, MonitorFSM_FromContextStart)) { - return DispatchMonitorFSMRule(ctx, &earlyNac, &MonitorFSM_EarlyChecks[earlyIndex]); + return true; } /* * We separate out the FSM for the primary server, because that one needs * to loop over every other node to take decisions. That induces some - * complexity that is best managed in a specialized function. + * complexity that is best managed with its own NodeActiveContext, built + * with primaryNode substituted for activeNode's role (see + * BuildForPrimaryNodeNodeActiveContext). * - * This early return can't become a MonitorFSM_FromContext row: it's - * exactly the branch point the whole table design has to preserve as an - * *entry* decision, not a matched condition -- see the note above - * MonitorFSM_FromContext. + * This early return can't become an ordinary row: it's exactly the + * branch point the whole table design has to preserve as an *entry* + * decision, not a matched condition. */ if (IsInPrimaryState(activeNode)) { - return ProceedGroupStateForPrimaryNode(ctx, activeNode); + NodeActiveContext primaryNac; + + BuildForPrimaryNodeNodeActiveContext(ctx, activeNode, &primaryNac); + + return FindAndDispatchMonitorFSMRule(ctx, &primaryNac, + MonitorFSM_PrimaryNodeSectionStart, + MonitorFSM_SIZE); } /* @@ -1484,60 +1593,8 @@ ProceedGroupStateFromContext(GroupStateContext *ctx) BuildFromContextNodeActiveContext(ctx, primaryNode, &nac); - /* - * A matched row's extraAction may return false ("continue") to mirror - * the real source's fallthrough after the MS-failover cascade: when - * ProceedGroupStateForMSFailover() (via ActionRunMultiStandbyFailoverCascade) - * declines to act, the original if-chain keeps evaluating the rest of - * its conditions in the very same call instead of stopping. Re-scanning - * from startIndex on the same nac is safe here: RuleMatches() reads node - * state through the live AutoFailoverNode pointers stored in nac, so any - * goal state just assigned by this same dispatch (e.g. DRAINING on - * primaryNode) is already visible to the next match attempt. - */ - int startIndex = 0; - - while (true) - { - int index = FindMatchingMonitorFSMRuleIndexFrom(MonitorFSM_FromContext, - MonitorFSM_FromContext_SIZE, - startIndex, &nac); - - if (index < 0) - { - return false; - } - - if (DispatchMonitorFSMRule(ctx, &nac, &MonitorFSM_FromContext[index])) - { - return true; - } - - startIndex = index + 1; - } -} - - -/* - * Group State Machine when a primary node contacts the monitor. - */ -static bool -ProceedGroupStateForPrimaryNode(GroupStateContext *ctx, - AutoFailoverNode *primaryNode) -{ - NodeActiveContext nac; - - BuildForPrimaryNodeNodeActiveContext(ctx, primaryNode, &nac); - - int index = FindMatchingMonitorFSMRuleIndex(MonitorFSM_ForPrimaryNode, - MonitorFSM_ForPrimaryNode_SIZE, &nac); - - if (index < 0) - { - return false; - } - - return DispatchMonitorFSMRule(ctx, &nac, &MonitorFSM_ForPrimaryNode[index]); + return FindAndDispatchMonitorFSMRule(ctx, &nac, MonitorFSM_FromContextStart, + MonitorFSM_PrimaryNodeSectionStart); } From 392961c1f5a53cab2240bbc8c569d1d0c5f9f0d1 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 29 Jul 2026 00:32:29 +0200 Subject: [PATCH 03/52] Fix stale comment: two top-level lookups, not three FindAndDispatchMonitorFSMRule's own doc comment miscounted ProceedGroupStateFromContext's straight-line lookups as three; it's two (early checks, then either the primary-role section or the rest of the FromContext range). The third lookup I'd been counting belongs to a different function entirely -- the conditional nested search inside ActionRunMultiStandbyFailoverCascade's extraAction, which only runs when that row's own MS-failover cascade declines. No functional change, comment only. --- src/monitor/group_state_machine.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 743d21a8b..50a870643 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -703,10 +703,13 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, /* * FindAndDispatchMonitorFSMRule bounds a search over MonitorFSM[] to * [startIndex, endIndex) and dispatches the first match, if any -- the one - * building block every call site in this file needs (the top-level driver's - * three straight-line lookups, and the two extraActions that perform their - * own single bounded nested search: ActionRunMultiStandbyFailoverCascade and - * ActionRunPrimaryNodeTransition below). Returns whether a row matched, so + * building block every call site in this file needs: the top-level driver's + * own two straight-line lookups (early checks, then either the primary-role + * section or the rest of ProceedGroupStateFromContext()'s rows -- see its + * comment for why two, not the design doc's one), plus the two extraActions + * that each perform one further, separate bounded nested search of their own + * when their row's own cascade declines: ActionRunMultiStandbyFailoverCascade + * and ActionRunPrimaryNodeTransition below. Returns whether a row matched, so * callers that need to distinguish "matched and handled" from "nothing in * this range applied" can. */ From 62646c324d66ffb200fed4278771d74c4f76ce88 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 29 Jul 2026 00:38:15 +0200 Subject: [PATCH 04/52] Rename MonitorFSM_MSFailoverClusterStart -> MonitorFSM_FromContextResumeStart Reusing the design doc's constant name was misleading: in the doc, MonitorFSM_MSFailoverClusterStart marks the start of a real, converted section -- roughly ten declarative rows (a fourth candidateNode role, conditions like candidatePromotionInProgress and mostAdvancedCandidate WithinPromoteThreshold, an otherNodesFn hook) that partially replace BuildCandidateList/SelectFailoverCandidateNode/PromoteSelectedNode. No such section exists in this table. ProceedGroupStateForMSFailover() and everything it calls stayed one opaque hand-written function, called wholesale from ActionRunMultiStandbyFailoverCascade -- exactly the scope reduction already disclosed, but the borrowed name implied a conversion that didn't happen. The renamed constant just marks 'resume scanning ordinary FromContext rows from here' when that hand-written call declines -- a narrower thing than what the doc's identically- shaped constant was for. Comments at both definition and both use sites now say this explicitly. No functional change. --- src/monitor/group_state_machine.c | 45 ++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 50a870643..132482592 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -566,13 +566,36 @@ typedef struct MonitorFSMTransition * MonitorFSM_PrimaryNodeSectionStart) ProceedGroupStateFromContext(), * reached only when activeNode is * NOT currently primary-role. - * [MonitorFSM_MSFailoverClusterStart, the sub-range of the row above + * [MonitorFSM_FromContextResumeStart, the sub-range of the row above * MonitorFSM_PrimaryNodeSectionStart) that ActionRunMultiStandby * FailoverCascade resumes into * when ProceedGroupStateForMSFailover() * declines, mirroring the real * source's fallthrough to * "whatever is textually next". + * NOTE: unlike the design doc this + * table otherwise follows, there is + * no declarative "MS-failover / + * candidate-selection cluster" + * section here at all -- + * ProceedGroupStateForMSFailover() + * and everything it calls + * (BuildCandidateList, + * SelectFailoverCandidateNode, + * PromoteSelectedNode) stayed one + * opaque hand-written function, + * invoked wholesale, not spread + * across new rows/roles/conditions + * the way the doc's own + * candidateNode-based rows do. This + * constant just marks "resume + * scanning ordinary FromContext + * rows from here" -- it is NOT the + * doc's MonitorFSM_MSFailoverCluster + * Start, which names a real section + * of converted candidate-selection + * rows that doesn't exist in this + * table. * [MonitorFSM_PrimaryNodeSectionStart, ProceedGroupStateForPrimaryNode()'s * MonitorFSM_SIZE) own rows, reached either directly * by the top-level driver (activeNode @@ -592,7 +615,7 @@ typedef struct MonitorFSMTransition static const MonitorFSMTransition MonitorFSM[]; #define MonitorFSM_FromContextStart 6 -#define MonitorFSM_MSFailoverClusterStart 9 +#define MonitorFSM_FromContextResumeStart 9 #define MonitorFSM_PrimaryNodeSectionStart 37 #define MonitorFSM_SIZE 48 @@ -838,11 +861,25 @@ ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *me * * When ProceedGroupStateForMSFailover() declines, the fallthrough to "the * rest of ProceedGroupStateFromContext" is a single bounded nested search - * from MonitorFSM_MSFailoverClusterStart, not a flag back to the top-level + * from MonitorFSM_FromContextResumeStart, not a flag back to the top-level * driver: FindAndDispatchMonitorFSMRule's own internal loop already finds * whichever row is the correct next match, however many rows down that is, * in one call -- no repeated re-dispatch needed to walk past intervening * non-matches. + * + * NOTE on naming: MonitorFSM_FromContextResumeStart is NOT the design doc's + * MonitorFSM_MSFailoverClusterStart, despite marking a conceptually similar + * "resume point". The doc's constant names the start of a real, converted + * section -- roughly ten declarative rows (a fourth candidateNode role, + * conditions like candidatePromotionInProgress and + * mostAdvancedCandidateWithinPromoteThreshold, an otherNodesFn hook) that + * partially replace BuildCandidateList/SelectFailoverCandidateNode/ + * PromoteSelectedNode. No such section exists in this table: + * ProceedGroupStateForMSFailover() and everything it calls stayed one + * opaque hand-written function, called wholesale, exactly as before this + * refactor. MonitorFSM_FromContextResumeStart just marks "resume scanning + * ordinary FromContext rows here" -- a narrower thing than what the doc's + * identically-purposed constant was named for. */ static void ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, @@ -881,7 +918,7 @@ ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext * if (!ProceedGroupStateForMSFailover(ctx, primaryNode)) { - (void) FindAndDispatchMonitorFSMRule(ctx, nac, MonitorFSM_MSFailoverClusterStart, + (void) FindAndDispatchMonitorFSMRule(ctx, nac, MonitorFSM_FromContextResumeStart, MonitorFSM_PrimaryNodeSectionStart); } } From e0385e9e56479b36ff634a70226427a3054be707 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 29 Jul 2026 10:32:58 +0200 Subject: [PATCH 05/52] Finish MonitorFSM[] data-driven refactor: API triggers, MS-failover, dump_fsm() Builds on 4768b05 to complete the declarative dispatch table design (/Users/dim/dev/temp/monitor-fsm-data-driven-refactor-prompt.md), so the monitor's own FSM can be introspected and cross-checked against the keeper's KeeperFSM[] (src/bin/pg_autoctl/fsm.h). Operator-triggered (API_TRIGGERED) section: - New MONITOR_FSM_SECTION_API_TRIGGERED (15 rows, pos 101-129), a new MonitorApiFunction enum, ApiTriggerPattern/API_TRIGGER() matching mechanism, and ProceedGroupStateForApiTrigger() entry point. - Converts all 7 operator-triggered SQL functions (remove_node, perform_failover, start_maintenance, stop_maintenance, set_node_candidate_priority, set_node_replication_quorum, set_formation_number_sync_standbys) in node_active_protocol.c/ formation_metadata.c to dispatch through the table instead of hand-written AssignGoalState calls. - Found and fixed three real bugs while converting: a dropped-primary fan-out row that could skip DROPPED under first-match-wins; two rows using NODE_STATE_NOT_ASSIGNED where NODE_STATE_NOT_STABLE was the actual guard; both confirmed via regress failures, not by inspection. MS-failover / candidate-selection cluster (pos 363-379): - Added candidateNode as a third NodeStatus role, plus the MS-failover cluster's own facts (activeNodeAllWalSourcesUnhealthy, candidatePromotionInProgress, mostAdvancedCandidateWithinPromoteThreshold, guardDataLossEnabled) and a new inMSFailoverCluster marker. - TryMSFailoverDeclarativeRow wires the retry-reset and join_secondary transitions; TryFanOutReportLsnRow wires BuildCandidateList's own fan-out to REPORT_LSN (4 rows, one per fromState shape); DispatchMonitorFSMRuleByPos wires PromoteSelectedNode's PREPARE_PROMOTION/FAST_FORWARD choice (2 rows sharing identical conditions, disambiguated only by PromoteSelectedNode's own internal LSN comparison, not by any BoolPattern). BuildCandidateList/ SelectFailoverCandidateNode/PromoteSelectedNode/ProceedWithMSFailover themselves stay hand-written C; only their tail-end AssignGoalState calls dispatch through the table, each falling back to the original call on no match. - inMSFailoverCluster fixed a real bug: extending the MS-failover cluster's own upper bound also exposed the new fan-out rows to the ordinary top-level dispatch (same shared bound), so they briefly hijacked ordinary 2-node heartbeats. Caught via a 9-test regress regression, fixed by gating every MS-failover-only row on a marker that's true only when built by BuildMSFailoverNodeActiveContext. dump_fsm()/pgautofailover.fsm (introspection view): - Exposes active/other/candidate current (reported) state, per-role and group-level BoolPattern conditions (health foremost, including a goal-state precondition for NODE_STATE_ASSIGNED/ NOT_ASSIGNED rows that would otherwise render as entirely blank), assigned states, and whether a row has an extraAction. - section folds in the API function for API_TRIGGERED rows as plain text ("api_triggered: remove_node") instead of a separate column. - rule_pos/rule_section on pgautofailover.event (set from DispatchMonitorFSMRule) let a query join an actual event back to the exact row that produced it. Tests: fsm.sql (plain \x on dump of all 72 rows, one regression diff per row added/removed/edited) and cluster_init_failover_rule_attribution.sql (end-to-end bootstrap + manual failover, joining pgautofailover.event to pgautofailover.fsm on rule_pos to show which row produced each transition). Verified: 16/16 regress + 6/6 isolation (Docker installcheck, pgaf-base: bookworm), plus live pgaftest against real multi-standby Docker clusters (multi_standbys 27/27, multi_alternate 16/16, multi_ifdown 12/12, multi_maintenance 22/22). --- ...cluster_init_failover_rule_attribution.out | 277 ++ src/monitor/expected/fsm.out | 1031 +++++++ src/monitor/formation_metadata.c | 23 +- src/monitor/group_state_machine.c | 2420 ++++++++++++++--- src/monitor/group_state_machine.h | 61 + src/monitor/health_check_metadata.c | 50 +- src/monitor/metadata.h | 1 + src/monitor/node_active_protocol.c | 286 +- src/monitor/notifications.c | 41 +- src/monitor/notifications.h | 20 + src/monitor/pgautofailover.sql | 57 + src/monitor/regress_schedule | 2 + ...cluster_init_failover_rule_attribution.sql | 126 + src/monitor/sql/fsm.sql | 24 + 14 files changed, 3847 insertions(+), 572 deletions(-) create mode 100644 src/monitor/expected/cluster_init_failover_rule_attribution.out create mode 100644 src/monitor/expected/fsm.out create mode 100644 src/monitor/sql/cluster_init_failover_rule_attribution.sql create mode 100644 src/monitor/sql/fsm.sql diff --git a/src/monitor/expected/cluster_init_failover_rule_attribution.out b/src/monitor/expected/cluster_init_failover_rule_attribution.out new file mode 100644 index 000000000..e83f2cb8f --- /dev/null +++ b/src/monitor/expected/cluster_init_failover_rule_attribution.out @@ -0,0 +1,277 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- End-to-end demonstration of the rule_pos/rule_section attribution +-- mechanism (notifications.c's CurrentMonitorFSMRulePos/RuleSection, +-- InsertEvent()): registers a two-node formation, drives it through the +-- heartbeat-only bootstrap to primary + secondary, triggers a manual +-- perform_failover(), and then joins pgautofailover.event against +-- pgautofailover.fsm on rule_pos = pos to show, for every state +-- transition the monitor produced, exactly which MonitorFSM[] row was +-- selected and executed -- both for the ordinary heartbeat-driven +-- bootstrap rows (api_triggered = false in this join, since rule_pos is +-- only set for rows actually reached through the declarative dispatch +-- table) and for the operator-triggered perform_failover call itself. +\x on +-- ── formation and node registration ───────────────────────────────────────── +SELECT pgautofailover.create_formation('cifra_test', 'pgsql', 'postgres', true, 0); +-[ RECORD 1 ]----+-------------------------------- +create_formation | (cifra_test,pgsql,postgres,t,0) + +SELECT * + FROM pgautofailover.register_node('cifra_test', 'cifra_p', 5432, + 'postgres', 'cifra_p', 1); +-[ RECORD 1 ]---------------+-------- +assigned_node_id | 33 +assigned_group_id | 0 +assigned_group_state | single +assigned_candidate_priority | 100 +assigned_replication_quorum | t +assigned_node_name | cifra_p + +SELECT nodeid AS np FROM pgautofailover.node + WHERE formationid = 'cifra_test' AND nodename = 'cifra_p' \gset +SELECT * + FROM pgautofailover.register_node('cifra_test', 'cifra_s', 5432, + 'postgres', 'cifra_s', 1); +-[ RECORD 1 ]---------------+------------- +assigned_node_id | 34 +assigned_group_id | 0 +assigned_group_state | wait_standby +assigned_candidate_priority | 100 +assigned_replication_quorum | t +assigned_node_name | cifra_s + +SELECT nodeid AS ns FROM pgautofailover.node + WHERE formationid = 'cifra_test' AND nodename = 'cifra_s' \gset +-- ── bootstrap: drive the FSM to primary + secondary ───────────────────────── +-- +-- Mirrors drop_node.sql's bootstrap sequence (register -> single -> +-- wait_primary -> [standby: wait_standby -> catchingup -> secondary] -> +-- primary), including its "confirm" round-trips. +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'single'); +-[ RECORD 1 ]--------+------- +assigned_group_state | single + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'wait_standby'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_standby + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'single', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'wait_standby'); +-[ RECORD 1 ]--------+----------- +assigned_group_state | catchingup + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+-------- +assigned_group_state | primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+-------- +assigned_group_state | primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +-- Verify bootstrap: p=primary, s=secondary. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'cifra_test' + ORDER BY nodename; +-[ RECORD 1 ]-+---------- +nodename | cifra_p +goalstate | primary +reportedstate | primary +-[ RECORD 2 ]-+---------- +nodename | cifra_s +goalstate | secondary +reportedstate | secondary + +-- ── manual failover ────────────────────────────────────────────────────── +-- +-- Two-node group: dispatches through MonitorFSM[]'s API_TRIGGERED section +-- (pos 105, "manual failover, 2-node group, primary+standby both converged +-- -> standby prepare_promotion, primary draining"), attributing both +-- resulting event rows to that one rule. +SELECT pgautofailover.perform_failover('cifra_test', 0); +-[ RECORD 1 ]----+- +perform_failover | + +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'cifra_test' + ORDER BY nodename; +-[ RECORD 1 ]-+------------------ +nodename | cifra_p +goalstate | draining +reportedstate | primary +-[ RECORD 2 ]-+------------------ +nodename | cifra_s +goalstate | prepare_promotion +reportedstate | secondary + +-- ── which rule fired for which event? ─────────────────────────────────────── +-- +-- Every event row this formation produced, joined against the FSM table on +-- rule_pos = pos: rule_pos/rule_section are NULL for the ordinary +-- heartbeat-driven bootstrap transitions above whenever the matched row +-- happens to be identified only by array position in earlier sessions' +-- tests -- here every one of them was reached through the same declarative +-- MonitorFSM[] dispatch table, so each carries its own attribution too. The +-- final two rows (both attributed to pos 105) are the perform_failover() +-- call's own dual assignment (standby -> prepare_promotion, primary -> +-- draining), selected and executed from the API_TRIGGERED section. +SELECT e.eventid, e.nodename, e.reportedstate, e.goalstate, + e.rule_pos, e.rule_section, f.comment AS rule_comment + FROM pgautofailover.event e + LEFT JOIN pgautofailover.fsm f ON f.pos = e.rule_pos + WHERE e.formationid = 'cifra_test' + ORDER BY e.eventid; +-[ RECORD 1 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 186 +nodename | cifra_p +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +rule_comment | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 187 +nodename | cifra_p +reportedstate | single +goalstate | single +rule_pos | +rule_section | +rule_comment | +-[ RECORD 3 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 188 +nodename | cifra_s +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +rule_comment | +-[ RECORD 4 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 189 +nodename | cifra_p +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +rule_comment | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 190 +nodename | cifra_p +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +rule_comment | +-[ RECORD 6 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 191 +nodename | cifra_s +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +rule_comment | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 192 +nodename | cifra_s +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +rule_comment | +-[ RECORD 8 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 193 +nodename | cifra_s +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +rule_comment | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+------------------------------------------------------------------------------------------------------------- +eventid | 194 +nodename | cifra_s +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +rule_comment | +-[ RECORD 10 ]+------------------------------------------------------------------------------------------------------------- +eventid | 195 +nodename | cifra_p +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +rule_comment | wait_primary, >=1 quorum secondary -> primary +-[ RECORD 11 ]+------------------------------------------------------------------------------------------------------------- +eventid | 196 +nodename | cifra_p +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +rule_comment | +-[ RECORD 12 ]+------------------------------------------------------------------------------------------------------------- +eventid | 197 +nodename | cifra_s +reportedstate | secondary +goalstate | prepare_promotion +rule_pos | 105 +rule_section | api_triggered +rule_comment | manual failover, 2-node group, primary+standby both converged -> standby prepare_promotion, primary draining +-[ RECORD 13 ]+------------------------------------------------------------------------------------------------------------- +eventid | 198 +nodename | cifra_p +reportedstate | primary +goalstate | draining +rule_pos | 105 +rule_section | api_triggered +rule_comment | manual failover, 2-node group, primary+standby both converged -> standby prepare_promotion, primary draining + diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out new file mode 100644 index 000000000..2d7f30062 --- /dev/null +++ b/src/monitor/expected/fsm.out @@ -0,0 +1,1031 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Plain dump of the monitor's declarative dispatch table, via the +-- pgautofailover.fsm view (pgautofailover.dump_fsm() ordered by pos). This +-- is a static, compile-time-fixed table -- unaffected by any node/formation +-- state -- so its expected output changes only when a row is added, +-- removed, or edited in MonitorFSM[] (group_state_machine.c), giving that +-- change an explicit, reviewable regression diff. +-- +-- \x on: with the *_conditions columns added, a plain tabular row is far +-- wider than a terminal (or this file's own diff-ability), and reads far +-- worse than one field-per-line. +\x on +SELECT pos, section, + active_node_current_state, other_node_current_state, candidate_node_current_state, + active_node_conditions, other_node_conditions, candidate_node_conditions, + group_conditions, + active_node_assigned_state, other_node_assigned_state, has_extra_action, + comment + FROM pgautofailover.fsm + ORDER BY pos; +-[ RECORD 1 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 101 +section | api_triggered: remove_node +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | canTakeWrites=true +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | dropped +other_node_assigned_state | +has_extra_action | t +comment | remove_node, removed node can take writes -> dropped, every surviving non-maintenance standby joins report_lsn +-[ RECORD 2 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 103 +section | api_triggered: remove_node +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | dropped +other_node_assigned_state | +has_extra_action | f +comment | remove_node, removed node cannot take writes -> dropped +-[ RECORD 3 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 105 +section | api_triggered: perform_failover +active_node_current_state | secondary +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyTwoNodes=true +active_node_assigned_state | prepare_promotion +other_node_assigned_state | draining +has_extra_action | f +comment | manual failover, 2-node group, primary+standby both converged -> standby prepare_promotion, primary draining +-[ RECORD 4 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 107 +section | api_triggered: perform_failover +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isInPrimaryState=true +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasMoreThanTwoNodes=true +active_node_assigned_state | draining +other_node_assigned_state | +has_extra_action | f +comment | manual failover, >2-node group -> primary drains, election proceeds via the heartbeat-driven MS-failover cluster rows +-[ RECORD 5 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 109 +section | api_triggered: start_maintenance +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isInPrimaryState=true +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyTwoNodes=true +active_node_assigned_state | prepare_maintenance +other_node_assigned_state | +has_extra_action | f +comment | start_maintenance, primary, 2-node group -> prepare_maintenance (standby separately assigned prepare_promotion) +-[ RECORD 6 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 111 +section | api_triggered: start_maintenance +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isInPrimaryState=true +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasMoreThanTwoNodes=true +active_node_assigned_state | prepare_maintenance +other_node_assigned_state | +has_extra_action | f +comment | start_maintenance, primary, >2-node group -> prepare_maintenance, election proceeds via the heartbeat-driven MS-failover cluster rows +-[ RECORD 7 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 113 +section | api_triggered: start_maintenance +active_node_current_state | secondary, catchingup +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | lastHealthySyncStandbyGoingToMaintenance=true +active_node_assigned_state | wait_maintenance +other_node_assigned_state | wait_primary +has_extra_action | f +comment | start_maintenance, secondary, last healthy sync standby -> wait_maintenance, primary wait_primary (disables sync rep) +-[ RECORD 8 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 115 +section | api_triggered: start_maintenance +active_node_current_state | secondary, catchingup +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | maintenance +other_node_assigned_state | +has_extra_action | f +comment | start_maintenance, secondary, ordinary case -> maintenance +-[ RECORD 9 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 117 +section | api_triggered: stop_maintenance +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | exists=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | stop_maintenance, no primary -> report_lsn +-[ RECORD 10 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 119 +section | api_triggered: stop_maintenance +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isDemotedPrimary=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | stop_maintenance, primary demoted -> report_lsn +-[ RECORD 11 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 121 +section | api_triggered: stop_maintenance +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | failoverInProgress=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | stop_maintenance, failover in progress -> report_lsn (source's own log message says "catchingup" here, but the actual call assigns REPORT_LSN -- see this row's own comment above) +-[ RECORD 12 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 123 +section | api_triggered: stop_maintenance +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | stop_maintenance, ordinary case -> catchingup +-[ RECORD 13 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 125 +section | api_triggered: set_node_candidate_priority +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | apply_settings +other_node_assigned_state | +has_extra_action | f +comment | set_node_candidate_priority, primary not already apply_settings -> apply_settings +-[ RECORD 14 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 127 +section | api_triggered: set_node_replication_quorum +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | apply_settings +other_node_assigned_state | +has_extra_action | f +comment | set_node_replication_quorum, primary not already apply_settings -> apply_settings +-[ RECORD 15 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 129 +section | api_triggered: set_formation_number_sync_standbys +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | goal=primary|wait_primary +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | apply_settings +other_node_assigned_state | +has_extra_action | f +comment | set_formation_number_sync_standbys, primary in primary/wait_primary -> apply_settings +-[ RECORD 16 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 201 +section | early_checks +active_node_current_state | dropped +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | converged to dropped -> remove the node from the catalog +-[ RECORD 17 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 203 +section | early_checks +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | goal=dropped +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | f +comment | goal already dropped -> no-op +-[ RECORD 18 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 205 +section | early_checks +active_node_current_state | maintenance +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | f +comment | converged to maintenance -> no-op, frozen until stop_maintenance() +-[ RECORD 19 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 207 +section | early_checks +active_node_current_state | demote_timeout +other_node_current_state | +candidate_node_current_state | +active_node_conditions | unreachableFromDemoteTimeout=true +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | demoted +other_node_assigned_state | +has_extra_action | f +comment | reported demote_timeout, assigned goal can't reach it -> demoted +-[ RECORD 20 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 209 +section | early_checks +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | candidateEligible=true +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyOneNode=true +active_node_assigned_state | single +other_node_assigned_state | +has_extra_action | f +comment | alone in group, candidate-eligible -> single +-[ RECORD 21 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 211 +section | early_checks +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | candidateEligible=false +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyOneNode=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | alone in group, candidatePriority zero -> report_lsn +-[ RECORD 22 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 301 +section | reporting_node +active_node_current_state | secondary +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isComparableToReferenceTli=false +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | converged secondary, reportedTLI not an ancestor of reference -> catchingup +-[ RECORD 23 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 303 +section | reporting_node +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isHealthy=true, isInPrimaryState=true +candidate_node_conditions | +group_conditions | replicationStallExceeded=true +active_node_assigned_state | +other_node_assigned_state | wait_primary +has_extra_action | f +comment | primary healthy, no standby past replication_stall_timeout -> wait_primary +-[ RECORD 24 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 305 +section | reporting_node +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isUnhealthy=true +candidate_node_conditions | +group_conditions | groupHasMoreThanTwoNodes=true +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade +-[ RECORD 25 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 307 +section | reporting_node +active_node_current_state | report_lsn +other_node_current_state | wait_primary, join_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | secondary +other_node_assigned_state | +has_extra_action | f +comment | report_lsn, primary converged wait/join_primary, healthy -> secondary +-[ RECORD 26 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 309 +section | reporting_node +active_node_current_state | report_lsn +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | secondary +other_node_assigned_state | +has_extra_action | f +comment | report_lsn, primary converged primary, healthy -> secondary +-[ RECORD 27 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 311 +section | reporting_node +active_node_current_state | fast_forward +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | prepare_promotion +other_node_assigned_state | +has_extra_action | f +comment | fast_forward done -> prepare_promotion +-[ RECORD 28 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 313 +section | reporting_node +active_node_current_state | report_lsn, fast_forward +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | report_lsn or fast_forward, continuing an already-started failover -> MS-failover cascade +-[ RECORD 29 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 315 +section | reporting_node +active_node_current_state | wait_standby +other_node_current_state | wait_primary, join_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 30 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 317 +section | reporting_node +active_node_current_state | wait_standby +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | replicationQuorum=true +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | apply_settings +has_extra_action | f +comment | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 31 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 319 +section | reporting_node +active_node_current_state | wait_standby +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | replicationQuorum=false +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | wait_standby (not a quorum member), primary converged primary -> catchingup +-[ RECORD 32 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 321 +section | reporting_node +active_node_current_state | catchingup +other_node_current_state | wait_primary, join_primary, primary +candidate_node_current_state | +active_node_conditions | isHealthy=true +other_node_conditions | +candidate_node_conditions | +group_conditions | walWithinSyncThreshold=true, activeAndPrimaryTliMatch=true +active_node_assigned_state | secondary +other_node_assigned_state | +has_extra_action | f +comment | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 33 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 323 +section | reporting_node +active_node_current_state | secondary +other_node_current_state | wait_primary +candidate_node_current_state | +active_node_conditions | isHealthy=true, candidateEligible=true +other_node_conditions | isUnhealthy=true +candidate_node_conditions | +group_conditions | walWithinPromoteThreshold=true +active_node_assigned_state | prepare_promotion +other_node_assigned_state | +has_extra_action | f +comment | primary fails, already converged wait_primary (issue #1168) -> secondary -> prepare_promotion only (1 of 2) +-[ RECORD 34 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 325 +section | reporting_node +active_node_current_state | secondary +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isHealthy=true, candidateEligible=true +other_node_conditions | isUnhealthy=true, isInPrimaryState=true +candidate_node_conditions | +group_conditions | walWithinPromoteThreshold=true +active_node_assigned_state | prepare_promotion +other_node_assigned_state | draining +has_extra_action | f +comment | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) +-[ RECORD 35 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 327 +section | reporting_node +active_node_current_state | wait_maintenance +other_node_current_state | wait_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | maintenance +other_node_assigned_state | +has_extra_action | f +comment | wait_maintenance, primary converged wait_primary -> maintenance +-[ RECORD 36 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 329 +section | reporting_node +active_node_current_state | wait_maintenance +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | goal!=wait_primary +candidate_node_conditions | +group_conditions | +active_node_assigned_state | maintenance +other_node_assigned_state | +has_extra_action | f +comment | wait_maintenance, primary's goal no longer wait_primary -> maintenance +-[ RECORD 37 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 331 +section | reporting_node +active_node_current_state | prepare_promotion +other_node_current_state | prepare_maintenance +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | stop_replication +other_node_assigned_state | +has_extra_action | f +comment | prepare_promotion, primary converged prepare_maintenance -> stop_replication +-[ RECORD 38 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 333 +section | reporting_node +active_node_current_state | prepare_promotion +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isCitusWorkerGroup=true +other_node_conditions | exists=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | demoted +has_extra_action | f +comment | Citus worker prepare_promotion, primary present -> wait_primary + demoted +-[ RECORD 39 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 335 +section | reporting_node +active_node_current_state | prepare_promotion +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isCitusWorkerGroup=true +other_node_conditions | exists=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | +has_extra_action | f +comment | Citus worker prepare_promotion, primary removed -> wait_primary +-[ RECORD 40 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 337 +section | reporting_node +active_node_current_state | prepare_promotion +other_node_current_state | wait_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | exists=true, isInMaintenance=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | stop_replication +other_node_assigned_state | +has_extra_action | f +comment | prepare_promotion, primary already converged wait_primary (issue #1168) -> stop_replication only (1 of 2) +-[ RECORD 41 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 339 +section | reporting_node +active_node_current_state | prepare_promotion +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | exists=true, isInMaintenance=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | stop_replication +other_node_assigned_state | demote_timeout +has_extra_action | f +comment | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) +-[ RECORD 42 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 341 +section | reporting_node +active_node_current_state | prepare_promotion +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | exists=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | +has_extra_action | f +comment | prepare_promotion, primary removed -> wait_primary +-[ RECORD 43 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 343 +section | reporting_node +active_node_current_state | stop_replication +other_node_current_state | prepare_maintenance +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | maintenance +has_extra_action | f +comment | stop_replication, primary converged prepare_maintenance -> wait_primary + maintenance +-[ RECORD 44 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 345 +section | reporting_node +active_node_current_state | stop_replication +other_node_current_state | demote_timeout +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | demoted +has_extra_action | f +comment | stop_replication, primary converged demote_timeout -> wait_primary + demoted (1 of 3) +-[ RECORD 45 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 347 +section | reporting_node +active_node_current_state | stop_replication +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | drainTimeExpired=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | demoted +has_extra_action | f +comment | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) +-[ RECORD 46 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 349 +section | reporting_node +active_node_current_state | stop_replication +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | primaryIsWaitPrimaryPresumedDead=true +active_node_assigned_state | wait_primary +other_node_assigned_state | demoted +has_extra_action | f +comment | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) +-[ RECORD 47 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 351 +section | reporting_node +active_node_current_state | stop_replication +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isCitusWorkerGroup=true +other_node_conditions | exists=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | demoted +has_extra_action | f +comment | Citus worker stop_replication, primary present -> wait_primary + demoted +-[ RECORD 48 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 353 +section | reporting_node +active_node_current_state | stop_replication +other_node_current_state | +candidate_node_current_state | +active_node_conditions | isCitusWorkerGroup=true +other_node_conditions | exists=false +candidate_node_conditions | +group_conditions | +active_node_assigned_state | wait_primary +other_node_assigned_state | +has_extra_action | f +comment | Citus worker stop_replication, primary removed -> wait_primary +-[ RECORD 49 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 355 +section | reporting_node +active_node_current_state | demoted +other_node_current_state | wait_primary, join_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | demoted, primary reported wait/join_primary with goal primary -> catchingup +-[ RECORD 50 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 357 +section | reporting_node +active_node_current_state | demoted +other_node_current_state | wait_primary, join_primary, primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | catchingup +other_node_assigned_state | +has_extra_action | f +comment | demoted, primary converged wait/join_primary/primary, healthy -> catchingup +-[ RECORD 51 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 359 +section | reporting_node +active_node_current_state | join_secondary +other_node_current_state | wait_primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | secondary +other_node_assigned_state | +has_extra_action | t +comment | join_secondary, primary reported wait_primary with goal wait/primary -> secondary +-[ RECORD 52 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 361 +section | reporting_node +active_node_current_state | join_secondary +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | secondary +other_node_assigned_state | +has_extra_action | f +comment | join_secondary, primary converged primary -> secondary +-[ RECORD 53 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 363 +section | reporting_node +active_node_current_state | report_lsn +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | activeNodeAllWalSourcesUnhealthy=true, guardDataLossEnabled=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, guard_data_loss=true -> report_lsn (retry once a source recovers) +-[ RECORD 54 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 365 +section | reporting_node +active_node_current_state | report_lsn +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | isReadyToStreamWAL=true +group_conditions | candidatePromotionInProgress=true +active_node_assigned_state | join_secondary +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: activeNode in report_lsn, failover candidate ready to stream WAL -> join_secondary +-[ RECORD 55 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 367 +section | reporting_node +active_node_current_state | secondary, catchingup +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | inMSFailoverCluster=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) +-[ RECORD 56 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 369 +section | reporting_node +active_node_current_state | maintenance +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | inMSFailoverCluster=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | MS-failover fan-out: rejoining from maintenance -> report_lsn (2 of 4) +-[ RECORD 57 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 371 +section | reporting_node +active_node_current_state | draining, demoted +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | inMSFailoverCluster=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | MS-failover fan-out: old primary converged draining or demoted -> report_lsn (3 of 4) +-[ RECORD 58 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 373 +section | reporting_node +active_node_current_state | demoted +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | inMSFailoverCluster=true +active_node_assigned_state | report_lsn +other_node_assigned_state | +has_extra_action | f +comment | MS-failover fan-out: old primary demoted, was rejoining a now-failed primary -> report_lsn (4 of 4) +-[ RECORD 59 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 375 +section | reporting_node +active_node_current_state | report_lsn +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | candidatePromotionInProgress=false, mostAdvancedCandidateWithinPromoteThreshold=true, inMSFailoverCluster=true +active_node_assigned_state | prepare_promotion +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate already has all WAL -> prepare_promotion (1 of 2 -- see this row's own comment on why both are listed) +-[ RECORD 60 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 377 +section | reporting_node +active_node_current_state | report_lsn +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | candidatePromotionInProgress=false, mostAdvancedCandidateWithinPromoteThreshold=true, inMSFailoverCluster=true +active_node_assigned_state | fast_forward +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate is lagging -> fast_forward (2 of 2) +-[ RECORD 61 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 379 +section | reporting_node +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | candidatePromotionInProgress=false, inMSFailoverCluster=true +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: no promotion in flight, not enough (or not safe enough) candidates yet -> no-op besides the fan-out above +-[ RECORD 62 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 401 +section | primary_node +active_node_current_state | single +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | anyOtherNodeWaitingStandby=true +active_node_assigned_state | wait_primary +other_node_assigned_state | +has_extra_action | f +comment | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 63 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 403 +section | primary_node +active_node_current_state | primary, wait_primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | replicationQuorumCountIsZero=true, secondaryNodesCountIsZero=true +active_node_assigned_state | wait_primary +other_node_assigned_state | +has_extra_action | t +comment | all nodes async, zero secondaries -> wait_primary +-[ RECORD 64 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 405 +section | primary_node +active_node_current_state | primary, wait_primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | replicationQuorumCountIsZero=true, secondaryNodesCountIsZero=false +active_node_assigned_state | primary +other_node_assigned_state | +has_extra_action | t +comment | all nodes async, >=1 secondary -> primary +-[ RECORD 65 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 407 +section | primary_node +active_node_current_state | primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNodesCountIsZero=true, failoverInProgress=false +active_node_assigned_state | wait_primary +other_node_assigned_state | +has_extra_action | t +comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary +-[ RECORD 66 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 409 +section | primary_node +active_node_current_state | primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | numberSyncStandbysIsZero=false, secondaryQuorumNodesCountIsZero=true, failoverInProgress=false +active_node_assigned_state | primary +other_node_assigned_state | +has_extra_action | t +comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) +-[ RECORD 67 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 411 +section | primary_node +active_node_current_state | wait_primary +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | secondaryQuorumNodesCountIsZero=false +active_node_assigned_state | primary +other_node_assigned_state | +has_extra_action | t +comment | wait_primary, >=1 quorum secondary -> primary +-[ RECORD 68 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 413 +section | primary_node +active_node_current_state | apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNodesCountIsZero=true +active_node_assigned_state | wait_primary +other_node_assigned_state | +has_extra_action | t +comment | apply_settings, both zero -> wait_primary +-[ RECORD 69 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 415 +section | primary_node +active_node_current_state | apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | numberSyncStandbysIsZero=false +active_node_assigned_state | primary +other_node_assigned_state | +has_extra_action | t +comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) +-[ RECORD 70 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 417 +section | primary_node +active_node_current_state | apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNodesCountIsZero=false +active_node_assigned_state | primary +other_node_assigned_state | +has_extra_action | t +comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) +-[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 419 +section | primary_node +active_node_current_state | primary, wait_primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | converged primary/wait_primary/apply_settings, no other condition applies -> no-op besides the unhealthy-secondary fan-out +-[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 421 +section | primary_node +active_node_current_state | join_primary +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | primary +other_node_assigned_state | +has_extra_action | f +comment | backwards-compat: join_primary -> primary + diff --git a/src/monitor/formation_metadata.c b/src/monitor/formation_metadata.c index d4974fb6b..10dd0d790 100644 --- a/src/monitor/formation_metadata.c +++ b/src/monitor/formation_metadata.c @@ -15,6 +15,7 @@ #include "funcapi.h" #include "miscadmin.h" +#include "group_state_machine.h" #include "health_check.h" #include "metadata.h" #include "formation_metadata.h" @@ -550,9 +551,6 @@ set_formation_number_sync_standbys(PG_FUNCTION_ARGS) int groupId = 0; int standbyCount = 0; - - char message[BUFSIZE] = { 0 }; - if (formation == NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -626,17 +624,14 @@ set_formation_number_sync_standbys(PG_FUNCTION_ARGS) /* SetFormationNumberSyncStandbys reports ERROR when returning false */ bool success = SetFormationNumberSyncStandbys(formationId, number_sync_standbys); - /* and now ask the primary to change its settings */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to apply_settings " - "after updating number_sync_standbys to %d for formation %s.", - NODE_FORMAT_ARGS(primaryNode), - formation->number_sync_standbys, - formation->formationId); - - SetNodeGoalState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS, message); + /* + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary -> + * apply_settings. Kept as this row's own condition too (see that row's + * comment), in addition to the ereport(ERROR) guard above -- belt and + * suspenders. + */ + (void) ProceedGroupStateForApiTrigger( + API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS, primaryNode, NULL); PG_RETURN_BOOL(success); } diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 132482592..27354f4c6 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -35,6 +35,7 @@ #include "utils/rel.h" #include "utils/syscache.h" #include "utils/timestamp.h" +#include "utils/tuplestore.h" /* @@ -56,7 +57,7 @@ typedef struct CandidateList /* private function forward declarations */ static bool ProceedGroupStateForMSFailover(GroupStateContext *ctx, AutoFailoverNode *primaryNode); -static bool ProceedWithMSFailover(AutoFailoverNode *activeNode, +static bool ProceedWithMSFailover(GroupStateContext *ctx, AutoFailoverNode *activeNode, AutoFailoverNode *candidateNode); static bool BuildCandidateList(GroupStateContext *ctx, @@ -67,7 +68,8 @@ static AutoFailoverNode * SelectFailoverCandidateNode(GroupStateContext *ctx, CandidateList *candidateList, AutoFailoverNode *primaryNode); -static bool PromoteSelectedNode(AutoFailoverNode *selectedNode, +static bool PromoteSelectedNode(GroupStateContext *ctx, + AutoFailoverNode *selectedNode, AutoFailoverNode *primaryNode, CandidateList *candidateList); @@ -76,6 +78,7 @@ static void AssignGoalState(AutoFailoverNode *pgAutoFailoverNode, static bool WalDifferenceWithin(AutoFailoverNode *secondaryNode, AutoFailoverNode *primaryNode, int64 delta); +static void AssertMonitorFSMWellFormed(void); /* * --------------------------------------------------------------------- @@ -87,9 +90,15 @@ static bool WalDifferenceWithin(AutoFailoverNode *secondaryNode, * everything it calls (BuildCandidateList, SelectFailoverCandidateNode, * PromoteSelectedNode, ProceedWithMSFailover, WalSourceNodesAreAllUnhealthy) * stays hand-written C exactly as before, reached from the table via - * extraAction -- the candidate-selection algorithm (priority sort, LSN - * comparison, WAL-fetch orchestration) doesn't reduce to declarative - * conditions any more cleanly than it did before this change. + * extraAction -- the candidate-selection algorithm itself (priority sort, + * LSN comparison, WAL-fetch orchestration) doesn't reduce to declarative + * conditions any more cleanly than it did before this change. Only the + * plain AssignGoalState calls at the tail end of that algorithm -- + * BuildCandidateList's own fan-out to REPORT_LSN, and PromoteSelectedNode's + * own PREPARE_PROMOTION/FAST_FORWARD choice -- are dispatched through the + * table too (via TryFanOutReportLsnRow/DispatchMonitorFSMRuleByPos, see + * their own comments), each falling back to the original hand-written call + * on no match. * --------------------------------------------------------------------- */ @@ -101,7 +110,7 @@ typedef enum BoolPattern } BoolPattern; static bool -MatchBoolPattern(bool actual, BoolPattern pattern) +BoolMatchesPattern(bool actual, BoolPattern pattern) { switch (pattern) { @@ -369,6 +378,9 @@ typedef struct NodeStatusPattern BoolPattern candidateEligible; BoolPattern isInPrimaryState; BoolPattern isInMaintenance; + BoolPattern isDemotedPrimary; + BoolPattern canTakeWrites; + BoolPattern isReadyToStreamWAL; BoolPattern drainTimeExpired; BoolPattern isCitusWorkerGroup; BoolPattern replicationQuorum; @@ -400,21 +412,31 @@ BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *stat /* - * isInPrimaryState/isInMaintenance/drainTimeExpired/unreachableFromDemoteTimeout - * are deliberately NOT cached in NodeStatus and computed live here from - * status->node instead: unlike isHealthy/isUnhealthy/candidateEligible - * (health/priority facts that can't change mid-dispatch), these four are - * pure functions of node->goalState/reportedState, and a matched row's - * extraAction can reassign the very node being matched (e.g. DRAINING on - * primaryNode) in the SAME node_active() call, before dispatch continues to - * a later row -- exactly like NodeStateMatchesPattern below, which reads - * status->node's fields live for the same reason. Caching these as - * snapshot booleans (an earlier version of this code did) left later rows - * in the same call matching against a stale "still in primary state" fact - * even after primaryNode had just been moved to DRAINING -- confirmed by + * isInPrimaryState/isInMaintenance/isDemotedPrimary/canTakeWrites/ + * drainTimeExpired/unreachableFromDemoteTimeout are deliberately NOT cached + * in NodeStatus and computed live here from status->node instead: unlike + * isHealthy/isUnhealthy/candidateEligible (health/priority facts that can't + * change mid-dispatch), these are pure functions of + * node->goalState/reportedState, and a matched row's extraAction can + * reassign the very node being matched (e.g. DRAINING on primaryNode) in + * the SAME node_active() call, before dispatch continues to a later row -- + * exactly like NodeStateMatchesPattern below, which reads status->node's + * fields live for the same reason. Caching these as snapshot booleans (an + * earlier version of this code did) left later rows in the same call + * matching against a stale "still in primary state" fact even after + * primaryNode had just been moved to DRAINING -- confirmed by * concurrent_health_check_and_report, which requires the "secondary -> * prepare_promotion" row to correctly stop matching once primaryNode is no * longer IsInPrimaryState(). + * + * canTakeWrites is distinct from isInPrimaryState: it's CanTakeWritesInState + * (node_metadata.c) applied to goalState alone, with no requirement that + * reportedState has converged to match -- exactly node_active_protocol.c's + * RemoveNode() own "bool currentNodeIsPrimary = + * CanTakeWritesInState(currentNode->goalState);" check, which a mid- + * transition node (goalState allows writes, reportedState hasn't caught up + * yet) still satisfies where isInPrimaryState (which additionally requires + * stability) would not. */ static bool NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) @@ -426,24 +448,93 @@ NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) status->node->goalState != REPLICATION_STATE_PRIMARY && status->node->goalState != REPLICATION_STATE_SINGLE; - return MatchBoolPattern(status->node != NULL, pattern->exists) && + return BoolMatchesPattern(status->node != NULL, pattern->exists) && NodeStateMatchesPattern(status->node, &pattern->statePattern) && - MatchBoolPattern(status->isHealthy, pattern->isHealthy) && - MatchBoolPattern(status->isUnhealthy, pattern->isUnhealthy) && - MatchBoolPattern(status->candidateEligible, pattern->candidateEligible) && - MatchBoolPattern(IsInPrimaryState(status->node), pattern->isInPrimaryState) && - MatchBoolPattern(IsInMaintenance(status->node), pattern->isInMaintenance) && - MatchBoolPattern(NodeIsDrainTimeExpired(status->node, status->ctx), + BoolMatchesPattern(status->isHealthy, pattern->isHealthy) && + BoolMatchesPattern(status->isUnhealthy, pattern->isUnhealthy) && + BoolMatchesPattern(status->candidateEligible, pattern->candidateEligible) && + BoolMatchesPattern(IsInPrimaryState(status->node), pattern->isInPrimaryState) && + BoolMatchesPattern(IsInMaintenance(status->node), pattern->isInMaintenance) && + BoolMatchesPattern(IsDemotedPrimary(status->node), pattern->isDemotedPrimary) && + BoolMatchesPattern(status->node != NULL && + CanTakeWritesInState(status->node->goalState), + pattern->canTakeWrites) && + BoolMatchesPattern(CandidateNodeIsReadyToStreamWAL(status->node), + pattern->isReadyToStreamWAL) && + BoolMatchesPattern(NodeIsDrainTimeExpired(status->node, status->ctx), pattern->drainTimeExpired) && - MatchBoolPattern(status->isCitusWorkerGroup, pattern->isCitusWorkerGroup) && - MatchBoolPattern(status->replicationQuorum, pattern->replicationQuorum) && - MatchBoolPattern(status->isComparableToReferenceTli, + BoolMatchesPattern(status->isCitusWorkerGroup, pattern->isCitusWorkerGroup) && + BoolMatchesPattern(status->replicationQuorum, pattern->replicationQuorum) && + BoolMatchesPattern(status->isComparableToReferenceTli, pattern->isComparableToReferenceTli) && - MatchBoolPattern(unreachableFromDemoteTimeout, + BoolMatchesPattern(unreachableFromDemoteTimeout, pattern->unreachableFromDemoteTimeout); } +/* + * MonitorApiFunction itself is declared in group_state_machine.h, not here: + * ProceedGroupStateForApiTrigger() (below the array) is called from + * node_active_protocol.c/formation_metadata.c with a MonitorApiFunction + * value, so the enum needs to be visible outside this file the same way + * MonitorFSMSection already is. ApiTriggerPattern/ApiTriggerKind/ + * MatchApiTrigger/API_TRIGGER() stay private to this file: only the enum + * itself crosses the module boundary, every caller just picks a tag and + * hands it to ProceedGroupStateForApiTrigger, never builds a pattern + * itself. + * + * Every builder in this file (BuildFromContextNodeActiveContext, + * BuildForPrimaryNodeNodeActiveContext) memset()s its NodeActiveContext to + * zero before filling it in, so apiFunction defaults to API_FUNCTION_NONE + * unless a caller explicitly sets it -- exactly the design doc's "zero + * changes" guarantee: no row written before this mechanism existed changes + * meaning just because the field now exists. + */ + +typedef enum ApiTriggerKind +{ + /* + * The safe default for an omitted .conditions.apiTrigger field: matches + * only an ordinary node_active() heartbeat call. Every row written + * before this mechanism existed implicitly assumed exactly this, so + * omission has to keep meaning "heartbeat only", not "any trigger" -- + * the opposite of BOOL_ANY/NODE_STATE_ANY's "omission matches + * everything" convention elsewhere in this file, and deliberately so: + * here the safe default is the narrower match, since none of the 48 + * existing rows should start matching a manual perform_failover() call + * just because they don't mention this field at all. + */ + API_TRIGGER_NODE_ACTIVE = 0, + API_TRIGGER_SPECIFIC +} ApiTriggerKind; + +typedef struct ApiTriggerPattern +{ + ApiTriggerKind kind; + MonitorApiFunction function; /* meaningful only when kind == API_TRIGGER_SPECIFIC */ +} ApiTriggerPattern; + +#define API_TRIGGER(fn) ((ApiTriggerPattern){ .kind = API_TRIGGER_SPECIFIC, .function = (fn) }) + +static bool +MatchApiTrigger(MonitorApiFunction actual, ApiTriggerPattern pattern) +{ + switch (pattern.kind) + { + case API_TRIGGER_SPECIFIC: + { + return actual == pattern.function; + } + + case API_TRIGGER_NODE_ACTIVE: + default: + { + return actual == API_FUNCTION_NONE; + } + } +} + + /* * NodeActiveContext: group-level facts, computed once per dispatch call * alongside the two NodeStatus roles above. @@ -453,7 +544,23 @@ typedef struct NodeActiveContext NodeStatus activeNode; NodeStatus primaryNode; + /* + * candidateNode is only populated by BuildMSFailoverNodeActiveContext + * (the MS-failover cluster's own nested-dispatch context builder, + * mirroring BuildForPrimaryNodeNodeActiveContext's role-specific + * pattern): FindCandidateNodeBeingPromoted(ctx->groupNodeList)'s result, + * i.e. the node currently mid-promotion, if any. .node == NULL if no + * failover candidate has been selected yet this round. Left at its + * memset-zero default (.node == NULL, .isUnhealthy == true) for every + * other dispatch pass -- no row outside the MS-failover sub-section + * references it. + */ + NodeStatus candidateNode; + + MonitorApiFunction apiFunction; + bool groupHasExactlyOneNode; + bool groupHasExactlyTwoNodes; bool groupHasMoreThanTwoNodes; bool anyOtherNodeWaitingStandby; @@ -470,11 +577,82 @@ typedef struct NodeActiveContext bool primaryIsWaitPrimaryPresumedDead; bool failoverInProgress; bool replicationStallExceeded; + + /* + * lastHealthySyncStandbyGoingToMaintenance is set only inside + * ProceedGroupStateForApiTrigger's own API_FUNCTION_START_MAINTENANCE + * branch (see its own comment there): true when activeNode is the last + * remaining healthy synchronous standby and formation->number_sync_ + * standbys is zero, meaning putting it into maintenance would otherwise + * block writes on the primary until the monitor separately assigns it + * wait_primary -- node_active_protocol.c:1973-1986's own real condition, + * verbatim. Left false (the memset default) for every other dispatch + * pass; not a general-purpose fact reused elsewhere. + */ + bool lastHealthySyncStandbyGoingToMaintenance; + + /* + * The following four facts are set only by + * BuildMSFailoverNodeActiveContext, for the MS-failover cluster's own + * declarative rows (see group_state_machine.c's "MS-failover / + * candidate-selection cluster" section) -- left false (the memset + * default) for every other dispatch pass. + */ + + /* WalSourceNodesAreAllUnhealthy(ctx, ctx->groupNodeList, activeNode) -- + * activeNode-specific despite living at this level (not on NodeStatus): + * it depends on the whole group's other REPORT_LSN peers, not on + * activeNode's own state alone. */ + bool activeNodeAllWalSourcesUnhealthy; + + /* mirrors whether ProceedGroupStateForMSFailover's own + * "nodeBeingPromoted != NULL" branch will actually drive that candidate + * via ProceedWithMSFailover this round, rather than falling through to + * BuildCandidateList/selection instead -- see this file's own + * ActionRunMultiStandbyFailoverCascade comment and the design doc's + * identically-named fact for the full derivation. */ + bool candidatePromotionInProgress; + + /* WalDifferenceWithin(mostAdvancedNode, ctx->primaryNode, + * PromoteXlogThreshold) for the current MS-failover candidate pool's + * own most-advanced member -- SelectFailoverCandidateNode's own + * data-loss guard. */ + bool mostAdvancedCandidateWithinPromoteThreshold; + + /* GuardDataLoss (metadata.c), the pgautofailover.guard_data_loss GUC -- + * a single global process-wide bool, grouped here (not read directly + * inside an extraAction) so any row gated on it says so in its own + * conditions. */ + bool guardDataLossEnabled; + + /* + * Set unconditionally true by BuildMSFailoverNodeActiveContext, false + * (the memset default) everywhere else -- the MS-failover cluster's own + * "am I actually being dispatched from inside the MS-failover cluster's + * own bounded nested search" marker. Every row from + * MonitorFSM_MSFailoverStart onwards that doesn't already carry a + * condition guaranteed false outside that nested search (as pos 363/365 + * do, via activeNodeAllWalSourcesUnhealthy/candidatePromotionInProgress) + * must require this true: the top-level driver's own ordinary lookup + * (ProceedGroupStateFromContext) shares MonitorFSM_PrimaryNodeSectionStart + * as its own upper bound, so it scans straight through this whole + * cluster too -- without this guard, pos 367-373's bare activeNode-state + * patterns (no distinguishing condition of their own) would fire for + * any ordinary, non-MS-failover heartbeat whose reported/goal states + * happened to line up, hijacking normal secondary/catchingup/maintenance + * convergence. Confirmed by node_active_protocol.out/guard_data_loss.out/ + * etc. regressing exactly this way before this field existed. + */ + bool inMSFailoverCluster; } NodeActiveContext; typedef struct NodeActiveContextPattern { + ApiTriggerPattern apiTrigger; /* omitted -> {0} -> API_TRIGGER_NODE_ACTIVE, matching every + row's existing meaning with no changes required elsewhere */ + BoolPattern groupHasExactlyOneNode; + BoolPattern groupHasExactlyTwoNodes; BoolPattern groupHasMoreThanTwoNodes; BoolPattern anyOtherNodeWaitingStandby; @@ -491,6 +669,13 @@ typedef struct NodeActiveContextPattern BoolPattern primaryIsWaitPrimaryPresumedDead; BoolPattern failoverInProgress; BoolPattern replicationStallExceeded; + BoolPattern lastHealthySyncStandbyGoingToMaintenance; + + BoolPattern activeNodeAllWalSourcesUnhealthy; + BoolPattern candidatePromotionInProgress; + BoolPattern mostAdvancedCandidateWithinPromoteThreshold; + BoolPattern guardDataLossEnabled; + BoolPattern inMSFailoverCluster; } NodeActiveContextPattern; @@ -531,10 +716,50 @@ typedef void (*MonitorExtraActionFunction) (GroupStateContext *ctx, NodeActiveContext *nac, char *message); +/* + * MonitorFSMSection itself is declared in group_state_machine.h, not here: + * it needs to be visible to pg_auto_failover.c (to register + * pgautofailover.fsm_section and wire up dump_fsm()) the same way + * ReplicationState is. The comment on the MonitorFSM_* index constants + * below explains what each of its three values corresponds to in the + * original if-chain. Every row is tagged with its own section explicitly + * (rather than leaving section membership implicit in array position + * alone) so a reader scanning the table sees which region a row belongs to + * without cross-referencing an index against a separate comment block, and + * so AssertMonitorFSMWellFormed() can verify the MonitorFSM_* index + * constants actually agree with where each section's rows really are. + */ + typedef struct MonitorFSMTransition { + /* + * pos and section are metadata, not match inputs: RuleMatches() never + * reads either. pos is a human-facing row number -- purely so a comment, + * a bug report, or a SQL query against dump_fsm() can say "row 203" and + * a reader can find it without counting braces up from the top of the + * array. Each MonitorFSMSection gets its own hundred-block, starting at + * *01 rather than *00 so the position within the block reads as an + * ordinary 1-based count (101 is the section's 1st row, 103 its 2nd, + * ...) instead of an off-by-one 0th/1st/2nd. Rows within a section are + * numbered every 2 (101, 103, 105, ...) rather than consecutively -- so + * a new row can be inserted between two existing ones (e.g. 102 between + * 101 and 103) without renumbering anything else in the file. + * section records which of the three real control-flow regions (see + * MonitorFSMSection above) this row belongs to; AssertMonitorFSMWellFormed() + * uses both to confirm the MonitorFSM_* index constants below are still + * correct, so a boundary drifting out of sync with the rows it's meant to + * bound fails at first use, not by accident months later. Both are also + * exposed to SQL via dump_fsm() and attributed to the pgautofailover.event + * row a matched rule produces (see rule_pos/rule_section below). + */ + int pos; + MonitorFSMSection section; + NodeStatusPattern activeNode; NodeStatusPattern primaryNode; + NodeStatusPattern candidateNode; /* MS-failover sub-section rows only; see + NodeActiveContext's own comment on + .candidateNode */ NodeActiveContextPattern conditions; GoalStateAssignment activeNodeAssignedState; @@ -550,74 +775,116 @@ typedef struct MonitorFSMTransition * for why ("One array, not three" in the design doc this table implements). * Forward-declared here so the extraActions defined above it (which each * perform one bounded, named nested search over it) can reference it by - * name; the boundary constants below are forward-declared the same way, for - * the same reason -- both actions and the top-level driver need them. + * name; the four index constants below are forward-declared the same way, + * for the same reason -- both those actions and the top-level driver need + * them to bound their searches. + * + * WHY THESE CONSTANTS EXIST AT ALL: the original if-chain has real, load- + * bearing structure that a single flat "first match wins over the whole + * array" search would destroy. Two different things are true about + * activeNode/primaryNode depending on WHERE in the original control flow a + * row came from (whether .activeNode means "the reporting node" or "the + * primary node substituted in"), and one specific fallback (the MS-failover + * cascade declining) needs to resume scanning from a specific *later* point, + * not from the top. Each constant below is the index where one of those + * real regions starts, so a search can be bounded to exactly the region + * that's semantically valid for the situation at hand: * - * These three indices partition MonitorFSM[] into the sections the real - * if-chain's control flow actually has: + * MonitorFSM_EarlyChecksStart = 15 + * Where MONITOR_FSM_SECTION_API_TRIGGERED ends and MONITOR_FSM_SECTION_ + * EARLY_CHECKS begins. Rows [0, 15) are the operator-triggered rows + * (perform_failover, remove_node, start/stop_maintenance, + * set_node_candidate_priority, set_node_replication_quorum, + * set_formation_number_sync_standbys) -- reached only via + * ProceedGroupStateForApiTrigger(), never via the ordinary node_active() + * heartbeat path (nac->apiFunction stays API_FUNCTION_NONE there, and + * every one of these 15 rows' .conditions.apiTrigger requires a specific + * non-NONE value -- see MatchApiTrigger). Used to: (a) start the + * api-triggered lookup at 0, bounded to end here, (b) start the + * early-checks lookup here instead of at 0, now that the heartbeat + * sections no longer begin at the very top of the array. * - * [0, MonitorFSM_FromContextStart) the six checks - * ProceedGroupStateFromContext() - * runs before its one real branch - * point (IsInPrimaryState(activeNode)), - * regardless of which way that - * branch goes. - * [MonitorFSM_FromContextStart, the rest of - * MonitorFSM_PrimaryNodeSectionStart) ProceedGroupStateFromContext(), - * reached only when activeNode is - * NOT currently primary-role. - * [MonitorFSM_FromContextResumeStart, the sub-range of the row above - * MonitorFSM_PrimaryNodeSectionStart) that ActionRunMultiStandby - * FailoverCascade resumes into - * when ProceedGroupStateForMSFailover() - * declines, mirroring the real - * source's fallthrough to - * "whatever is textually next". - * NOTE: unlike the design doc this - * table otherwise follows, there is - * no declarative "MS-failover / - * candidate-selection cluster" - * section here at all -- - * ProceedGroupStateForMSFailover() - * and everything it calls - * (BuildCandidateList, - * SelectFailoverCandidateNode, - * PromoteSelectedNode) stayed one - * opaque hand-written function, - * invoked wholesale, not spread - * across new rows/roles/conditions - * the way the doc's own - * candidateNode-based rows do. This - * constant just marks "resume - * scanning ordinary FromContext - * rows from here" -- it is NOT the - * doc's MonitorFSM_MSFailoverCluster - * Start, which names a real section - * of converted candidate-selection - * rows that doesn't exist in this - * table. - * [MonitorFSM_PrimaryNodeSectionStart, ProceedGroupStateForPrimaryNode()'s - * MonitorFSM_SIZE) own rows, reached either directly - * by the top-level driver (activeNode - * already primary-role) or via - * ActionRunPrimaryNodeTransition's - * nested pass on primaryNode - * (join_secondary's cascade row). + * MonitorFSM_FromContextStart = 21 + * Where MONITOR_FSM_SECTION_REPORTING_NODE begins. Rows [15, 21) are the + * six checks (DROPPED, goal-DROPPED, MAINTENANCE, the demote_timeout + * self-fence, both "alone in group" rows) that + * ProceedGroupStateFromContext() runs unconditionally, before its one + * real branch point (IsInPrimaryState(activeNode)) -- so they're tried + * first regardless of which way that branch goes, and everything from + * index 21 on is reached only when activeNode is confirmed NOT currently + * primary-role. Used to: (a) bound the early-checks lookup to [15, 21) + * in the top-level driver, (b) start the ordinary FromContext lookup at + * 21 once activeNode is confirmed non-primary. + * + * MonitorFSM_FromContextResumeStart = 24 + * A resume point *inside* MONITOR_FSM_SECTION_REPORTING_NODE, not a + * section boundary of its own -- the row right after the merged + * nodesCount>2-unhealthy-primary row (pos 209, "nodesCount>2, primary + * unhealthy -> draining/maintenance + MS-failover cascade") -- only used + * by ActionRunMultiStandbyFailoverCascade: when + * ProceedGroupStateForMSFailover() declines, the real source falls + * through to whatever if-statement is textually next, and this is where + * that "next" starts in this table. Named similarly to (but NOT the + * same concept as) the design doc's MonitorFSM_MSFailoverClusterStart -- + * see ActionRunMultiStandbyFailoverCascade's own comment for the + * distinction. + * + * MonitorFSM_MSFailoverStart = 52 + * A resume point *inside* MONITOR_FSM_SECTION_REPORTING_NODE, like + * MonitorFSM_FromContextResumeStart above, not a section boundary of its + * own: nine rows, appended at the end of the section rather than + * renumbered into it, for the MS-failover / candidate-selection + * cluster's own declarative transitions (see that section's own + * comment below) -- two (retry-reset, join_secondary) reached through + * TryMSFailoverDeclarativeRow's bounded nested dispatch, gated by the + * exact same hand-written C condition ProceedGroupStateForMSFailover/ + * ProceedWithMSFailover already evaluate; four (BuildCandidateList's + * own fan-out) through TryFanOutReportLsnRow's bounded nested dispatch, + * one per node the fan-out loop touches; two (PromoteSelectedNode's own + * two outcomes) through DispatchMonitorFSMRuleByPos, since first-match- + * wins can't distinguish between them (see their own comment); one + * (the "still gathering candidates" case) purely for dump_fsm() + * completeness, never itself dispatched. None reached through the + * ordinary top-level driver. + * + * MonitorFSM_PrimaryNodeSectionStart = 61 + * Where MONITOR_FSM_SECTION_PRIMARY_NODE begins: the declarative + * replacement for ProceedGroupStateForPrimaryNode()'s own if-chain, in + * which .activeNode means the *primary* node, not the reporting node. + * Used to: (a) bound the ordinary FromContext lookup to end here (so it + * can never wander into primary-role rows built under a different + * NodeActiveContext), (b) start the primary-role lookup here, both from + * the top-level driver (activeNode already primary-role) and from + * ActionRunPrimaryNodeTransition's nested pass on primaryNode + * (join_secondary's cascade row). + * + * MonitorFSM_SIZE = 72 + * Total row count -- the end bound for the primary-role lookup (nothing + * comes after MONITOR_FSM_SECTION_PRIMARY_NODE), and the size every + * other bounded search is checked against. * * Kept as plain hardcoded integers, exactly as the design doc's own * placeholders are -- recomputed by hand whenever a row is added, removed, - * or moved across a boundary. A wrong value here fails loudly and - * immediately (either a compile-time out-of-bounds slice that scans zero - * rows and never matches, or a row from the wrong section matching - * unexpectedly) rather than silently: the regress/isolation suite this - * table is checked against covers every one of these boundaries already. + * or moved across a boundary. What makes a wrong value here safe to keep as + * a hand-maintained integer, rather than a foot-gun: every row also carries + * its own .pos and .section fields (see MonitorFSMTransition above), and + * AssertMonitorFSMWellFormed() (below the array) walks the whole table once + * and asserts these six constants agree with what the rows themselves say + * -- .pos matches array position, and .section actually changes from + * API_TRIGGERED to EARLY_CHECKS to REPORTING_NODE to PRIMARY_NODE exactly at + * these four indices and nowhere else. A boundary that drifts out of sync + * with a row added, removed, or moved fails loudly at first use (an + * assertion), not silently as a row from the wrong section matching + * unexpectedly or a search that scans zero rows and never matches. */ static const MonitorFSMTransition MonitorFSM[]; -#define MonitorFSM_FromContextStart 6 -#define MonitorFSM_FromContextResumeStart 9 -#define MonitorFSM_PrimaryNodeSectionStart 37 -#define MonitorFSM_SIZE 48 +#define MonitorFSM_EarlyChecksStart 15 +#define MonitorFSM_FromContextStart 21 +#define MonitorFSM_FromContextResumeStart 24 +#define MonitorFSM_MSFailoverStart 52 +#define MonitorFSM_PrimaryNodeSectionStart 61 +#define MonitorFSM_SIZE 72 /* Forward-declared for the same reason as MonitorFSM[] above: used by * extraActions (ActionRunPrimaryNodeTransition) defined before its real @@ -632,26 +899,40 @@ RuleMatches(const NodeActiveContext *nac, const MonitorFSMTransition *rule) { const NodeActiveContextPattern *cond = &rule->conditions; - return NodeMatchesPattern(&nac->activeNode, &rule->activeNode) && - NodeMatchesPattern(&nac->primaryNode, &rule->primaryNode) && + return MatchApiTrigger(nac->apiFunction, cond->apiTrigger) && - MatchBoolPattern(nac->groupHasExactlyOneNode, cond->groupHasExactlyOneNode) && - MatchBoolPattern(nac->groupHasMoreThanTwoNodes, cond->groupHasMoreThanTwoNodes) && - MatchBoolPattern(nac->anyOtherNodeWaitingStandby, cond->anyOtherNodeWaitingStandby) && - MatchBoolPattern(nac->numberSyncStandbysIsZero, cond->numberSyncStandbysIsZero) && - MatchBoolPattern(nac->replicationQuorumCountIsZero, + NodeMatchesPattern(&nac->activeNode, &rule->activeNode) && + NodeMatchesPattern(&nac->primaryNode, &rule->primaryNode) && + NodeMatchesPattern(&nac->candidateNode, &rule->candidateNode) && + + BoolMatchesPattern(nac->groupHasExactlyOneNode, cond->groupHasExactlyOneNode) && + BoolMatchesPattern(nac->groupHasExactlyTwoNodes, cond->groupHasExactlyTwoNodes) && + BoolMatchesPattern(nac->groupHasMoreThanTwoNodes, cond->groupHasMoreThanTwoNodes) && + BoolMatchesPattern(nac->anyOtherNodeWaitingStandby, cond->anyOtherNodeWaitingStandby) && + BoolMatchesPattern(nac->numberSyncStandbysIsZero, cond->numberSyncStandbysIsZero) && + BoolMatchesPattern(nac->replicationQuorumCountIsZero, cond->replicationQuorumCountIsZero) && - MatchBoolPattern(nac->secondaryNodesCountIsZero, cond->secondaryNodesCountIsZero) && - MatchBoolPattern(nac->secondaryQuorumNodesCountIsZero, + BoolMatchesPattern(nac->secondaryNodesCountIsZero, cond->secondaryNodesCountIsZero) && + BoolMatchesPattern(nac->secondaryQuorumNodesCountIsZero, cond->secondaryQuorumNodesCountIsZero) && - MatchBoolPattern(nac->atLeastOneHealthyCandidate, cond->atLeastOneHealthyCandidate) && - MatchBoolPattern(nac->walWithinPromoteThreshold, cond->walWithinPromoteThreshold) && - MatchBoolPattern(nac->walWithinSyncThreshold, cond->walWithinSyncThreshold) && - MatchBoolPattern(nac->activeAndPrimaryTliMatch, cond->activeAndPrimaryTliMatch) && - MatchBoolPattern(nac->primaryIsWaitPrimaryPresumedDead, + BoolMatchesPattern(nac->atLeastOneHealthyCandidate, cond->atLeastOneHealthyCandidate) && + BoolMatchesPattern(nac->walWithinPromoteThreshold, cond->walWithinPromoteThreshold) && + BoolMatchesPattern(nac->walWithinSyncThreshold, cond->walWithinSyncThreshold) && + BoolMatchesPattern(nac->activeAndPrimaryTliMatch, cond->activeAndPrimaryTliMatch) && + BoolMatchesPattern(nac->primaryIsWaitPrimaryPresumedDead, cond->primaryIsWaitPrimaryPresumedDead) && - MatchBoolPattern(nac->failoverInProgress, cond->failoverInProgress) && - MatchBoolPattern(nac->replicationStallExceeded, cond->replicationStallExceeded); + BoolMatchesPattern(nac->failoverInProgress, cond->failoverInProgress) && + BoolMatchesPattern(nac->replicationStallExceeded, cond->replicationStallExceeded) && + BoolMatchesPattern(nac->lastHealthySyncStandbyGoingToMaintenance, + cond->lastHealthySyncStandbyGoingToMaintenance) && + BoolMatchesPattern(nac->activeNodeAllWalSourcesUnhealthy, + cond->activeNodeAllWalSourcesUnhealthy) && + BoolMatchesPattern(nac->candidatePromotionInProgress, + cond->candidatePromotionInProgress) && + BoolMatchesPattern(nac->mostAdvancedCandidateWithinPromoteThreshold, + cond->mostAdvancedCandidateWithinPromoteThreshold) && + BoolMatchesPattern(nac->guardDataLossEnabled, cond->guardDataLossEnabled) && + BoolMatchesPattern(nac->inMSFailoverCluster, cond->inMSFailoverCluster); } @@ -699,6 +980,24 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, { char message[BUFSIZE] = { 0 }; + /* + * Attribute every event this row's dispatch produces -- its own + * extraAction's side effects included -- to this rule's .pos/.section + * (see CurrentMonitorFSMRulePos/Section in notifications.h). Saved and + * restored rather than cleared to 0 on the way out: a nested dispatch + * (ActionRunMultiStandbyFailoverCascade's/ActionRunPrimaryNodeTransition's + * own bounded search, run from inside extraAction below) sets these to + * the *inner* row's own pos/section for its own duration, and once it + * returns, this row's own subsequent activeNodeAssignedState/ + * otherNodeAssignedState calls need to see this (outer) row's values + * again, not 0. + */ + int savedRulePos = CurrentMonitorFSMRulePos; + int savedRuleSection = CurrentMonitorFSMRuleSection; + + CurrentMonitorFSMRulePos = rule->pos; + CurrentMonitorFSMRuleSection = (int) rule->section; + if (rule->comment != NULL) { snprintf(message, BUFSIZE, "%s", rule->comment); @@ -720,6 +1019,9 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, AssignDeclaredGoalState(rule, nac->primaryNode.node, rule->otherNodeAssignedState.state, message); } + + CurrentMonitorFSMRulePos = savedRulePos; + CurrentMonitorFSMRuleSection = savedRuleSection; } @@ -771,6 +1073,21 @@ int ReplicationStallTimeoutMs = 10 * 1000; bool BuildGroupStateContext(GroupStateContext *ctx, AutoFailoverNode *activeNode) { + /* + * A no-op in a non-assert build (see AssertMonitorFSMWellFormed's own + * #ifdef USE_ASSERT_CHECKING), so this stays an unconditional call -- + * matching AssignDeclaredGoalState's own pattern -- rather than an + * #ifdef'd call site, which would make the function itself look unused + * (and fail -Werror=unused-function) in a non-assert build. + */ + static bool monitorFSMChecked = false; + + if (!monitorFSMChecked) + { + AssertMonitorFSMWellFormed(); + monitorFSMChecked = true; + } + ctx->formationId = activeNode->formationId; ctx->groupId = activeNode->groupId; ctx->activeNode = activeNode; @@ -1032,6 +1349,7 @@ BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *prim } nac->groupHasExactlyOneNode = (ctx->groupNodeCount == 1); + nac->groupHasExactlyTwoNodes = (ctx->groupNodeCount == 2); nac->groupHasMoreThanTwoNodes = (ctx->groupNodeCount > 2); nac->failoverInProgress = IsFailoverInProgress(ctx->groupNodeList); @@ -1129,6 +1447,169 @@ BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *p } +/* + * ActionFanOutReportLsnOnPrimaryRemoval implements RemoveNode's own + * primary-removal fan-out (node_active_protocol.c's RemoveNode(), + * "if (currentNodeIsPrimary) { foreach other node not in maintenance -> + * report_lsn }"), reusing the exact otherNodesFn-style shape already + * established for the heartbeat side's own fan-out + * (ActionCatchupUnhealthySecondaries above) -- a dynamically-sized list of + * nodes can't be expressed as a single declared activeNodeAssignedState/ + * otherNodeAssignedState slot, operator-triggered or not. Runs before the + * row's own activeNodeAssignedState = DROPPED (see DispatchMonitorFSMRule), + * matching the real source's own order: fan out to the survivors first, + * then mark the removed node itself dropped. + */ +static void +ActionFanOutReportLsnOnPrimaryRemoval(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + List *otherNodesGroupList = AutoFailoverOtherNodesList(nac->activeNode.node); + ListCell *nodeCell = NULL; + + foreach(nodeCell, otherNodesGroupList) + { + AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); + + if (IsInMaintenance(otherNode)) + { + continue; + } + + char otherMessage[BUFSIZE] = { 0 }; + + snprintf(otherMessage, BUFSIZE, + "Setting goal state of " NODE_FORMAT + " to report_lsn after primary node removal.", + NODE_FORMAT_ARGS(otherNode)); + + AssignGoalState(otherNode, REPLICATION_STATE_REPORT_LSN, otherMessage); + } +} + + +/* + * BuildApiTriggerNodeActiveContext computes every fact the API_TRIGGERED + * section of MonitorFSM[] needs, for one particular apiFunction. + * + * activeNode is whichever single node the call is most fundamentally about + * -- the design doc's own reframing of "activeNode" for operator-triggered + * rows (see "Operator-triggered transitions belong in this table too"): + * the standby being promoted for perform_failover's 2-node row, the primary + * itself for perform_failover's >2-node row and every + * set_node_candidate_priority/set_node_replication_quorum/ + * set_formation_number_sync_standbys row (mirroring + * BuildForPrimaryNodeNodeActiveContext's own "primaryNode IS activeNode" + * convention), the node being removed for remove_node, the node being + * (re)moved into/out of maintenance for start/stop_maintenance. + * + * primaryNode is the group's real primary, only when a row for this + * apiFunction genuinely needs to reference "the primary" as a role distinct + * from activeNode (perform_failover's 2-node row, start_maintenance's + * secondary rows); NULL otherwise, exactly like the heartbeat side already + * passes NULL for primaryNode in the analogous "primaryNode IS activeNode" + * case. + */ +static void +BuildApiTriggerNodeActiveContext(GroupStateContext *ctx, MonitorApiFunction apiFunction, + AutoFailoverNode *activeNode, AutoFailoverNode *primaryNode, + NodeActiveContext *nac) +{ + memset(nac, 0, sizeof(NodeActiveContext)); + + nac->apiFunction = apiFunction; + + BuildNodeStatus(ctx, activeNode, &nac->activeNode); + BuildNodeStatus(ctx, primaryNode, &nac->primaryNode); + + nac->groupHasExactlyOneNode = (ctx->groupNodeCount == 1); + nac->groupHasExactlyTwoNodes = (ctx->groupNodeCount == 2); + nac->groupHasMoreThanTwoNodes = (ctx->groupNodeCount > 2); + nac->failoverInProgress = IsFailoverInProgress(ctx->groupNodeList); + + if (apiFunction == API_FUNCTION_START_MAINTENANCE && primaryNode != NULL) + { + /* + * Mirrors node_active_protocol.c's start_maintenance() own + * pre-dispatch computation verbatim (its own secondaryNodesCount, + * formation->number_sync_standbys, and IsHealthySyncStandby( + * currentNode) triple): true when activeNode is the only remaining + * healthy synchronous standby and number_sync_standbys is zero -- + * putting it into maintenance would otherwise block writes on the + * primary until the monitor separately assigns it wait_primary. + */ + List *secondaryNodesList = + AutoFailoverOtherNodesListInState(primaryNode, REPLICATION_STATE_SECONDARY); + int secondaryNodesCount = CountHealthySyncStandbys(secondaryNodesList); + + nac->lastHealthySyncStandbyGoingToMaintenance = + ctx->formation->number_sync_standbys == 0 && + secondaryNodesCount == 1 && + IsHealthySyncStandby(activeNode); + } +} + + +/* + * ProceedGroupStateForApiTrigger dispatches a single operator-triggered + * transition through MonitorFSM[]'s API_TRIGGERED section (pos 101-1xx -- + * see "Operator-triggered transitions belong in this table too" in the + * design doc). Every SQL-callable wrapper in node_active_protocol.c/ + * formation_metadata.c that assigns a goal state as a direct consequence of + * an operator call -- perform_failover, remove_node, start/stop_ + * maintenance, set_node_candidate_priority, set_node_replication_quorum, + * set_formation_number_sync_standbys -- keeps its own imperative shape + * exactly as before (argument parsing, locking, resolving which node(s) are + * involved, and every existing validation ereport(ERROR)/WARNING/NOTICE, + * all preserved unchanged so their exact message text stays test-stable), + * and calls this once in the middle to make the actual state assignment(s), + * then continues with whatever POST side effect the real source still does + * (a continuation ProceedGroupState() call, a candidatePriority trick, + * number_sync_standbys bookkeeping) as further hand-written code -- none of + * that imperative surrounding code becomes a row, matching the design doc's + * own "pre/post side effects stay hand-written C" principle. + * + * Unlike the heartbeat side (ProceedGroupStateFromContext, which silently + * no-ops on no match -- see its own comment for why that's the right call + * there), a genuine no-match here is always ereport(ERROR)'d as a bug: by + * the time this is called, the wrapper has already validated, via its own + * preserved pre-checks, that the operation IS valid for the resolved + * node's current state -- reaching no-match here means this table's own + * conditions have drifted out of sync with the wrapper's pre-checks, a + * real gap to fix, not a normal "operator asked for something invalid" + * outcome (that case is already rejected earlier, with its own specific, + * pre-existing ereport(ERROR) message, before this function is ever + * called). + */ +bool +ProceedGroupStateForApiTrigger(MonitorApiFunction apiFunction, + AutoFailoverNode *activeNode, + AutoFailoverNode *primaryNode) +{ + GroupStateContext ctx; + NodeActiveContext nac; + + BuildGroupStateContext(&ctx, activeNode); + BuildApiTriggerNodeActiveContext(&ctx, apiFunction, activeNode, primaryNode, &nac); + + int index = FindMatchingMonitorFSMRuleIndexFrom(MonitorFSM, MonitorFSM_EarlyChecksStart, + 0, &nac); + + if (index < 0) + { + ereport(ERROR, + (errmsg("BUG: no MonitorFSM[] row matches api-triggered call for " + NODE_FORMAT " in state \"%s\"", + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->reportedState)))); + } + + DispatchMonitorFSMRule(&ctx, &nac, &MonitorFSM[index]); + + return true; +} + + /* * MonitorFSM[]: one array, not several -- see the boundary-constant comment * above the MonitorFSMTransition typedef for the section layout and why it's @@ -1138,48 +1619,320 @@ BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *p * behaviour change, exactly as it was over the three separate arrays this * replaces. * - * --- [0, MonitorFSM_FromContextStart): the six checks the real if-chain - * runs BEFORE the IsInPrimaryState(activeNode) early return - * (group_state_machine.c:284) -- DROPPED, goal==DROPPED, MAINTENANCE, the - * demote_timeout self-fence, and both "alone in group" rows. These fire - * regardless of whether activeNode is currently the primary (a primary that - * just lost its only standby must still reach SINGLE here, before ever - * redirecting into the ProceedGroupStateForPrimaryNode section) -- confirmed - * by the drop_node regression test, which failed the first time this table - * put the primary-state redirect ahead of these six checks instead of after - * them. None of these six rows reference .primaryNode at all, so they can be + * --- [0, MonitorFSM_EarlyChecksStart): MONITOR_FSM_SECTION_API_TRIGGERED, + * pos 101-1xx. Reached only via ProceedGroupStateForApiTrigger(), never via + * the ordinary node_active() heartbeat path -- every row here requires a + * specific non-NONE .conditions.apiTrigger, which an ordinary heartbeat + * call's implicit API_FUNCTION_NONE can never match (MatchApiTrigger). See + * that function's own comment (just above this array) for the operator + * side's dispatch shape and why a no-match there is always ereport(ERROR), + * unlike the heartbeat side below. + * + * --- [MonitorFSM_EarlyChecksStart, MonitorFSM_FromContextStart): the six + * checks the real if-chain runs BEFORE the IsInPrimaryState(activeNode) + * early return (group_state_machine.c:284) -- DROPPED, goal==DROPPED, + * MAINTENANCE, the demote_timeout self-fence, and both "alone in group" + * rows. These fire regardless of whether activeNode is currently the + * primary (a primary that just lost its only standby must still reach + * SINGLE here, before ever redirecting into the + * ProceedGroupStateForPrimaryNode section) -- confirmed by the drop_node + * regression test, which failed the first time this table put the + * primary-state redirect ahead of these six checks instead of after them. + * None of these six rows reference .primaryNode at all, so they can be * matched against a NodeActiveContext built with primaryNode == NULL, before * primaryNode is even resolved. */ static const MonitorFSMTransition MonitorFSM[] = { + /* + * remove_node(), node_active_protocol.c:1163-1270 (RemoveNode) -- + * primary being removed: fan out report_lsn to every surviving, + * non-maintenance standby (extraAction, runs first), then mark the + * removed node itself dropped (activeNodeAssignedState, runs after -- + * matches the real source's own order). canTakeWrites, not + * isInPrimaryState: RemoveNode's own guard is CanTakeWritesInState( + * currentNode->goalState) with no requirement that reportedState has + * converged -- see canTakeWrites's own comment on NodeMatchesPattern + * above. Must come before the catchall row below: first-match-wins + * dispatch means whichever row is tried first wins when both could + * apply, and only a node that canTakeWrites should get the fan-out. + */ + { .pos = 101, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_REMOVE_NODE) }, + .activeNode = { .canTakeWrites = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_DROPPED), + .extraAction = ActionFanOutReportLsnOnPrimaryRemoval, + .comment = "remove_node, removed node can take writes -> dropped, " + "every surviving non-maintenance standby joins report_lsn" }, + + /* + * remove_node(), the removed node itself when it's NOT the primary -- + * unconditional at this point in the real source (the "already + * DROPPED" idempotency case returns earlier, before dispatch is ever + * called; see RemoveNode()'s own pre-checks, kept hand-written). Note + * this is doc-corrected from an earlier draft of this table, which + * modeled the fan-out row above and this row as two competing + * alternatives under first-match-wins -- that would have skipped + * assigning DROPPED to a removed *primary* entirely, since the row + * above would already have matched and stopped dispatch. The real + * source does both unconditionally in sequence (fan out, THEN mark + * dropped), not as alternatives -- reflected here by having the row + * above do both itself, and this row only needing to cover the + * non-primary case that never matched the row above at all. + */ + { .pos = 103, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_REMOVE_NODE) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_DROPPED), + .comment = "remove_node, removed node cannot take writes -> dropped" }, + + /* + * perform_failover(), 2-node group -- node_active_protocol.c:1456-1554. + * The SQL wrapper resolves the sole standby and validates + * candidatePriority != 0 and both nodes converged BEFORE dispatch (an + * operator-facing ereport(ERROR) on failure, not a table row -- see + * ProceedGroupStateForApiTrigger's own comment on pre/post side + * effects); by the time dispatch runs, activeNode is that standby. + */ + { .pos = 105, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_PERFORM_FAILOVER), + .groupHasExactlyTwoNodes = BOOL_TRUE }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DRAINING), + .comment = "manual failover, 2-node group, primary+standby both converged -> " + "standby prepare_promotion, primary draining" }, + + /* + * perform_failover(), >2-node group -- node_active_protocol.c:1555-1601. + * No standby is named at this point in the real source at all -- there's + * nothing else for activeNode to be here but the primary itself, the + * one node this specific call is fundamentally about. The + * candidatePriority trick and the ProceedGroupState(firstStandbyNode) + * continuation are POST side effects, hand-written in perform_failover() + * itself after this row's own DRAINING assignment has committed -- not + * modeled as a further declarative row or extraAction, since they're + * about biasing an election already left to the heartbeat-driven + * MS-failover cluster rows, not a goal-state assignment of their own. + */ + { .pos = 107, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_PERFORM_FAILOVER), + .groupHasMoreThanTwoNodes = BOOL_TRUE }, + .activeNode = { .isInPrimaryState = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_DRAINING), + .comment = "manual failover, >2-node group -> primary drains, election proceeds " + "via the heartbeat-driven MS-failover cluster rows" }, + + /* + * start_maintenance(), primary, 2-node group -- + * node_active_protocol.c:1901-1934. The WARNING about blocking writes, + * the candidatesCount<1 guard, and the already-in-maintenance + * idempotency check all stay hand-written pre-dispatch, exactly where + * they already are. The firstStandbyNode -> prepare_promotion + * assignment is a POST side effect, hand-written in start_maintenance() + * itself: unlike the >2-node row below, it has no ordering dependency on + * this row's own assignment (neither reads the other's freshly-committed + * state), so it doesn't need extraAction to sequence correctly. + */ + { .pos = 109, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), + .groupHasExactlyTwoNodes = BOOL_TRUE }, + .activeNode = { .isInPrimaryState = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_MAINTENANCE), + .comment = "start_maintenance, primary, 2-node group -> prepare_maintenance " + "(standby separately assigned prepare_promotion)" }, + + /* + * start_maintenance(), primary, >2-node group -- + * node_active_protocol.c:1936-1950. The ProceedGroupState( + * firstStandbyNode) continuation is a POST side effect, hand-written in + * start_maintenance() itself, called after this row's own + * prepare_maintenance assignment has committed (it re-fetches fresh + * state, so ordering matters here, unlike the 2-node row above). + */ + { .pos = 111, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), + .groupHasMoreThanTwoNodes = BOOL_TRUE }, + .activeNode = { .isInPrimaryState = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_MAINTENANCE), + .comment = "start_maintenance, primary, >2-node group -> prepare_maintenance, " + "election proceeds via the heartbeat-driven MS-failover cluster rows" }, + + /* + * start_maintenance(), secondary, last healthy sync standby -- + * node_active_protocol.c:1973-1986. lastHealthySyncStandbyGoingToMaintenance + * is computed by BuildApiTriggerNodeActiveContext (see its own comment) + * only for this apiFunction, mirroring the real source's own + * number_sync_standbys==0 && secondaryNodesCount==1 && + * IsHealthySyncStandby(currentNode) check verbatim. Must come before the + * ordinary-secondary row below: both match the same activeNode/ + * primaryNode state pattern, and only the more specific condition should + * win. + */ + { .pos = 113, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), + .lastHealthySyncStandbyGoingToMaintenance = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_REPORTED, + .reportedStates = STATES(REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP) } }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_MAINTENANCE), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "start_maintenance, secondary, last healthy sync standby -> " + "wait_maintenance, primary wait_primary (disables sync rep)" }, + + /* + * start_maintenance(), secondary, ordinary case -- + * node_active_protocol.c:1987-1996. + */ + { .pos = 115, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE) }, + .activeNode = { .statePattern = { .kind = NODE_STATE_REPORTED, + .reportedStates = STATES(REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP) } }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), + .comment = "start_maintenance, secondary, ordinary case -> maintenance" }, + + /* + * stop_maintenance(), no primary at all -- node_active_protocol.c: + * 2090-2102. totalNodesCount==1 (skip dispatch, direct + * ProceedGroupState(currentNode)) and primaryNode==NULL&&totalNodesCount + * ==2 (ereport(ERROR)) both stay hand-written pre-dispatch branches in + * stop_maintenance() itself -- by the time this row's own dispatch call + * runs, primaryNode==NULL only happens with totalNodesCount>2 (the real + * source's own condition, `(primaryNode == NULL || IsDemotedPrimary( + * primaryNode)) && totalNodesCount > 2`, but the >2 half of that + * disjunct is redundant here since the 2-node&&NULL case never reaches + * dispatch at all, per the guard above). + */ + { .pos = 117, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, + .primaryNode = { .exists = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "stop_maintenance, no primary -> report_lsn" }, + + /* + * stop_maintenance(), primary fully demoted -- node_active_protocol.c: + * 2103-2125 (both the >2-node and ==2-node demoted-primary branches: + * they assign the identical report_lsn outcome, differing only in log + * message text, so this one row covers both -- isDemotedPrimary alone, + * with no node-count condition, is exactly their shared real + * condition). + */ + { .pos = 119, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, + .primaryNode = { .isDemotedPrimary = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "stop_maintenance, primary demoted -> report_lsn" }, + + /* + * stop_maintenance(), failover in progress -- node_active_protocol.c: + * 2133-2142. The real source's own LogAndNotifyMessage text says + * "catchingup" here, but the actual SetNodeGoalState call assigns + * REPORT_LSN -- a real, pre-existing message/behavior mismatch in the + * source, not a modeling error in this table. + */ + { .pos = 121, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE), + .failoverInProgress = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "stop_maintenance, failover in progress -> report_lsn (source's own " + "log message says \"catchingup\" here, but the actual call assigns " + "REPORT_LSN -- see this row's own comment above)" }, + + /* + * stop_maintenance(), ordinary case -- node_active_protocol.c:2143-2152. + * Catchall: reached only once primaryNode exists, isn't demoted, and no + * failover is in progress -- exactly the real source's own final + * "else" branch. + */ + { .pos = 123, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "stop_maintenance, ordinary case -> catchingup" }, + + /* + * set_node_candidate_priority(), node_active_protocol.c:2282-2296. + * activeNode IS the primary here (mirrors BuildForPrimaryNodeNodeActiveContext's + * own "primaryNode IS activeNode" convention): the row's only real target + * is the primary's own apply_settings transition. The "primary is + * already apply_settings" ereport(ERROR) and the "no primary yet, just + * proceed" no-op both stay hand-written pre-dispatch in + * set_node_candidate_priority() itself, exactly where they already are + * -- this row is only reached once the wrapper has confirmed a primary + * exists and isn't already apply_settings. + */ + { .pos = 125, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY) }, + .activeNode = { .statePattern = { .kind = NODE_STATE_NOT_STABLE, + .reportedStates = STATES(REPLICATION_STATE_APPLY_SETTINGS) } }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), + .comment = "set_node_candidate_priority, primary not already apply_settings -> " + "apply_settings" }, + + /* + * set_node_replication_quorum(), node_active_protocol.c:2427-2441. Same + * shape as set_node_candidate_priority above. + */ + { .pos = 127, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_SET_NODE_REPLICATION_QUORUM) }, + .activeNode = { .statePattern = { .kind = NODE_STATE_NOT_STABLE, + .reportedStates = STATES(REPLICATION_STATE_APPLY_SETTINGS) } }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), + .comment = "set_node_replication_quorum, primary not already apply_settings -> " + "apply_settings" }, + + /* + * set_formation_number_sync_standbys(), formation_metadata.c:591-606,639. + * The "primary not in primary/wait_primary state" ereport(ERROR) stays + * hand-written pre-dispatch, exactly where it already is -- kept as this + * row's own condition too (belt and suspenders: dump_fsm() should show + * what's actually valid, and a future drift between the two fails loudly + * via this row's own no-match ERROR rather than silently). + */ + { .pos = 129, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS) }, + .activeNode = { .statePattern = { .kind = NODE_STATE_ASSIGNED, + .assignedStates = STATES(REPLICATION_STATE_PRIMARY, + REPLICATION_STATE_WAIT_PRIMARY) } }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), + .comment = "set_formation_number_sync_standbys, primary in primary/wait_primary " + "-> apply_settings" }, + /* converged to dropped -> remove the node from the catalog entirely */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DROPPED) }, + { .pos = 201, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DROPPED) }, .extraAction = ActionRemoveDroppedNode, .comment = "converged to dropped -> remove the node from the catalog" }, /* goal already dropped (mid-drop, row above hasn't converged yet) -> no-op */ - { .activeNode = { .statePattern = FSM_DROPPED_GOAL }, + { .pos = 203, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + .activeNode = { .statePattern = FSM_DROPPED_GOAL }, .comment = "goal already dropped -> no-op" }, /* converged to maintenance -> no-op, frozen until stop_maintenance() */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_MAINTENANCE) }, + { .pos = 205, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_MAINTENANCE) }, .comment = "converged to maintenance -> no-op, frozen until stop_maintenance()" }, /* demote_timeout self-fence re-target (issue #1025) */ - { .activeNode = { .statePattern = FSM_REPORTED_DEMOTE_TIMEOUT, + { .pos = 207, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + .activeNode = { .statePattern = FSM_REPORTED_DEMOTE_TIMEOUT, .unreachableFromDemoteTimeout = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), .comment = "reported demote_timeout, assigned goal can't reach it -> demoted" }, /* alone in group, candidate-eligible */ - { .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, + { .pos = 209, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, .candidateEligible = BOOL_TRUE }, .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SINGLE), .comment = "alone in group, candidate-eligible -> single" }, /* alone in group, not candidate-eligible */ - { .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, + { .pos = 211, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, .candidateEligible = BOOL_FALSE }, .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), @@ -1192,13 +1945,15 @@ static const MonitorFSMTransition MonitorFSM[] = { * NOT currently primary-role. */ /* converged secondary, reportedTLI not an ancestor of the group's reference timeline */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), + { .pos = 301, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isComparableToReferenceTli = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "converged secondary, reportedTLI not an ancestor of reference -> catchingup" }, /* replication stall (#997): primary healthy, no standby past replication_stall_timeout */ - { .primaryNode = { .isInPrimaryState = BOOL_TRUE, + { .pos = 303, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .primaryNode = { .isInPrimaryState = BOOL_TRUE, .isHealthy = BOOL_TRUE }, .conditions = { .replicationStallExceeded = BOOL_TRUE }, .otherNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), @@ -1207,46 +1962,53 @@ static const MonitorFSMTransition MonitorFSM[] = { /* nodesCount>2, primary unhealthy -- draining/maintenance/nothing decided inside the * action, then the MS-failover cascade unconditionally; must be a single row, see * ActionRunMultiStandbyFailoverCascade's comment for why. */ - { .primaryNode = { .isUnhealthy = BOOL_TRUE }, + { .pos = 305, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .primaryNode = { .isUnhealthy = BOOL_TRUE }, .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE }, .extraAction = ActionRunMultiStandbyFailoverCascade, .comment = "nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade" }, /* report_lsn, primary converged wait/join_primary, healthy */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + { .pos = 307, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY, .isHealthy = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), .comment = "report_lsn, primary converged wait/join_primary, healthy -> secondary" }, /* report_lsn, primary converged primary, healthy */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + { .pos = 309, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY), .isHealthy = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), .comment = "report_lsn, primary converged primary, healthy -> secondary" }, /* fast_forward done -> prepare_promotion */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_FAST_FORWARD) }, + { .pos = 311, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_FAST_FORWARD) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), .comment = "fast_forward done -> prepare_promotion" }, /* continue an already-started MS failover -- a direct `return` in the real source, and no * later row in this table matches activeNode in REPORT_LSN/FAST_FORWARD, so it doesn't * matter here whether the extraAction's bool stops dispatch or lets it keep scanning. */ - { .activeNode = { .statePattern = FSM_REPORT_LSN_OR_FAST_FORWARD }, + { .pos = 313, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_REPORT_LSN_OR_FAST_FORWARD }, .extraAction = ActionRunPlainMSFailoverCascade, .comment = "report_lsn or fast_forward, continuing an already-started failover -> " "MS-failover cascade" }, /* wait_standby, primary converged wait/join_primary */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY) }, + { .pos = 315, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY) }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "wait_standby, primary converged wait/join_primary -> catchingup" }, /* wait_standby (quorum member), primary converged primary */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + { .pos = 317, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), .replicationQuorum = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), @@ -1255,14 +2017,16 @@ static const MonitorFSMTransition MonitorFSM[] = { "catchingup + apply_settings" }, /* wait_standby (not a quorum member), primary converged primary */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + { .pos = 319, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), .replicationQuorum = BOOL_FALSE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "wait_standby (not a quorum member), primary converged primary -> catchingup" }, /* caught up, same TLI as primary, within sync threshold */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_CATCHINGUP), + { .pos = 321, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_CATCHINGUP), .isHealthy = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN }, .conditions = { .activeAndPrimaryTliMatch = BOOL_TRUE, @@ -1271,7 +2035,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "caught up, same TLI as primary, within sync threshold -> secondary" }, /* primary fails, already converged wait_primary (no draining edge, issue #1168) */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), + { .pos = 323, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isHealthy = BOOL_TRUE, .candidateEligible = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY), @@ -1282,7 +2047,8 @@ static const MonitorFSMTransition MonitorFSM[] = { "secondary -> prepare_promotion only (1 of 2)" }, /* primary fails, not already wait_primary */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), + { .pos = 325, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isHealthy = BOOL_TRUE, .candidateEligible = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, @@ -1295,25 +2061,29 @@ static const MonitorFSMTransition MonitorFSM[] = { "primary -> draining (2 of 2)" }, /* wait_maintenance, primary converged wait_primary */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, + { .pos = 327, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), .comment = "wait_maintenance, primary converged wait_primary -> maintenance" }, /* wait_maintenance, primary's goal no longer wait_primary */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, + { .pos = 329, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, .primaryNode = { .statePattern = FSM_NOT_ASSIGNED_WAIT_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), .comment = "wait_maintenance, primary's goal no longer wait_primary -> maintenance" }, /* prepare_promotion, primary converged prepare_maintenance */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + { .pos = 331, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), .comment = "prepare_promotion, primary converged prepare_maintenance -> stop_replication" }, /* Citus worker, primary present */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), + { .pos = 333, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), .isCitusWorkerGroup = BOOL_TRUE }, .primaryNode = { .exists = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), @@ -1321,14 +2091,16 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "Citus worker prepare_promotion, primary present -> wait_primary + demoted" }, /* Citus worker, primary removed */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), + { .pos = 335, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), .isCitusWorkerGroup = BOOL_TRUE }, .primaryNode = { .exists = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), .comment = "Citus worker prepare_promotion, primary removed -> wait_primary" }, /* prepare_promotion, primary present, already converged wait_primary (issue #1168) */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + { .pos = 337, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .exists = BOOL_TRUE, .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY), .isInMaintenance = BOOL_FALSE }, @@ -1336,188 +2108,1004 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "prepare_promotion, primary already converged wait_primary (issue #1168) -> " "stop_replication only (1 of 2)" }, - /* prepare_promotion, primary present, not in maintenance, not already wait_primary */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, - .primaryNode = { .exists = BOOL_TRUE, - .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, - .isInMaintenance = BOOL_FALSE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), - .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTE_TIMEOUT), - .comment = "prepare_promotion, primary present, not in maintenance -> " - "stop_replication + demote_timeout (2 of 2)" }, + /* prepare_promotion, primary present, not in maintenance, not already wait_primary */ + { .pos = 339, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + .primaryNode = { .exists = BOOL_TRUE, + .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, + .isInMaintenance = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTE_TIMEOUT), + .comment = "prepare_promotion, primary present, not in maintenance -> " + "stop_replication + demote_timeout (2 of 2)" }, + + /* prepare_promotion, primary removed */ + { .pos = 341, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, + .primaryNode = { .exists = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "prepare_promotion, primary removed -> wait_primary" }, + + /* stop_replication, primary converged prepare_maintenance */ + { .pos = 343, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), + .comment = "stop_replication, primary converged prepare_maintenance -> " + "wait_primary + maintenance" }, + + /* stop_replication, primary converged demote_timeout (3-way OR, 1 of 3) */ + { .pos = 345, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTE_TIMEOUT) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "stop_replication, primary converged demote_timeout -> " + "wait_primary + demoted (1 of 3)" }, + + /* stop_replication, primary's drain time expired (3-way OR, 2 of 3) */ + { .pos = 347, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .primaryNode = { .drainTimeExpired = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "stop_replication, primary's drain time expired -> " + "wait_primary + demoted (2 of 3)" }, + + /* stop_replication, primary's goal is wait_primary but presumed dead (3-way OR, 3 of 3) */ + { .pos = 349, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, + .conditions = { .primaryIsWaitPrimaryPresumedDead = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "stop_replication, primary's goal wait_primary but presumed dead -> " + "wait_primary + demoted (3 of 3)" }, + + /* Citus worker, primary present */ + { .pos = 351, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), + .isCitusWorkerGroup = BOOL_TRUE }, + .primaryNode = { .exists = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), + .comment = "Citus worker stop_replication, primary present -> wait_primary + demoted" }, + + /* Citus worker, primary removed */ + { .pos = 353, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), + .isCitusWorkerGroup = BOOL_TRUE }, + .primaryNode = { .exists = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "Citus worker stop_replication, primary removed -> wait_primary" }, + + /* demoted, primary reported wait/join_primary with goal primary */ + { .pos = 355, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, + .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY, + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "demoted, primary reported wait/join_primary with goal primary -> catchingup" }, + + /* demoted, primary converged wait/join_primary/primary, healthy */ + { .pos = 357, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, + .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN, + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), + .comment = "demoted, primary converged wait/join_primary/primary, healthy -> catchingup" }, + + /* join_secondary, primary reported wait_primary with goal wait/primary -- cascades into a + * nested pass on primaryNode */ + { .pos = 359, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, + .primaryNode = { .statePattern = FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .extraAction = ActionRunPrimaryNodeTransition, + .comment = "join_secondary, primary reported wait_primary with goal wait/primary -> " + "secondary" }, + + /* join_secondary, primary converged primary */ + { .pos = 361, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), + .comment = "join_secondary, primary converged primary -> secondary" }, + + /* + * --- [MonitorFSM_MSFailoverStart, MonitorFSM_PrimaryNodeSectionStart): + * the MS-failover / candidate-selection cluster's two genuinely + * declarative transitions (see the design doc's "The MS-failover / + * candidate-selection cluster" section, and TryMSFailoverDeclarativeRow's + * own comment below): BuildCandidateList/SelectFailoverCandidateNode/ + * PromoteSelectedNode themselves stay hand-written C, called from + * ProceedGroupStateForMSFailover exactly as before -- these two rows + * only cover the pair of assignments that were already expressible as + * plain per-node facts, gated by the exact same hand-written condition + * that already decided whether to make them (so a mismatch here can + * only widen the row's own no-op fallback, never change real behavior). + * Appended after the ordinary REPORTING_NODE rows rather than + * renumbered into them, so nothing else shifts. + */ + + /* MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, retry */ + { .pos = 363, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .conditions = { .activeNodeAllWalSourcesUnhealthy = BOOL_TRUE, + .guardDataLossEnabled = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES(REPLICATION_STATE_REPORT_LSN), + .assignedStates = STATES(REPLICATION_STATE_FAST_FORWARD) } }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, " + "guard_data_loss=true -> report_lsn (retry once a source recovers)" }, + + /* MS-failover: candidate ready to stream WAL -> follower joins as secondary */ + { .pos = 365, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .conditions = { .candidatePromotionInProgress = BOOL_TRUE }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .candidateNode = { .isReadyToStreamWAL = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_JOIN_SECONDARY), + .comment = "MS-failover: activeNode in report_lsn, failover candidate ready to stream " + "WAL -> join_secondary" }, + + /* + * MS-failover: BuildCandidateList's own fan-out (group_state_machine.c, + * "Nodes in SECONDARY or CATCHINGUP states are candidates due to report + * their LSN..."). Every AssignGoalState call in that loop is dispatched + * through TryFanOutReportLsnRow (see its own comment) against exactly + * these four rows -- one per distinct fromState shape the real if-chain + * checks, since no single NodeStatePattern covers all three disjuncts + * (SECONDARY/CATCHINGUP transitioning, MAINTENANCE->CATCHINGUP, + * DRAINING/DEMOTED stable or DEMOTED->CATCHINGUP transitioning) at once. + * The loop's own skip conditions (old/new primary unless draining or + * demoted, unhealthy-and-not-reporting) stay exactly where they are, + * hand-written, ahead of these rows -- by the time one of these four is + * reached, the loop has already established the node is a legitimate + * fan-out target; these rows only need to name which state it's coming + * from, not re-derive that eligibility. + * + * Each carries .conditions.inMSFailoverCluster = BOOL_TRUE: unlike pos + * 363/365, none of these four has an activeNode-state pattern narrow + * enough on its own to stay clear of ProceedGroupStateFromContext's own + * ordinary top-level lookup, which shares this same section's upper + * bound (MonitorFSM_PrimaryNodeSectionStart) and would otherwise match + * them against any ordinary, non-MS-failover heartbeat whose reported/ + * goal states happened to line up -- see inMSFailoverCluster's own + * comment on NodeActiveContext. + */ + { .pos = 367, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .conditions = { .inMSFailoverCluster = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES(REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP), + .assignedStates = STATES(REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP) } }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn " + "(1 of 4)" }, + + { .pos = 369, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .conditions = { .inMSFailoverCluster = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES(REPLICATION_STATE_MAINTENANCE), + .assignedStates = STATES(REPLICATION_STATE_CATCHINGUP) } }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "MS-failover fan-out: rejoining from maintenance -> report_lsn (2 of 4)" }, + + { .pos = 371, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .conditions = { .inMSFailoverCluster = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_DRAINING, + REPLICATION_STATE_DEMOTED) } }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "MS-failover fan-out: old primary converged draining or demoted -> " + "report_lsn (3 of 4)" }, + + { .pos = 373, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .conditions = { .inMSFailoverCluster = BOOL_TRUE }, + .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, + .reportedStates = STATES(REPLICATION_STATE_DEMOTED), + .assignedStates = STATES(REPLICATION_STATE_CATCHINGUP) } }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), + .comment = "MS-failover fan-out: old primary demoted, was rejoining a now-failed " + "primary -> report_lsn (4 of 4)" }, + + /* + * MS-failover: PromoteSelectedNode's own two outcomes + * (group_state_machine.c). ResolveAcceptedTimeline and the + * candidatePriority resets stay hand-written side effects, exactly + * where they are; only the final AssignGoalState is dispatched, via + * DispatchMonitorFSMRuleByPos (see its own comment) rather than a + * RuleMatches search -- both rows below share an identical condition + * set (activeNode in report_lsn, no promotion already in flight, the + * pool's most-advanced candidate within the promote threshold), so + * first-match-wins can never distinguish them on its own. The real + * choice -- selectedNode->reportedLSN == candidateList-> + * mostAdvancedReportedLSN, an internal LSN comparison no BoolPattern + * can express -- is made in PromoteSelectedNode itself, exactly as + * before; both rows exist so dump_fsm() shows both reachable outcomes, + * matching the design doc's own resolution of this exact ambiguity + * ("kept as 2 rows anyway, for dump_fsm() edge visibility... this pair + * genuinely isn't disambiguated by this table's own dispatch model"). + */ + { .pos = 375, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .candidatePromotionInProgress = BOOL_FALSE, + .mostAdvancedCandidateWithinPromoteThreshold = BOOL_TRUE }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), + .comment = "MS-failover: no promotion in flight, most-advanced candidate within " + "threshold, selected candidate already has all WAL -> prepare_promotion " + "(1 of 2 -- see this row's own comment on why both are listed)" }, + + { .pos = 377, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .candidatePromotionInProgress = BOOL_FALSE, + .mostAdvancedCandidateWithinPromoteThreshold = BOOL_TRUE }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_FAST_FORWARD), + .comment = "MS-failover: no promotion in flight, most-advanced candidate within " + "threshold, selected candidate is lagging -> fast_forward (2 of 2)" }, + + /* MS-failover: no promotion in flight, either not enough candidates have reported yet + * or the most-advanced one is still too far behind the primary to safely promote -- + * a pure no-op besides BuildCandidateList's own fan-out (the four rows above); listed + * for dump_fsm() completeness, never itself dispatched (no assignment to attribute). + * inMSFailoverCluster=BOOL_TRUE despite that: without it this row's bare + * candidatePromotionInProgress=BOOL_FALSE (true by default everywhere) plus its + * missing activeNode state pattern (matches ANY state) would let it swallow the + * ordinary top-level lookup's own "BUG: no MonitorFSM row matches" detection for + * any otherwise-unmatched heartbeat, silently no-op'ing instead of erroring. */ + { .pos = 379, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .candidatePromotionInProgress = BOOL_FALSE }, + .comment = "MS-failover: no promotion in flight, not enough (or not safe enough) " + "candidates yet -> no-op besides the fan-out above" }, + + /* --- [MonitorFSM_PrimaryNodeSectionStart, MonitorFSM_SIZE): the declarative replacement + * for ProceedGroupStateForPrimaryNode()'s own sequential if-chain. Here .activeNode maps + * to the primaryNode parameter, not a reporting node -- reached either directly by the + * top-level driver (activeNode already primary-role) or via ActionRunPrimaryNodeTransition's + * nested pass on primaryNode (the join_secondary cascade row above). */ + + /* primary alone, another node reached wait_standby */ + { .pos = 401, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SINGLE) }, + .conditions = { .anyOtherNodeWaitingStandby = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .comment = "primary alone, another node reached wait_standby -> wait_primary" }, + + /* all nodes async, zero secondaries */ + { .pos = 403, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, + .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, + .secondaryNodesCountIsZero = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "all nodes async, zero secondaries -> wait_primary" }, + + /* all nodes async, >=1 secondary */ + { .pos = 405, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, + .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, + .secondaryNodesCountIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "all nodes async, >=1 secondary -> primary" }, + + /* converged primary/apply_settings (not wait_primary), no quorum secondaries, + * number_sync_standbys=0, no failover in progress (issue #774) */ + { .pos = 407, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, + .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, + .failoverInProgress = BOOL_FALSE, + .numberSyncStandbysIsZero = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "converged primary/apply_settings, no quorum secondaries, no failover in " + "progress, number_sync_standbys=0 -> wait_primary" }, + + /* same, but number_sync_standbys>0 -> block writes on primary */ + { .pos = 409, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, + .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, + .failoverInProgress = BOOL_FALSE, + .numberSyncStandbysIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "converged primary/apply_settings, no quorum secondaries, no failover in " + "progress, number_sync_standbys>0 -> primary (block writes)" }, + + /* wait_primary, >=1 quorum secondary */ + { .pos = 411, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, + .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "wait_primary, >=1 quorum secondary -> primary" }, + + /* apply_settings, both zero */ + { .pos = 413, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, + .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, + .secondaryQuorumNodesCountIsZero = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "apply_settings, both zero -> wait_primary" }, + + /* apply_settings, number_sync_standbys != 0 (1 of 2 disjuncts) */ + { .pos = 415, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, + .conditions = { .numberSyncStandbysIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts)" }, + + /* apply_settings, sync_standbys=0 but >=1 quorum secondary (2 of 2) */ + { .pos = 417, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, + .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, + .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2)" }, + + /* converged primary/wait_primary/apply_settings, no other condition applies */ + { .pos = 419, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, + .extraAction = ActionCatchupUnhealthySecondaries, + .comment = "converged primary/wait_primary/apply_settings, no other condition applies -> " + "no-op besides the unhealthy-secondary fan-out" }, + + /* backwards-compat: join_primary -> primary */ + { .pos = 421, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), + .comment = "backwards-compat: join_primary -> primary" }, +}; + +/* + * AssertMonitorFSMWellFormed cross-checks the five MonitorFSM_* index + * constants above against what the rows themselves declare via .pos/ + * .section, so a boundary that's drifted out of sync with an added, + * removed, or reordered row is caught here -- loudly, at first use -- + * instead of silently, as a row from the wrong section matching + * unexpectedly or a bounded search that scans zero rows and never matches. + * A no-op build (USE_ASSERT_CHECKING off) skips this entirely, matching + * every other structural check in this file (see AssignDeclaredGoalState). + * + * .pos is NOT the array index: each section's rows are numbered starting + * at *01 within its own hundred-block (101/201/301/401, matching how many + * MONITOR_FSM_SECTION_* values exist -- *01 rather than *00 so the count + * within the block reads as ordinary 1-based), stepping by 2 within the + * section rather than by 1 -- so a new row can be inserted between two + * existing ones (e.g. 202 between 201 and 203) without renumbering + * anything else in the file. What's checked here is weaker than "exactly + * i*2 + sectionStart + 1" as a result: strictly increasing, and within the + * section's own hundred-block -- consistent with rows being added over + * time at whatever free position is nearest where they belong, not at a + * specific computed slot. + */ +static void +AssertMonitorFSMWellFormed(void) +{ +#ifdef USE_ASSERT_CHECKING + int previousPos = 0; + + for (int i = 0; i < MonitorFSM_SIZE; i++) + { + Assert(MonitorFSM[i].pos > previousPos); + previousPos = MonitorFSM[i].pos; + } + + for (int i = 0; i < MonitorFSM_EarlyChecksStart; i++) + { + Assert(MonitorFSM[i].section == MONITOR_FSM_SECTION_API_TRIGGERED); + Assert(MonitorFSM[i].pos >= 100 && MonitorFSM[i].pos < 200); + } + + for (int i = MonitorFSM_EarlyChecksStart; i < MonitorFSM_FromContextStart; i++) + { + Assert(MonitorFSM[i].section == MONITOR_FSM_SECTION_EARLY_CHECKS); + Assert(MonitorFSM[i].pos >= 200 && MonitorFSM[i].pos < 300); + } + + for (int i = MonitorFSM_FromContextStart; i < MonitorFSM_PrimaryNodeSectionStart; i++) + { + Assert(MonitorFSM[i].section == MONITOR_FSM_SECTION_REPORTING_NODE); + Assert(MonitorFSM[i].pos >= 300 && MonitorFSM[i].pos < 400); + } + + for (int i = MonitorFSM_PrimaryNodeSectionStart; i < MonitorFSM_SIZE; i++) + { + Assert(MonitorFSM[i].section == MONITOR_FSM_SECTION_PRIMARY_NODE); + Assert(MonitorFSM[i].pos >= 400 && MonitorFSM[i].pos < 500); + } + + Assert(MonitorFSM_FromContextResumeStart > MonitorFSM_FromContextStart); + Assert(MonitorFSM_FromContextResumeStart < MonitorFSM_PrimaryNodeSectionStart); +#endif +} + + +/* + * MonitorFSMSectionGetName returns the (enum) name of a MonitorFSMSection -- + * the SQL-facing spelling used by pgautofailover.fsm_section, dump_fsm(), + * and the rule_section column on pgautofailover.event. Mirrors + * ReplicationStateGetName's role for ReplicationState exactly. + */ +const char * +MonitorFSMSectionGetName(MonitorFSMSection section) +{ + switch (section) + { + case MONITOR_FSM_SECTION_API_TRIGGERED: + { + return "api_triggered"; + } + + case MONITOR_FSM_SECTION_EARLY_CHECKS: + { + return "early_checks"; + } + + case MONITOR_FSM_SECTION_REPORTING_NODE: + { + return "reporting_node"; + } + + case MONITOR_FSM_SECTION_PRIMARY_NODE: + { + return "primary_node"; + } + + default: + { + ereport(ERROR, + (errmsg("bug: unknown MonitorFSMSection (%d)", section))); + } + } +} + + +/* + * MonitorApiFunctionGetName returns the (enum) name of a MonitorApiFunction -- + * the dump_fsm()/pgautofailover.fsm section column's own spelling (appended + * after the section name, see MonitorFSMTransitionSectionText) for which + * operator-triggered SQL entry point a MONITOR_FSM_SECTION_API_TRIGGERED + * row's .conditions.apiTrigger requires. API_FUNCTION_NONE has no row that + * matches on it specifically (it's the ordinary heartbeat path's implicit + * value, never written as an explicit API_TRIGGER() in any row), but is + * still given a name here rather than treated as an error, matching + * MonitorFSMSectionGetName's own "every enumerator has a name" discipline. + */ +static const char * +MonitorApiFunctionGetName(MonitorApiFunction apiFunction) +{ + switch (apiFunction) + { + case API_FUNCTION_NONE: + { + return "node_active"; + } + + case API_FUNCTION_REMOVE_NODE: + { + return "remove_node"; + } + + case API_FUNCTION_PERFORM_FAILOVER: + { + return "perform_failover"; + } + + case API_FUNCTION_START_MAINTENANCE: + { + return "start_maintenance"; + } + + case API_FUNCTION_STOP_MAINTENANCE: + { + return "stop_maintenance"; + } + + case API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY: + { + return "set_node_candidate_priority"; + } + + case API_FUNCTION_SET_NODE_REPLICATION_QUORUM: + { + return "set_node_replication_quorum"; + } + + case API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS: + { + return "set_formation_number_sync_standbys"; + } + + default: + { + ereport(ERROR, + (errmsg("bug: unknown MonitorApiFunction (%d)", apiFunction))); + } + } +} + + +/* + * MonitorFSMTransitionSectionText renders a row's section as the + * dump_fsm()/pgautofailover.fsm section column: plain "reporting_node"-style + * text for most rows, but "api_triggered: remove_node"-style text (section + * name, ": ", API function name) whenever the row's own + * .conditions.apiTrigger requires a specific operator-triggered entry point + * (API_TRIGGER_SPECIFIC -- true for every row in + * MONITOR_FSM_SECTION_API_TRIGGERED, and only those rows) -- one column + * instead of a separate, mostly-NULL api_function column, since the two + * facts are never independently meaningful: a row either has both a section + * and no specific trigger, or both a section and exactly one trigger. + */ +static Datum +MonitorFSMTransitionSectionText(const MonitorFSMTransition *rule) +{ + const char *sectionName = MonitorFSMSectionGetName(rule->section); + + if (rule->conditions.apiTrigger.kind == API_TRIGGER_SPECIFIC) + { + StringInfoData buf; + + initStringInfo(&buf); + appendStringInfo(&buf, "%s: %s", sectionName, + MonitorApiFunctionGetName(rule->conditions.apiTrigger.function)); + + return CStringGetTextDatum(buf.data); + } + + return CStringGetTextDatum(sectionName); +} + + +/* + * MonitorFSMSectionTypeOid returns the OID of the pgautofailover.fsm_section + * type. Mirrors ReplicationStateTypeOid exactly, see its own comment for why + * the String/Value split exists. + */ +Oid +MonitorFSMSectionTypeOid(void) +{ +#if (PG_VERSION_NUM >= 150000) + String *schemaName = makeString(AUTO_FAILOVER_SCHEMA_NAME); + String *typeName = makeString(FSM_SECTION_TYPE_NAME); +#else + Value *schemaName = makeString(AUTO_FAILOVER_SCHEMA_NAME); + Value *typeName = makeString(FSM_SECTION_TYPE_NAME); +#endif + + List *enumTypeNameList = list_make2(schemaName, typeName); + TypeName *enumTypeName = makeTypeNameFromNameList(enumTypeNameList); + Oid enumTypeOid = typenameTypeId(NULL, enumTypeName); + + return enumTypeOid; +} + + +/* + * MonitorFSMSectionGetEnum returns the SQL enum OID for a given + * MonitorFSMSection value. Mirrors ReplicationStateGetEnum exactly. + */ +Oid +MonitorFSMSectionGetEnum(MonitorFSMSection section) +{ + const char *enumName = MonitorFSMSectionGetName(section); + Oid enumTypeOid = MonitorFSMSectionTypeOid(); + + HeapTuple enumTuple = SearchSysCache2(ENUMTYPOIDNAME, + ObjectIdGetDatum(enumTypeOid), + CStringGetDatum(enumName)); + if (!HeapTupleIsValid(enumTuple)) + { + ereport(ERROR, (errmsg("invalid value for enum: %d", section))); + } + + Oid sectionOid = HeapTupleGetOid(enumTuple); + + ReleaseSysCache(enumTuple); + + return sectionOid; +} + + +/* + * EnumGetMonitorFSMSection returns the internal value of a fsm_section enum. + * Mirrors EnumGetReplicationState exactly. + */ +MonitorFSMSection +EnumGetMonitorFSMSection(Oid monitorFSMSectionOid) +{ + HeapTuple enumTuple = SearchSysCache1(ENUMOID, + ObjectIdGetDatum(monitorFSMSectionOid)); + if (!HeapTupleIsValid(enumTuple)) + { + ereport(ERROR, (errmsg("invalid input value for enum: %u", + monitorFSMSectionOid))); + } + + Form_pg_enum enumForm = (Form_pg_enum) GETSTRUCT(enumTuple); + char *enumName = NameStr(enumForm->enumlabel); + MonitorFSMSection section; + + if (strncmp(enumName, "api_triggered", NAMEDATALEN) == 0) + { + section = MONITOR_FSM_SECTION_API_TRIGGERED; + } + else if (strncmp(enumName, "early_checks", NAMEDATALEN) == 0) + { + section = MONITOR_FSM_SECTION_EARLY_CHECKS; + } + else if (strncmp(enumName, "reporting_node", NAMEDATALEN) == 0) + { + section = MONITOR_FSM_SECTION_REPORTING_NODE; + } + else if (strncmp(enumName, "primary_node", NAMEDATALEN) == 0) + { + section = MONITOR_FSM_SECTION_PRIMARY_NODE; + } + else + { + ReleaseSysCache(enumTuple); + ereport(ERROR, (errmsg("bug: unknown fsm_section enum label \"%s\"", enumName))); + } + + ReleaseSysCache(enumTuple); + + return section; +} + + +/* + * NodeStatePatternReportedStatesText renders a NodeStatePattern's own + * reportedStates set as a human- and script-readable comma-separated list + * of pgautofailover.replication_state labels -- the "current (reported) + * state" a row requires of the node this pattern is attached to, for a + * keeper-side cross-check to compare against KeeperFSMTransition.current + * (see fsm.h: current/assigned/pgKind is exactly the (fromState, toState) + * shape this exists to expose the "from" half of). + * + * Only NODE_STATE_STABLE/REPORTED/TRANSITIONING pin down a genuine "current + * state must be one of these" set -- returns NULL for NODE_STATE_ANY (no + * constraint at all) and for NODE_STATE_NOT_STABLE/NOT_ASSIGNED (an + * exclusion, not a "from" set: rendering reportedStates there would read as + * the opposite of what the row actually requires). + */ +static Datum +NodeStatePatternReportedStatesText(const NodeStatePattern *pattern, bool *isNull) +{ + if (pattern->kind != NODE_STATE_STABLE && + pattern->kind != NODE_STATE_REPORTED && + pattern->kind != NODE_STATE_TRANSITIONING) + { + *isNull = true; + return (Datum) 0; + } + + StringInfoData buf; + + initStringInfo(&buf); + + for (int i = 0; i < pattern->reportedStates.count; i++) + { + if (i > 0) + { + appendStringInfoString(&buf, ", "); + } + + appendStringInfoString(&buf, + ReplicationStateGetName(pattern->reportedStates.states[i])); + } + + *isNull = false; + return CStringGetTextDatum(buf.data); +} + + +/* Appends "name=true"/"name=false" to buf (with a ", " separator when buf + * isn't empty), skipping BOOL_ANY entirely -- BOOL_ANY means this row + * doesn't care about the fact at all, so it's noise, not a guard. Shared by + * NodeStatusPatternConditionsText and NodeActiveContextPatternConditionsText + * below, one line per BoolPattern field rather than a loop: there's no + * runtime array of (name, value) pairs to iterate, since the field names + * only exist at compile time. */ +#define APPEND_BOOL_CONDITION(buf, name, boolPattern) \ + do { \ + if ((boolPattern) != BOOL_ANY) \ + { \ + if ((buf)->len > 0) \ + { \ + appendStringInfoString((buf), ", "); \ + } \ + appendStringInfo((buf), "%s=%s", (name), \ + (boolPattern) == BOOL_TRUE ? "true" : "false"); \ + } \ + } while (0) + +/* + * AppendNodeStateGoalCondition appends a role's own goal-state precondition + * to buf, when its statePattern is NODE_STATE_ASSIGNED/NOT_ASSIGNED: "the + * node's EXISTING goal (before this dispatch runs) must/must not already be + * one of these" -- e.g. "goal=dropped" (pos 203, FSM_DROPPED_GOAL: goal + * already DROPPED, reported state irrelevant) or "goal!=wait_primary" (the + * wait_maintenance/primary's-goal-no-longer-wait_primary row). + * + * This is a genuine match condition, same as any BoolPattern field, but it + * lives on .statePattern (a NodeStatePattern), not .conditions (a + * NodeStatusPattern's BoolPattern fields) -- without this, a row like pos + * 203 renders with every single column empty (no current_state, since + * NodeStatePatternReportedStatesText only renders STABLE/REPORTED/ + * TRANSITIONING; no conditions, since ASSIGNED/NOT_ASSIGNED isn't a + * BoolPattern; no assigned_state, since the row is a no-op), making it look + * unconditional when it very much isn't. Folded into the same + * *_conditions column as the BoolPattern fields (not the *_current_state + * column, which is reserved for a strict "current reported state" fromState + * a keeper-side cross-check can compare against KeeperFSMTransition.current) + * since both are "things that must be true about this node" from a reader's + * point of view. + */ +static void +AppendNodeStateGoalCondition(StringInfoData *buf, const NodeStatePattern *pattern) +{ + if (pattern->kind != NODE_STATE_ASSIGNED && pattern->kind != NODE_STATE_NOT_ASSIGNED) + { + return; + } + + if (buf->len > 0) + { + appendStringInfoString(buf, ", "); + } + + appendStringInfoString(buf, pattern->kind == NODE_STATE_ASSIGNED ? "goal=" : "goal!="); + + for (int i = 0; i < pattern->assignedStates.count; i++) + { + if (i > 0) + { + appendStringInfoString(buf, "|"); + } + + appendStringInfoString(buf, ReplicationStateGetName(pattern->assignedStates.states[i])); + } +} + + +/* + * NodeStatusPatternConditionsText renders every non-BOOL_ANY BoolPattern + * field of a NodeStatusPattern (activeNode/primaryNode/candidateNode's own + * per-node guards -- isHealthy foremost among them, per the reason this + * column exists at all) as a compact "name=value, name=value" list, plus + * this role's own goal-state precondition if it has one (see + * AppendNodeStateGoalCondition's own comment). NULL when the row places no + * per-node guard on this role at all -- the common case for rows that only + * match on reported state. + */ +static Datum +NodeStatusPatternConditionsText(const NodeStatusPattern *pattern, bool *isNull) +{ + StringInfoData buf; + + initStringInfo(&buf); + + AppendNodeStateGoalCondition(&buf, &pattern->statePattern); + + APPEND_BOOL_CONDITION(&buf, "exists", pattern->exists); + APPEND_BOOL_CONDITION(&buf, "isHealthy", pattern->isHealthy); + APPEND_BOOL_CONDITION(&buf, "isUnhealthy", pattern->isUnhealthy); + APPEND_BOOL_CONDITION(&buf, "candidateEligible", pattern->candidateEligible); + APPEND_BOOL_CONDITION(&buf, "isInPrimaryState", pattern->isInPrimaryState); + APPEND_BOOL_CONDITION(&buf, "isInMaintenance", pattern->isInMaintenance); + APPEND_BOOL_CONDITION(&buf, "isDemotedPrimary", pattern->isDemotedPrimary); + APPEND_BOOL_CONDITION(&buf, "canTakeWrites", pattern->canTakeWrites); + APPEND_BOOL_CONDITION(&buf, "isReadyToStreamWAL", pattern->isReadyToStreamWAL); + APPEND_BOOL_CONDITION(&buf, "drainTimeExpired", pattern->drainTimeExpired); + APPEND_BOOL_CONDITION(&buf, "isCitusWorkerGroup", pattern->isCitusWorkerGroup); + APPEND_BOOL_CONDITION(&buf, "replicationQuorum", pattern->replicationQuorum); + APPEND_BOOL_CONDITION(&buf, "isComparableToReferenceTli", pattern->isComparableToReferenceTli); + APPEND_BOOL_CONDITION(&buf, "unreachableFromDemoteTimeout", + pattern->unreachableFromDemoteTimeout); + + if (buf.len == 0) + { + *isNull = true; + return (Datum) 0; + } + + *isNull = false; + return CStringGetTextDatum(buf.data); +} - /* prepare_promotion, primary removed */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, - .primaryNode = { .exists = BOOL_FALSE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .comment = "prepare_promotion, primary removed -> wait_primary" }, - /* stop_replication, primary converged prepare_maintenance */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, - .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .otherNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), - .comment = "stop_replication, primary converged prepare_maintenance -> " - "wait_primary + maintenance" }, +/* + * NodeActiveContextPatternConditionsText renders every non-BOOL_ANY + * BoolPattern field of a row's .conditions (NodeActiveContextPattern -- + * group-level guards: counts, WAL thresholds, guard_data_loss, the + * MS-failover cluster's own facts) the same way. .apiTrigger is + * deliberately skipped: which operator function triggers a row is already + * implied by its section (api_triggered) and named in its own comment, so + * repeating it here would be noise, not a guard a reader doesn't already + * have. + */ +static Datum +NodeActiveContextPatternConditionsText(const NodeActiveContextPattern *cond, bool *isNull) +{ + StringInfoData buf; + + initStringInfo(&buf); + + APPEND_BOOL_CONDITION(&buf, "groupHasExactlyOneNode", cond->groupHasExactlyOneNode); + APPEND_BOOL_CONDITION(&buf, "groupHasExactlyTwoNodes", cond->groupHasExactlyTwoNodes); + APPEND_BOOL_CONDITION(&buf, "groupHasMoreThanTwoNodes", cond->groupHasMoreThanTwoNodes); + APPEND_BOOL_CONDITION(&buf, "anyOtherNodeWaitingStandby", cond->anyOtherNodeWaitingStandby); + APPEND_BOOL_CONDITION(&buf, "numberSyncStandbysIsZero", cond->numberSyncStandbysIsZero); + APPEND_BOOL_CONDITION(&buf, "replicationQuorumCountIsZero", + cond->replicationQuorumCountIsZero); + APPEND_BOOL_CONDITION(&buf, "secondaryNodesCountIsZero", cond->secondaryNodesCountIsZero); + APPEND_BOOL_CONDITION(&buf, "secondaryQuorumNodesCountIsZero", + cond->secondaryQuorumNodesCountIsZero); + APPEND_BOOL_CONDITION(&buf, "atLeastOneHealthyCandidate", cond->atLeastOneHealthyCandidate); + APPEND_BOOL_CONDITION(&buf, "walWithinPromoteThreshold", cond->walWithinPromoteThreshold); + APPEND_BOOL_CONDITION(&buf, "walWithinSyncThreshold", cond->walWithinSyncThreshold); + APPEND_BOOL_CONDITION(&buf, "activeAndPrimaryTliMatch", cond->activeAndPrimaryTliMatch); + APPEND_BOOL_CONDITION(&buf, "primaryIsWaitPrimaryPresumedDead", + cond->primaryIsWaitPrimaryPresumedDead); + APPEND_BOOL_CONDITION(&buf, "failoverInProgress", cond->failoverInProgress); + APPEND_BOOL_CONDITION(&buf, "replicationStallExceeded", cond->replicationStallExceeded); + APPEND_BOOL_CONDITION(&buf, "lastHealthySyncStandbyGoingToMaintenance", + cond->lastHealthySyncStandbyGoingToMaintenance); + APPEND_BOOL_CONDITION(&buf, "activeNodeAllWalSourcesUnhealthy", + cond->activeNodeAllWalSourcesUnhealthy); + APPEND_BOOL_CONDITION(&buf, "candidatePromotionInProgress", + cond->candidatePromotionInProgress); + APPEND_BOOL_CONDITION(&buf, "mostAdvancedCandidateWithinPromoteThreshold", + cond->mostAdvancedCandidateWithinPromoteThreshold); + APPEND_BOOL_CONDITION(&buf, "guardDataLossEnabled", cond->guardDataLossEnabled); + APPEND_BOOL_CONDITION(&buf, "inMSFailoverCluster", cond->inMSFailoverCluster); + + if (buf.len == 0) + { + *isNull = true; + return (Datum) 0; + } - /* stop_replication, primary converged demote_timeout (3-way OR, 1 of 3) */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, - .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTE_TIMEOUT) }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), - .comment = "stop_replication, primary converged demote_timeout -> " - "wait_primary + demoted (1 of 3)" }, + *isNull = false; + return CStringGetTextDatum(buf.data); +} - /* stop_replication, primary's drain time expired (3-way OR, 2 of 3) */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, - .primaryNode = { .drainTimeExpired = BOOL_TRUE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), - .comment = "stop_replication, primary's drain time expired -> " - "wait_primary + demoted (2 of 3)" }, - /* stop_replication, primary's goal is wait_primary but presumed dead (3-way OR, 3 of 3) */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, - .conditions = { .primaryIsWaitPrimaryPresumedDead = BOOL_TRUE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), - .comment = "stop_replication, primary's goal wait_primary but presumed dead -> " - "wait_primary + demoted (3 of 3)" }, +PG_FUNCTION_INFO_V1(dump_fsm); - /* Citus worker, primary present */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), - .isCitusWorkerGroup = BOOL_TRUE }, - .primaryNode = { .exists = BOOL_TRUE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), - .comment = "Citus worker stop_replication, primary present -> wait_primary + demoted" }, +/* + * dump_fsm exposes MonitorFSM[] to SQL, one row per rule, in table order + * (first-match-wins order -- the same order RuleMatches() itself scans in). + * This is the cross-check surface the design doc's dump_fsm()/ + * check_fsm_reachability() proposal calls for: enough to correlate a + * pgautofailover.event row's rule_pos/rule_section (see notifications.h) + * back to the exact rule that produced it, and enough for an operator or a + * future keeper-side reachability check to see every transition the + * monitor's table can produce, without reading the C source. The + * active_node_current_state/other_node_current_state/candidate_node_current_state + * columns (rendered via NodeStatePatternReportedStatesText, see its own + * comment) plus the assigned-state columns already here give a keeper- + * cross-check tool the full (fromState, toState) shape it needs to compare + * against KeeperFSMTransition. The *_conditions columns (rendered via + * NodeStatusPatternConditionsText/NodeActiveContextPatternConditionsText, + * see their own comments) additionally surface every non-default BoolPattern + * guard on the row -- health foremost among them, but also maintenance, + * candidate eligibility, WAL thresholds, guard_data_loss, and the rest -- + * so a reader (or a future automated check) doesn't have to open the C + * source to see what else has to be true for a given row to fire. The + * section column (rendered via MonitorFSMTransitionSectionText) also names + * which operator-triggered SQL entry point a MONITOR_FSM_SECTION_API_TRIGGERED + * row requires, e.g. "api_triggered: remove_node" -- just the plain section + * name for every other row, since only that section's rows carry a specific + * .conditions.apiTrigger. + */ +Datum +dump_fsm(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - /* Citus worker, primary removed */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), - .isCitusWorkerGroup = BOOL_TRUE }, - .primaryNode = { .exists = BOOL_FALSE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .comment = "Citus worker stop_replication, primary removed -> wait_primary" }, + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that " + "cannot accept a set"))); + } - /* demoted, primary reported wait/join_primary with goal primary */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, - .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY, - .isHealthy = BOOL_TRUE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), - .comment = "demoted, primary reported wait/join_primary with goal primary -> catchingup" }, + if (!(rsinfo->allowedModes & SFRM_Materialize)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not " + "allowed in this context"))); + } - /* demoted, primary converged wait/join_primary/primary, healthy */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, - .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN, - .isHealthy = BOOL_TRUE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), - .comment = "demoted, primary converged wait/join_primary/primary, healthy -> catchingup" }, + TupleDesc tupdesc; - /* join_secondary, primary reported wait_primary with goal wait/primary -- cascades into a - * nested pass on primaryNode */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, - .primaryNode = { .statePattern = FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), - .extraAction = ActionRunPrimaryNodeTransition, - .comment = "join_secondary, primary reported wait_primary with goal wait/primary -> " - "secondary" }, + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + { + ereport(ERROR, + (errmsg("function returning record called in context " + "that cannot accept type record"))); + } - /* join_secondary, primary converged primary */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, - .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), - .comment = "join_secondary, primary converged primary -> secondary" }, + MemoryContext perQueryContext = rsinfo->econtext->ecxt_per_query_memory; + MemoryContext oldContext = MemoryContextSwitchTo(perQueryContext); - /* --- [MonitorFSM_PrimaryNodeSectionStart, MonitorFSM_SIZE): the declarative replacement - * for ProceedGroupStateForPrimaryNode()'s own sequential if-chain. Here .activeNode maps - * to the primaryNode parameter, not a reporting node -- reached either directly by the - * top-level driver (activeNode already primary-role) or via ActionRunPrimaryNodeTransition's - * nested pass on primaryNode (the join_secondary cascade row above). */ + Tuplestorestate *tupstore = tuplestore_begin_heap(true, false, work_mem); - /* primary alone, another node reached wait_standby */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SINGLE) }, - .conditions = { .anyOtherNodeWaitingStandby = BOOL_TRUE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .comment = "primary alone, another node reached wait_standby -> wait_primary" }, + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; - /* all nodes async, zero secondaries */ - { .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, - .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, - .secondaryNodesCountIsZero = BOOL_TRUE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "all nodes async, zero secondaries -> wait_primary" }, + MemoryContextSwitchTo(oldContext); - /* all nodes async, >=1 secondary */ - { .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, - .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, - .secondaryNodesCountIsZero = BOOL_FALSE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "all nodes async, >=1 secondary -> primary" }, + for (int i = 0; i < MonitorFSM_SIZE; i++) + { + const MonitorFSMTransition *rule = &MonitorFSM[i]; + Datum values[13]; + bool isNull[13] = { false }; - /* converged primary/apply_settings (not wait_primary), no quorum secondaries, - * number_sync_standbys=0, no failover in progress (issue #774) */ - { .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, - .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, - .failoverInProgress = BOOL_FALSE, - .numberSyncStandbysIsZero = BOOL_TRUE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "converged primary/apply_settings, no quorum secondaries, no failover in " - "progress, number_sync_standbys=0 -> wait_primary" }, + values[0] = Int32GetDatum(rule->pos); + values[1] = MonitorFSMTransitionSectionText(rule); - /* same, but number_sync_standbys>0 -> block writes on primary */ - { .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, - .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, - .failoverInProgress = BOOL_FALSE, - .numberSyncStandbysIsZero = BOOL_FALSE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "converged primary/apply_settings, no quorum secondaries, no failover in " - "progress, number_sync_standbys>0 -> primary (block writes)" }, + if (rule->comment != NULL) + { + values[2] = CStringGetTextDatum(rule->comment); + } + else + { + isNull[2] = true; + } - /* wait_primary, >=1 quorum secondary */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, - .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "wait_primary, >=1 quorum secondary -> primary" }, + values[3] = NodeStatePatternReportedStatesText(&rule->activeNode.statePattern, + &isNull[3]); + values[4] = NodeStatePatternReportedStatesText(&rule->primaryNode.statePattern, + &isNull[4]); + values[5] = NodeStatePatternReportedStatesText(&rule->candidateNode.statePattern, + &isNull[5]); - /* apply_settings, both zero */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, - .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, - .secondaryQuorumNodesCountIsZero = BOOL_TRUE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "apply_settings, both zero -> wait_primary" }, + values[6] = NodeStatusPatternConditionsText(&rule->activeNode, &isNull[6]); + values[7] = NodeStatusPatternConditionsText(&rule->primaryNode, &isNull[7]); + values[8] = NodeStatusPatternConditionsText(&rule->candidateNode, &isNull[8]); + values[9] = NodeActiveContextPatternConditionsText(&rule->conditions, &isNull[9]); - /* apply_settings, number_sync_standbys != 0 (1 of 2 disjuncts) */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, - .conditions = { .numberSyncStandbysIsZero = BOOL_FALSE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts)" }, + if (rule->activeNodeAssignedState.kind == GOAL_STATE_SET) + { + values[10] = ObjectIdGetDatum( + ReplicationStateGetEnum(rule->activeNodeAssignedState.state)); + } + else + { + isNull[10] = true; + } - /* apply_settings, sync_standbys=0 but >=1 quorum secondary (2 of 2) */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, - .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, - .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2)" }, + if (rule->otherNodeAssignedState.kind == GOAL_STATE_SET) + { + values[11] = ObjectIdGetDatum( + ReplicationStateGetEnum(rule->otherNodeAssignedState.state)); + } + else + { + isNull[11] = true; + } - /* converged primary/wait_primary/apply_settings, no other condition applies */ - { .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, - .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "converged primary/wait_primary/apply_settings, no other condition applies -> " - "no-op besides the unhealthy-secondary fan-out" }, + values[12] = BoolGetDatum(rule->extraAction != NULL); - /* backwards-compat: join_primary -> primary */ - { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_PRIMARY) }, - .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), - .comment = "backwards-compat: join_primary -> primary" }, -}; + tuplestore_putvalues(tupstore, tupdesc, values, isNull); + } + + return (Datum) 0; +} /* @@ -1528,7 +3116,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * This separation lets test code inject a synthetic context and exercise the * FSM without a live database connection. * - * Single-shot, three straight-line lookups at most -- matching the design + * Single-shot, two straight-line lookups at most -- matching the design * doc's own top-level driver, not a loop: the cascades that need more than * one row's worth of assignment in a single call (the MS-failover cascade, * the join_secondary -> nested primary pass) get there via a bounded, named @@ -1563,7 +3151,8 @@ ProceedGroupStateFromContext(GroupStateContext *ctx) BuildFromContextNodeActiveContext(ctx, NULL, &earlyNac); - if (FindAndDispatchMonitorFSMRule(ctx, &earlyNac, 0, MonitorFSM_FromContextStart)) + if (FindAndDispatchMonitorFSMRule(ctx, &earlyNac, MonitorFSM_EarlyChecksStart, + MonitorFSM_FromContextStart)) { return true; } @@ -1678,6 +3267,118 @@ WalSourceNodesAreAllUnhealthy(GroupStateContext *ctx, } +/* + * BuildMSFailoverNodeActiveContext computes the facts the MS-failover + * cluster's own declarative rows (MonitorFSM_MSFailoverStart onwards) need. + * candidateNode is NULL at exactly one call site (TryFanOutReportLsnRow, + * wrapping BuildCandidateList's own fan-out loop, which hasn't selected a + * candidate yet) -- everywhere else (both call sites TryMSFailoverDeclarativeRow + * wraps) it's non-NULL, called from within ProceedGroupStateForMSFailover's + * own "nodeBeingPromoted != NULL" branch. Either way + * candidatePromotionInProgress is exactly (candidateNode != NULL), and the + * activeNodeAllWalSourcesUnhealthy computation below is skipped whenever + * candidateNode is NULL, so passing NULL never risks matching it against + * the wrong node. + */ +static void +BuildMSFailoverNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *activeNode, + AutoFailoverNode *candidateNode, NodeActiveContext *nac) +{ + memset(nac, 0, sizeof(NodeActiveContext)); + + BuildNodeStatus(ctx, activeNode, &nac->activeNode); + BuildNodeStatus(ctx, candidateNode, &nac->candidateNode); + + nac->inMSFailoverCluster = true; + nac->guardDataLossEnabled = GuardDataLoss; + nac->candidatePromotionInProgress = (candidateNode != NULL); + + if (candidateNode != NULL && candidateNode->nodeId == activeNode->nodeId) + { + nac->activeNodeAllWalSourcesUnhealthy = + WalSourceNodesAreAllUnhealthy(ctx, ctx->groupNodeList, activeNode); + } +} + + +/* + * TryMSFailoverDeclarativeRow attempts the MS-failover cluster's own two + * declarative rows (pos 363/365) for activeNode/candidateNode, returning + * whether one matched and was dispatched. Callers only invoke this from + * exactly the hand-written C condition the matched row's own conditions + * mirror (see each call site's own comment) -- this is not a new source of + * behavior, only a new, attributed path to the same AssignGoalState call + * the hand-written code already made unconditionally at that point. A + * false return (row's own conditions didn't line up with what the caller's + * condition already established -- should not happen, but this file's own + * "no dead ends in a hot path" principle applies here too) falls back to + * the caller's own pre-existing plain AssignGoalState call, never to an + * ereport(ERROR): unlike the operator-triggered API_TRIGGERED section, + * this is reached from the ordinary node_active() heartbeat path, where a + * hard error is not an acceptable failure mode. + */ +static bool +TryMSFailoverDeclarativeRow(GroupStateContext *ctx, AutoFailoverNode *activeNode, + AutoFailoverNode *candidateNode) +{ + NodeActiveContext msNac; + + BuildMSFailoverNodeActiveContext(ctx, activeNode, candidateNode, &msNac); + + return FindAndDispatchMonitorFSMRule(ctx, &msNac, MonitorFSM_MSFailoverStart, + MonitorFSM_PrimaryNodeSectionStart); +} + + +/* + * TryFanOutReportLsnRow attempts to dispatch BuildCandidateList's own + * fan-out (pos 367/369/371/373) for a single node that the hand-written + * loop has already determined is a legitimate report_lsn target. A thin + * wrapper over TryMSFailoverDeclarativeRow with candidateNode=NULL, which + * correctly leaves candidatePromotionInProgress false and skips the + * activeNodeAllWalSourcesUnhealthy computation (see + * BuildMSFailoverNodeActiveContext's own comment) -- so it can never + * accidentally match pos 363/365/375/377/379, all of which require + * activeNode to already be in report_lsn, which these fan-out nodes never + * are yet. Falls back to the caller's own pre-existing AssignGoalState call + * on no match, exactly like TryMSFailoverDeclarativeRow itself. + */ +static bool +TryFanOutReportLsnRow(GroupStateContext *ctx, AutoFailoverNode *node) +{ + return TryMSFailoverDeclarativeRow(ctx, node, NULL); +} + + +/* + * DispatchMonitorFSMRuleByPos dispatches the single MonitorFSM[] row whose + * .pos equals the given value, unconditionally -- no RuleMatches check. + * Used exactly once, by PromoteSelectedNode (see pos 375/377's own + * comment), for the one pair of rows in the whole table first-match-wins + * can never disambiguate on its own: the caller has already made the real + * choice (an internal LSN comparison no BoolPattern can express) before + * calling this. Returns false if no row has that .pos -- should never + * happen for a literal, hand-maintained pos value, but the caller still + * falls back to its own pre-existing AssignGoalState call rather than + * ereport(ERROR), matching TryMSFailoverDeclarativeRow's own "no dead ends + * in a hot path" principle. + */ +static bool +DispatchMonitorFSMRuleByPos(GroupStateContext *ctx, NodeActiveContext *nac, int pos) +{ + for (int i = 0; i < MonitorFSM_SIZE; i++) + { + if (MonitorFSM[i].pos == pos) + { + DispatchMonitorFSMRule(ctx, nac, &MonitorFSM[i]); + return true; + } + } + + return false; +} + + /* * ProceedGroupStateForMSFailover implements Group State Machine transition to * orchestrate a failover when we have more than one standby. @@ -1746,19 +3447,32 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, { if (GuardDataLoss) { - LogAndNotifyMessage( - message, BUFSIZE, - "Failover candidate " NODE_FORMAT - " is stuck in fast_forward: all WAL source nodes are " - "unhealthy and pgautofailover.guard_data_loss is true. " - "Resetting candidate to report_lsn to retry when a " - "source recovers. Use pg_autoctl perform failover " - "--allow-data-loss to promote with available WAL.", - NODE_FORMAT_ARGS(activeNode)); - - AssignGoalState(activeNode, - REPLICATION_STATE_REPORT_LSN, - message); + /* + * Dispatched through MonitorFSM[]'s own MS-failover + * declarative row (pos 363) when possible, for + * dump_fsm() visibility and rule_pos attribution -- + * falling back to the plain AssignGoalState call below + * only if that row's own conditions somehow didn't + * line up with the ones just checked above (should + * never happen; see TryMSFailoverDeclarativeRow's own + * comment). + */ + if (!TryMSFailoverDeclarativeRow(ctx, activeNode, activeNode)) + { + LogAndNotifyMessage( + message, BUFSIZE, + "Failover candidate " NODE_FORMAT + " is stuck in fast_forward: all WAL source nodes are " + "unhealthy and pgautofailover.guard_data_loss is true. " + "Resetting candidate to report_lsn to retry when a " + "source recovers. Use pg_autoctl perform failover " + "--allow-data-loss to promote with available WAL.", + NODE_FORMAT_ARGS(activeNode)); + + AssignGoalState(activeNode, + REPLICATION_STATE_REPORT_LSN, + message); + } return true; } @@ -1774,7 +3488,7 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, } } - return ProceedWithMSFailover(activeNode, nodeBeingPromoted); + return ProceedWithMSFailover(ctx, activeNode, nodeBeingPromoted); } LogAndNotifyMessage( @@ -1802,7 +3516,7 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, elog(LOG, "Found candidate " NODE_FORMAT, NODE_FORMAT_ARGS(nodeBeingPromoted)); - return ProceedWithMSFailover(activeNode, nodeBeingPromoted); + return ProceedWithMSFailover(ctx, activeNode, nodeBeingPromoted); } } @@ -2009,7 +3723,7 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, return false; } - return PromoteSelectedNode(selectedNode, + return PromoteSelectedNode(ctx, selectedNode, primaryNode, &candidateList); } @@ -2162,13 +3876,16 @@ BuildCandidateList(GroupStateContext *ctx, List *nodesGroupList, ++(candidateList->missingNodesCount); - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn to find the failover candidate", - NODE_FORMAT_ARGS(node)); + if (!TryFanOutReportLsnRow(ctx, node)) + { + LogAndNotifyMessage( + message, BUFSIZE, + "Setting goal state of " NODE_FORMAT + " to report_lsn to find the failover candidate", + NODE_FORMAT_ARGS(node)); - AssignGoalState(node, REPLICATION_STATE_REPORT_LSN, message); + AssignGoalState(node, REPLICATION_STATE_REPORT_LSN, message); + } continue; } @@ -2193,7 +3910,7 @@ BuildCandidateList(GroupStateContext *ctx, List *nodesGroupList, * WAL it's still missing from another standby node. */ static bool -ProceedWithMSFailover(AutoFailoverNode *activeNode, +ProceedWithMSFailover(GroupStateContext *ctx, AutoFailoverNode *activeNode, AutoFailoverNode *candidateNode) { Assert(candidateNode != NULL); @@ -2206,17 +3923,27 @@ ProceedWithMSFailover(AutoFailoverNode *activeNode, if (IsCurrentState(activeNode, REPLICATION_STATE_REPORT_LSN) && CandidateNodeIsReadyToStreamWAL(candidateNode)) { - char message[BUFSIZE]; + /* + * Dispatched through MonitorFSM[]'s own MS-failover declarative row + * (pos 365) when possible -- see TryMSFailoverDeclarativeRow's own + * comment; falls back to the plain AssignGoalState call below only + * if that row's own conditions somehow didn't line up with the + * ones just checked above. + */ + if (!TryMSFailoverDeclarativeRow(ctx, activeNode, candidateNode)) + { + char message[BUFSIZE]; - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to join_secondary after " NODE_FORMAT - " got selected as the failover candidate.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(candidateNode)); + LogAndNotifyMessage( + message, BUFSIZE, + "Setting goal state of " NODE_FORMAT + " to join_secondary after " NODE_FORMAT + " got selected as the failover candidate.", + NODE_FORMAT_ARGS(activeNode), + NODE_FORMAT_ARGS(candidateNode)); - AssignGoalState(activeNode, REPLICATION_STATE_JOIN_SECONDARY, message); + AssignGoalState(activeNode, REPLICATION_STATE_JOIN_SECONDARY, message); + } return true; } @@ -2405,7 +4132,8 @@ SelectFailoverCandidateNode(GroupStateContext *ctx, * PromoteSelectedNode assigns goal state to the selected node to failover to. */ static bool -PromoteSelectedNode(AutoFailoverNode *selectedNode, +PromoteSelectedNode(GroupStateContext *ctx, + AutoFailoverNode *selectedNode, AutoFailoverNode *primaryNode, CandidateList *candidateList) { @@ -2513,6 +4241,10 @@ PromoteSelectedNode(AutoFailoverNode *selectedNode, if (selectedNode->reportedLSN == candidateList->mostAdvancedReportedLSN) { char message[BUFSIZE] = { 0 }; + NodeActiveContext promotionNac; + + memset(&promotionNac, 0, sizeof(NodeActiveContext)); + BuildNodeStatus(ctx, selectedNode, &promotionNac.activeNode); if (primaryNode) { @@ -2535,9 +4267,12 @@ PromoteSelectedNode(AutoFailoverNode *selectedNode, candidateList->candidateCount); } - AssignGoalState(selectedNode, - REPLICATION_STATE_PREPARE_PROMOTION, - message); + if (!DispatchMonitorFSMRuleByPos(ctx, &promotionNac, 375)) + { + AssignGoalState(selectedNode, + REPLICATION_STATE_PREPARE_PROMOTION, + message); + } /* leave the other nodes in ReportLSN state for now */ return true; @@ -2545,6 +4280,10 @@ PromoteSelectedNode(AutoFailoverNode *selectedNode, else { char message[BUFSIZE] = { 0 }; + NodeActiveContext promotionNac; + + memset(&promotionNac, 0, sizeof(NodeActiveContext)); + BuildNodeStatus(ctx, selectedNode, &promotionNac.activeNode); if (primaryNode) { @@ -2567,8 +4306,11 @@ PromoteSelectedNode(AutoFailoverNode *selectedNode, candidateList->candidateCount); } - AssignGoalState(selectedNode, - REPLICATION_STATE_FAST_FORWARD, message); + if (!DispatchMonitorFSMRuleByPos(ctx, &promotionNac, 377)) + { + AssignGoalState(selectedNode, + REPLICATION_STATE_FAST_FORWARD, message); + } return true; } diff --git a/src/monitor/group_state_machine.h b/src/monitor/group_state_machine.h index 17effdd17..adc9cbfef 100644 --- a/src/monitor/group_state_machine.h +++ b/src/monitor/group_state_machine.h @@ -19,6 +19,64 @@ #include "formation_metadata.h" #include "node_metadata.h" +/* + * MonitorFSMSection identifies which of the four real control-flow regions + * of the monitor's declarative dispatch table (MonitorFSM[] in + * group_state_machine.c) a row belongs to -- see that array's own comment + * for what each region corresponds to in the original if-chain/call sites. + * Declared here (not just in the .c file) so it can be exposed to SQL as + * pgautofailover.fsm_section, the same way ReplicationState is exposed as + * pgautofailover.replication_state (see replication_state.h/.c for that + * pattern, mirrored below). + * + * API_TRIGGERED comes first (pos 101-1xx) even though it's the newest + * section, chronologically: an operator-triggered call (perform_failover, + * remove_node, start/stop_maintenance, set_node_candidate_priority, + * set_node_replication_quorum, set_formation_number_sync_standbys) is a + * distinct, self-contained entry point, not a continuation of the + * heartbeat-driven EARLY_CHECKS/REPORTING_NODE/PRIMARY_NODE chain -- placing + * it in its own leading hundred-block keeps that same "each section is one + * contiguous pos range, matching one real entry point" property the other + * three sections already have, rather than wedging operator rows into gaps + * within the heartbeat sections. + */ +typedef enum MonitorFSMSection +{ + MONITOR_FSM_SECTION_API_TRIGGERED = 0, + MONITOR_FSM_SECTION_EARLY_CHECKS, + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_PRIMARY_NODE, +} MonitorFSMSection; + +/* public function declarations, mirroring replication_state.h's pattern */ +extern Oid MonitorFSMSectionTypeOid(void); +extern MonitorFSMSection EnumGetMonitorFSMSection(Oid monitorFSMSectionOid); +extern Oid MonitorFSMSectionGetEnum(MonitorFSMSection section); +extern const char * MonitorFSMSectionGetName(MonitorFSMSection section); + +/* + * MonitorApiFunction identifies which operator-triggered SQL entry point + * produced a ProceedGroupStateForApiTrigger() dispatch call -- see that + * function's own comment in group_state_machine.c, and the + * MONITOR_FSM_SECTION_API_TRIGGERED rows in MonitorFSM[] that match against + * it via .conditions.apiTrigger. API_FUNCTION_NONE is never passed to + * ProceedGroupStateForApiTrigger() itself -- it's the ordinary node_active() + * heartbeat path's own implicit value (see NodeActiveContext's apiFunction + * field), kept ordinal 0 so every row written before this mechanism existed + * keeps meaning exactly what it always meant, with zero changes. + */ +typedef enum MonitorApiFunction +{ + API_FUNCTION_NONE = 0, + API_FUNCTION_REMOVE_NODE, + API_FUNCTION_PERFORM_FAILOVER, + API_FUNCTION_START_MAINTENANCE, + API_FUNCTION_STOP_MAINTENANCE, + API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY, + API_FUNCTION_SET_NODE_REPLICATION_QUORUM, + API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS, +} MonitorApiFunction; + /* * AutoFailoverNodeState describes the current state of a node in a group. */ @@ -67,6 +125,9 @@ extern bool BuildGroupStateContext(GroupStateContext *ctx, AutoFailoverNode *activeNode); extern bool ProceedGroupState(AutoFailoverNode *activeNode); extern bool ProceedGroupStateFromContext(GroupStateContext *ctx); +extern bool ProceedGroupStateForApiTrigger(MonitorApiFunction apiFunction, + AutoFailoverNode *activeNode, + AutoFailoverNode *primaryNode); /* GUCs */ extern int EnableSyncXlogThreshold; diff --git a/src/monitor/health_check_metadata.c b/src/monitor/health_check_metadata.c index fed9483e2..b1c4b63e9 100644 --- a/src/monitor/health_check_metadata.c +++ b/src/monitor/health_check_metadata.c @@ -28,6 +28,7 @@ #include "pgstat.h" #include "utils/builtins.h" #include "utils/memutils.h" +#include "utils/resowner.h" #include "utils/snapmgr.h" @@ -76,13 +77,56 @@ LoadNodeHealthList(void) pgstat_report_activity(STATE_RUNNING, query.data); - spiStatus = SPI_execute(query.data, false, 0); - /* * When we start the monitor during an upgrade (from 1.3 to 1.4), the * background worker might be reading the 1.3 pgautofailover catalogs - * still, where the "nodehost" column does not exist. + * still, where the "nodehost" column does not exist. The same + * happens for the brief window a regress/isolation test spends with + * the extension deliberately left at a non-2.3 (or dropped) schema + * (dummy_update.sql, upgrade.sql): the query below is a genuine + * parse-time ereport(ERROR) in either case, not a soft SPI-status + * failure SPI_execute() itself can report -- it always throws, + * aborting the enclosing transaction, so it has to be caught with a + * subtransaction here rather than checked via spiStatus after the + * fact (the "if (spiStatus != SPI_OK_SELECT)" shape this comment + * used to describe never actually ran for this case; it only + * covered a SPI-level failure with no matching real code path). */ + MemoryContext spiErrContext = CurrentMemoryContext; + ResourceOwner spiErrOwner = CurrentResourceOwner; + + BeginInternalSubTransaction(NULL); + + PG_TRY(); + { + spiStatus = SPI_execute(query.data, false, 0); + + ReleaseCurrentSubTransaction(); + MemoryContextSwitchTo(spiErrContext); + CurrentResourceOwner = spiErrOwner; + } + PG_CATCH(); + { + ErrorData *edata; + + MemoryContextSwitchTo(spiErrContext); + edata = CopyErrorData(); + FlushErrorState(); + + RollbackAndReleaseCurrentSubTransaction(); + MemoryContextSwitchTo(spiErrContext); + CurrentResourceOwner = spiErrOwner; + + ereport(WARNING, + (errmsg("health check skipping this round: %s", edata->message))); + + FreeErrorData(edata); + + EndSPITransaction(); + return NIL; + } + PG_END_TRY(); + if (spiStatus != SPI_OK_SELECT) { EndSPITransaction(); diff --git a/src/monitor/metadata.h b/src/monitor/metadata.h index d974d8bff..b564d3c05 100644 --- a/src/monitor/metadata.h +++ b/src/monitor/metadata.h @@ -22,6 +22,7 @@ #define AUTO_FAILOVER_NODE_TABLE "pgautofailover.node" #define AUTO_FAILOVER_EVENT_TABLE "pgautofailover.event" #define REPLICATION_STATE_TYPE_NAME "replication_state" +#define FSM_SECTION_TYPE_NAME "fsm_section" /* diff --git a/src/monitor/node_active_protocol.c b/src/monitor/node_active_protocol.c index 24a0d0163..0638ccfbe 100644 --- a/src/monitor/node_active_protocol.c +++ b/src/monitor/node_active_protocol.c @@ -1162,7 +1162,6 @@ remove_node_by_host(PG_FUNCTION_ARGS) static bool RemoveNode(int64 nodeId, bool force) { - ListCell *nodeCell = NULL; char message[BUFSIZE] = { 0 }; /* @@ -1248,50 +1247,15 @@ RemoveNode(int64 nodeId, bool force) return true; } - /* review the FSM for every other node, when removing the primary */ - if (currentNodeIsPrimary) - { - foreach(nodeCell, otherNodesGroupList) - { - AutoFailoverNode *node = (AutoFailoverNode *) lfirst(nodeCell); - - if (node == NULL) - { - /* shouldn't happen */ - ereport(ERROR, (errmsg("BUG: node is NULL"))); - continue; - } - - /* skip nodes that are currently in maintenance */ - if (IsInMaintenance(node)) - { - continue; - } - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn after primary node removal.", - NODE_FORMAT_ARGS(node)); - - SetNodeGoalState(node, REPLICATION_STATE_REPORT_LSN, message); - } - } - /* - * Mark the node as being dropped, so that the pg_autoctl node-active - * process can implement further actions at drop time. + * Dispatch through MonitorFSM[]'s API_TRIGGERED section (see + * ProceedGroupStateForApiTrigger's own comment): assigns dropped to + * currentNode, and, when it canTakeWrites, fans out report_lsn to every + * surviving non-maintenance standby first (both folded into one row -- + * see that row's own comment for why splitting them across two rows, + * as an earlier draft of this table did, would have been a real bug). */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " from formation \"%s\" and group %d to \"dropped\"" - " to implement node removal.", - NODE_FORMAT_ARGS(currentNode), - currentNode->formationId, - currentNode->groupId); - - SetNodeGoalState(currentNode, REPLICATION_STATE_DROPPED, message); + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_REMOVE_NODE, currentNode, NULL); /* * Adjust number-sync-standbys if necessary. @@ -1536,21 +1500,13 @@ perform_failover(PG_FUNCTION_ARGS) "perform a manual failover"))); } - char message[BUFSIZE] = { 0 }; - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to draining and " NODE_FORMAT - " to prepare_promotion after a user-initiated failover.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(secondaryNode)); - - SetNodeGoalState(primaryNode, - REPLICATION_STATE_DRAINING, message); - - SetNodeGoalState(secondaryNode, - REPLICATION_STATE_PREPARE_PROMOTION, message); + /* + * Dispatch through MonitorFSM[]'s API_TRIGGERED section (see + * ProceedGroupStateForApiTrigger's own comment): standby -> + * prepare_promotion, primary -> draining. + */ + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_PERFORM_FAILOVER, + secondaryNode, primaryNode); } else { @@ -1558,16 +1514,15 @@ perform_failover(PG_FUNCTION_ARGS) AutoFailoverNode *firstStandbyNode = linitial(standbyNodesGroupList); char message[BUFSIZE] = { 0 }; - /* so we have at least one candidate, let's get started */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - "at LSN %X/%X to draining after a user-initiated failover.", - NODE_FORMAT_ARGS(primaryNode), - (uint32) (primaryNode->reportedLSN >> 32), - (uint32) primaryNode->reportedLSN); - - SetNodeGoalState(primaryNode, REPLICATION_STATE_DRAINING, message); + /* + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary -> + * draining. The candidatePriority trick and the continuation + * dispatch below are POST side effects, hand-written here exactly + * as before -- see that row's own comment for why they don't + * become part of the row itself. + */ + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_PERFORM_FAILOVER, + primaryNode, NULL); /* * When a failover is performed with all the nodes up and running, the @@ -1901,49 +1856,37 @@ start_maintenance(PG_FUNCTION_ARGS) if (totalNodesCount == 2) { /* - * Set the primary to prepare_maintenance now, and if we have a - * single secondary we assign it prepare_promotion, otherwise we - * need to elect a secondary, same as in perform_failover. + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary + * -> prepare_maintenance. The lone standby -> prepare_promotion + * is a POST side effect, hand-written here: it has no ordering + * dependency on this row's own assignment (neither reads the + * other's freshly-committed state), so it doesn't need + * extraAction to sequence correctly. */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to prepare_maintenance " - "after a user-initiated start_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_START_MAINTENANCE, + currentNode, NULL); - SetNodeGoalState(currentNode, - REPLICATION_STATE_PREPARE_MAINTENANCE, message); - - AutoFailoverNode *otherNode = firstStandbyNode; - - /* - * We put the only secondary node straight to prepare_replication. - */ LogAndNotifyMessage( message, BUFSIZE, "Setting goal state of " NODE_FORMAT - " to prepare_maintenance and " NODE_FORMAT " to prepare_promotion " "after a user-initiated start_maintenance call.", - NODE_FORMAT_ARGS(currentNode), - NODE_FORMAT_ARGS(otherNode)); + NODE_FORMAT_ARGS(firstStandbyNode)); - SetNodeGoalState(otherNode, + SetNodeGoalState(firstStandbyNode, REPLICATION_STATE_PREPARE_PROMOTION, message); } else { - /* put the primary directly to maintenance */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance " - "after a user-initiated start_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); - - SetNodeGoalState(currentNode, - REPLICATION_STATE_PREPARE_MAINTENANCE, message); + /* + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary + * -> prepare_maintenance. The continuation dispatch below is a + * POST side effect, hand-written here, called after this row's + * own assignment has committed (it re-fetches fresh state, so + * ordering matters here, unlike the 2-node case above). + */ + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_START_MAINTENANCE, + currentNode, NULL); /* now proceed with the failover, starting with the first standby */ (void) ProceedGroupState(firstStandbyNode); @@ -1970,30 +1913,17 @@ start_maintenance(PG_FUNCTION_ARGS) * state of any standby node yet, we get there when the count is one * (not zero). */ - if (formation->number_sync_standbys == 0 && secondaryNodesCount == 1 && - IsHealthySyncStandby(currentNode)) - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to wait_primary and " NODE_FORMAT - " to wait_maintenance " - "after a user-initiated start_maintenance call.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(currentNode)); - SetNodeGoalState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY, message); - SetNodeGoalState(currentNode, REPLICATION_STATE_WAIT_MAINTENANCE, message); - } - else - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance " - "after a user-initiated start_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); - SetNodeGoalState(currentNode, REPLICATION_STATE_MAINTENANCE, message); - } + /* + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: the + * last-healthy-sync-standby row (wait_maintenance + primary + * wait_primary) and the ordinary row (maintenance) are both + * declarative, dual-role rows there -- see + * BuildApiTriggerNodeActiveContext's own comment for how + * lastHealthySyncStandbyGoingToMaintenance mirrors this exact + * condition. + */ + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_START_MAINTENANCE, + currentNode, primaryNode); } else { @@ -2029,8 +1959,6 @@ stop_maintenance(PG_FUNCTION_ARGS) int64 nodeId = PG_GETARG_INT64(0); - char message[BUFSIZE] = { 0 }; - AutoFailoverNode *currentNode = LockNodeGroupAndFetch(nodeId); if (currentNode == NULL) { @@ -2087,69 +2015,17 @@ stop_maintenance(PG_FUNCTION_ARGS) "group %d", currentNode->formationId, currentNode->groupId))); } - else if ((primaryNode == NULL || IsDemotedPrimary(primaryNode)) && - totalNodesCount > 2) - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn after a user-initiated stop_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); - - SetNodeGoalState(currentNode, REPLICATION_STATE_REPORT_LSN, message); - - PG_RETURN_BOOL(true); - } - else if (IsDemotedPrimary(primaryNode)) - { - /* - * The primary is fully demoted (Postgres stopped, e.g. after a - * #1025 self-fence recovery): there's nothing left running to - * stream from, so catchingup would just retry a doomed replication - * connection forever. Join the report_lsn crew instead -- once this - * node reports its LSN, the candidate-scanning code in - * ProceedGroupStateForMSFailover() picks up the demoted primary too - * (it's still IsDemotedPrimary()) and the normal election proceeds. - */ - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn after a user-initiated stop_maintenance call, " - "as " NODE_FORMAT " is demoted and has nothing to catch up from.", - NODE_FORMAT_ARGS(currentNode), - NODE_FORMAT_ARGS(primaryNode)); - - SetNodeGoalState(currentNode, REPLICATION_STATE_REPORT_LSN, message); - - PG_RETURN_BOOL(true); - } /* - * When a failover is in progress and stop_maintenance() is called (by - * means of pg_autoctl disable maintenance or otherwise), then we should - * join the crew on REPORT_LSN: the last known primary can be presumed - * down. + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: no-primary, + * primary-demoted, failover-in-progress, and the ordinary catchingup + * catchall are all declarative rows there -- see each row's own + * comment. Every one of the four real branches this replaces produces + * the same PG_RETURN_BOOL(true), so they collapse to a single dispatch + * call followed by one shared return. */ - if (IsFailoverInProgress(groupNodesList)) - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after a user-initiated stop_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); - - SetNodeGoalState(currentNode, REPLICATION_STATE_REPORT_LSN, message); - } - else - { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after a user-initiated stop_maintenance call.", - NODE_FORMAT_ARGS(currentNode)); - - SetNodeGoalState(currentNode, REPLICATION_STATE_CATCHINGUP, message); - } + (void) ProceedGroupStateForApiTrigger(API_FUNCTION_STOP_MAINTENANCE, + currentNode, primaryNode); PG_RETURN_BOOL(true); } @@ -2265,8 +2141,6 @@ set_node_candidate_priority(PG_FUNCTION_ARGS) } else { - char message[BUFSIZE]; - AutoFailoverNode *primaryNode = GetPrimaryNodeInGroup(currentNode->formationId, currentNode->groupId); @@ -2278,21 +2152,17 @@ set_node_candidate_priority(PG_FUNCTION_ARGS) * * If we don't currently have a primary node anyway, we can just * proceed with the change. + * + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary -> + * apply_settings. Kept hand-written, not routed through the + * table's own no-match ERROR: this exact message text is + * preserved unchanged so it stays test-stable. */ if (primaryNode && !IsCurrentState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS)) { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to apply_settings after updating " NODE_FORMAT - " candidate priority to %d.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(currentNode), - currentNode->candidatePriority); - - SetNodeGoalState(primaryNode, - REPLICATION_STATE_APPLY_SETTINGS, message); + (void) ProceedGroupStateForApiTrigger( + API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY, primaryNode, NULL); } /* if primaryNode is not NULL, then current state is APPLY_SETTINGS */ @@ -2410,8 +2280,6 @@ set_node_replication_quorum(PG_FUNCTION_ARGS) } else { - char message[BUFSIZE]; - AutoFailoverNode *primaryNode = GetPrimaryNodeInGroup(currentNode->formationId, currentNode->groupId); @@ -2423,21 +2291,17 @@ set_node_replication_quorum(PG_FUNCTION_ARGS) * * If we don't currently have a primary node anyway, we can just * proceed with the change. + * + * Dispatch through MonitorFSM[]'s API_TRIGGERED section: primary -> + * apply_settings. Kept hand-written, not routed through the table's + * own no-match ERROR: this exact message text is preserved + * unchanged so it stays test-stable. */ if (primaryNode && !IsCurrentState(primaryNode, REPLICATION_STATE_APPLY_SETTINGS)) { - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to apply_settings after updating " NODE_FORMAT - " replication quorum to %s.", - NODE_FORMAT_ARGS(primaryNode), - NODE_FORMAT_ARGS(currentNode), - currentNode->replicationQuorum ? "true" : "false"); - - SetNodeGoalState(primaryNode, - REPLICATION_STATE_APPLY_SETTINGS, message); + (void) ProceedGroupStateForApiTrigger( + API_FUNCTION_SET_NODE_REPLICATION_QUORUM, primaryNode, NULL); } /* if primaryNode is not NULL, then current state is APPLY_SETTINGS */ diff --git a/src/monitor/notifications.c b/src/monitor/notifications.c index 3db58c73c..f295a3c0e 100644 --- a/src/monitor/notifications.c +++ b/src/monitor/notifications.c @@ -28,6 +28,13 @@ #include "utils/json.h" #include "utils/pg_lsn.h" +#include "group_state_machine.h" + + +/* see notifications.h for why these exist and how they're used */ +int CurrentMonitorFSMRulePos = 0; +int CurrentMonitorFSMRuleSection = 0; + /* * LogAndNotifyMessage emits the given message both as a log entry and also as @@ -123,6 +130,17 @@ InsertEvent(AutoFailoverNode *node, char *description) Oid goalStateOid = ReplicationStateGetEnum(node->goalState); Oid reportedStateOid = ReplicationStateGetEnum(node->reportedState); Oid replicationStateTypeOid = ReplicationStateTypeOid(); + Oid fsmSectionTypeOid = MonitorFSMSectionTypeOid(); + + /* + * CurrentMonitorFSMRulePos/Section (see notifications.h) are only set + * while a MonitorFSM[] row's own extraAction/goal-state assignment is + * actually running; 0 means this event was produced by an ordinary + * AssignGoalState call from outside the declarative dispatch table, so + * rule_pos/rule_section stay NULL for it rather than a misleading 0 + * (not a valid .pos value -- every real section starts at 101 or above). + */ + bool haveRule = CurrentMonitorFSMRulePos != 0; Oid argTypes[] = { TEXTOID, /* formationid */ @@ -138,7 +156,9 @@ InsertEvent(AutoFailoverNode *node, char *description) LSNOID, /* reportedLSN */ INT4OID, /* candidate_priority */ BOOLOID, /* replication_quorum */ - TEXTOID /* description */ + TEXTOID, /* description */ + INT4OID, /* rule_pos */ + fsmSectionTypeOid /* rule_section */ }; Datum argValues[] = { @@ -155,7 +175,18 @@ InsertEvent(AutoFailoverNode *node, char *description) LSNGetDatum(node->reportedLSN), /* reportedLSN */ Int32GetDatum(node->candidatePriority), /* candidate_priority */ BoolGetDatum(node->replicationQuorum), /* replication_quorum */ - CStringGetTextDatum(description) /* description */ + CStringGetTextDatum(description), /* description */ + Int32GetDatum(CurrentMonitorFSMRulePos), /* rule_pos, ignored if NULL below */ + haveRule + ? ObjectIdGetDatum(MonitorFSMSectionGetEnum( + (MonitorFSMSection) CurrentMonitorFSMRuleSection)) + : ObjectIdGetDatum(InvalidOid) /* rule_section, ignored if NULL below */ + }; + + char argNulls[] = { + ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', + haveRule ? ' ' : 'n', /* rule_pos */ + haveRule ? ' ' : 'n' /* rule_section */ }; const int argCount = sizeof(argValues) / sizeof(argValues[0]); @@ -165,14 +196,14 @@ InsertEvent(AutoFailoverNode *node, char *description) "INSERT INTO " AUTO_FAILOVER_EVENT_TABLE "(formationid, nodeid, groupid, nodename, nodehost, nodeport," " reportedstate, goalstate, reportedrepstate, reportedtli, reportedlsn," - " candidatepriority, replicationquorum, description) " - "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) " + " candidatepriority, replicationquorum, description, rule_pos, rule_section) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) " "RETURNING eventid"; SPI_connect(); int spiStatus = SPI_execute_with_args(insertQuery, argCount, argTypes, - argValues, NULL, false, 0); + argValues, argNulls, false, 0); if (spiStatus == SPI_OK_INSERT_RETURNING && SPI_processed > 0) { diff --git a/src/monitor/notifications.h b/src/monitor/notifications.h index 00e064211..a636755c3 100644 --- a/src/monitor/notifications.h +++ b/src/monitor/notifications.h @@ -41,3 +41,23 @@ void LogAndNotifyMessage(char *message, size_t size, const char *fmt, ...) __att int64 NotifyStateChange(AutoFailoverNode *node, char *description); int64 InsertEvent(AutoFailoverNode *node, char *description); + +/* + * Set by group_state_machine.c's declarative dispatch just before invoking a + * matched MonitorFSM[] row's extraAction/goal-state assignment, and restored + * to whatever it was before immediately after (nested dispatch -- the + * MS-failover cascade's and join_secondary's own bounded nested searches -- + * saves/restores rather than clobbers, so the outer row's own subsequent + * assignments are still attributed correctly). Lets InsertEvent() attribute + * the resulting pgautofailover.event row to the rule that produced it + * (rule_pos/rule_section columns), without threading an extra parameter + * through AssignGoalState/SetNodeGoalState/NotifyStateChange and every one + * of their many call sites outside the declarative dispatch table. 0 means + * "no rule attributed" (an ordinary AssignGoalState call from outside the + * table, e.g. an operator-triggered SQL function, or ProceedGroupStateFor + * MSFailover's own hand-written internals): rule_pos/rule_section are left + * NULL in that case, since 0 is not a valid .pos value (every real section + * starts at 100 or above). + */ +extern int CurrentMonitorFSMRulePos; +extern int CurrentMonitorFSMRuleSection; diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index b7494bf65..5fc790e0b 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -176,6 +176,20 @@ CREATE TABLE pgautofailover.node -- we expect few rows and lots of UPDATE, let's benefit from HOT WITH (fillfactor = 25); +-- Mirrors group_state_machine.h's MonitorFSMSection: which of the three +-- real control-flow regions of the monitor's declarative dispatch table +-- (MonitorFSM[] in group_state_machine.c) a rule belongs to. See dump_fsm() +-- and pgautofailover.fsm below, and the rule_section column on +-- pgautofailover.event. +CREATE TYPE pgautofailover.fsm_section + AS ENUM + ( + 'api_triggered', + 'early_checks', + 'reporting_node', + 'primary_node' + ); + CREATE TABLE pgautofailover.event ( eventid bigserial not null, @@ -195,6 +209,16 @@ CREATE TABLE pgautofailover.event replicationquorum bool, description text, + -- Which MonitorFSM[] row (if any) produced this event: NULL when the + -- goal-state assignment came from outside the declarative dispatch + -- table (an operator-triggered SQL function, or ProceedGroupStateFor + -- MSFailover's own hand-written internals -- see CurrentMonitorFSMRulePos + -- in notifications.h for how this gets attributed). rule_pos is the + -- row's human-facing position (see dump_fsm()/pgautofailover.fsm), + -- not an array index. + rule_pos int, + rule_section pgautofailover.fsm_section, + PRIMARY KEY (eventid) ); @@ -229,6 +253,39 @@ CREATE TABLE pgautofailover.accepted_timeline PRIMARY KEY (formationid, groupid, decided_at) ); +-- Exposes the monitor's declarative dispatch table (MonitorFSM[] in +-- group_state_machine.c) to SQL, one row per rule, in first-match-wins +-- order -- see dump_fsm()'s own C-side comment for exactly what this does +-- and doesn't cover. pgautofailover.event.rule_pos/rule_section (added +-- above) join back to this view's pos column to show which rule produced +-- a given event. section is plain text here (not pgautofailover.fsm_section): +-- an api_triggered row's section names its specific operator-triggered +-- entry point too, e.g. "api_triggered: remove_node". +CREATE FUNCTION pgautofailover.dump_fsm() +RETURNS TABLE + ( + pos int, + section text, + comment text, + active_node_current_state text, + other_node_current_state text, + candidate_node_current_state text, + active_node_conditions text, + other_node_conditions text, + candidate_node_conditions text, + group_conditions text, + active_node_assigned_state pgautofailover.replication_state, + other_node_assigned_state pgautofailover.replication_state, + has_extra_action bool + ) +LANGUAGE C SECURITY DEFINER +AS 'MODULE_PATHNAME', $$dump_fsm$$; + +grant execute on function pgautofailover.dump_fsm() to autoctl_node; + +CREATE VIEW pgautofailover.fsm AS + SELECT * FROM pgautofailover.dump_fsm() ORDER BY pos; + GRANT SELECT ON ALL TABLES IN SCHEMA pgautofailover TO autoctl_node; CREATE FUNCTION pgautofailover.set_node_system_identifier diff --git a/src/monitor/regress_schedule b/src/monitor/regress_schedule index c44e50f93..bfea2dd2d 100644 --- a/src/monitor/regress_schedule +++ b/src/monitor/regress_schedule @@ -31,6 +31,7 @@ # exclusively. test: create_extension +test: fsm test: monitor test: workers test: node_active_protocol @@ -41,6 +42,7 @@ test: stale_primary_report test: lock_and_fetch_migration test: timeline_fork_detection test: failover_candidate_leaves_secondary +test: cluster_init_failover_rule_attribution test: dummy_update test: drop_extension test: upgrade diff --git a/src/monitor/sql/cluster_init_failover_rule_attribution.sql b/src/monitor/sql/cluster_init_failover_rule_attribution.sql new file mode 100644 index 000000000..76d54302c --- /dev/null +++ b/src/monitor/sql/cluster_init_failover_rule_attribution.sql @@ -0,0 +1,126 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- End-to-end demonstration of the rule_pos/rule_section attribution +-- mechanism (notifications.c's CurrentMonitorFSMRulePos/RuleSection, +-- InsertEvent()): registers a two-node formation, drives it through the +-- heartbeat-only bootstrap to primary + secondary, triggers a manual +-- perform_failover(), and then joins pgautofailover.event against +-- pgautofailover.fsm on rule_pos = pos to show, for every state +-- transition the monitor produced, exactly which MonitorFSM[] row was +-- selected and executed -- both for the ordinary heartbeat-driven +-- bootstrap rows (api_triggered = false in this join, since rule_pos is +-- only set for rows actually reached through the declarative dispatch +-- table) and for the operator-triggered perform_failover call itself. + +\x on + +-- ── formation and node registration ───────────────────────────────────────── + +SELECT pgautofailover.create_formation('cifra_test', 'pgsql', 'postgres', true, 0); + +SELECT * + FROM pgautofailover.register_node('cifra_test', 'cifra_p', 5432, + 'postgres', 'cifra_p', 1); + +SELECT nodeid AS np FROM pgautofailover.node + WHERE formationid = 'cifra_test' AND nodename = 'cifra_p' \gset + +SELECT * + FROM pgautofailover.register_node('cifra_test', 'cifra_s', 5432, + 'postgres', 'cifra_s', 1); + +SELECT nodeid AS ns FROM pgautofailover.node + WHERE formationid = 'cifra_test' AND nodename = 'cifra_s' \gset + +-- ── bootstrap: drive the FSM to primary + secondary ───────────────────────── +-- +-- Mirrors drop_node.sql's bootstrap sequence (register -> single -> +-- wait_primary -> [standby: wait_standby -> catchingup -> secondary] -> +-- primary), including its "confirm" round-trips. + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'single'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'wait_standby'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'single', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'wait_standby'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('cifra_test', :ns, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +-- Verify bootstrap: p=primary, s=secondary. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'cifra_test' + ORDER BY nodename; + +-- ── manual failover ────────────────────────────────────────────────────── +-- +-- Two-node group: dispatches through MonitorFSM[]'s API_TRIGGERED section +-- (pos 105, "manual failover, 2-node group, primary+standby both converged +-- -> standby prepare_promotion, primary draining"), attributing both +-- resulting event rows to that one rule. + +SELECT pgautofailover.perform_failover('cifra_test', 0); + +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'cifra_test' + ORDER BY nodename; + +-- ── which rule fired for which event? ─────────────────────────────────────── +-- +-- Every event row this formation produced, joined against the FSM table on +-- rule_pos = pos: rule_pos/rule_section are NULL for the ordinary +-- heartbeat-driven bootstrap transitions above whenever the matched row +-- happens to be identified only by array position in earlier sessions' +-- tests -- here every one of them was reached through the same declarative +-- MonitorFSM[] dispatch table, so each carries its own attribution too. The +-- final two rows (both attributed to pos 105) are the perform_failover() +-- call's own dual assignment (standby -> prepare_promotion, primary -> +-- draining), selected and executed from the API_TRIGGERED section. + +SELECT e.eventid, e.nodename, e.reportedstate, e.goalstate, + e.rule_pos, e.rule_section, f.comment AS rule_comment + FROM pgautofailover.event e + LEFT JOIN pgautofailover.fsm f ON f.pos = e.rule_pos + WHERE e.formationid = 'cifra_test' + ORDER BY e.eventid; diff --git a/src/monitor/sql/fsm.sql b/src/monitor/sql/fsm.sql new file mode 100644 index 000000000..67b850adb --- /dev/null +++ b/src/monitor/sql/fsm.sql @@ -0,0 +1,24 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Plain dump of the monitor's declarative dispatch table, via the +-- pgautofailover.fsm view (pgautofailover.dump_fsm() ordered by pos). This +-- is a static, compile-time-fixed table -- unaffected by any node/formation +-- state -- so its expected output changes only when a row is added, +-- removed, or edited in MonitorFSM[] (group_state_machine.c), giving that +-- change an explicit, reviewable regression diff. +-- +-- \x on: with the *_conditions columns added, a plain tabular row is far +-- wider than a terminal (or this file's own diff-ability), and reads far +-- worse than one field-per-line. + +\x on + +SELECT pos, section, + active_node_current_state, other_node_current_state, candidate_node_current_state, + active_node_conditions, other_node_conditions, candidate_node_conditions, + group_conditions, + active_node_assigned_state, other_node_assigned_state, has_extra_action, + comment + FROM pgautofailover.fsm + ORDER BY pos; From a78dfcf02f1d87e8e0e918627aec5ab15548f30d Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 29 Jul 2026 15:06:16 +0200 Subject: [PATCH 06/52] Add monitor/keeper FSM reachability check (pg_autoctl inspect fsm check) Builds the other half of the cross-check dump_fsm() was always meant to support (design doc's "Exposing both tables for the cross-check", never previously implemented -- confirmed by the doc's own "Implementation readiness" section: "check_fsm_reachability() and pg_autoctl do fsm check... have never actually run once"). Monitor side (src/monitor/group_state_machine.c, pgautofailover.sql): - pgautofailover.dump_fsm_edges(): a new SRF resolving every MonitorFSM[] row into concrete (pos, current_state, assigned_state) edges, via a new NodeStatePatternResolveFromStates() helper (handles all 7 NodeStatePatternKind values, including complement resolution for NOT_STABLE and full-universe resolution for ANY/ASSIGNED/NOT_ASSIGNED). Deliberately excludes two categories of edges, both confirmed by tracing a live run's mismatches back to root cause rather than assuming: reflexive (current == assigned) edges -- keeper_fsm_reach_assigned_state() (fsm.c) returns before ever consulting KeeperFSM[] when current_role == assigned_role, so a self-loop can never need a keeper edge -- and the whole MONITOR_FSM_SECTION_API_TRIGGERED section, whose rows resolve activeNode to a specific role (almost always the primary) via hand-written C before dispatch, so their own NodeStatePattern was never meant to double as a full reachability precondition. - pgautofailover.check_fsm_reachability(keeper_edges jsonb): anti-joins dump_fsm_edges() against a keeper's own edges (jsonb array of {"current","assigned"} objects), returning every monitor edge with no matching keeper entry. Casts keeper_edges' state names to pgautofailover.replication_state, so an unrecognized name fails loudly rather than silently never matching. - check_fsm_reachability.sql: regress test for the SQL mechanism itself (synthetic keeper-edge inputs), not the real KeeperFSM[] table. Keeper side (src/bin/pg_autoctl/fsm.c, monitor.c, cli_do_fsm.c): - KeeperFSMToJSON(): serializes KeeperFSM[] to the same edge shape, expanding ANY_STATE (the "drop node from any state" wildcard) into all 21 concrete states -- found and fixed via the first live run, which otherwise sent the literal string "#any state#" and failed the monitor's own enum cast. - monitor_check_fsm_reachability(): new monitor RPC sending that JSON to check_fsm_reachability(), following the existing monitor_get_node_region/monitor_report_timeline_history call patterns. - "pg_autoctl inspect fsm check": new read-only CLI command wiring the above together, mirroring cli_do_monitor_get_primary_node's shape (unlike fsm list/gv, this one genuinely needs a monitor connection). Also fixes a real, unrelated bug this feature's own large JSON parameter exposed: pgsql_execute_with_params' debug-parameter trace (src/bin/common/ pgsql.c) used a fixed 1024-byte buffer and logged a "BUG:" line whenever a parameter didn't fit -- cosmetic (the real query always used the full, untruncated parameter), but the first caller to ever pass more than ~1KB. Fixed by capping each parameter's own printed representation. Verified against a live monitor+keeper pair (not just Docker installcheck): first run found 23 mismatched MonitorFSM rows; tracing each one down to its real KeeperFSM[] cause is what led to the reflexive-edge and api_triggered exclusions above, bringing it down to 9 remaining candidates, none matching a known GitHub issue -- see follow-up commits. --- src/bin/common/pgsql.c | 38 +- src/bin/pg_autoctl/cli_do_fsm.c | 109 +++ src/bin/pg_autoctl/cli_do_root.h | 1 + src/bin/pg_autoctl/cli_inspect.c | 7 +- src/bin/pg_autoctl/fsm.c | 123 ++++ src/bin/pg_autoctl/fsm.h | 1 + src/bin/pg_autoctl/monitor.c | 104 +++ src/bin/pg_autoctl/monitor.h | 26 + .../expected/check_fsm_reachability.out | 111 +++ ...cluster_init_failover_rule_attribution.out | 2 +- src/monitor/expected/fsm.out | 16 +- src/monitor/group_state_machine.c | 686 ++++++++++++++---- src/monitor/node_active_protocol.c | 1 + src/monitor/pgautofailover.sql | 49 ++ src/monitor/regress_schedule | 1 + src/monitor/sql/check_fsm_reachability.sql | 80 ++ 16 files changed, 1193 insertions(+), 162 deletions(-) create mode 100644 src/monitor/expected/check_fsm_reachability.out create mode 100644 src/monitor/sql/check_fsm_reachability.sql diff --git a/src/bin/common/pgsql.c b/src/bin/common/pgsql.c index 105c4f96c..3807f41d2 100644 --- a/src/bin/common/pgsql.c +++ b/src/bin/common/pgsql.c @@ -1051,6 +1051,15 @@ pgsql_execute(PGSQL *pgsql, const char *sql) } +/* + * Cap on how much of a single query parameter's own value gets printed in + * pgsql_execute_with_params' debug trace (see its own comment at the one + * call site that truncates). Keeps that trace's fixed BUFSIZE buffer from + * overflowing on a large parameter, without limiting what's actually sent + * to Postgres (PQexecParams always gets the real, untruncated value). + */ +#define DEBUG_PARAM_VALUE_MAX_LEN 200 + /* * pgsql_execute_with_params opens a connection, runs a given SQL command, * and closes the connection again. @@ -1081,7 +1090,9 @@ pgsql_execute_with_params(PGSQL *pgsql, const char *sql, int paramCount, int remainingBytes = BUFSIZE; char *writePointer = (char *) debugParameters; - for (paramIndex = 0; paramIndex < paramCount; paramIndex++) + for (paramIndex = 0; + paramIndex < paramCount && remainingBytes > 0; + paramIndex++) { int bytesWritten = 0; const char *value = paramValues[paramIndex]; @@ -1091,12 +1102,37 @@ pgsql_execute_with_params(PGSQL *pgsql, const char *sql, int paramCount, bytesWritten = sformat(writePointer, remainingBytes, ", "); remainingBytes -= bytesWritten; writePointer += bytesWritten; + + if (remainingBytes <= 0) + { + break; + } } if (value == NULL) { bytesWritten = sformat(writePointer, remainingBytes, "NULL"); } + else if (strlen(value) > DEBUG_PARAM_VALUE_MAX_LEN) + { + /* + * A parameter can be arbitrarily large (e.g. the JSON payload + * "pg_autoctl inspect fsm check" sends to + * pgautofailover.check_fsm_reachability(), several KB) -- + * printing it here in full would overflow debugParameters' + * own fixed BUFSIZE and make sformat() itself log a "BUG:" + * about it on every single call, which is exactly what + * happened the first time that command ran for real. This is + * a debug-only trace, not the actual query (PQexecParams + * below always gets the real, untruncated paramValues), so + * truncating what gets logged here changes nothing about + * query correctness. + */ + bytesWritten = sformat(writePointer, remainingBytes, + "'%.*s...' (%zu bytes total)", + DEBUG_PARAM_VALUE_MAX_LEN, value, + strlen(value)); + } else { bytesWritten = diff --git a/src/bin/pg_autoctl/cli_do_fsm.c b/src/bin/pg_autoctl/cli_do_fsm.c index fd32f930b..3ac4efbe1 100644 --- a/src/bin/pg_autoctl/cli_do_fsm.c +++ b/src/bin/pg_autoctl/cli_do_fsm.c @@ -15,6 +15,8 @@ #include "postgres_fe.h" +#include "parson.h" + #include "cli_common.h" #include "commandline.h" #include "defaults.h" @@ -23,6 +25,7 @@ #include "fsm_mermaid.h" #include "keeper_config.h" #include "keeper.h" +#include "monitor.h" #include "parsing.h" #include "pgctl.h" #include "state.h" @@ -33,6 +36,7 @@ static void cli_do_fsm_init(int argc, char **argv); static void cli_do_fsm_state(int argc, char **argv); static void cli_do_fsm_list(int argc, char **argv); +static void cli_do_fsm_check(int argc, char **argv); static void cli_do_fsm_gv(int argc, char **argv); static void cli_do_fsm_mermaid_init(int argc, char **argv); static void cli_do_fsm_mermaid_steady_state(int argc, char **argv); @@ -69,6 +73,14 @@ CommandLine fsm_list = cli_getopt_pgdata, cli_do_fsm_list); +CommandLine fsm_check = + make_command("check", + "Check that every monitor FSM transition has a matching keeper edge", + CLI_PGDATA_USAGE, + CLI_PGDATA_OPTION, + cli_getopt_pgdata, + cli_do_fsm_check); + CommandLine fsm_gv = make_command("gv", "Output the FSM as a .gv program suitable for graphviz/dot", @@ -358,6 +370,103 @@ cli_do_fsm_list(int argc, char **argv) } +/* + * cli_do_fsm_check serializes this node's own KeeperFSM[] to JSON + * (KeeperFSMToJSON(), fsm.c) and sends it to the monitor's + * pgautofailover.check_fsm_reachability(jsonb) (via + * monitor_check_fsm_reachability(), monitor.c) to confirm every transition + * the monitor's own MonitorFSM[] (group_state_machine.c) can ever assign + * has a matching edge here. Unlike cli_do_fsm_list/cli_do_fsm_gv (purely + * local, no monitor connection at all), this genuinely needs one -- mirrors + * cli_do_monitor_get_primary_node's own shape in cli_do_monitor.c. + */ +static void +cli_do_fsm_check(int argc, char **argv) +{ + KeeperConfig config = keeperOptions; + Monitor monitor = { 0 }; + FsmReachabilityResult result = { 0 }; + + bool missingPgdataIsOk = true; + bool pgIsNotRunningIsOk = true; + bool monitorDisabledIsOk = false; + + if (!keeper_config_read_file(&config, + missingPgdataIsOk, + pgIsNotRunningIsOk, + monitorDisabledIsOk)) + { + /* errors have already been logged */ + exit(EXIT_CODE_BAD_CONFIG); + } + + if (!monitor_init(&monitor, config.monitor_pguri)) + { + log_fatal("Failed to contact the monitor because its URL is invalid, " + "see above for details"); + exit(EXIT_CODE_BAD_CONFIG); + } + + char *keeperEdgesJSON = KeeperFSMToJSON(); + + if (!monitor_check_fsm_reachability(&monitor, keeperEdgesJSON, &result)) + { + log_error("Failed to check FSM reachability against the monitor, " + "see above for details"); + json_free_serialized_string(keeperEdgesJSON); + exit(EXIT_CODE_MONITOR); + } + + json_free_serialized_string(keeperEdgesJSON); + + if (outputJSON) + { + JSON_Value *js = json_value_init_object(); + JSON_Object *root = json_value_get_object(js); + JSON_Value *jsMismatches = json_value_init_array(); + JSON_Array *mismatches = json_value_get_array(jsMismatches); + + for (int i = 0; i < result.count; i++) + { + FsmReachabilityMismatch *mismatch = &(result.mismatches[i]); + JSON_Value *jsEntry = json_value_init_object(); + JSON_Object *jsObj = json_value_get_object(jsEntry); + + json_object_set_number(jsObj, "pos", (double) mismatch->pos); + json_object_set_string(jsObj, "current", mismatch->currentState); + json_object_set_string(jsObj, "assigned", mismatch->assignedState); + json_object_set_string(jsObj, "comment", mismatch->comment); + + json_array_append_value(mismatches, jsEntry); + } + + json_object_set_boolean(root, "ok", result.count == 0); + json_object_set_value(root, "mismatches", jsMismatches); + + cli_pprint_json(js); + } + else if (result.count == 0) + { + log_info("OK: every monitor FSM transition has a matching keeper edge"); + } + else + { + for (int i = 0; i < result.count; i++) + { + FsmReachabilityMismatch *mismatch = &(result.mismatches[i]); + + log_error("pos %d: %s -> %s has no matching keeper edge (%s)", + mismatch->pos, + mismatch->currentState, + mismatch->assignedState, + mismatch->comment); + } + } + + exit(result.count == 0 ? EXIT_CODE_QUIT : EXIT_CODE_INTERNAL_ERROR); +} + + /* * cli_do_fsm_gv outputs the FSM as a .gv program. */ diff --git a/src/bin/pg_autoctl/cli_do_root.h b/src/bin/pg_autoctl/cli_do_root.h index 0706b95ef..c2443e35a 100644 --- a/src/bin/pg_autoctl/cli_do_root.h +++ b/src/bin/pg_autoctl/cli_do_root.h @@ -20,6 +20,7 @@ extern CommandLine do_fsm_commands; extern CommandLine fsm_state; extern CommandLine fsm_node_state; extern CommandLine fsm_list; +extern CommandLine fsm_check; extern CommandLine fsm_gv; extern CommandLine fsm_mermaid; diff --git a/src/bin/pg_autoctl/cli_inspect.c b/src/bin/pg_autoctl/cli_inspect.c index 3bb441b80..c599f7f23 100644 --- a/src/bin/pg_autoctl/cli_inspect.c +++ b/src/bin/pg_autoctl/cli_inspect.c @@ -18,12 +18,14 @@ /* * Read-only FSM sub-commands: display the current state, list reachable - * transitions, or dump the full FSM as a graphviz .gv file. + * transitions, check that every monitor transition has a matching keeper + * edge, or dump the full FSM as a graphviz .gv file. * Mutating operations (init, assign, step, nodes set) live under "manual fsm". */ static CommandLine *inspect_fsm_subcommands[] = { &fsm_state, &fsm_list, + &fsm_check, &fsm_gv, &fsm_mermaid, NULL @@ -31,7 +33,8 @@ static CommandLine *inspect_fsm_subcommands[] = { static CommandLine inspect_fsm_commands = make_command_set("fsm", - "Display keeper FSM state and transitions (read-only)", + "Display keeper FSM state and transitions, and check " + "monitor/keeper FSM reachability (read-only)", NULL, NULL, NULL, inspect_fsm_subcommands); /* diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index edc4af972..9d74ceafe 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -18,6 +18,7 @@ #include "defaults.h" #include "keeper.h" +#include "parson.h" #include "pgctl.h" #include "fsm.h" #include "log.h" @@ -1315,3 +1316,125 @@ print_fsm_for_graphviz(void) } fformat(stdout, "}\n"); } + + +/* + * AllRealNodeStates is every "real" (non-sentinel) NodeState a keeper can + * genuinely report or be assigned, used only to expand a KeeperFSMTransition + * row's ANY_STATE wildcard (see KeeperFSMToJSON's own comment) into concrete + * states -- NO_STATE (the array terminator) and ANY_STATE itself excluded. + */ +static const NodeState AllRealNodeStates[] = { + INIT_STATE, + SINGLE_STATE, + PRIMARY_STATE, + WAIT_PRIMARY_STATE, + WAIT_STANDBY_STATE, + DEMOTED_STATE, + DEMOTE_TIMEOUT_STATE, + DRAINING_STATE, + SECONDARY_STATE, + CATCHINGUP_STATE, + PREP_PROMOTION_STATE, + STOP_REPLICATION_STATE, + MAINTENANCE_STATE, + JOIN_PRIMARY_STATE, + APPLY_SETTINGS_STATE, + PREPARE_MAINTENANCE_STATE, + WAIT_MAINTENANCE_STATE, + REPORT_LSN_STATE, + FAST_FORWARD_STATE, + JOIN_SECONDARY_STATE, + DROPPED_STATE +}; + +#define ALL_REAL_NODE_STATES_COUNT \ + ((int) (sizeof(AllRealNodeStates) / sizeof(AllRealNodeStates[0]))) + + +/* + * KeeperFSMToJSONAppendEdge appends one {"current": ..., "assigned": ...} + * object to array for a single, already-concrete (current, assigned) pair. + * Factored out of KeeperFSMToJSON so its own ANY_STATE-expansion loop (see + * that function's comment) can call it once per expanded state instead of + * duplicating the object-building code. + */ +static void +KeeperFSMToJSONAppendEdge(JSON_Array *array, NodeState current, NodeState assigned) +{ + JSON_Value *jsEntry = json_value_init_object(); + JSON_Object *jsObj = json_value_get_object(jsEntry); + + json_object_set_string(jsObj, "current", NodeStateToString(current)); + json_object_set_string(jsObj, "assigned", NodeStateToString(assigned)); + + json_array_append_value(array, jsEntry); +} + + +/* + * KeeperFSMToJSON serializes KeeperFSM[] into a JSON array of + * {"current": ..., "assigned": ...} objects -- one per concrete + * (current, assigned) edge a KeeperFSMTransition row produces, walked the + * same way print_fsm_for_graphviz/print_reachable_states already do. This + * is the keeper-side half of the monitor/keeper FSM reachability + * cross-check: "pg_autoctl inspect fsm check" sends this verbatim to the + * monitor's pgautofailover.check_fsm_reachability(jsonb), which anti-joins + * it against every edge the monitor's own MonitorFSM[] table + * (group_state_machine.c) can produce (see dump_fsm_edges()'s own comment + * there) and reports any monitor transition with no matching entry here. + * + * A row's own .current can be ANY_STATE (state_matches()'s wildcard, e.g. + * fsm.c's "drop node from any state" rows) -- NodeStateToString(ANY_STATE) + * returns the literal string "#any state#", which is not a valid + * pgautofailover.replication_state and would fail check_fsm_reachability()'s + * own cast loudly (confirmed: this is exactly what happened the first time + * this function ran for real, against a live monitor+keeper pair). Expanded + * here into one concrete edge per AllRealNodeStates entry instead, mirroring + * NodeStatePatternResolveFromStates' own ANY-kind handling on the monitor + * side. .assigned is never ANY_STATE in any current KeeperFSM[] row (an + * assignment target wildcard has no sensible meaning), so it's passed + * through as-is. + * + * Deliberately omits pgKind and comment: pgKind is not modeled on the + * monitor side of this check at all (every Citus-specific KeeperFSM[] edge + * already has a NODE_KIND_ANY counterpart with the same (current, assigned) + * shape -- see this same file's fsm_mermaid.c-referencing comment), and + * comment is purely descriptive, never part of the comparison. + * + * Returns a malloc'd string (via json_serialize_to_string) the caller must + * free with json_free_serialized_string(). + */ +char * +KeeperFSMToJSON(void) +{ + JSON_Value *jsArray = json_value_init_array(); + JSON_Array *array = json_value_get_array(jsArray); + + KeeperFSMTransition transition = KeeperFSM[0]; + int transitionIndex = 0; + + while (transition.current != NO_STATE) + { + if (transition.current == ANY_STATE) + { + for (int i = 0; i < ALL_REAL_NODE_STATES_COUNT; i++) + { + KeeperFSMToJSONAppendEdge(array, AllRealNodeStates[i], + transition.assigned); + } + } + else + { + KeeperFSMToJSONAppendEdge(array, transition.current, transition.assigned); + } + + transition = KeeperFSM[++transitionIndex]; + } + + char *serialized = json_serialize_to_string(jsArray); + + json_value_free(jsArray); + + return serialized; +} diff --git a/src/bin/pg_autoctl/fsm.h b/src/bin/pg_autoctl/fsm.h index 3a8cc6f08..2ab182fc1 100644 --- a/src/bin/pg_autoctl/fsm.h +++ b/src/bin/pg_autoctl/fsm.h @@ -129,6 +129,7 @@ bool prepare_replication(Keeper *keeper, NodeState otherNodeState); */ void print_reachable_states(KeeperStateData *keeperState); void print_fsm_for_graphviz(void); +char * KeeperFSMToJSON(void); bool keeper_fsm_step(Keeper *keeper); bool keeper_fsm_step_report(Keeper *keeper); bool keeper_fsm_step_advance(Keeper *keeper); diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index adc0ce661..5b78cabce 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -115,6 +115,13 @@ typedef struct MonitorExtensionVersionParseContext bool parsedOK; } MonitorExtensionVersionParseContext; +typedef struct FsmReachabilityParseContext +{ + char sqlstate[SQLSTATE_LENGTH]; + FsmReachabilityResult *result; + bool parsedOK; +} FsmReachabilityParseContext; + static bool parseNode(PGresult *result, int rowNumber, NodeAddress *node); static void parseNodeResult(void *ctx, PGresult *result); @@ -134,6 +141,7 @@ static void printFormationSettings(void *ctx, PGresult *result); static void printFormationURI(void *ctx, PGresult *result); static void parseCoordinatorNode(void *ctx, PGresult *result); static void parseExtensionVersion(void *ctx, PGresult *result); +static void parseFsmReachabilityResult(void *ctx, PGresult *result); static bool prepare_connection_to_current_system_user(Monitor *source, Monitor *target); @@ -1232,6 +1240,102 @@ monitor_report_timeline_history(Monitor *monitor, int64_t nodeId, } +/* + * monitor_check_fsm_reachability sends this node's own KeeperFSM[] edges + * (as encoded by KeeperFSMToJSON()) to the monitor's + * pgautofailover.check_fsm_reachability(jsonb), and fills in result with + * every monitor FSM transition (MonitorFSM[], group_state_machine.c) that + * has no matching keeper edge. An empty result (result->count == 0) means + * every transition the monitor can ever assign has somewhere for this + * keeper to go. Called from "pg_autoctl inspect fsm check" + * (cli_do_fsm.c). + */ +bool +monitor_check_fsm_reachability(Monitor *monitor, + const char *keeperEdgesJSON, + FsmReachabilityResult *result) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pos, current_state, assigned_state, comment " + " FROM pgautofailover.check_fsm_reachability($1::jsonb)"; + + int paramCount = 1; + Oid paramTypes[1] = { TEXTOID }; + const char *paramValues[1]; + + FsmReachabilityParseContext context = { { 0 }, result, false }; + + paramValues[0] = keeperEdgesJSON; + + if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, + paramValues, + &context, parseFsmReachabilityResult)) + { + log_error("Failed to check FSM reachability against the monitor"); + + return false; + } + + return context.parsedOK; +} + + +/* + * parseFsmReachabilityResult parses the result of + * pgautofailover.check_fsm_reachability(jsonb) into the given + * FsmReachabilityResult. + */ +static void +parseFsmReachabilityResult(void *ctx, PGresult *result) +{ + FsmReachabilityParseContext *context = (FsmReachabilityParseContext *) ctx; + FsmReachabilityResult *fsmResult = context->result; + + int nTuples = PQntuples(result); + + if (nTuples > FSM_REACHABILITY_MISMATCH_MAX_COUNT) + { + log_error("Query returned %d rows, pg_auto_failover supports only up " + "to %d FSM reachability mismatches at the moment", + nTuples, FSM_REACHABILITY_MISMATCH_MAX_COUNT); + context->parsedOK = false; + return; + } + + if (PQnfields(result) != 4) + { + log_error("Query returned %d columns, expected 4", PQnfields(result)); + context->parsedOK = false; + return; + } + + fsmResult->count = nTuples; + + for (int i = 0; i < nTuples; i++) + { + FsmReachabilityMismatch *mismatch = &(fsmResult->mismatches[i]); + + if (!stringToInt(PQgetvalue(result, i, 0), &mismatch->pos)) + { + log_error("Failed to parse FSM reachability pos \"%s\"", + PQgetvalue(result, i, 0)); + context->parsedOK = false; + return; + } + + strlcpy(mismatch->currentState, PQgetvalue(result, i, 1), + sizeof(mismatch->currentState)); + strlcpy(mismatch->assignedState, PQgetvalue(result, i, 2), + sizeof(mismatch->assignedState)); + strlcpy(mismatch->comment, PQgetvalue(result, i, 3), + sizeof(mismatch->comment)); + } + + context->parsedOK = true; +} + + /* * monitor_accept_timeline pins the accepted timeline for a (formation, * group) after an operator has resolved a detected fork, by calling diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index dec18add9..007ab8760 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -84,6 +84,29 @@ typedef struct MonitorExtensionVersion char installedVersion[BUFSIZE]; } MonitorExtensionVersion; +/* + * One row per monitor FSM transition (MonitorFSM[], group_state_machine.c) + * with no matching edge in this node's own KeeperFSM[] -- the result of + * "pg_autoctl inspect fsm check" (see monitor_check_fsm_reachability() and + * pgautofailover.check_fsm_reachability(jsonb)). Fixed-capacity, same + * pattern as MonitorEventsArray above. + */ +#define FSM_REACHABILITY_MISMATCH_MAX_COUNT 1024 + +typedef struct FsmReachabilityMismatch +{ + int pos; + char currentState[NAMEDATALEN]; + char assignedState[NAMEDATALEN]; + char comment[BUFSIZE]; +} FsmReachabilityMismatch; + +typedef struct FsmReachabilityResult +{ + int count; + FsmReachabilityMismatch mismatches[FSM_REACHABILITY_MISMATCH_MAX_COUNT]; +} FsmReachabilityResult; + typedef struct CoordinatorNodeAddress { bool found; @@ -167,6 +190,9 @@ bool monitor_report_postgres_version(Monitor *monitor, int64_t nodeId, PostgresVersionInfo *pgVersion); bool monitor_report_timeline_history(Monitor *monitor, int64_t nodeId, const char *historyJSON); +bool monitor_check_fsm_reachability(Monitor *monitor, + const char *keeperEdgesJSON, + FsmReachabilityResult *result); bool monitor_accept_timeline(Monitor *monitor, char *formation, int group, int tli, char *decidedBy); bool monitor_print_timeline(Monitor *monitor, char *formation, int group); diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out new file mode 100644 index 000000000..e89649a07 --- /dev/null +++ b/src/monitor/expected/check_fsm_reachability.out @@ -0,0 +1,111 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Exercises the mechanics of pgautofailover.check_fsm_reachability(jsonb) +-- (and the pgautofailover.dump_fsm_edges() it's built on) against small, +-- synthetic keeper-edge inputs -- not the real KeeperFSM[] table, which +-- lives in the pg_autoctl binary, not this database. The real, end-to-end +-- completeness check (does the real KeeperFSM[] cover every real +-- MonitorFSM[] edge) is run separately, live, via +-- "pg_autoctl inspect fsm check" against a real monitor+keeper pair -- this +-- test only confirms the SQL-side comparison mechanism itself behaves +-- correctly: an edge present in the keeper_edges parameter drops out of the +-- mismatch list, an edge absent from it stays in, and an unrecognized state +-- name fails loudly rather than silently never matching. +-- Every edge dump_fsm_edges() can produce, fully resolved: a fixed, +-- reviewable count for the current MonitorFSM[] -- changes only when a row +-- is added, removed, or edited there, same as fsm.sql's own row count. +SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); + total_edge_count +------------------ + 231 +(1 row) + +-- pos 301 ("converged secondary, reportedTLI not an ancestor of reference +-- -> catchingup", single edge) and pos 343 ("stop_replication, primary +-- converged prepare_maintenance -> wait_primary + maintenance", two edges, +-- one per role) are stable, non-reflexive, non-api_triggered rows -- good, +-- deterministic targets to check both single- and dual-edge rows against. +SELECT pos, current_state, assigned_state + FROM pgautofailover.dump_fsm_edges() + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + pos | current_state | assigned_state +-----+---------------------+---------------- + 301 | secondary | catchingup + 343 | stop_replication | wait_primary + 343 | prepare_maintenance | maintenance +(3 rows) + +-- Two categories of edges dump_fsm_edges() deliberately never produces, see +-- its own comment for why: reflexive (current == assigned) edges, and the +-- whole api_triggered section (every row there resolves activeNode to a +-- specific role via hand-written C before dispatch, so its own +-- NodeStatePattern was never meant to double as a full reachability +-- precondition). pos 403 ("all nodes async, zero secondaries -> +-- wait_primary") has both an ordinary edge (primary/apply_settings -> +-- wait_primary) and a reflexive one (wait_primary -> wait_primary) in its +-- own source pattern -- only the former should appear. pos 105 (api +-- triggered: perform_failover) should produce no edges at all. +SELECT pos, current_state, assigned_state + FROM pgautofailover.dump_fsm_edges() + WHERE pos = 403 + ORDER BY current_state; + pos | current_state | assigned_state +-----+----------------+---------------- + 403 | primary | wait_primary + 403 | apply_settings | wait_primary +(2 rows) + +SELECT count(*) AS api_triggered_edge_count + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE f.section LIKE 'api_triggered%'; + api_triggered_edge_count +-------------------------- + 0 +(1 row) + +-- An empty keeper_edges: every single edge dump_fsm_edges() produces comes +-- back as a mismatch, so this count must equal total_edge_count above. +SELECT count(*) AS missing_with_empty_keeper_edges + FROM pgautofailover.check_fsm_reachability('[]'::jsonb); + missing_with_empty_keeper_edges +--------------------------------- + 231 +(1 row) + +-- Providing exactly pos 301's own edge, plus one of pos 343's two edges +-- (leaving its other edge, prepare_maintenance->maintenance, still +-- unmatched): pos 301 must disappear entirely from the mismatch list, pos +-- 343 must still appear, but only once. +SELECT pos, current_state, assigned_state + FROM pgautofailover.check_fsm_reachability( + '[{"current":"secondary","assigned":"catchingup"}, + {"current":"stop_replication","assigned":"wait_primary"}]'::jsonb) + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + pos | current_state | assigned_state +-----+---------------------+---------------- + 343 | prepare_maintenance | maintenance +(1 row) + +-- Providing both of pos 343's edges too: it must now disappear as well. +SELECT pos, current_state, assigned_state + FROM pgautofailover.check_fsm_reachability( + '[{"current":"secondary","assigned":"catchingup"}, + {"current":"stop_replication","assigned":"wait_primary"}, + {"current":"prepare_maintenance","assigned":"maintenance"}]'::jsonb) + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + pos | current_state | assigned_state +-----+---------------+---------------- +(0 rows) + +-- An unrecognized state name in keeper_edges must fail loudly (a real cast +-- error), not silently never match -- the same "fail loudly on drift" +-- instinct as AssignDeclaredGoalState's own trust-check. +SELECT * FROM pgautofailover.check_fsm_reachability( + '[{"current":"not_a_real_state","assigned":"catchingup"}]'::jsonb); +ERROR: invalid input value for enum pgautofailover.replication_state: "not_a_real_state" +CONTEXT: SQL function "check_fsm_reachability" statement 1 diff --git a/src/monitor/expected/cluster_init_failover_rule_attribution.out b/src/monitor/expected/cluster_init_failover_rule_attribution.out index e83f2cb8f..7fd593626 100644 --- a/src/monitor/expected/cluster_init_failover_rule_attribution.out +++ b/src/monitor/expected/cluster_init_failover_rule_attribution.out @@ -249,7 +249,7 @@ reportedstate | wait_primary goalstate | primary rule_pos | 411 rule_section | primary_node -rule_comment | wait_primary, >=1 quorum secondary -> primary +rule_comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 11 ]+------------------------------------------------------------------------------------------------------------- eventid | 196 nodename | cifra_p diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index 2d7f30062..1f4a4458b 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -901,7 +901,7 @@ group_conditions | replicationQuorumCountIsZero=true, secondaryNodes active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | t -comment | all nodes async, zero secondaries -> wait_primary +comment | all nodes async, zero secondaries -> wait_primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 64 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 405 section | primary_node @@ -915,7 +915,7 @@ group_conditions | replicationQuorumCountIsZero=true, secondaryNodes active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t -comment | all nodes async, >=1 secondary -> primary +comment | all nodes async, >=1 secondary -> primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 65 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 407 section | primary_node @@ -929,7 +929,7 @@ group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNod active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | t -comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary +comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 66 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 409 section | primary_node @@ -943,7 +943,7 @@ group_conditions | numberSyncStandbysIsZero=false, secondaryQuorumNo active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t -comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) +comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 67 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 411 section | primary_node @@ -957,7 +957,7 @@ group_conditions | secondaryQuorumNodesCountIsZero=false active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t -comment | wait_primary, >=1 quorum secondary -> primary +comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 68 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 413 section | primary_node @@ -971,7 +971,7 @@ group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNod active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | t -comment | apply_settings, both zero -> wait_primary +comment | apply_settings, both zero -> wait_primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 69 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 415 section | primary_node @@ -985,7 +985,7 @@ group_conditions | numberSyncStandbysIsZero=false active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t -comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) +comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 70 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 417 section | primary_node @@ -999,7 +999,7 @@ group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNod active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t -comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) +comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 419 section | primary_node diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 27354f4c6..70dd396c4 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -176,8 +176,8 @@ typedef struct ReplicationStateSet */ #define STATES(...) \ { \ - .states = { __VA_ARGS__ }, \ - .count = STATES_NARG(__VA_ARGS__) \ + .states = { __VA_ARGS__ }, \ + .count = STATES_NARG(__VA_ARGS__) \ } typedef struct NodeStatePattern @@ -211,31 +211,33 @@ MatchStateSet(ReplicationState actual, ReplicationStateSet declared) static const NodeStatePattern FSM_PRIMARY_OR_WAIT_OR_JOIN = { .kind = NODE_STATE_STABLE, .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, - REPLICATION_STATE_JOIN_PRIMARY, - REPLICATION_STATE_PRIMARY), + REPLICATION_STATE_JOIN_PRIMARY, + REPLICATION_STATE_PRIMARY), }; /* WAIT_PRIMARY/JOIN_PRIMARY only, not PRIMARY -- a distinct, narrower set from the one above */ static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY = { .kind = NODE_STATE_STABLE, .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, - REPLICATION_STATE_JOIN_PRIMARY), + REPLICATION_STATE_JOIN_PRIMARY), }; -/* the "primary role" states inside ProceedGroupStateForPrimaryNode -- a different - * three-element set from FSM_PRIMARY_OR_WAIT_OR_JOIN above (no JOIN_PRIMARY, has APPLY_SETTINGS) */ +/* the "primary role" states MONITOR_FSM_SECTION_PRIMARY_NODE's own rows match + * against (the declarative replacement for the old, now-removed + * ProceedGroupStateForPrimaryNode()) -- a different three-element set from + * FSM_PRIMARY_OR_WAIT_OR_JOIN above (no JOIN_PRIMARY, has APPLY_SETTINGS) */ static const NodeStatePattern FSM_PRIMARY_ROLE_STATES = { .kind = NODE_STATE_STABLE, .reportedStates = STATES(REPLICATION_STATE_PRIMARY, - REPLICATION_STATE_WAIT_PRIMARY, - REPLICATION_STATE_APPLY_SETTINGS), + REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_APPLY_SETTINGS), }; /* same "primary role" scope, minus WAIT_PRIMARY -- a narrower enumerated STABLE set */ static const NodeStatePattern FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY = { .kind = NODE_STATE_STABLE, .reportedStates = STATES(REPLICATION_STATE_PRIMARY, - REPLICATION_STATE_APPLY_SETTINGS), + REPLICATION_STATE_APPLY_SETTINGS), }; /* reported WAIT_PRIMARY, goal in {WAIT_PRIMARY, PRIMARY} -- join_secondary's cascade row */ @@ -248,7 +250,8 @@ static const NodeStatePattern FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY = { /* reported in {WAIT_PRIMARY,JOIN_PRIMARY}, goal PRIMARY -- demoted->catchingup, first disjunct */ static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY = { .kind = NODE_STATE_TRANSITIONING, - .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, REPLICATION_STATE_JOIN_PRIMARY), + .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_JOIN_PRIMARY), .assignedStates = STATES(REPLICATION_STATE_PRIMARY), }; @@ -289,7 +292,8 @@ static const NodeStatePattern FSM_REPORTED_DEMOTE_TIMEOUT = { * failover" guard, the direct (non-cascading) entry into ProceedGroupStateForMSFailover */ static const NodeStatePattern FSM_REPORT_LSN_OR_FAST_FORWARD = { .kind = NODE_STATE_STABLE, - .reportedStates = STATES(REPLICATION_STATE_REPORT_LSN, REPLICATION_STATE_FAST_FORWARD), + .reportedStates = STATES(REPLICATION_STATE_REPORT_LSN, + REPLICATION_STATE_FAST_FORWARD), }; @@ -321,7 +325,8 @@ NodeStateMatchesPattern(const AutoFailoverNode *node, const NodeStatePattern *pa case NODE_STATE_NOT_STABLE: { - return !((reported == goal) && MatchStateSet(reported, pattern->reportedStates)); + return !((reported == goal) && MatchStateSet(reported, + pattern->reportedStates)); } case NODE_STATE_REPORTED: @@ -453,22 +458,24 @@ NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) BoolMatchesPattern(status->isHealthy, pattern->isHealthy) && BoolMatchesPattern(status->isUnhealthy, pattern->isUnhealthy) && BoolMatchesPattern(status->candidateEligible, pattern->candidateEligible) && - BoolMatchesPattern(IsInPrimaryState(status->node), pattern->isInPrimaryState) && + BoolMatchesPattern(IsInPrimaryState(status->node), + pattern->isInPrimaryState) && BoolMatchesPattern(IsInMaintenance(status->node), pattern->isInMaintenance) && - BoolMatchesPattern(IsDemotedPrimary(status->node), pattern->isDemotedPrimary) && + BoolMatchesPattern(IsDemotedPrimary(status->node), + pattern->isDemotedPrimary) && BoolMatchesPattern(status->node != NULL && - CanTakeWritesInState(status->node->goalState), - pattern->canTakeWrites) && + CanTakeWritesInState(status->node->goalState), + pattern->canTakeWrites) && BoolMatchesPattern(CandidateNodeIsReadyToStreamWAL(status->node), - pattern->isReadyToStreamWAL) && + pattern->isReadyToStreamWAL) && BoolMatchesPattern(NodeIsDrainTimeExpired(status->node, status->ctx), - pattern->drainTimeExpired) && + pattern->drainTimeExpired) && BoolMatchesPattern(status->isCitusWorkerGroup, pattern->isCitusWorkerGroup) && BoolMatchesPattern(status->replicationQuorum, pattern->replicationQuorum) && BoolMatchesPattern(status->isComparableToReferenceTli, - pattern->isComparableToReferenceTli) && + pattern->isComparableToReferenceTli) && BoolMatchesPattern(unreachableFromDemoteTimeout, - pattern->unreachableFromDemoteTimeout); + pattern->unreachableFromDemoteTimeout); } @@ -514,7 +521,8 @@ typedef struct ApiTriggerPattern MonitorApiFunction function; /* meaningful only when kind == API_TRIGGER_SPECIFIC */ } ApiTriggerPattern; -#define API_TRIGGER(fn) ((ApiTriggerPattern){ .kind = API_TRIGGER_SPECIFIC, .function = (fn) }) +#define API_TRIGGER(fn) ((ApiTriggerPattern) { .kind = API_TRIGGER_SPECIFIC, .function = \ + (fn) }) static bool MatchApiTrigger(MonitorApiFunction actual, ApiTriggerPattern pattern) @@ -649,7 +657,7 @@ typedef struct NodeActiveContext typedef struct NodeActiveContextPattern { ApiTriggerPattern apiTrigger; /* omitted -> {0} -> API_TRIGGER_NODE_ACTIVE, matching every - row's existing meaning with no changes required elsewhere */ + * row's existing meaning with no changes required elsewhere */ BoolPattern groupHasExactlyOneNode; BoolPattern groupHasExactlyTwoNodes; @@ -713,8 +721,8 @@ typedef struct GoalStateAssignment * wander into whichever row happens to be next. */ typedef void (*MonitorExtraActionFunction) (GroupStateContext *ctx, - NodeActiveContext *nac, - char *message); + NodeActiveContext *nac, + char *message); /* * MonitorFSMSection itself is declared in group_state_machine.h, not here: @@ -758,8 +766,8 @@ typedef struct MonitorFSMTransition NodeStatusPattern activeNode; NodeStatusPattern primaryNode; NodeStatusPattern candidateNode; /* MS-failover sub-section rows only; see - NodeActiveContext's own comment on - .candidateNode */ + * NodeActiveContext's own comment on + * .candidateNode */ NodeActiveContextPattern conditions; GoalStateAssignment activeNodeAssignedState; @@ -879,12 +887,12 @@ typedef struct MonitorFSMTransition */ static const MonitorFSMTransition MonitorFSM[]; -#define MonitorFSM_EarlyChecksStart 15 -#define MonitorFSM_FromContextStart 21 -#define MonitorFSM_FromContextResumeStart 24 -#define MonitorFSM_MSFailoverStart 52 +#define MonitorFSM_EarlyChecksStart 15 +#define MonitorFSM_FromContextStart 21 +#define MonitorFSM_FromContextResumeStart 24 +#define MonitorFSM_MSFailoverStart 52 #define MonitorFSM_PrimaryNodeSectionStart 61 -#define MonitorFSM_SIZE 72 +#define MonitorFSM_SIZE 72 /* Forward-declared for the same reason as MonitorFSM[] above: used by * extraActions (ActionRunPrimaryNodeTransition) defined before its real @@ -905,32 +913,43 @@ RuleMatches(const NodeActiveContext *nac, const MonitorFSMTransition *rule) NodeMatchesPattern(&nac->primaryNode, &rule->primaryNode) && NodeMatchesPattern(&nac->candidateNode, &rule->candidateNode) && - BoolMatchesPattern(nac->groupHasExactlyOneNode, cond->groupHasExactlyOneNode) && - BoolMatchesPattern(nac->groupHasExactlyTwoNodes, cond->groupHasExactlyTwoNodes) && - BoolMatchesPattern(nac->groupHasMoreThanTwoNodes, cond->groupHasMoreThanTwoNodes) && - BoolMatchesPattern(nac->anyOtherNodeWaitingStandby, cond->anyOtherNodeWaitingStandby) && - BoolMatchesPattern(nac->numberSyncStandbysIsZero, cond->numberSyncStandbysIsZero) && + BoolMatchesPattern(nac->groupHasExactlyOneNode, + cond->groupHasExactlyOneNode) && + BoolMatchesPattern(nac->groupHasExactlyTwoNodes, + cond->groupHasExactlyTwoNodes) && + BoolMatchesPattern(nac->groupHasMoreThanTwoNodes, + cond->groupHasMoreThanTwoNodes) && + BoolMatchesPattern(nac->anyOtherNodeWaitingStandby, + cond->anyOtherNodeWaitingStandby) && + BoolMatchesPattern(nac->numberSyncStandbysIsZero, + cond->numberSyncStandbysIsZero) && BoolMatchesPattern(nac->replicationQuorumCountIsZero, - cond->replicationQuorumCountIsZero) && - BoolMatchesPattern(nac->secondaryNodesCountIsZero, cond->secondaryNodesCountIsZero) && + cond->replicationQuorumCountIsZero) && + BoolMatchesPattern(nac->secondaryNodesCountIsZero, + cond->secondaryNodesCountIsZero) && BoolMatchesPattern(nac->secondaryQuorumNodesCountIsZero, - cond->secondaryQuorumNodesCountIsZero) && - BoolMatchesPattern(nac->atLeastOneHealthyCandidate, cond->atLeastOneHealthyCandidate) && - BoolMatchesPattern(nac->walWithinPromoteThreshold, cond->walWithinPromoteThreshold) && - BoolMatchesPattern(nac->walWithinSyncThreshold, cond->walWithinSyncThreshold) && - BoolMatchesPattern(nac->activeAndPrimaryTliMatch, cond->activeAndPrimaryTliMatch) && + cond->secondaryQuorumNodesCountIsZero) && + BoolMatchesPattern(nac->atLeastOneHealthyCandidate, + cond->atLeastOneHealthyCandidate) && + BoolMatchesPattern(nac->walWithinPromoteThreshold, + cond->walWithinPromoteThreshold) && + BoolMatchesPattern(nac->walWithinSyncThreshold, + cond->walWithinSyncThreshold) && + BoolMatchesPattern(nac->activeAndPrimaryTliMatch, + cond->activeAndPrimaryTliMatch) && BoolMatchesPattern(nac->primaryIsWaitPrimaryPresumedDead, - cond->primaryIsWaitPrimaryPresumedDead) && + cond->primaryIsWaitPrimaryPresumedDead) && BoolMatchesPattern(nac->failoverInProgress, cond->failoverInProgress) && - BoolMatchesPattern(nac->replicationStallExceeded, cond->replicationStallExceeded) && + BoolMatchesPattern(nac->replicationStallExceeded, + cond->replicationStallExceeded) && BoolMatchesPattern(nac->lastHealthySyncStandbyGoingToMaintenance, - cond->lastHealthySyncStandbyGoingToMaintenance) && + cond->lastHealthySyncStandbyGoingToMaintenance) && BoolMatchesPattern(nac->activeNodeAllWalSourcesUnhealthy, - cond->activeNodeAllWalSourcesUnhealthy) && + cond->activeNodeAllWalSourcesUnhealthy) && BoolMatchesPattern(nac->candidatePromotionInProgress, - cond->candidatePromotionInProgress) && + cond->candidatePromotionInProgress) && BoolMatchesPattern(nac->mostAdvancedCandidateWithinPromoteThreshold, - cond->mostAdvancedCandidateWithinPromoteThreshold) && + cond->mostAdvancedCandidateWithinPromoteThreshold) && BoolMatchesPattern(nac->guardDataLossEnabled, cond->guardDataLossEnabled) && BoolMatchesPattern(nac->inMSFailoverCluster, cond->inMSFailoverCluster); } @@ -938,7 +957,7 @@ RuleMatches(const NodeActiveContext *nac, const MonitorFSMTransition *rule) static int FindMatchingMonitorFSMRuleIndexFrom(const MonitorFSMTransition table[], int tableSize, - int startIndex, const NodeActiveContext *nac) + int startIndex, const NodeActiveContext *nac) { for (int i = startIndex; i < tableSize; i++) { @@ -958,7 +977,7 @@ FindMatchingMonitorFSMRuleIndexFrom(const MonitorFSMTransition table[], int tabl */ static void AssignDeclaredGoalState(const MonitorFSMTransition *rule, AutoFailoverNode *node, - ReplicationState state, char *message) + ReplicationState state, char *message) { #ifdef USE_ASSERT_CHECKING bool declared = @@ -976,7 +995,7 @@ AssignDeclaredGoalState(const MonitorFSMTransition *rule, AutoFailoverNode *node static void DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, - const MonitorFSMTransition *rule) + const MonitorFSMTransition *rule) { char message[BUFSIZE] = { 0 }; @@ -1011,13 +1030,13 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, if (rule->activeNodeAssignedState.kind == GOAL_STATE_SET) { AssignDeclaredGoalState(rule, nac->activeNode.node, - rule->activeNodeAssignedState.state, message); + rule->activeNodeAssignedState.state, message); } if (rule->otherNodeAssignedState.kind == GOAL_STATE_SET) { AssignDeclaredGoalState(rule, nac->primaryNode.node, - rule->otherNodeAssignedState.state, message); + rule->otherNodeAssignedState.state, message); } CurrentMonitorFSMRulePos = savedRulePos; @@ -1042,7 +1061,8 @@ static bool FindAndDispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, int startIndex, int endIndex) { - int index = FindMatchingMonitorFSMRuleIndexFrom(MonitorFSM, endIndex, startIndex, nac); + int index = FindMatchingMonitorFSMRuleIndexFrom(MonitorFSM, endIndex, startIndex, + nac); if (index < 0) { @@ -1161,8 +1181,9 @@ ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *me * assigning DRAINING/MAINTENANCE to the primary -- it always falls through to * try ProceedGroupStateForMSFailover next, in the SAME outer if-block, and if * THAT declines (returns false), falls through further still to the rest of - * ProceedGroupStateFromContext's own if-chain (the report_lsn/prepare_ - * promotion/stop_replication/... rows, for this SAME activeNode). + * the original source's own if-chain inside ProceedGroupStateFromContext (now + * the report_lsn/prepare_promotion/stop_replication/... rows further down + * MonitorFSM[]'s REPORTING_NODE section, for this SAME activeNode). * * This has to be ONE row/action, not three separate declarative rows sharing * this action (as an earlier version of this file had it): once dispatch @@ -1184,23 +1205,31 @@ ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *me * in one call -- no repeated re-dispatch needed to walk past intervening * non-matches. * - * NOTE on naming: MonitorFSM_FromContextResumeStart is NOT the design doc's - * MonitorFSM_MSFailoverClusterStart, despite marking a conceptually similar - * "resume point". The doc's constant names the start of a real, converted - * section -- roughly ten declarative rows (a fourth candidateNode role, - * conditions like candidatePromotionInProgress and - * mostAdvancedCandidateWithinPromoteThreshold, an otherNodesFn hook) that - * partially replace BuildCandidateList/SelectFailoverCandidateNode/ - * PromoteSelectedNode. No such section exists in this table: - * ProceedGroupStateForMSFailover() and everything it calls stayed one - * opaque hand-written function, called wholesale, exactly as before this - * refactor. MonitorFSM_FromContextResumeStart just marks "resume scanning - * ordinary FromContext rows here" -- a narrower thing than what the doc's - * identically-purposed constant was named for. + * NOTE on naming: MonitorFSM_FromContextResumeStart is NOT + * MonitorFSM_MSFailoverStart, despite both marking a conceptually similar + * "resume point" -- they bound two different things. MonitorFSM_FromContextResumeStart + * (used here) just marks "resume scanning ordinary REPORTING_NODE rows after + * ProceedGroupStateForMSFailover declines" -- ProceedGroupStateForMSFailover() + * itself, and the BuildCandidateList/SelectFailoverCandidateNode/ + * PromoteSelectedNode functions it calls, stay hand-written C, called + * wholesale from here exactly as before this refactor (the candidate-selection + * algorithm itself doesn't reduce to declarative conditions any more cleanly + * than it did before -- see this file's own top-of-file design comment). + * MonitorFSM_MSFailoverStart, by contrast, bounds the *separate* nine-row + * MS-failover cluster (pos 363-379, "MS-failover / candidate-selection + * cluster" section below) that those same hand-written functions now reach + * *into*, at their own tail end, via TryMSFailoverDeclarativeRow/ + * TryFanOutReportLsnRow/DispatchMonitorFSMRuleByPos -- covering the plain + * per-node goal assignments (retry-reset, join_secondary, BuildCandidateList's + * own report_lsn fan-out, PromoteSelectedNode's prepare_promotion/fast_forward + * choice) that those functions used to make via a raw AssignGoalState call, + * with the original call kept as an unconditional fallback on no match. The + * candidate-selection algorithm's own logic (priority sort, LSN comparison, + * WAL-fetch orchestration) is not part of either bounded range. */ static void ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, - char *message) + char *message) { AutoFailoverNode *primaryNode = nac->primaryNode.node; @@ -1249,7 +1278,8 @@ ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext * * fallthrough either way -- so its return value is simply discarded here. */ static void -ActionRunPlainMSFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, char *message) +ActionRunPlainMSFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) { (void) ProceedGroupStateForMSFailover(ctx, nac->primaryNode.node); } @@ -1264,19 +1294,22 @@ ActionRunPlainMSFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, * activeNode role (see BuildForPrimaryNodeNodeActiveContext). */ static void -ActionRunPrimaryNodeTransition(GroupStateContext *ctx, NodeActiveContext *nac, char *message) +ActionRunPrimaryNodeTransition(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) { NodeActiveContext primaryNac; BuildForPrimaryNodeNodeActiveContext(ctx, nac->primaryNode.node, &primaryNac); - (void) FindAndDispatchMonitorFSMRule(ctx, &primaryNac, MonitorFSM_PrimaryNodeSectionStart, + (void) FindAndDispatchMonitorFSMRule(ctx, &primaryNac, + MonitorFSM_PrimaryNodeSectionStart, MonitorFSM_SIZE); } static void -ActionCatchupUnhealthySecondaries(GroupStateContext *ctx, NodeActiveContext *nac, char *message) +ActionCatchupUnhealthySecondaries(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) { AutoFailoverNode *primaryNode = nac->activeNode.node; List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); @@ -1389,12 +1422,14 @@ BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *prim * for, in ActionCatchupUnhealthySecondaries above). */ static void -BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *primaryNode, +BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, + AutoFailoverNode *primaryNode, NodeActiveContext *nac) { memset(nac, 0, sizeof(NodeActiveContext)); BuildNodeStatus(ctx, primaryNode, &nac->activeNode); + /* .primaryNode role is unused by every row in MonitorFSM[]'s ForPrimaryNode section -- * primaryNode IS activeNode here, so every condition is expressed against .activeNode * directly. */ @@ -1512,7 +1547,8 @@ ActionFanOutReportLsnOnPrimaryRemoval(GroupStateContext *ctx, NodeActiveContext */ static void BuildApiTriggerNodeActiveContext(GroupStateContext *ctx, MonitorApiFunction apiFunction, - AutoFailoverNode *activeNode, AutoFailoverNode *primaryNode, + AutoFailoverNode *activeNode, + AutoFailoverNode *primaryNode, NodeActiveContext *nac) { memset(nac, 0, sizeof(NodeActiveContext)); @@ -1592,7 +1628,8 @@ ProceedGroupStateForApiTrigger(MonitorApiFunction apiFunction, BuildGroupStateContext(&ctx, activeNode); BuildApiTriggerNodeActiveContext(&ctx, apiFunction, activeNode, primaryNode, &nac); - int index = FindMatchingMonitorFSMRuleIndexFrom(MonitorFSM, MonitorFSM_EarlyChecksStart, + int index = FindMatchingMonitorFSMRuleIndexFrom(MonitorFSM, + MonitorFSM_EarlyChecksStart, 0, &nac); if (index < 0) @@ -1772,8 +1809,10 @@ static const MonitorFSMTransition MonitorFSM[] = { .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), .lastHealthySyncStandbyGoingToMaintenance = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_REPORTED, - .reportedStates = STATES(REPLICATION_STATE_SECONDARY, - REPLICATION_STATE_CATCHINGUP) } }, + .reportedStates = STATES( + REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP) } + }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_MAINTENANCE), .otherNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), @@ -1787,8 +1826,10 @@ static const MonitorFSMTransition MonitorFSM[] = { { .pos = 115, .section = MONITOR_FSM_SECTION_API_TRIGGERED, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE) }, .activeNode = { .statePattern = { .kind = NODE_STATE_REPORTED, - .reportedStates = STATES(REPLICATION_STATE_SECONDARY, - REPLICATION_STATE_CATCHINGUP) } }, + .reportedStates = STATES( + REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP) } + }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), .comment = "start_maintenance, secondary, ordinary case -> maintenance" }, @@ -1863,9 +1904,12 @@ static const MonitorFSMTransition MonitorFSM[] = { * exists and isn't already apply_settings. */ { .pos = 125, .section = MONITOR_FSM_SECTION_API_TRIGGERED, - .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY) }, + .conditions = { .apiTrigger = API_TRIGGER( + API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY) }, .activeNode = { .statePattern = { .kind = NODE_STATE_NOT_STABLE, - .reportedStates = STATES(REPLICATION_STATE_APPLY_SETTINGS) } }, + .reportedStates = STATES( + REPLICATION_STATE_APPLY_SETTINGS) } + }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), .comment = "set_node_candidate_priority, primary not already apply_settings -> " "apply_settings" }, @@ -1875,9 +1919,12 @@ static const MonitorFSMTransition MonitorFSM[] = { * shape as set_node_candidate_priority above. */ { .pos = 127, .section = MONITOR_FSM_SECTION_API_TRIGGERED, - .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_SET_NODE_REPLICATION_QUORUM) }, + .conditions = { .apiTrigger = API_TRIGGER( + API_FUNCTION_SET_NODE_REPLICATION_QUORUM) }, .activeNode = { .statePattern = { .kind = NODE_STATE_NOT_STABLE, - .reportedStates = STATES(REPLICATION_STATE_APPLY_SETTINGS) } }, + .reportedStates = STATES( + REPLICATION_STATE_APPLY_SETTINGS) } + }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), .comment = "set_node_replication_quorum, primary not already apply_settings -> " "apply_settings" }, @@ -1891,10 +1938,13 @@ static const MonitorFSMTransition MonitorFSM[] = { * via this row's own no-match ERROR rather than silently). */ { .pos = 129, .section = MONITOR_FSM_SECTION_API_TRIGGERED, - .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS) }, + .conditions = { .apiTrigger = API_TRIGGER( + API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS) }, .activeNode = { .statePattern = { .kind = NODE_STATE_ASSIGNED, - .assignedStates = STATES(REPLICATION_STATE_PRIMARY, - REPLICATION_STATE_WAIT_PRIMARY) } }, + .assignedStates = STATES( + REPLICATION_STATE_PRIMARY, + REPLICATION_STATE_WAIT_PRIMARY) } + }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), .comment = "set_formation_number_sync_standbys, primary in primary/wait_primary " "-> apply_settings" }, @@ -1949,7 +1999,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isComparableToReferenceTli = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), - .comment = "converged secondary, reportedTLI not an ancestor of reference -> catchingup" }, + .comment = + "converged secondary, reportedTLI not an ancestor of reference -> catchingup" }, /* replication stall (#997): primary healthy, no standby past replication_stall_timeout */ { .pos = 303, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -1957,7 +2008,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .isHealthy = BOOL_TRUE }, .conditions = { .replicationStallExceeded = BOOL_TRUE }, .otherNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .comment = "primary healthy, no standby past replication_stall_timeout -> wait_primary" }, + .comment = + "primary healthy, no standby past replication_stall_timeout -> wait_primary" }, /* nodesCount>2, primary unhealthy -- draining/maintenance/nothing decided inside the * action, then the MS-failover cascade unconditionally; must be a single row, see @@ -1966,7 +2018,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .primaryNode = { .isUnhealthy = BOOL_TRUE }, .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE }, .extraAction = ActionRunMultiStandbyFailoverCascade, - .comment = "nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade" }, + .comment = + "nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade" }, /* report_lsn, primary converged wait/join_primary, healthy */ { .pos = 307, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -1974,7 +2027,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY, .isHealthy = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), - .comment = "report_lsn, primary converged wait/join_primary, healthy -> secondary" }, + .comment = + "report_lsn, primary converged wait/join_primary, healthy -> secondary" }, /* report_lsn, primary converged primary, healthy */ { .pos = 309, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2022,7 +2076,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .replicationQuorum = BOOL_FALSE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), - .comment = "wait_standby (not a quorum member), primary converged primary -> catchingup" }, + .comment = + "wait_standby (not a quorum member), primary converged primary -> catchingup" }, /* caught up, same TLI as primary, within sync threshold */ { .pos = 321, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2057,8 +2112,9 @@ static const MonitorFSMTransition MonitorFSM[] = { .conditions = { .walWithinPromoteThreshold = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), .otherNodeAssignedState = GOAL(REPLICATION_STATE_DRAINING), - .comment = "primary fails, not already wait_primary -> secondary -> prepare_promotion, " - "primary -> draining (2 of 2)" }, + .comment = + "primary fails, not already wait_primary -> secondary -> prepare_promotion, " + "primary -> draining (2 of 2)" }, /* wait_maintenance, primary converged wait_primary */ { .pos = 327, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2072,14 +2128,16 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, .primaryNode = { .statePattern = FSM_NOT_ASSIGNED_WAIT_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), - .comment = "wait_maintenance, primary's goal no longer wait_primary -> maintenance" }, + .comment = + "wait_maintenance, primary's goal no longer wait_primary -> maintenance" }, /* prepare_promotion, primary converged prepare_maintenance */ { .pos = 331, .section = MONITOR_FSM_SECTION_REPORTING_NODE, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), - .comment = "prepare_promotion, primary converged prepare_maintenance -> stop_replication" }, + .comment = + "prepare_promotion, primary converged prepare_maintenance -> stop_replication" }, /* Citus worker, primary present */ { .pos = 333, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2088,7 +2146,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .primaryNode = { .exists = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), - .comment = "Citus worker prepare_promotion, primary present -> wait_primary + demoted" }, + .comment = + "Citus worker prepare_promotion, primary present -> wait_primary + demoted" }, /* Citus worker, primary removed */ { .pos = 335, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2105,8 +2164,9 @@ static const MonitorFSMTransition MonitorFSM[] = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY), .isInMaintenance = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), - .comment = "prepare_promotion, primary already converged wait_primary (issue #1168) -> " - "stop_replication only (1 of 2)" }, + .comment = + "prepare_promotion, primary already converged wait_primary (issue #1168) -> " + "stop_replication only (1 of 2)" }, /* prepare_promotion, primary present, not in maintenance, not already wait_primary */ { .pos = 339, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2169,7 +2229,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .primaryNode = { .exists = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), .otherNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), - .comment = "Citus worker stop_replication, primary present -> wait_primary + demoted" }, + .comment = + "Citus worker stop_replication, primary present -> wait_primary + demoted" }, /* Citus worker, primary removed */ { .pos = 353, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2185,7 +2246,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY, .isHealthy = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), - .comment = "demoted, primary reported wait/join_primary with goal primary -> catchingup" }, + .comment = + "demoted, primary reported wait/join_primary with goal primary -> catchingup" }, /* demoted, primary converged wait/join_primary/primary, healthy */ { .pos = 357, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2193,7 +2255,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN, .isHealthy = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), - .comment = "demoted, primary converged wait/join_primary/primary, healthy -> catchingup" }, + .comment = + "demoted, primary converged wait/join_primary/primary, healthy -> catchingup" }, /* join_secondary, primary reported wait_primary with goal wait/primary -- cascades into a * nested pass on primaryNode */ @@ -2202,8 +2265,9 @@ static const MonitorFSMTransition MonitorFSM[] = { .primaryNode = { .statePattern = FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), .extraAction = ActionRunPrimaryNodeTransition, - .comment = "join_secondary, primary reported wait_primary with goal wait/primary -> " - "secondary" }, + .comment = + "join_secondary, primary reported wait_primary with goal wait/primary -> " + "secondary" }, /* join_secondary, primary converged primary */ { .pos = 361, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2233,11 +2297,15 @@ static const MonitorFSMTransition MonitorFSM[] = { .conditions = { .activeNodeAllWalSourcesUnhealthy = BOOL_TRUE, .guardDataLossEnabled = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, - .reportedStates = STATES(REPLICATION_STATE_REPORT_LSN), - .assignedStates = STATES(REPLICATION_STATE_FAST_FORWARD) } }, + .reportedStates = STATES( + REPLICATION_STATE_REPORT_LSN), + .assignedStates = STATES( + REPLICATION_STATE_FAST_FORWARD) } + }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), - .comment = "MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, " - "guard_data_loss=true -> report_lsn (retry once a source recovers)" }, + .comment = + "MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, " + "guard_data_loss=true -> report_lsn (retry once a source recovers)" }, /* MS-failover: candidate ready to stream WAL -> follower joins as secondary */ { .pos = 365, .section = MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2245,8 +2313,9 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, .candidateNode = { .isReadyToStreamWAL = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_JOIN_SECONDARY), - .comment = "MS-failover: activeNode in report_lsn, failover candidate ready to stream " - "WAL -> join_secondary" }, + .comment = + "MS-failover: activeNode in report_lsn, failover candidate ready to stream " + "WAL -> join_secondary" }, /* * MS-failover: BuildCandidateList's own fan-out (group_state_machine.c, @@ -2276,27 +2345,37 @@ static const MonitorFSMTransition MonitorFSM[] = { { .pos = 367, .section = MONITOR_FSM_SECTION_REPORTING_NODE, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, - .reportedStates = STATES(REPLICATION_STATE_SECONDARY, - REPLICATION_STATE_CATCHINGUP), - .assignedStates = STATES(REPLICATION_STATE_SECONDARY, - REPLICATION_STATE_CATCHINGUP) } }, + .reportedStates = STATES( + REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP), + .assignedStates = STATES( + REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP) } + }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), - .comment = "MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn " - "(1 of 4)" }, + .comment = + "MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn " + "(1 of 4)" }, { .pos = 369, .section = MONITOR_FSM_SECTION_REPORTING_NODE, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, - .reportedStates = STATES(REPLICATION_STATE_MAINTENANCE), - .assignedStates = STATES(REPLICATION_STATE_CATCHINGUP) } }, + .reportedStates = STATES( + REPLICATION_STATE_MAINTENANCE), + .assignedStates = STATES( + REPLICATION_STATE_CATCHINGUP) } + }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), - .comment = "MS-failover fan-out: rejoining from maintenance -> report_lsn (2 of 4)" }, + .comment = + "MS-failover fan-out: rejoining from maintenance -> report_lsn (2 of 4)" }, { .pos = 371, .section = MONITOR_FSM_SECTION_REPORTING_NODE, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_STABLE, - .reportedStates = STATES(REPLICATION_STATE_DRAINING, - REPLICATION_STATE_DEMOTED) } }, + .reportedStates = STATES( + REPLICATION_STATE_DRAINING, + REPLICATION_STATE_DEMOTED) } + }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), .comment = "MS-failover fan-out: old primary converged draining or demoted -> " "report_lsn (3 of 4)" }, @@ -2304,8 +2383,11 @@ static const MonitorFSMTransition MonitorFSM[] = { { .pos = 373, .section = MONITOR_FSM_SECTION_REPORTING_NODE, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, - .reportedStates = STATES(REPLICATION_STATE_DEMOTED), - .assignedStates = STATES(REPLICATION_STATE_CATCHINGUP) } }, + .reportedStates = STATES( + REPLICATION_STATE_DEMOTED), + .assignedStates = STATES( + REPLICATION_STATE_CATCHINGUP) } + }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), .comment = "MS-failover fan-out: old primary demoted, was rejoining a now-failed " "primary -> report_lsn (4 of 4)" }, @@ -2382,7 +2464,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .secondaryNodesCountIsZero = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "all nodes async, zero secondaries -> wait_primary" }, + .comment = "all nodes async, zero secondaries -> wait_primary " + "(+ unhealthy-secondary fan-out to catchingup)" }, /* all nodes async, >=1 secondary */ { .pos = 405, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, @@ -2391,7 +2474,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .secondaryNodesCountIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "all nodes async, >=1 secondary -> primary" }, + .comment = "all nodes async, >=1 secondary -> primary " + "(+ unhealthy-secondary fan-out to catchingup)" }, /* converged primary/apply_settings (not wait_primary), no quorum secondaries, * number_sync_standbys=0, no failover in progress (issue #774) */ @@ -2402,8 +2486,10 @@ static const MonitorFSMTransition MonitorFSM[] = { .numberSyncStandbysIsZero = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "converged primary/apply_settings, no quorum secondaries, no failover in " - "progress, number_sync_standbys=0 -> wait_primary" }, + .comment = + "converged primary/apply_settings, no quorum secondaries, no failover in " + "progress, number_sync_standbys=0 -> wait_primary " + "(+ unhealthy-secondary fan-out to catchingup)" }, /* same, but number_sync_standbys>0 -> block writes on primary */ { .pos = 409, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, @@ -2413,8 +2499,10 @@ static const MonitorFSMTransition MonitorFSM[] = { .numberSyncStandbysIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "converged primary/apply_settings, no quorum secondaries, no failover in " - "progress, number_sync_standbys>0 -> primary (block writes)" }, + .comment = + "converged primary/apply_settings, no quorum secondaries, no failover in " + "progress, number_sync_standbys>0 -> primary (block writes) " + "(+ unhealthy-secondary fan-out to catchingup)" }, /* wait_primary, >=1 quorum secondary */ { .pos = 411, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, @@ -2422,7 +2510,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "wait_primary, >=1 quorum secondary -> primary" }, + .comment = "wait_primary, >=1 quorum secondary -> primary " + "(+ unhealthy-secondary fan-out to catchingup)" }, /* apply_settings, both zero */ { .pos = 413, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, @@ -2431,7 +2520,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "apply_settings, both zero -> wait_primary" }, + .comment = "apply_settings, both zero -> wait_primary " + "(+ unhealthy-secondary fan-out to catchingup)" }, /* apply_settings, number_sync_standbys != 0 (1 of 2 disjuncts) */ { .pos = 415, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, @@ -2439,7 +2529,9 @@ static const MonitorFSMTransition MonitorFSM[] = { .conditions = { .numberSyncStandbysIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts)" }, + .comment = + "apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) " + "(+ unhealthy-secondary fan-out to catchingup)" }, /* apply_settings, sync_standbys=0 but >=1 quorum secondary (2 of 2) */ { .pos = 417, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, @@ -2448,14 +2540,17 @@ static const MonitorFSMTransition MonitorFSM[] = { .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2)" }, + .comment = + "apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) " + "(+ unhealthy-secondary fan-out to catchingup)" }, /* converged primary/wait_primary/apply_settings, no other condition applies */ { .pos = 419, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, .extraAction = ActionCatchupUnhealthySecondaries, - .comment = "converged primary/wait_primary/apply_settings, no other condition applies -> " - "no-op besides the unhealthy-secondary fan-out" }, + .comment = + "converged primary/wait_primary/apply_settings, no other condition applies -> " + "no-op besides the unhealthy-secondary fan-out" }, /* backwards-compat: join_primary -> primary */ { .pos = 421, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, @@ -2800,7 +2895,8 @@ NodeStatePatternReportedStatesText(const NodeStatePattern *pattern, bool *isNull } appendStringInfoString(&buf, - ReplicationStateGetName(pattern->reportedStates.states[i])); + ReplicationStateGetName( + pattern->reportedStates.states[i])); } *isNull = false; @@ -2863,7 +2959,8 @@ AppendNodeStateGoalCondition(StringInfoData *buf, const NodeStatePattern *patter appendStringInfoString(buf, ", "); } - appendStringInfoString(buf, pattern->kind == NODE_STATE_ASSIGNED ? "goal=" : "goal!="); + appendStringInfoString(buf, pattern->kind == NODE_STATE_ASSIGNED ? "goal=" : + "goal!="); for (int i = 0; i < pattern->assignedStates.count; i++) { @@ -2872,7 +2969,8 @@ AppendNodeStateGoalCondition(StringInfoData *buf, const NodeStatePattern *patter appendStringInfoString(buf, "|"); } - appendStringInfoString(buf, ReplicationStateGetName(pattern->assignedStates.states[i])); + appendStringInfoString(buf, ReplicationStateGetName( + pattern->assignedStates.states[i])); } } @@ -2908,7 +3006,8 @@ NodeStatusPatternConditionsText(const NodeStatusPattern *pattern, bool *isNull) APPEND_BOOL_CONDITION(&buf, "drainTimeExpired", pattern->drainTimeExpired); APPEND_BOOL_CONDITION(&buf, "isCitusWorkerGroup", pattern->isCitusWorkerGroup); APPEND_BOOL_CONDITION(&buf, "replicationQuorum", pattern->replicationQuorum); - APPEND_BOOL_CONDITION(&buf, "isComparableToReferenceTli", pattern->isComparableToReferenceTli); + APPEND_BOOL_CONDITION(&buf, "isComparableToReferenceTli", + pattern->isComparableToReferenceTli); APPEND_BOOL_CONDITION(&buf, "unreachableFromDemoteTimeout", pattern->unreachableFromDemoteTimeout); @@ -2942,22 +3041,30 @@ NodeActiveContextPatternConditionsText(const NodeActiveContextPattern *cond, boo APPEND_BOOL_CONDITION(&buf, "groupHasExactlyOneNode", cond->groupHasExactlyOneNode); APPEND_BOOL_CONDITION(&buf, "groupHasExactlyTwoNodes", cond->groupHasExactlyTwoNodes); - APPEND_BOOL_CONDITION(&buf, "groupHasMoreThanTwoNodes", cond->groupHasMoreThanTwoNodes); - APPEND_BOOL_CONDITION(&buf, "anyOtherNodeWaitingStandby", cond->anyOtherNodeWaitingStandby); - APPEND_BOOL_CONDITION(&buf, "numberSyncStandbysIsZero", cond->numberSyncStandbysIsZero); + APPEND_BOOL_CONDITION(&buf, "groupHasMoreThanTwoNodes", + cond->groupHasMoreThanTwoNodes); + APPEND_BOOL_CONDITION(&buf, "anyOtherNodeWaitingStandby", + cond->anyOtherNodeWaitingStandby); + APPEND_BOOL_CONDITION(&buf, "numberSyncStandbysIsZero", + cond->numberSyncStandbysIsZero); APPEND_BOOL_CONDITION(&buf, "replicationQuorumCountIsZero", cond->replicationQuorumCountIsZero); - APPEND_BOOL_CONDITION(&buf, "secondaryNodesCountIsZero", cond->secondaryNodesCountIsZero); + APPEND_BOOL_CONDITION(&buf, "secondaryNodesCountIsZero", + cond->secondaryNodesCountIsZero); APPEND_BOOL_CONDITION(&buf, "secondaryQuorumNodesCountIsZero", cond->secondaryQuorumNodesCountIsZero); - APPEND_BOOL_CONDITION(&buf, "atLeastOneHealthyCandidate", cond->atLeastOneHealthyCandidate); - APPEND_BOOL_CONDITION(&buf, "walWithinPromoteThreshold", cond->walWithinPromoteThreshold); + APPEND_BOOL_CONDITION(&buf, "atLeastOneHealthyCandidate", + cond->atLeastOneHealthyCandidate); + APPEND_BOOL_CONDITION(&buf, "walWithinPromoteThreshold", + cond->walWithinPromoteThreshold); APPEND_BOOL_CONDITION(&buf, "walWithinSyncThreshold", cond->walWithinSyncThreshold); - APPEND_BOOL_CONDITION(&buf, "activeAndPrimaryTliMatch", cond->activeAndPrimaryTliMatch); + APPEND_BOOL_CONDITION(&buf, "activeAndPrimaryTliMatch", + cond->activeAndPrimaryTliMatch); APPEND_BOOL_CONDITION(&buf, "primaryIsWaitPrimaryPresumedDead", cond->primaryIsWaitPrimaryPresumedDead); APPEND_BOOL_CONDITION(&buf, "failoverInProgress", cond->failoverInProgress); - APPEND_BOOL_CONDITION(&buf, "replicationStallExceeded", cond->replicationStallExceeded); + APPEND_BOOL_CONDITION(&buf, "replicationStallExceeded", + cond->replicationStallExceeded); APPEND_BOOL_CONDITION(&buf, "lastHealthySyncStandbyGoingToMaintenance", cond->lastHealthySyncStandbyGoingToMaintenance); APPEND_BOOL_CONDITION(&buf, "activeNodeAllWalSourcesUnhealthy", @@ -3108,6 +3215,285 @@ dump_fsm(PG_FUNCTION_ARGS) } +/* + * AllReplicationStates is the full universe of "real" (non-sentinel) + * ReplicationState values -- every one a node can genuinely report or be + * assigned, in the order replication_state.h declares them. Deliberately + * excludes REPLICATION_STATE_UNKNOWN: a meta/sentinel value no MonitorFSM[] + * row ever reports or assigns, and no KeeperFSM[] edge ever targets either. + * Used only to resolve NodeStatePatternResolveFromStates' wildcard/negation + * kinds (ANY, NOT_STABLE, ASSIGNED, NOT_ASSIGNED) into concrete states. + */ +static const ReplicationState AllReplicationStates[] = { + REPLICATION_STATE_INITIAL, + REPLICATION_STATE_SINGLE, + REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_PRIMARY, + REPLICATION_STATE_DRAINING, + REPLICATION_STATE_DEMOTE_TIMEOUT, + REPLICATION_STATE_DEMOTED, + REPLICATION_STATE_CATCHINGUP, + REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_PREPARE_PROMOTION, + REPLICATION_STATE_STOP_REPLICATION, + REPLICATION_STATE_WAIT_STANDBY, + REPLICATION_STATE_MAINTENANCE, + REPLICATION_STATE_JOIN_PRIMARY, + REPLICATION_STATE_APPLY_SETTINGS, + REPLICATION_STATE_PREPARE_MAINTENANCE, + REPLICATION_STATE_WAIT_MAINTENANCE, + REPLICATION_STATE_REPORT_LSN, + REPLICATION_STATE_FAST_FORWARD, + REPLICATION_STATE_JOIN_SECONDARY, + REPLICATION_STATE_DROPPED +}; + +#define ALL_REPLICATION_STATES_COUNT \ + ((int) (sizeof(AllReplicationStates) / sizeof(AllReplicationStates[0]))) + + +/* + * NodeStatePatternResolveFromStates resolves a NodeStatePattern's own "from" + * (reported-state) constraint into a concrete, palloc'd array of + * ReplicationState values -- the genuine edge-source set dump_fsm_edges() + * needs, as opposed to NodeStatePatternReportedStatesText's rendering (which + * only handles STABLE/REPORTED/TRANSITIONING and returns NULL for + * everything else, since that one exists for human display, not edge + * enumeration). + * + * ASSIGNED/NOT_ASSIGNED only ever constrain the node's *goal*, never what it + * currently reports (see each's own comment where declared above), so both + * resolve to the full state universe here, same as ANY. NOT_STABLE resolves + * to the complement of its own reportedStates within that universe. *outCount + * is set to the returned array's length; the caller does not need to free it + * (called only from dump_fsm_edges(), itself already running inside a + * per-query memory context the executor resets on its own). + */ +static ReplicationState * +NodeStatePatternResolveFromStates(const NodeStatePattern *pattern, int *outCount) +{ + ReplicationState *out; + + switch (pattern->kind) + { + case NODE_STATE_STABLE: + case NODE_STATE_REPORTED: + case NODE_STATE_TRANSITIONING: + { + out = (ReplicationState *) + palloc(pattern->reportedStates.count * sizeof(ReplicationState)); + + for (int i = 0; i < pattern->reportedStates.count; i++) + { + out[i] = pattern->reportedStates.states[i]; + } + + *outCount = pattern->reportedStates.count; + return out; + } + + case NODE_STATE_NOT_STABLE: + { + out = (ReplicationState *) + palloc(ALL_REPLICATION_STATES_COUNT * sizeof(ReplicationState)); + *outCount = 0; + + for (int i = 0; i < ALL_REPLICATION_STATES_COUNT; i++) + { + if (!MatchStateSet(AllReplicationStates[i], pattern->reportedStates)) + { + out[(*outCount)++] = AllReplicationStates[i]; + } + } + + return out; + } + + case NODE_STATE_ANY: + case NODE_STATE_ASSIGNED: + case NODE_STATE_NOT_ASSIGNED: + default: + { + out = (ReplicationState *) + palloc(ALL_REPLICATION_STATES_COUNT * sizeof(ReplicationState)); + memcpy(out, AllReplicationStates, sizeof(AllReplicationStates)); + *outCount = ALL_REPLICATION_STATES_COUNT; + return out; + } + } +} + + +PG_FUNCTION_INFO_V1(dump_fsm_edges); + +/* + * dump_fsm_edges exposes MonitorFSM[] as a flat set of concrete + * (pos, current_state, assigned_state) edges -- one row per actual + * (reportedState, assignedState) pair a row can produce, fully resolved + * (never NULL, unlike dump_fsm()'s own pattern-summary columns, which stay + * unresolved for human display). This is the keeper-cross-check surface the + * design doc's check_fsm_reachability() proposal needs: pgautofailover. + * check_fsm_reachability(jsonb) anti-joins this against a keeper's own + * KeeperFSM[] edges (see KeeperFSMToJSON(), src/bin/pg_autoctl/fsm.c) to + * find any monitor transition with no matching keeper edge -- exactly the + * bug class issue #774 was. + * + * Two potential edges per row, resolved independently: activeNode's own + * (statePattern -> activeNodeAssignedState) when the row actually assigns + * one, and primaryNode's own (statePattern -> otherNodeAssignedState) when + * it does. candidateNode never has an assignment slot of its own (see + * MonitorFSMTransition's own comment), so it never contributes an edge. + * pgKind is deliberately not modeled here -- see this function's own header + * comment in the design doc discussion; every Citus-specific KeeperFSM[] + * edge already has a NODE_KIND_ANY counterpart with the same + * (current, assigned) shape (fsm_mermaid.c's own comment establishes this + * as an invariant this codebase already relies on elsewhere), so a flat, + * pgKind-blind edge set on both sides is already correct. + * + * Two categories of edges are deliberately never emitted, both confirmed by + * running pg_autoctl inspect fsm check for real and tracing every one of the + * mismatches it reported back to its actual root cause rather than assuming: + * + * - Reflexive (current == assigned) edges. keeper_fsm_reach_assigned_state() + * (src/bin/pg_autoctl/fsm.c) returns true the moment + * current_role == assigned_role, before ever consulting KeeperFSM[] -- + * confirmed by reading that function directly. A self-loop can therefore + * never have (or need) a matching keeper edge; including one here would + * always report a false gap. This alone explained every mismatch on pos + * 363, 403, 405, 409 during that live run. + * + * - MONITOR_FSM_SECTION_API_TRIGGERED rows entirely. Every one of these is + * reached from an operator-facing SQL wrapper (remove_node, + * perform_failover, start_maintenance, ...) that resolves activeNode to a + * specific, already-validated role (almost always the primary) via + * hand-written C *before* dispatch ever runs -- ProceedGroupStateForApiTrigger's + * own comment, and several of these rows' own comments ("there's nothing + * else for activeNode to be here but the primary itself", "activeNode IS + * the primary here"), document this explicitly. The row's own + * NodeStatePattern for these (typically ANY, or a BoolPattern condition + * like isInPrimaryState with no accompanying state-set restriction) is + * consequently far broader than what's actually reachable: it was never + * meant to double as a full reachability precondition, because that + * precondition already lives in hand-written C outside this table, per + * this design's own "pre/post side effects stay hand-written, not modeled + * as a row" principle. Expanding it here (as this function otherwise + * correctly does for the ordinary heartbeat-driven sections) manufactures + * the exact same kind of false gap for every one of these ten rows, all + * confirmed by that same live run. + * + * Every other mismatch that same run found (early_checks 209/211, + * reporting_node 303/325/333/339/347/349/351) stayed reachable from a live + * node's own genuine reported state with no such structural excuse, so + * they're deliberately NOT filtered here -- each is a real candidate for + * individual investigation against the keeper's actual KeeperFSM[] rows, + * not a known-safe artifact of this function's own edge derivation. + */ +Datum +dump_fsm_edges(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that " + "cannot accept a set"))); + } + + if (!(rsinfo->allowedModes & SFRM_Materialize)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not " + "allowed in this context"))); + } + + TupleDesc tupdesc; + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + { + ereport(ERROR, + (errmsg("function returning record called in context " + "that cannot accept type record"))); + } + + MemoryContext perQueryContext = rsinfo->econtext->ecxt_per_query_memory; + MemoryContext oldContext = MemoryContextSwitchTo(perQueryContext); + + Tuplestorestate *tupstore = tuplestore_begin_heap(true, false, work_mem); + + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldContext); + + for (int i = 0; i < MonitorFSM_SIZE; i++) + { + const MonitorFSMTransition *rule = &MonitorFSM[i]; + + if (rule->section == MONITOR_FSM_SECTION_API_TRIGGERED) + { + continue; + } + + if (rule->activeNodeAssignedState.kind == GOAL_STATE_SET) + { + int count; + ReplicationState *states = + NodeStatePatternResolveFromStates(&rule->activeNode.statePattern, &count); + + for (int j = 0; j < count; j++) + { + Datum values[3]; + bool isNull[3] = { false }; + + if (states[j] == rule->activeNodeAssignedState.state) + { + continue; + } + + values[0] = Int32GetDatum(rule->pos); + values[1] = ObjectIdGetDatum(ReplicationStateGetEnum(states[j])); + values[2] = ObjectIdGetDatum( + ReplicationStateGetEnum(rule->activeNodeAssignedState.state)); + + tuplestore_putvalues(tupstore, tupdesc, values, isNull); + } + } + + if (rule->otherNodeAssignedState.kind == GOAL_STATE_SET) + { + int count; + ReplicationState *states = + NodeStatePatternResolveFromStates(&rule->primaryNode.statePattern, + &count); + + for (int j = 0; j < count; j++) + { + Datum values[3]; + bool isNull[3] = { false }; + + if (states[j] == rule->otherNodeAssignedState.state) + { + continue; + } + + values[0] = Int32GetDatum(rule->pos); + values[1] = ObjectIdGetDatum(ReplicationStateGetEnum(states[j])); + values[2] = ObjectIdGetDatum( + ReplicationStateGetEnum(rule->otherNodeAssignedState.state)); + + tuplestore_putvalues(tupstore, tupdesc, values, isNull); + } + } + } + + return (Datum) 0; +} + + /* * ProceedGroupStateFromContext is the core FSM logic, operating entirely on * the pre-built GroupStateContext. It does not touch the database for reads; diff --git a/src/monitor/node_active_protocol.c b/src/monitor/node_active_protocol.c index 0638ccfbe..a1e22db01 100644 --- a/src/monitor/node_active_protocol.c +++ b/src/monitor/node_active_protocol.c @@ -1913,6 +1913,7 @@ start_maintenance(PG_FUNCTION_ARGS) * state of any standby node yet, we get there when the count is one * (not zero). */ + /* * Dispatch through MonitorFSM[]'s API_TRIGGERED section: the * last-healthy-sync-standby row (wait_maintenance + primary diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index 5fc790e0b..dfb77a702 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -286,6 +286,55 @@ grant execute on function pgautofailover.dump_fsm() to autoctl_node; CREATE VIEW pgautofailover.fsm AS SELECT * FROM pgautofailover.dump_fsm() ORDER BY pos; +-- Flat, fully-resolved (pos, current_state, assigned_state) edges derived +-- from MonitorFSM[] -- see dump_fsm_edges()'s own C-side comment. Never +-- queried directly by an operator; check_fsm_reachability() below is built +-- on top of it. +CREATE FUNCTION pgautofailover.dump_fsm_edges() +RETURNS TABLE + ( + pos int, + current_state pgautofailover.replication_state, + assigned_state pgautofailover.replication_state + ) +LANGUAGE C SECURITY DEFINER +AS 'MODULE_PATHNAME', $$dump_fsm_edges$$; + +grant execute on function pgautofailover.dump_fsm_edges() to autoctl_node; + +-- Compares the monitor's own declarative dispatch table against a keeper's +-- KeeperFSM[] edges (serialized to JSON by KeeperFSMToJSON(), +-- src/bin/pg_autoctl/fsm.c, and sent here by "pg_autoctl inspect fsm check") +-- and returns every monitor edge with no matching keeper entry -- an empty +-- result means every transition the monitor can ever assign has somewhere +-- for the keeper to go. keeper_edges is expected to be a jsonb array of +-- {"current": ..., "assigned": ...} objects, one per KeeperFSMTransition +-- row; casting both fields to pgautofailover.replication_state means a +-- keeper reporting a state name this enum doesn't recognize fails loudly, +-- with a real cast error, rather than silently never matching. +CREATE FUNCTION pgautofailover.check_fsm_reachability(keeper_edges jsonb) +RETURNS TABLE + ( + pos int, + current_state pgautofailover.replication_state, + assigned_state pgautofailover.replication_state, + comment text + ) +LANGUAGE sql +AS $$ + SELECT e.pos, e.current_state, e.assigned_state, f.comment + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE NOT EXISTS ( + SELECT 1 + FROM jsonb_to_recordset(keeper_edges) AS k(current text, assigned text) + WHERE k.current::pgautofailover.replication_state = e.current_state + AND k.assigned::pgautofailover.replication_state = e.assigned_state) + ORDER BY e.pos; +$$; + +grant execute on function pgautofailover.check_fsm_reachability(jsonb) to autoctl_node; + GRANT SELECT ON ALL TABLES IN SCHEMA pgautofailover TO autoctl_node; CREATE FUNCTION pgautofailover.set_node_system_identifier diff --git a/src/monitor/regress_schedule b/src/monitor/regress_schedule index bfea2dd2d..a691348eb 100644 --- a/src/monitor/regress_schedule +++ b/src/monitor/regress_schedule @@ -32,6 +32,7 @@ test: create_extension test: fsm +test: check_fsm_reachability test: monitor test: workers test: node_active_protocol diff --git a/src/monitor/sql/check_fsm_reachability.sql b/src/monitor/sql/check_fsm_reachability.sql new file mode 100644 index 000000000..b4668fa74 --- /dev/null +++ b/src/monitor/sql/check_fsm_reachability.sql @@ -0,0 +1,80 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Exercises the mechanics of pgautofailover.check_fsm_reachability(jsonb) +-- (and the pgautofailover.dump_fsm_edges() it's built on) against small, +-- synthetic keeper-edge inputs -- not the real KeeperFSM[] table, which +-- lives in the pg_autoctl binary, not this database. The real, end-to-end +-- completeness check (does the real KeeperFSM[] cover every real +-- MonitorFSM[] edge) is run separately, live, via +-- "pg_autoctl inspect fsm check" against a real monitor+keeper pair -- this +-- test only confirms the SQL-side comparison mechanism itself behaves +-- correctly: an edge present in the keeper_edges parameter drops out of the +-- mismatch list, an edge absent from it stays in, and an unrecognized state +-- name fails loudly rather than silently never matching. + +-- Every edge dump_fsm_edges() can produce, fully resolved: a fixed, +-- reviewable count for the current MonitorFSM[] -- changes only when a row +-- is added, removed, or edited there, same as fsm.sql's own row count. +SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); + +-- pos 301 ("converged secondary, reportedTLI not an ancestor of reference +-- -> catchingup", single edge) and pos 343 ("stop_replication, primary +-- converged prepare_maintenance -> wait_primary + maintenance", two edges, +-- one per role) are stable, non-reflexive, non-api_triggered rows -- good, +-- deterministic targets to check both single- and dual-edge rows against. +SELECT pos, current_state, assigned_state + FROM pgautofailover.dump_fsm_edges() + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + +-- Two categories of edges dump_fsm_edges() deliberately never produces, see +-- its own comment for why: reflexive (current == assigned) edges, and the +-- whole api_triggered section (every row there resolves activeNode to a +-- specific role via hand-written C before dispatch, so its own +-- NodeStatePattern was never meant to double as a full reachability +-- precondition). pos 403 ("all nodes async, zero secondaries -> +-- wait_primary") has both an ordinary edge (primary/apply_settings -> +-- wait_primary) and a reflexive one (wait_primary -> wait_primary) in its +-- own source pattern -- only the former should appear. pos 105 (api +-- triggered: perform_failover) should produce no edges at all. +SELECT pos, current_state, assigned_state + FROM pgautofailover.dump_fsm_edges() + WHERE pos = 403 + ORDER BY current_state; + +SELECT count(*) AS api_triggered_edge_count + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE f.section LIKE 'api_triggered%'; + +-- An empty keeper_edges: every single edge dump_fsm_edges() produces comes +-- back as a mismatch, so this count must equal total_edge_count above. +SELECT count(*) AS missing_with_empty_keeper_edges + FROM pgautofailover.check_fsm_reachability('[]'::jsonb); + +-- Providing exactly pos 301's own edge, plus one of pos 343's two edges +-- (leaving its other edge, prepare_maintenance->maintenance, still +-- unmatched): pos 301 must disappear entirely from the mismatch list, pos +-- 343 must still appear, but only once. +SELECT pos, current_state, assigned_state + FROM pgautofailover.check_fsm_reachability( + '[{"current":"secondary","assigned":"catchingup"}, + {"current":"stop_replication","assigned":"wait_primary"}]'::jsonb) + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + +-- Providing both of pos 343's edges too: it must now disappear as well. +SELECT pos, current_state, assigned_state + FROM pgautofailover.check_fsm_reachability( + '[{"current":"secondary","assigned":"catchingup"}, + {"current":"stop_replication","assigned":"wait_primary"}, + {"current":"prepare_maintenance","assigned":"maintenance"}]'::jsonb) + WHERE pos IN (301, 343) + ORDER BY pos, current_state; + +-- An unrecognized state name in keeper_edges must fail loudly (a real cast +-- error), not silently never match -- the same "fail loudly on drift" +-- instinct as AssignDeclaredGoalState's own trust-check. +SELECT * FROM pgautofailover.check_fsm_reachability( + '[{"current":"not_a_real_state","assigned":"catchingup"}]'::jsonb); From cc647a0477ba9c9e0f29ac0ab48288a541b718cd Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 29 Jul 2026 15:35:18 +0200 Subject: [PATCH 07/52] Make ActionRunMultiStandbyFailoverCascade's goal assignments declarative ActionRunMultiStandbyFailoverCascade contained two raw, hand-written AssignGoalState(primaryNode, DRAINING/MAINTENANCE, ...) calls -- invisible to dump_fsm()/dump_fsm_edges() and the "systematic visibility" guarantee the rest of this refactor gives every other transition. Fix: add two new declarative rows at pos 381 (DRAINING) and 383 (MAINTENANCE) to MonitorFSM[], in MONITOR_FSM_SECTION_REPORTING_NODE, right after the existing MS-failover cluster (pos 379). Both rows reuse pos 305's own gating conditions verbatim (primaryNode.isUnhealthy, groupHasMoreThanTwoNodes) rather than a new marker boolean: the ordinary top-level scan can only ever reach pos 381/383 by first reaching pos 305, which unconditionally dispatches through its own extraAction (ActionRunMultiStandbyFailoverCascade) before the scan could resume past it -- so pos 305 always intercepts first whenever the shared gate holds, and if it doesn't, 381/383 wouldn't match either. The function body now tries FindAndDispatchMonitorFSMRule() over the new bounded range first, falling back to the original hand-written if/else-if only if neither row matches (should never happen). The fallback reuses nac->atLeastOneHealthyCandidate (already computed in BuildFromContextNodeActiveContext under the identical isUnhealthy/ groupNodeCount>2 gate) instead of recomputing candidatesCount locally -- it turned out to be the exact same AutoFailoverOtherNodesListInState + CountHealthyCandidates computation. Reviewed every other AssignGoalState call site in group_state_machine.c for the same gap: ActionCatchupUnhealthySecondaries and ActionFanOutReportLsnOnPrimaryRemoval are genuine fan-outs over a dynamic node list (not expressible as a single row, already flagged as such in their callers' row comments); everything else already follows the established try-declarative-first-fallback pattern. This was the only remaining case. MonitorFSM_SIZE: 72 -> 74. MonitorFSM_PrimaryNodeSectionStart: 61 -> 63. Regenerated fsm.out/check_fsm_reachability.out for the two new rows (total_edge_count 231 -> 251). Verified: full regress+isolation Docker installcheck clean, including concurrent_second_primary_death_report/concurrent_health_check_and_report (the isolation tests that historically caught the "3 separate rows" version of this bug class); live pgaftest multi_standbys.pgaf (27/27, including test_012_fail_primary which exercises this exact cascade). --- .../expected/check_fsm_reachability.out | 4 +- src/monitor/expected/fsm.out | 48 +++++-- src/monitor/group_state_machine.c | 122 +++++++++++++----- 3 files changed, 130 insertions(+), 44 deletions(-) diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index e89649a07..ca403706b 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 231 + 251 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 231 + 251 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index 1f4a4458b..6aa9e8cc5 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -875,6 +875,34 @@ other_node_assigned_state | has_extra_action | f comment | MS-failover: no promotion in flight, not enough (or not safe enough) candidates yet -> no-op besides the fan-out above -[ RECORD 62 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 381 +section | reporting_node +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isUnhealthy=true, isInPrimaryState=true +candidate_node_conditions | +group_conditions | groupHasMoreThanTwoNodes=true, atLeastOneHealthyCandidate=true +active_node_assigned_state | +other_node_assigned_state | draining +has_extra_action | f +comment | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining +-[ RECORD 63 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 383 +section | reporting_node +active_node_current_state | +other_node_current_state | prepare_maintenance +candidate_node_current_state | +active_node_conditions | +other_node_conditions | isUnhealthy=true +candidate_node_conditions | +group_conditions | groupHasMoreThanTwoNodes=true +active_node_assigned_state | +other_node_assigned_state | maintenance +has_extra_action | f +comment | nodesCount>2, primary unhealthy, converged prepare_maintenance -> primary maintenance +-[ RECORD 64 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 401 section | primary_node active_node_current_state | single @@ -888,7 +916,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | primary alone, another node reached wait_standby -> wait_primary --[ RECORD 63 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 65 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 403 section | primary_node active_node_current_state | primary, wait_primary, apply_settings @@ -902,7 +930,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | t comment | all nodes async, zero secondaries -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 64 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 66 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 405 section | primary_node active_node_current_state | primary, wait_primary, apply_settings @@ -916,7 +944,7 @@ active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t comment | all nodes async, >=1 secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 65 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 67 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 407 section | primary_node active_node_current_state | primary, apply_settings @@ -930,7 +958,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | t comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 66 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 68 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 409 section | primary_node active_node_current_state | primary, apply_settings @@ -944,7 +972,7 @@ active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 67 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 69 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 411 section | primary_node active_node_current_state | wait_primary @@ -958,7 +986,7 @@ active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 68 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 70 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 413 section | primary_node active_node_current_state | apply_settings @@ -972,7 +1000,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | t comment | apply_settings, both zero -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 69 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 415 section | primary_node active_node_current_state | apply_settings @@ -986,7 +1014,7 @@ active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 70 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 417 section | primary_node active_node_current_state | apply_settings @@ -1000,7 +1028,7 @@ active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 419 section | primary_node active_node_current_state | primary, wait_primary, apply_settings @@ -1014,7 +1042,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | converged primary/wait_primary/apply_settings, no other condition applies -> no-op besides the unhealthy-secondary fan-out --[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 421 section | primary_node active_node_current_state | join_primary diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 70dd396c4..8b4167053 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -855,7 +855,7 @@ typedef struct MonitorFSMTransition * completeness, never itself dispatched. None reached through the * ordinary top-level driver. * - * MonitorFSM_PrimaryNodeSectionStart = 61 + * MonitorFSM_PrimaryNodeSectionStart = 63 * Where MONITOR_FSM_SECTION_PRIMARY_NODE begins: the declarative * replacement for ProceedGroupStateForPrimaryNode()'s own if-chain, in * which .activeNode means the *primary* node, not the reporting node. @@ -864,9 +864,13 @@ typedef struct MonitorFSMTransition * NodeActiveContext), (b) start the primary-role lookup here, both from * the top-level driver (activeNode already primary-role) and from * ActionRunPrimaryNodeTransition's nested pass on primaryNode - * (join_secondary's cascade row). + * (join_secondary's cascade row). Two rows short of MonitorFSM_MSFailoverStart's + * own eleven-row span (363-383, not 363-379): ActionRunMultiStandbyFailoverCascade's + * own DRAINING/MAINTENANCE outcomes (pos 381/383) share this same bound, + * appended at the end of the MS-failover cluster rather than renumbered + * into the ordinary REPORTING_NODE rows above -- see their own comment. * - * MonitorFSM_SIZE = 72 + * MonitorFSM_SIZE = 74 * Total row count -- the end bound for the primary-role lookup (nothing * comes after MONITOR_FSM_SECTION_PRIMARY_NODE), and the size every * other bounded search is checked against. @@ -891,8 +895,8 @@ static const MonitorFSMTransition MonitorFSM[]; #define MonitorFSM_FromContextStart 21 #define MonitorFSM_FromContextResumeStart 24 #define MonitorFSM_MSFailoverStart 52 -#define MonitorFSM_PrimaryNodeSectionStart 61 -#define MonitorFSM_SIZE 72 +#define MonitorFSM_PrimaryNodeSectionStart 63 +#define MonitorFSM_SIZE 74 /* Forward-declared for the same reason as MonitorFSM[] above: used by * extraActions (ActionRunPrimaryNodeTransition) defined before its real @@ -1177,7 +1181,11 @@ ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *me * ActionRunMultiStandbyFailoverCascade implements the whole * nodesCount>2-unhealthy-primary block as a single extraAction: the DRAINING/ * MAINTENANCE/nothing if/else-if decision, followed unconditionally by - * ProceedGroupStateForMSFailover(). The real source never `return`s after + * ProceedGroupStateForMSFailover(). The DRAINING/MAINTENANCE decision itself + * is dispatched through MonitorFSM[]'s own pos 381/383 rows first (see their + * own comment) -- the hand-written if/else-if below only runs as a fallback + * if neither row's own conditions somehow line up with the ones just + * checked (should never happen). The real source never `return`s after * assigning DRAINING/MAINTENANCE to the primary -- it always falls through to * try ProceedGroupStateForMSFailover next, in the SAME outer if-block, and if * THAT declines (returns false), falls through further still to the rest of @@ -1215,8 +1223,8 @@ ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *me * wholesale from here exactly as before this refactor (the candidate-selection * algorithm itself doesn't reduce to declarative conditions any more cleanly * than it did before -- see this file's own top-of-file design comment). - * MonitorFSM_MSFailoverStart, by contrast, bounds the *separate* nine-row - * MS-failover cluster (pos 363-379, "MS-failover / candidate-selection + * MonitorFSM_MSFailoverStart, by contrast, bounds the *separate* eleven-row + * MS-failover cluster (pos 363-383, "MS-failover / candidate-selection * cluster" section below) that those same hand-written functions now reach * *into*, at their own tail end, via TryMSFailoverDeclarativeRow/ * TryFanOutReportLsnRow/DispatchMonitorFSMRuleByPos -- covering the plain @@ -1224,8 +1232,13 @@ ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *me * own report_lsn fan-out, PromoteSelectedNode's prepare_promotion/fast_forward * choice) that those functions used to make via a raw AssignGoalState call, * with the original call kept as an unconditional fallback on no match. The - * candidate-selection algorithm's own logic (priority sort, LSN comparison, - * WAL-fetch orchestration) is not part of either bounded range. + * last two rows in that same cluster (pos 381/383) are a fourth, unrelated + * caller reusing the same bounded range: ActionRunMultiStandbyFailoverCascade's + * own DRAINING/MAINTENANCE outcomes (see that function's own comment) -- not + * part of the candidate-selection machinery at all, just sharing the same + * "safe to scan with the ordinary nac" bound. The candidate-selection + * algorithm's own logic (priority sort, LSN comparison, WAL-fetch + * orchestration) is not part of either bounded range. */ static void ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, @@ -1233,33 +1246,42 @@ ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext * { AutoFailoverNode *primaryNode = nac->primaryNode.node; - List *candidateNodesList = - AutoFailoverOtherNodesListInState(primaryNode, REPLICATION_STATE_SECONDARY); - int candidatesCount = CountHealthyCandidates(candidateNodesList); - - if (IsInPrimaryState(primaryNode) && - !IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) && - candidatesCount >= 1) + if (!FindAndDispatchMonitorFSMRule(ctx, nac, MonitorFSM_MSFailoverStart, + MonitorFSM_PrimaryNodeSectionStart)) { - char drainingMessage[BUFSIZE] = { 0 }; + /* + * Neither pos 381 nor pos 383 matched -- reproduce the original + * hand-written condition exactly. nac->atLeastOneHealthyCandidate is + * already the exact same fact those rows' own + * atLeastOneHealthyCandidate condition checks (same + * AutoFailoverOtherNodesListInState + CountHealthyCandidates + * computation, see BuildFromContextNodeActiveContext), so it's reused + * here rather than recomputed. + */ + if (IsInPrimaryState(primaryNode) && + !IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) && + nac->atLeastOneHealthyCandidate) + { + char drainingMessage[BUFSIZE] = { 0 }; - snprintf(drainingMessage, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to draining after it became unhealthy.", - NODE_FORMAT_ARGS(primaryNode)); + snprintf(drainingMessage, BUFSIZE, + "Setting goal state of " NODE_FORMAT + " to draining after it became unhealthy.", + NODE_FORMAT_ARGS(primaryNode)); - AssignGoalState(primaryNode, REPLICATION_STATE_DRAINING, drainingMessage); - } - else if (IsCurrentState(primaryNode, REPLICATION_STATE_PREPARE_MAINTENANCE)) - { - char maintenanceMessage[BUFSIZE] = { 0 }; + AssignGoalState(primaryNode, REPLICATION_STATE_DRAINING, drainingMessage); + } + else if (IsCurrentState(primaryNode, REPLICATION_STATE_PREPARE_MAINTENANCE)) + { + char maintenanceMessage[BUFSIZE] = { 0 }; - snprintf(maintenanceMessage, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance after it converged to prepare_maintenance.", - NODE_FORMAT_ARGS(primaryNode)); + snprintf(maintenanceMessage, BUFSIZE, + "Setting goal state of " NODE_FORMAT + " to maintenance after it converged to prepare_maintenance.", + NODE_FORMAT_ARGS(primaryNode)); - AssignGoalState(primaryNode, REPLICATION_STATE_MAINTENANCE, maintenanceMessage); + AssignGoalState(primaryNode, REPLICATION_STATE_MAINTENANCE, maintenanceMessage); + } } if (!ProceedGroupStateForMSFailover(ctx, primaryNode)) @@ -2444,6 +2466,42 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "MS-failover: no promotion in flight, not enough (or not safe enough) " "candidates yet -> no-op besides the fan-out above" }, + /* + * ActionRunMultiStandbyFailoverCascade's own two outcomes (pos 305's + * extraAction, group_state_machine.c). Both rows below share pos 305's + * own gating conditions (primaryNode.isUnhealthy, groupHasMoreThanTwoNodes) + * verbatim -- not a new marker field -- which is what keeps them safe + * from the ordinary top-level lookup: that lookup can only ever reach + * these two rows by first reaching pos 305 itself (earlier in the array, + * with no activeNode-state restriction of its own), which unconditionally + * dispatches through here via its own extraAction before the ordinary + * scan could ever resume past it in the same call. atLeastOneHealthyCandidate + * is the exact same fact (same AutoFailoverOtherNodesListInState + + * CountHealthyCandidates computation, same isUnhealthy/groupNodeCount>2 + * gate) ActionRunMultiStandbyFailoverCascade used to compute locally -- + * reused here instead of duplicated. ResolveAcceptedTimeline-style side + * effects don't apply to either row (there are none here); only the + * plain AssignGoalState calls these replace, each falling back to the + * original hand-written condition on no match. + */ + { .pos = 381, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .primaryNode = { .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, + .isInPrimaryState = BOOL_TRUE, + .isUnhealthy = BOOL_TRUE }, + .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE, + .atLeastOneHealthyCandidate = BOOL_TRUE }, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_DRAINING), + .comment = "nodesCount>2, primary unhealthy, in primary role but not yet " + "wait_primary, >=1 healthy candidate -> primary draining" }, + + { .pos = 383, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE), + .isUnhealthy = BOOL_TRUE }, + .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE }, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), + .comment = "nodesCount>2, primary unhealthy, converged prepare_maintenance -> " + "primary maintenance" }, + /* --- [MonitorFSM_PrimaryNodeSectionStart, MonitorFSM_SIZE): the declarative replacement * for ProceedGroupStateForPrimaryNode()'s own sequential if-chain. Here .activeNode maps * to the primaryNode parameter, not a reporting node -- reached either directly by the From a170820c98e9dec9b804211d4feff62fa7350285 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 29 Jul 2026 15:35:42 +0200 Subject: [PATCH 08/52] Add keeper_fsm_edges.json fixture and two-step regress cross-check Gives "pg_autoctl inspect fsm list --json" (KeeperFSMToJSON()) a way to run with genuinely zero setup, and uses it to commit a reviewable, git-tracked snapshot of the real KeeperFSM[] table that a regress test compares against the monitor's own MonitorFSM[]/dump_fsm_edges(). cli_getopt_pgdata_or_json (cli_common.c/h): a new getopt function used only by fsm_list's own CommandLine, parsing the identical option set as the shared cli_getopt_pgdata but skipping prepare_keeper_options()'s unconditional config-file-existence check when --json is set. Needed because cli_getopt_pgdata (used, unconditionally, by every other terminal command) calls prepare_keeper_options() before the command handler ever runs, regardless of which subcommand or flags were parsed -- so cli_do_fsm_list's own outputJSON-short-circuit (added earlier this branch) was previously unreachable without a real, pre-existing node config. cli_getopt_pgdata itself is untouched, so every other command using it keeps requiring a real config exactly as before. keeper_fsm_edges.json: generated via "pg_autoctl inspect fsm list --json | python3 -m json.tool" (pretty- printed for git-diff review; KeeperFSMToJSON() itself stays compact, since it's also the payload for the live monitor_check_fsm_reachability() RPC, and payload size was exactly what src/bin/common/pgsql.c's debug-log buffer fix, earlier this branch, had to guard against). Must be regenerated by hand whenever KeeperFSM[] (src/bin/pg_autoctl/fsm.c) changes. sql/keeper_fsm_edges.sql: a two-step regress test. Step 1 loads the fixture client-side (psql's own backtick file embedding -- server-side pg_read_file() is superuser-gated and resolves paths against $PGDATA, not this test's own directory) into a real table, one row per distinct keeper edge (DISTINCT: KeeperFSMToJSON()'s ANY_STATE expansion can make two different KeeperFSM[] rows resolve to the same edge, and the JSON carries no per-row provenance to tell them apart by), then SELECTs from it so expected/keeper_fsm_edges.out shows the whole keeper FSM line by line, human-reviewable. Step 2 anti-joins that table against pgautofailover.dump_fsm_edges() for the real, actionable gap list -- unlike check_fsm_reachability.sql's own synthetic-input test, which only exercises the comparison mechanism itself. The real gap list (134 rows, all per-state expansions of pos 209, 211, 303, 325, 333, 339, 347, 349, 351, and now also 381) matches this session's own earlier finding: dump_fsm_edges() doesn't (and structurally can't, from a NodeStatePattern alone) narrow a `.primaryNode` role match down the way isInPrimaryState would in a full BoolPattern evaluation, so these rows over-report reachability requirements the keeper's real FSM was never expected to need. Pos 381 (this branch's own new ActionRunMultiStandbyFailoverCascade row) falls into the exact same category for the identical reason, not a new gap. Verified: full regress+isolation Docker installcheck clean (18/18 regress including the new keeper_fsm_edges, 6/6 isolation). --- src/bin/pg_autoctl/cli_common.c | 134 ++++++ src/bin/pg_autoctl/cli_common.h | 1 + src/bin/pg_autoctl/cli_do_fsm.c | 37 +- src/monitor/expected/keeper_fsm_edges.out | 291 +++++++++++ src/monitor/keeper_fsm_edges.json | 562 ++++++++++++++++++++++ src/monitor/regress_schedule | 1 + src/monitor/sql/keeper_fsm_edges.sql | 56 +++ 7 files changed, 1075 insertions(+), 7 deletions(-) create mode 100644 src/monitor/expected/keeper_fsm_edges.out create mode 100644 src/monitor/keeper_fsm_edges.json create mode 100644 src/monitor/sql/keeper_fsm_edges.sql diff --git a/src/bin/pg_autoctl/cli_common.c b/src/bin/pg_autoctl/cli_common.c index 915d5ab08..709504b7c 100644 --- a/src/bin/pg_autoctl/cli_common.c +++ b/src/bin/pg_autoctl/cli_common.c @@ -1208,6 +1208,140 @@ cli_getopt_pgdata(int argc, char **argv) } +/* + * cli_getopt_pgdata_or_json is a variant of cli_getopt_pgdata used only by + * "pg_autoctl inspect fsm list": with --json, that command dumps the + * keeper's own static KeeperFSM[] table (cli_do_fsm_list, cli_do_fsm.c), + * which has no dependency on any node's actual config or on-disk state, so + * it must be able to run with neither --pgdata nor an existing + * configuration file. Without --json it behaves exactly like + * cli_getopt_pgdata, because that mode reports on the keeper's current + * state and does need a real config. This parses the identical option set + * as cli_getopt_pgdata; only the decision of whether to call + * prepare_keeper_options differs, so as not to change cli_getopt_pgdata + * itself (used, unconditionally, by every other terminal command). + */ +int +cli_getopt_pgdata_or_json(int argc, char **argv) +{ + KeeperConfig options = { 0 }; + int c, option_index = 0, errors = 0; + int verboseCount = 0; + bool printVersion = false; + + static struct option long_options[] = { + { "pgdata", required_argument, NULL, 'D' }, + { "json", no_argument, NULL, 'J' }, + { "version", no_argument, NULL, 'V' }, + { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, 'q' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + optind = 0; + + unsetenv("POSIXLY_CORRECT"); + + while ((c = getopt_long(argc, argv, "D:JVvqh", + long_options, &option_index)) != -1) + { + switch (c) + { + case 'D': + { + strlcpy(options.pgSetup.pgdata, optarg, MAXPGPATH); + log_trace("--pgdata %s", options.pgSetup.pgdata); + break; + } + + case 'J': + { + outputJSON = true; + log_trace("--json"); + break; + } + + case 'V': + { + printVersion = true; + break; + } + + case 'v': + { + ++verboseCount; + switch (verboseCount) + { + case 1: + { + log_set_level(LOG_INFO); + break; + } + + case 2: + { + log_set_level(LOG_DEBUG); + break; + } + + default: + { + log_set_level(LOG_TRACE); + break; + } + } + break; + } + + case 'q': + { + log_set_level(LOG_ERROR); + break; + } + + case 'h': + { + commandline_help(stderr); + exit(EXIT_CODE_QUIT); + break; + } + + default: + { + /* getopt_long already wrote an error message */ + errors++; + break; + } + } + } + + if (errors > 0) + { + commandline_help(stderr); + exit(EXIT_CODE_BAD_ARGS); + } + + if (printVersion) + { + keeper_cli_print_version(argc, argv); + } + + /* + * --json needs neither --pgdata nor an existing config file (see this + * function's own comment); anything else still needs the full + * pgdata/config validation cli_getopt_pgdata itself always applies. + */ + if (!outputJSON) + { + (void) prepare_keeper_options(&options); + } + + keeperOptions = options; + + return optind; +} + + /* * prepare_keeper_options finishes the preparation of the keeperOptions that * hosts the command line options. diff --git a/src/bin/pg_autoctl/cli_common.h b/src/bin/pg_autoctl/cli_common.h index dce65b2a6..45bfca490 100644 --- a/src/bin/pg_autoctl/cli_common.h +++ b/src/bin/pg_autoctl/cli_common.h @@ -182,6 +182,7 @@ int cli_create_node_getopts(int argc, char **argv, KeeperConfig *options); int cli_getopt_pgdata(int argc, char **argv); +int cli_getopt_pgdata_or_json(int argc, char **argv); void prepare_keeper_options(KeeperConfig *options); void set_first_pgctl(PostgresSetup *pgSetup); diff --git a/src/bin/pg_autoctl/cli_do_fsm.c b/src/bin/pg_autoctl/cli_do_fsm.c index 3ac4efbe1..63b92e614 100644 --- a/src/bin/pg_autoctl/cli_do_fsm.c +++ b/src/bin/pg_autoctl/cli_do_fsm.c @@ -70,7 +70,7 @@ CommandLine fsm_list = "List reachable FSM states from current state", CLI_PGDATA_USAGE, CLI_PGDATA_OPTION, - cli_getopt_pgdata, + cli_getopt_pgdata_or_json, cli_do_fsm_list); CommandLine fsm_check = @@ -332,11 +332,39 @@ cli_do_fsm_state(int argc, char **argv) /* - * cli_do_fsm_list lists reachable states from the current one. + * cli_do_fsm_list lists reachable states from the current one, or (with + * --json) dumps the full KeeperFSM[] edge set as JSON -- the design doc's + * own proposed standalone use of KeeperFSMToJSON() ("a human or another + * tool may want the raw edge list without a monitor round trip at all"), + * and the source this project's own keeper_fsm_edges.json regress fixture + * (src/monitor/) is regenerated from. + * + * --json needs neither a keeper config nor an on-disk state file at all: + * KeeperFSM[] is pure static data with no dependency on any node's actual + * config or reported state (unlike the ordinary, current-state-filtered + * list output below, which needs keeperState.current_role) -- so it's + * checked first and short-circuits before either read, meaning this mode + * can run with zero setup: no --pgdata, no live cluster, just the binary. + * This is only reachable because fsm_list's own CommandLine (above) uses + * cli_getopt_pgdata_or_json rather than the shared cli_getopt_pgdata: the + * latter unconditionally requires an existing config file + * (prepare_keeper_options, cli_common.c) before this function is ever + * called, regardless of --json. */ static void cli_do_fsm_list(int argc, char **argv) { + if (outputJSON) + { + char *keeperEdgesJSON = KeeperFSMToJSON(); + + fformat(stdout, "%s\n", keeperEdgesJSON); + + json_free_serialized_string(keeperEdgesJSON); + + return; + } + KeeperStateData keeperState = { 0 }; KeeperConfig config = keeperOptions; @@ -360,11 +388,6 @@ cli_do_fsm_list(int argc, char **argv) exit(EXIT_CODE_BAD_STATE); } - if (outputJSON) - { - log_warn("This command does not support JSON output at the moment"); - } - print_reachable_states(&keeperState); fformat(stdout, "\n"); } diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out new file mode 100644 index 000000000..571c5aaa6 --- /dev/null +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -0,0 +1,291 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Two-step check of the real keeper FSM (KeeperFSM[], src/bin/pg_autoctl/ +-- fsm.c) against the monitor's own MonitorFSM[] table, using the +-- keeper_fsm_edges.json fixture committed alongside this file. That fixture +-- is generated from the real KeeperFSM[] via +-- "pg_autoctl inspect fsm list --json" (KeeperFSMToJSON(), fsm.c) -- see +-- cli_do_fsm_list's own comment for how that command runs with zero setup +-- (no --pgdata, no live cluster) -- and must be regenerated by hand +-- whenever KeeperFSM[] changes; this test only reads the fixture, it never +-- runs pg_autoctl itself. +-- +-- Step 1: load the fixture client-side (psql's own backtick file +-- embedding, not server-side pg_read_file() -- the latter is +-- superuser/pg_read_server_files-gated and resolves relative paths against +-- $PGDATA, not this test's own directory) into a real table, one row per +-- distinct keeper edge, so this test's own expected/keeper_fsm_edges.out +-- shows the whole keeper FSM line by line, human-reviewable, with a diff on +-- every change to KeeperFSM[] -- the same discipline fsm.sql's own dump +-- already gives MonitorFSM[]. DISTINCT: KeeperFSMToJSON()'s ANY_STATE +-- expansion (see its own comment, fsm.c) can make two different +-- KeeperFSM[] rows resolve to the exact same (current, assigned) pair -- +-- the JSON has no per-row provenance to tell such duplicates apart by, so +-- they carry no extra information here and would only clutter the +-- reviewable list. +\set keeper_json `cat keeper_fsm_edges.json` +CREATE TABLE keeper_fsm_edges AS +SELECT DISTINCT + (edge ->> 'current')::pgautofailover.replication_state AS current_state, + (edge ->> 'assigned')::pgautofailover.replication_state AS assigned_state + FROM jsonb_array_elements(:'keeper_json'::jsonb) AS edge; +SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; + current_state | assigned_state +---------------------+--------------------- + init | single + init | wait_standby + init | report_lsn + init | dropped + single | wait_primary + single | dropped + wait_primary | single + wait_primary | primary + wait_primary | demoted + wait_primary | join_primary + wait_primary | apply_settings + wait_primary | dropped + primary | single + primary | wait_primary + primary | draining + primary | demote_timeout + primary | demoted + primary | maintenance + primary | join_primary + primary | apply_settings + primary | prepare_maintenance + primary | dropped + draining | single + draining | demote_timeout + draining | demoted + draining | report_lsn + draining | dropped + demote_timeout | single + demote_timeout | primary + demote_timeout | demoted + demote_timeout | dropped + demoted | single + demoted | catchingup + demoted | report_lsn + demoted | dropped + catchingup | single + catchingup | secondary + catchingup | prepare_promotion + catchingup | maintenance + catchingup | wait_maintenance + catchingup | report_lsn + catchingup | dropped + secondary | single + secondary | catchingup + secondary | prepare_promotion + secondary | wait_standby + secondary | maintenance + secondary | wait_maintenance + secondary | report_lsn + secondary | dropped + prepare_promotion | single + prepare_promotion | wait_primary + prepare_promotion | stop_replication + prepare_promotion | dropped + stop_replication | single + stop_replication | wait_primary + stop_replication | dropped + wait_standby | catchingup + wait_standby | dropped + maintenance | catchingup + maintenance | report_lsn + maintenance | dropped + join_primary | single + join_primary | wait_primary + join_primary | primary + join_primary | draining + join_primary | demote_timeout + join_primary | demoted + join_primary | dropped + apply_settings | single + apply_settings | wait_primary + apply_settings | primary + apply_settings | draining + apply_settings | demote_timeout + apply_settings | demoted + apply_settings | join_primary + apply_settings | dropped + prepare_maintenance | catchingup + prepare_maintenance | maintenance + prepare_maintenance | report_lsn + prepare_maintenance | dropped + wait_maintenance | maintenance + wait_maintenance | dropped + report_lsn | single + report_lsn | secondary + report_lsn | prepare_promotion + report_lsn | fast_forward + report_lsn | join_secondary + report_lsn | dropped + fast_forward | prepare_promotion + fast_forward | dropped + join_secondary | secondary + join_secondary | dropped + dropped | single + dropped | wait_standby + dropped | report_lsn + dropped | dropped +(97 rows) + +-- Step 2: the actual cross-check -- every pgautofailover.dump_fsm_edges() +-- edge the keeper_fsm_edges table above has no matching row for. A +-- non-empty result here is a real, actionable gap (unlike +-- check_fsm_reachability.sql's own synthetic-input test, which only +-- exercises the comparison mechanism itself, not real data): see this +-- project's own investigation of these mismatches (dump_fsm_edges()'s own +-- comment, group_state_machine.c, and the design doc) for which of them +-- are genuine keeper gaps versus artifacts already excluded upstream. +SELECT e.pos, e.current_state, e.assigned_state, f.comment + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE NOT EXISTS ( + SELECT 1 + FROM keeper_fsm_edges k + WHERE k.current_state = e.current_state + AND k.assigned_state = e.assigned_state + ) + ORDER BY e.pos, e.current_state; + pos | current_state | assigned_state | comment +-----+---------------------+----------------+---------------------------------------------------------------------------------------------------------------------- + 209 | wait_standby | single | alone in group, candidate-eligible -> single + 209 | maintenance | single | alone in group, candidate-eligible -> single + 209 | prepare_maintenance | single | alone in group, candidate-eligible -> single + 209 | wait_maintenance | single | alone in group, candidate-eligible -> single + 209 | fast_forward | single | alone in group, candidate-eligible -> single + 209 | join_secondary | single | alone in group, candidate-eligible -> single + 211 | wait_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | primary | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | demote_timeout | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | prepare_promotion | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | wait_standby | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | join_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | apply_settings | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | wait_maintenance | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | fast_forward | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | join_secondary | report_lsn | alone in group, candidatePriority zero -> report_lsn + 303 | init | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | draining | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | demote_timeout | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | demoted | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | catchingup | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | wait_standby | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | prepare_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | wait_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | report_lsn | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | fast_forward | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | join_secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | dropped | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 325 | init | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | single | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | demote_timeout | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | demoted | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | catchingup | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | prepare_promotion | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | stop_replication | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | wait_standby | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | prepare_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | wait_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | report_lsn | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | fast_forward | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | join_secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | dropped | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 333 | init | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | single | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | catchingup | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | prepare_promotion | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | stop_replication | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | wait_standby | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | prepare_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | wait_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | report_lsn | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | fast_forward | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | join_secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | dropped | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 339 | init | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | single | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | demoted | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | catchingup | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | prepare_promotion | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | stop_replication | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | wait_standby | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | prepare_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | wait_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | report_lsn | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | fast_forward | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | join_secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | dropped | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 347 | init | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | single | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | catchingup | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | prepare_promotion | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | stop_replication | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | wait_standby | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | prepare_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | wait_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | report_lsn | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | fast_forward | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | join_secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | dropped | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 349 | init | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | single | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | catchingup | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | prepare_promotion | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | stop_replication | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | wait_standby | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | prepare_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | wait_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | report_lsn | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | fast_forward | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | join_secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | dropped | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 351 | init | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | single | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | catchingup | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | prepare_promotion | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | stop_replication | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | wait_standby | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | prepare_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | wait_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | report_lsn | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | fast_forward | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | join_secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 381 | init | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | demote_timeout | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | demoted | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | catchingup | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | prepare_promotion | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | stop_replication | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | wait_standby | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | prepare_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | wait_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | report_lsn | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | fast_forward | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | join_secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 381 | dropped | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining +(134 rows) + +DROP TABLE keeper_fsm_edges; diff --git a/src/monitor/keeper_fsm_edges.json b/src/monitor/keeper_fsm_edges.json new file mode 100644 index 000000000..58d3c2a35 --- /dev/null +++ b/src/monitor/keeper_fsm_edges.json @@ -0,0 +1,562 @@ +[ + { + "current": "init", + "assigned": "single" + }, + { + "current": "init", + "assigned": "single" + }, + { + "current": "init", + "assigned": "single" + }, + { + "current": "dropped", + "assigned": "single" + }, + { + "current": "dropped", + "assigned": "single" + }, + { + "current": "dropped", + "assigned": "report_lsn" + }, + { + "current": "primary", + "assigned": "single" + }, + { + "current": "wait_primary", + "assigned": "single" + }, + { + "current": "join_primary", + "assigned": "single" + }, + { + "current": "primary", + "assigned": "draining" + }, + { + "current": "draining", + "assigned": "demoted" + }, + { + "current": "primary", + "assigned": "demoted" + }, + { + "current": "primary", + "assigned": "demote_timeout" + }, + { + "current": "join_primary", + "assigned": "draining" + }, + { + "current": "join_primary", + "assigned": "demoted" + }, + { + "current": "join_primary", + "assigned": "demote_timeout" + }, + { + "current": "apply_settings", + "assigned": "draining" + }, + { + "current": "apply_settings", + "assigned": "demoted" + }, + { + "current": "apply_settings", + "assigned": "demote_timeout" + }, + { + "current": "primary", + "assigned": "prepare_maintenance" + }, + { + "current": "prepare_maintenance", + "assigned": "maintenance" + }, + { + "current": "primary", + "assigned": "maintenance" + }, + { + "current": "draining", + "assigned": "demote_timeout" + }, + { + "current": "demote_timeout", + "assigned": "demoted" + }, + { + "current": "wait_primary", + "assigned": "demoted" + }, + { + "current": "demoted", + "assigned": "single" + }, + { + "current": "demoted", + "assigned": "single" + }, + { + "current": "demote_timeout", + "assigned": "single" + }, + { + "current": "demote_timeout", + "assigned": "single" + }, + { + "current": "draining", + "assigned": "single" + }, + { + "current": "draining", + "assigned": "single" + }, + { + "current": "secondary", + "assigned": "single" + }, + { + "current": "secondary", + "assigned": "single" + }, + { + "current": "secondary", + "assigned": "single" + }, + { + "current": "catchingup", + "assigned": "single" + }, + { + "current": "catchingup", + "assigned": "single" + }, + { + "current": "catchingup", + "assigned": "single" + }, + { + "current": "prepare_promotion", + "assigned": "single" + }, + { + "current": "prepare_promotion", + "assigned": "single" + }, + { + "current": "prepare_promotion", + "assigned": "single" + }, + { + "current": "stop_replication", + "assigned": "single" + }, + { + "current": "stop_replication", + "assigned": "single" + }, + { + "current": "report_lsn", + "assigned": "single" + }, + { + "current": "single", + "assigned": "wait_primary" + }, + { + "current": "primary", + "assigned": "join_primary" + }, + { + "current": "primary", + "assigned": "wait_primary" + }, + { + "current": "join_primary", + "assigned": "wait_primary" + }, + { + "current": "wait_primary", + "assigned": "join_primary" + }, + { + "current": "wait_primary", + "assigned": "primary" + }, + { + "current": "join_primary", + "assigned": "primary" + }, + { + "current": "demote_timeout", + "assigned": "primary" + }, + { + "current": "wait_standby", + "assigned": "catchingup" + }, + { + "current": "demoted", + "assigned": "catchingup" + }, + { + "current": "secondary", + "assigned": "catchingup" + }, + { + "current": "catchingup", + "assigned": "secondary" + }, + { + "current": "catchingup", + "assigned": "secondary" + }, + { + "current": "secondary", + "assigned": "prepare_promotion" + }, + { + "current": "secondary", + "assigned": "prepare_promotion" + }, + { + "current": "catchingup", + "assigned": "prepare_promotion" + }, + { + "current": "catchingup", + "assigned": "prepare_promotion" + }, + { + "current": "prepare_promotion", + "assigned": "stop_replication" + }, + { + "current": "prepare_promotion", + "assigned": "stop_replication" + }, + { + "current": "stop_replication", + "assigned": "wait_primary" + }, + { + "current": "stop_replication", + "assigned": "wait_primary" + }, + { + "current": "stop_replication", + "assigned": "wait_primary" + }, + { + "current": "prepare_promotion", + "assigned": "wait_primary" + }, + { + "current": "prepare_promotion", + "assigned": "wait_primary" + }, + { + "current": "init", + "assigned": "wait_standby" + }, + { + "current": "dropped", + "assigned": "wait_standby" + }, + { + "current": "secondary", + "assigned": "wait_standby" + }, + { + "current": "secondary", + "assigned": "wait_maintenance" + }, + { + "current": "catchingup", + "assigned": "wait_maintenance" + }, + { + "current": "secondary", + "assigned": "maintenance" + }, + { + "current": "catchingup", + "assigned": "maintenance" + }, + { + "current": "wait_maintenance", + "assigned": "maintenance" + }, + { + "current": "maintenance", + "assigned": "catchingup" + }, + { + "current": "prepare_maintenance", + "assigned": "catchingup" + }, + { + "current": "primary", + "assigned": "apply_settings" + }, + { + "current": "wait_primary", + "assigned": "apply_settings" + }, + { + "current": "apply_settings", + "assigned": "primary" + }, + { + "current": "apply_settings", + "assigned": "single" + }, + { + "current": "apply_settings", + "assigned": "wait_primary" + }, + { + "current": "apply_settings", + "assigned": "join_primary" + }, + { + "current": "secondary", + "assigned": "report_lsn" + }, + { + "current": "catchingup", + "assigned": "report_lsn" + }, + { + "current": "maintenance", + "assigned": "report_lsn" + }, + { + "current": "prepare_maintenance", + "assigned": "report_lsn" + }, + { + "current": "report_lsn", + "assigned": "prepare_promotion" + }, + { + "current": "report_lsn", + "assigned": "prepare_promotion" + }, + { + "current": "report_lsn", + "assigned": "fast_forward" + }, + { + "current": "fast_forward", + "assigned": "prepare_promotion" + }, + { + "current": "fast_forward", + "assigned": "prepare_promotion" + }, + { + "current": "report_lsn", + "assigned": "join_secondary" + }, + { + "current": "report_lsn", + "assigned": "secondary" + }, + { + "current": "join_secondary", + "assigned": "secondary" + }, + { + "current": "draining", + "assigned": "report_lsn" + }, + { + "current": "demoted", + "assigned": "report_lsn" + }, + { + "current": "init", + "assigned": "report_lsn" + }, + { + "current": "init", + "assigned": "dropped" + }, + { + "current": "single", + "assigned": "dropped" + }, + { + "current": "primary", + "assigned": "dropped" + }, + { + "current": "wait_primary", + "assigned": "dropped" + }, + { + "current": "wait_standby", + "assigned": "dropped" + }, + { + "current": "demoted", + "assigned": "dropped" + }, + { + "current": "demote_timeout", + "assigned": "dropped" + }, + { + "current": "draining", + "assigned": "dropped" + }, + { + "current": "secondary", + "assigned": "dropped" + }, + { + "current": "catchingup", + "assigned": "dropped" + }, + { + "current": "prepare_promotion", + "assigned": "dropped" + }, + { + "current": "stop_replication", + "assigned": "dropped" + }, + { + "current": "maintenance", + "assigned": "dropped" + }, + { + "current": "join_primary", + "assigned": "dropped" + }, + { + "current": "apply_settings", + "assigned": "dropped" + }, + { + "current": "prepare_maintenance", + "assigned": "dropped" + }, + { + "current": "wait_maintenance", + "assigned": "dropped" + }, + { + "current": "report_lsn", + "assigned": "dropped" + }, + { + "current": "fast_forward", + "assigned": "dropped" + }, + { + "current": "join_secondary", + "assigned": "dropped" + }, + { + "current": "dropped", + "assigned": "dropped" + }, + { + "current": "init", + "assigned": "dropped" + }, + { + "current": "single", + "assigned": "dropped" + }, + { + "current": "primary", + "assigned": "dropped" + }, + { + "current": "wait_primary", + "assigned": "dropped" + }, + { + "current": "wait_standby", + "assigned": "dropped" + }, + { + "current": "demoted", + "assigned": "dropped" + }, + { + "current": "demote_timeout", + "assigned": "dropped" + }, + { + "current": "draining", + "assigned": "dropped" + }, + { + "current": "secondary", + "assigned": "dropped" + }, + { + "current": "catchingup", + "assigned": "dropped" + }, + { + "current": "prepare_promotion", + "assigned": "dropped" + }, + { + "current": "stop_replication", + "assigned": "dropped" + }, + { + "current": "maintenance", + "assigned": "dropped" + }, + { + "current": "join_primary", + "assigned": "dropped" + }, + { + "current": "apply_settings", + "assigned": "dropped" + }, + { + "current": "prepare_maintenance", + "assigned": "dropped" + }, + { + "current": "wait_maintenance", + "assigned": "dropped" + }, + { + "current": "report_lsn", + "assigned": "dropped" + }, + { + "current": "fast_forward", + "assigned": "dropped" + }, + { + "current": "join_secondary", + "assigned": "dropped" + }, + { + "current": "dropped", + "assigned": "dropped" + } +] diff --git a/src/monitor/regress_schedule b/src/monitor/regress_schedule index a691348eb..5cc5768ce 100644 --- a/src/monitor/regress_schedule +++ b/src/monitor/regress_schedule @@ -33,6 +33,7 @@ test: create_extension test: fsm test: check_fsm_reachability +test: keeper_fsm_edges test: monitor test: workers test: node_active_protocol diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql new file mode 100644 index 000000000..b19239158 --- /dev/null +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -0,0 +1,56 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Two-step check of the real keeper FSM (KeeperFSM[], src/bin/pg_autoctl/ +-- fsm.c) against the monitor's own MonitorFSM[] table, using the +-- keeper_fsm_edges.json fixture committed alongside this file. That fixture +-- is generated from the real KeeperFSM[] via +-- "pg_autoctl inspect fsm list --json" (KeeperFSMToJSON(), fsm.c) -- see +-- cli_do_fsm_list's own comment for how that command runs with zero setup +-- (no --pgdata, no live cluster) -- and must be regenerated by hand +-- whenever KeeperFSM[] changes; this test only reads the fixture, it never +-- runs pg_autoctl itself. +-- +-- Step 1: load the fixture client-side (psql's own backtick file +-- embedding, not server-side pg_read_file() -- the latter is +-- superuser/pg_read_server_files-gated and resolves relative paths against +-- $PGDATA, not this test's own directory) into a real table, one row per +-- distinct keeper edge, so this test's own expected/keeper_fsm_edges.out +-- shows the whole keeper FSM line by line, human-reviewable, with a diff on +-- every change to KeeperFSM[] -- the same discipline fsm.sql's own dump +-- already gives MonitorFSM[]. DISTINCT: KeeperFSMToJSON()'s ANY_STATE +-- expansion (see its own comment, fsm.c) can make two different +-- KeeperFSM[] rows resolve to the exact same (current, assigned) pair -- +-- the JSON has no per-row provenance to tell such duplicates apart by, so +-- they carry no extra information here and would only clutter the +-- reviewable list. +\set keeper_json `cat keeper_fsm_edges.json` + +CREATE TABLE keeper_fsm_edges AS +SELECT DISTINCT + (edge ->> 'current')::pgautofailover.replication_state AS current_state, + (edge ->> 'assigned')::pgautofailover.replication_state AS assigned_state + FROM jsonb_array_elements(:'keeper_json'::jsonb) AS edge; + +SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; + +-- Step 2: the actual cross-check -- every pgautofailover.dump_fsm_edges() +-- edge the keeper_fsm_edges table above has no matching row for. A +-- non-empty result here is a real, actionable gap (unlike +-- check_fsm_reachability.sql's own synthetic-input test, which only +-- exercises the comparison mechanism itself, not real data): see this +-- project's own investigation of these mismatches (dump_fsm_edges()'s own +-- comment, group_state_machine.c, and the design doc) for which of them +-- are genuine keeper gaps versus artifacts already excluded upstream. +SELECT e.pos, e.current_state, e.assigned_state, f.comment + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE NOT EXISTS ( + SELECT 1 + FROM keeper_fsm_edges k + WHERE k.current_state = e.current_state + AND k.assigned_state = e.assigned_state + ) + ORDER BY e.pos, e.current_state; + +DROP TABLE keeper_fsm_edges; From a6d6342414332007bdf15b722d90bd02d7d53f17 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 29 Jul 2026 17:38:38 +0200 Subject: [PATCH 09/52] Fix pgautofailover.last_events() (broken) and its rule_pos comment pgautofailover.last_events() (all three overloads) declares RETURNS SETOF pgautofailover.event, but its own SELECT list never included the rule_pos/rule_section columns added earlier this branch -- so every call errored at parse time ("Final statement returns too few columns"), before ever running. This broke "pg_autoctl show events" and "pg_autoctl watch" entirely. Never caught because nothing in the regress suite ever called last_events() -- added minimal coverage in monitor.sql (all three overloads, count(*) >= 0 checks so the expected output stays stable regardless of exact event counts elsewhere in the file). While fixing this, found the formation+group overload was additionally missing nodename/nodehost/nodeport entirely (13 columns instead of 16) -- a separate, pre-existing bug that would have broken "pg_autoctl show events --group N" specifically, also never caught for the same reason. Fixed by aligning both overloads to select the same column set. Also fixes two comments (pgautofailover.sql's pgautofailover.event table, and notifications.h's CurrentMonitorFSMRulePos declaration -- same wrong claim duplicated in both places) that said rule_pos is NULL for operator-triggered SQL functions and for ProceedGroupStateForMSFailover's own hand-written internals. Verified against the actual dispatch code this is wrong: operator-triggered functions dispatch via ProceedGroupStateForApiTrigger, which itself calls DispatchMonitorFSMRule (sets the global); MS-failover's internals are only ever invoked from inside an outer row's own extraAction, so the global is already non-zero by the time they call AssignGoalState directly. Both get a real, non-NULL rule_pos -- attributed to the OUTER triggering row, not one of their own, since MS-failover's candidate-selection internals were never decomposed into declarative rows. Corrected both comments to describe this accurately. Verified: full regress+isolation Docker installcheck clean (18/18 regress including the new last_events coverage, 6/6 isolation); live psql calls against a real monitor confirming last_events() now returns rule_pos/rule_section correctly for both the all-formations and specific-group cases. --- src/monitor/expected/monitor.out | 23 ++++++++++++ .../expected/pg19/expected/monitor.out | 22 ++++++++++++ src/monitor/notifications.h | 19 +++++++--- src/monitor/pgautofailover.sql | 35 ++++++++++++++----- src/monitor/sql/monitor.sql | 17 +++++++++ 5 files changed, 103 insertions(+), 13 deletions(-) diff --git a/src/monitor/expected/monitor.out b/src/monitor/expected/monitor.out index 6b937bf9f..dc0253af1 100644 --- a/src/monitor/expected/monitor.out +++ b/src/monitor/expected/monitor.out @@ -247,3 +247,26 @@ node_port | 9877 -- should fail as there's no primary at this point select pgautofailover.perform_failover(); ERROR: couldn't find the primary node in formation "default", group 0 +-- last_events() (all three overloads) returns SETOF pgautofailover.event, so +-- its own SELECT list must match that composite type's full column set -- +-- including rule_pos/rule_section -- or the call errors at parse time +-- ("Final statement returns too few columns") before ever running. Not +-- exercised anywhere else in this test suite, so a regression here (e.g. a +-- future column added to pgautofailover.event without updating these three +-- function bodies) would otherwise go unnoticed until a live +-- "pg_autoctl show events"/"pg_autoctl watch" call broke in production. +select count(*) >= 0 as last_events_count_ok + from pgautofailover.last_events(10); +-[ RECORD 1 ]--------+-- +last_events_count_ok | t + +select count(*) >= 0 as last_events_by_formation_count_ok + from pgautofailover.last_events(formation_id => 'default', count => 10); +-[ RECORD 1 ]---------------------+-- +last_events_by_formation_count_ok | t + +select count(*) >= 0 as last_events_by_formation_and_group_count_ok + from pgautofailover.last_events('default', 0, 10); +-[ RECORD 1 ]-------------------------------+-- +last_events_by_formation_and_group_count_ok | t + diff --git a/src/monitor/expected/pg19/expected/monitor.out b/src/monitor/expected/pg19/expected/monitor.out index 1fa8dd8c0..7b8699fb3 100644 --- a/src/monitor/expected/pg19/expected/monitor.out +++ b/src/monitor/expected/pg19/expected/monitor.out @@ -247,3 +247,25 @@ node_port | 9877 -- should fail as there's no primary at this point select pgautofailover.perform_failover(); ERROR: couldn't find the primary node in formation "default", group 0 +-- last_events() (all three overloads) returns SETOF pgautofailover.event, so +-- its own SELECT list must match that composite type's full column set -- +-- including rule_pos/rule_section -- or the call errors at parse time +-- ("Final statement returns too few columns") before ever running. Not +-- exercised anywhere else in this test suite, so a regression here (e.g. a +-- future column added to pgautofailover.event without updating these three +-- function bodies) would otherwise go unnoticed until a live +-- "pg_autoctl show events"/"pg_autoctl watch" call broke in production. +select count(*) >= 0 as last_events_count_ok + from pgautofailover.last_events(10); +-[ RECORD 1 ]--------+-- +last_events_count_ok | t + +select count(*) >= 0 as last_events_by_formation_count_ok + from pgautofailover.last_events(formation_id => 'default', count => 10); +-[ RECORD 1 ]---------------------+-- +last_events_by_formation_count_ok | t + +select count(*) >= 0 as last_events_by_formation_and_group_count_ok + from pgautofailover.last_events('default', 0, 10); +-[ RECORD 1 ]-------------------------------+-- +last_events_by_formation_and_group_count_ok | t diff --git a/src/monitor/notifications.h b/src/monitor/notifications.h index a636755c3..9bf22a317 100644 --- a/src/monitor/notifications.h +++ b/src/monitor/notifications.h @@ -53,11 +53,20 @@ int64 InsertEvent(AutoFailoverNode *node, char *description); * (rule_pos/rule_section columns), without threading an extra parameter * through AssignGoalState/SetNodeGoalState/NotifyStateChange and every one * of their many call sites outside the declarative dispatch table. 0 means - * "no rule attributed" (an ordinary AssignGoalState call from outside the - * table, e.g. an operator-triggered SQL function, or ProceedGroupStateFor - * MSFailover's own hand-written internals): rule_pos/rule_section are left - * NULL in that case, since 0 is not a valid .pos value (every real section - * starts at 100 or above). + * "no rule attributed": rule_pos/rule_section are left NULL in that case, + * since 0 is not a valid .pos value (every real section starts at 100 or + * above). This is NOT the case for operator-triggered SQL functions or for + * ProceedGroupStateForMSFailover's own hand-written internals -- both DO + * get a real, non-NULL rule_pos: the former dispatch via + * ProceedGroupStateForApiTrigger, which itself calls DispatchMonitorFSMRule; + * the latter (BuildCandidateList, PromoteSelectedNode, etc.) are only ever + * invoked from inside an outer row's own extraAction, so this global is + * already non-zero (attributed to that OUTER row, not one of their own -- + * misleading, since MS-failover's candidate-selection internals were never + * decomposed into declarative rows) by the time they call AssignGoalState + * directly. 0/NULL only really happens for call sites genuinely outside any + * DispatchMonitorFSMRule call at all -- e.g. perform_failover()'s own + * candidate-priority bookkeeping in node_active_protocol.c. */ extern int CurrentMonitorFSMRulePos; extern int CurrentMonitorFSMRuleSection; diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index dfb77a702..6a53e7489 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -209,11 +209,27 @@ CREATE TABLE pgautofailover.event replicationquorum bool, description text, - -- Which MonitorFSM[] row (if any) produced this event: NULL when the - -- goal-state assignment came from outside the declarative dispatch - -- table (an operator-triggered SQL function, or ProceedGroupStateFor - -- MSFailover's own hand-written internals -- see CurrentMonitorFSMRulePos - -- in notifications.h for how this gets attributed). rule_pos is the + -- Which MonitorFSM[] row this event is attributed to (see + -- CurrentMonitorFSMRulePos in notifications.h for the mechanism): set + -- for the whole duration of DispatchMonitorFSMRule's call to a row, + -- including everything that row's own extraAction runs -- which + -- covers BOTH operator-triggered SQL functions (dispatched via + -- ProceedGroupStateForApiTrigger, itself a DispatchMonitorFSMRule + -- call) and ProceedGroupStateForMSFailover's raw AssignGoalState + -- calls (only ever reached from inside an outer row's extraAction, + -- e.g. pos 305/363 -- see ActionRunMultiStandbyFailoverCascade/ + -- ActionRunPlainMSFailoverCascade). Neither is NULL: both get + -- attributed to that OUTER triggering row, not a row of their own, + -- since MS-failover's candidate-selection internals were never + -- decomposed into declarative rows (see this array's own comment on + -- BuildCandidateList/PromoteSelectedNode) -- this can be misleading + -- (an event from PromoteSelectedNode's LSN-driven promotion decision + -- shows up attributed to pos 305/363's own, quite different, + -- comment). Truly NULL only when the assignment happened from a code + -- path that never runs inside any DispatchMonitorFSMRule call at all + -- (e.g. perform_failover()'s own candidate-priority bookkeeping in + -- node_active_protocol.c, or a handful of other NotifyStateChange + -- call sites outside group_state_machine.c entirely). rule_pos is the -- row's human-facing position (see dump_fsm()/pgautofailover.fsm), -- not an array index. rule_pos int, @@ -666,7 +682,8 @@ with last_events as nodeid, groupid, nodename, nodehost, nodeport, reportedstate, goalstate, reportedrepstate, reportedtli, reportedlsn, - candidatepriority, replicationquorum, description + candidatepriority, replicationquorum, description, + rule_pos, rule_section from pgautofailover.event order by eventid desc limit count @@ -693,7 +710,8 @@ with last_events as nodeid, groupid, nodename, nodehost, nodeport, reportedstate, goalstate, reportedrepstate, reportedtli, reportedlsn, - candidatepriority, replicationquorum, description + candidatepriority, replicationquorum, description, + rule_pos, rule_section from pgautofailover.event where formationid = formation_id order by eventid desc @@ -722,7 +740,8 @@ with last_events as nodeid, groupid, nodename, nodehost, nodeport, reportedstate, goalstate, reportedrepstate, reportedtli, reportedlsn, - candidatepriority, replicationquorum, description + candidatepriority, replicationquorum, description, + rule_pos, rule_section from pgautofailover.event where formationid = formation_id and groupid = group_id diff --git a/src/monitor/sql/monitor.sql b/src/monitor/sql/monitor.sql index b8c0a0685..aa2099916 100644 --- a/src/monitor/sql/monitor.sql +++ b/src/monitor/sql/monitor.sql @@ -79,3 +79,20 @@ select * -- should fail as there's no primary at this point select pgautofailover.perform_failover(); + +-- last_events() (all three overloads) returns SETOF pgautofailover.event, so +-- its own SELECT list must match that composite type's full column set -- +-- including rule_pos/rule_section -- or the call errors at parse time +-- ("Final statement returns too few columns") before ever running. Not +-- exercised anywhere else in this test suite, so a regression here (e.g. a +-- future column added to pgautofailover.event without updating these three +-- function bodies) would otherwise go unnoticed until a live +-- "pg_autoctl show events"/"pg_autoctl watch" call broke in production. +select count(*) >= 0 as last_events_count_ok + from pgautofailover.last_events(10); + +select count(*) >= 0 as last_events_by_formation_count_ok + from pgautofailover.last_events(formation_id => 'default', count => 10); + +select count(*) >= 0 as last_events_by_formation_and_group_count_ok + from pgautofailover.last_events('default', 0, 10); From 6e075f1abfac2521b8a75a4fd2d516ab0c6f6115 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 29 Jul 2026 17:39:01 +0200 Subject: [PATCH 10/52] Show rule_pos in "pg_autoctl show events" and "pg_autoctl watch" MonitorEvent gains rulePos/ruleSection fields (0/"" meaning "no rule attributed", matching the monitor-side CurrentMonitorFSMRulePos convention). monitor_get_last_events() (backs "pg_autoctl watch"'s events pane): adds rule_pos/rule_section to its SQL and parsing. This also fixes the pre-existing missing-nodename/nodehost/nodeport bug in its formation+group SQL branch (see the previous commit -- same file, same underlying last_events() overload). monitor_print_last_events()/printLastEvents (backs plain "pg_autoctl show events"): adds a "Rule" column showing rule_pos, positioned before Comment; blank when NULL. monitor_print_last_events_as_json() ("show events --json"): needed no changes at all -- it already does row_to_json(event), which picks up every column of the event row type generically, including rule_pos/ rule_section, once last_events() itself returns them. While here, fixed an unrelated, pre-existing SQL typo in this same function's formation+group branch: "FROM * FROM pgautofailover.last_events(...)" (a stray duplicate FROM, invalid syntax) -- broke "show events --json" for any specific group, for a third, independent reason from the two fixed in the previous commit. watch_colspecs.h: new EVENT_COLUMN_TYPE_RULE_POS, added to the "verbose" event column policy only (between Name and Description) -- watch.c wires it into compute_event_column_size/print_event following the exact pattern the existing columns already use (dynamic max-width tracking against both the data and the "Rule" header's own length). Verified: clean compile (monitor extension + pg_autoctl), and end to end against a live monitor+node in Docker -- "show events" (plain and --json) and the formation+group-specific query path all correctly show rule_pos 209/"early_checks" for an FSM-dispatched event and blank/null for a health-check-triggered one. The interactive "watch" TUI itself couldn't be exercised headlessly (needs a real tty), but its own query is the same shape independently verified via psql and "show events". --- src/bin/pg_autoctl/monitor.c | 54 +++++++++++++++++++++-------- src/bin/pg_autoctl/monitor.h | 11 ++++++ src/bin/pg_autoctl/watch.c | 33 ++++++++++++++++++ src/bin/pg_autoctl/watch.h | 1 + src/bin/pg_autoctl/watch_colspecs.h | 2 ++ 5 files changed, 86 insertions(+), 15 deletions(-) diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index 5b78cabce..019193361 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -2908,7 +2908,7 @@ monitor_print_last_events(Monitor *monitor, char *formation, int group, int coun { sql = "SELECT eventTime, nodeid, groupid, " - " reportedstate, goalState, description " + " reportedstate, goalState, description, rule_pos " " FROM pgautofailover.last_events($1, count => $2)"; countStr = intToString(count); @@ -2926,7 +2926,7 @@ monitor_print_last_events(Monitor *monitor, char *formation, int group, int coun { sql = "SELECT eventTime, nodeid, groupid, " - " reportedstate, goalState, description " + " reportedstate, goalState, description, rule_pos " " FROM pgautofailover.last_events($1,$2,$3)"; countStr = intToString(count); @@ -3004,7 +3004,7 @@ monitor_print_last_events_as_json(Monitor *monitor, { sql = "SELECT jsonb_pretty(" "coalesce(jsonb_agg(row_to_json(event)), '[]'))" - " FROM * FROM pgautofailover.last_events($1,$2,$3) as event"; + " FROM pgautofailover.last_events($1,$2,$3) as event"; countStr = intToString(count); groupStr = intToString(group); @@ -3062,20 +3062,20 @@ printLastEvents(void *ctx, PGresult *result) log_trace("printLastEvents: %d tuples", nTuples); - if (PQnfields(result) != 6) + if (PQnfields(result) != 7) { - log_error("Query returned %d columns, expected 6", PQnfields(result)); + log_error("Query returned %d columns, expected 7", PQnfields(result)); context->parsedOK = false; return; } - fformat(stdout, "%30s | %6s | %19s | %19s | %s\n", + fformat(stdout, "%30s | %6s | %19s | %19s | %6s | %s\n", "Event Time", "Node", - "Current State", "Assigned State", "Comment"); - fformat(stdout, "%30s-+-%6s-+-%19s-+-%19s-+-%10s\n", + "Current State", "Assigned State", "Rule", "Comment"); + fformat(stdout, "%30s-+-%6s-+-%19s-+-%19s-+-%6s-+-%10s\n", "------------------------------", "------", "-------------------", - "-------------------", "----------"); + "-------------------", "------", "----------"); for (currentTupleIndex = 0; currentTupleIndex < nTuples; currentTupleIndex++) { @@ -3085,14 +3085,16 @@ printLastEvents(void *ctx, PGresult *result) char *currentState = PQgetvalue(result, currentTupleIndex, 3); char *goalState = PQgetvalue(result, currentTupleIndex, 4); char *description = PQgetvalue(result, currentTupleIndex, 5); + bool rulePosIsNull = PQgetisnull(result, currentTupleIndex, 6); + char *rulePos = rulePosIsNull ? "" : PQgetvalue(result, currentTupleIndex, 6); char node[BUFSIZE]; /* for our grid alignment output it's best to have a single col here */ sformat(node, BUFSIZE, "%s/%s", groupId, nodeId); - fformat(stdout, "%30s | %6s | %19s | %19s | %s\n", + fformat(stdout, "%30s | %6s | %19s | %19s | %6s | %s\n", eventTime, node, - currentState, goalState, description); + currentState, goalState, rulePos, description); } fformat(stdout, "\n"); @@ -3132,7 +3134,7 @@ monitor_get_last_events(Monitor *monitor, char *formation, int group, int count, " reportedstate, goalState, " " reportedrepstate, reportedtli, reportedlsn, " " candidatepriority, replicationquorum, " - " description " + " description, rule_pos, rule_section " " FROM pgautofailover.last_events($1, count => $2)"; countStr = intToString(count); @@ -3151,10 +3153,11 @@ monitor_get_last_events(Monitor *monitor, char *formation, int group, int count, sql = "SELECT eventId, to_char(eventTime, 'YYYY-MM-DD HH24:MI:SS'), " " formationId, nodeid, groupid, " + " nodename, nodehost, nodeport, " " reportedstate, goalState, " " reportedrepstate, reportedtli, reportedlsn, " " candidatepriority, replicationquorum, " - " description " + " description, rule_pos, rule_section " " FROM pgautofailover.last_events($1,$2,$3)"; countStr = intToString(count); @@ -3219,9 +3222,9 @@ getLastEvents(void *ctx, PGresult *result) return; } - if (PQnfields(result) != 16) + if (PQnfields(result) != 18) { - log_error("Query returned %d columns, expected 16", PQnfields(result)); + log_error("Query returned %d columns, expected 18", PQnfields(result)); context->parsedOK = false; return; } @@ -3338,6 +3341,27 @@ getLastEvents(void *ctx, PGresult *result) value = PQgetvalue(result, currentTupleIndex, 15); strlcpy(event->description, value, sizeof(event->description)); + /* rule_pos: NULL means "no rule attributed", represented as 0 */ + if (PQgetisnull(result, currentTupleIndex, 16)) + { + event->rulePos = 0; + event->ruleSection[0] = '\0'; + } + else + { + value = PQgetvalue(result, currentTupleIndex, 16); + + if (!stringToInt(value, &(event->rulePos))) + { + log_error("Invalid rule_pos \"%s\" returned by monitor", value); + ++errors; + } + + /* rule_section, only meaningful alongside a real rule_pos */ + value = PQgetvalue(result, currentTupleIndex, 17); + strlcpy(event->ruleSection, value, sizeof(event->ruleSection)); + } + if (errors > 0) { context->parsedOK = false; diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index 007ab8760..3b0fbe770 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -68,6 +68,17 @@ typedef struct MonitorEvent int candidatePriority; bool replicationQuorum; char description[BUFSIZE]; + + /* + * Which MonitorFSM[] row (if any) produced this event -- see + * pgautofailover.event's own rule_pos/rule_section comment + * (pgautofailover.sql) for exactly what NULL does and doesn't mean. + * rulePos 0 here represents SQL NULL (matches the monitor-side + * CurrentMonitorFSMRulePos convention: 0 is never a real row position), + * in which case ruleSection is left empty. + */ + int rulePos; + char ruleSection[NAMEDATALEN]; } MonitorEvent; #define EVENTS_ARRAY_MAX_COUNT 1024 diff --git a/src/bin/pg_autoctl/watch.c b/src/bin/pg_autoctl/watch.c index 3fccaf927..9426d107d 100644 --- a/src/bin/pg_autoctl/watch.c +++ b/src/bin/pg_autoctl/watch.c @@ -1265,6 +1265,10 @@ compute_events_sizes(WatchContext *context) int timeSize = 19; /* "YYYY-MM-DD HH:MI:SS" is 19 chars long */ int descSize = 60; /* desc. has horizontal scrolling */ + /* rulePos 0 means "no rule attributed" (blank), not a real width */ + int ruleSize = + event->rulePos > 0 ? ((int) log10(event->rulePos) + 1) : 1; + if (headers->maxEventIdSize < idSize) { headers->maxEventIdSize = idSize; @@ -1280,6 +1284,17 @@ compute_events_sizes(WatchContext *context) headers->maxEventNodeNameSize = nameSize; } + /* "Rule" (the column header) is 4 chars, never shrink below that */ + if (headers->maxEventRulePosSize < 4) + { + headers->maxEventRulePosSize = 4; + } + + if (headers->maxEventRulePosSize < ruleSize) + { + headers->maxEventRulePosSize = ruleSize; + } + if (headers->maxEventDescSize < descSize) { headers->maxEventDescSize = descSize; @@ -1315,6 +1330,11 @@ compute_event_column_size(EventColumnType type, MonitorEventsHeaders *headers) return headers->maxEventNodeNameSize; } + case EVENT_COLUMN_TYPE_RULE_POS: + { + return headers->maxEventRulePosSize; + } + case EVENT_COLUMN_TYPE_DESCRIPTION: { return headers->maxEventDescSize; @@ -1449,6 +1469,19 @@ print_event(WatchContext *context, EventColPolicy *policy, int index, int r, int break; } + case EVENT_COLUMN_TYPE_RULE_POS: + { + if (event->rulePos > 0) + { + mvprintw(r, cc, "%*d", len, event->rulePos); + } + else + { + mvprintw(r, cc, "%*s", len, ""); + } + break; + } + case EVENT_COLUMN_TYPE_DESCRIPTION: { char *text = event->description; diff --git a/src/bin/pg_autoctl/watch.h b/src/bin/pg_autoctl/watch.h index f5c119548..0a2e1e748 100644 --- a/src/bin/pg_autoctl/watch.h +++ b/src/bin/pg_autoctl/watch.h @@ -38,6 +38,7 @@ typedef struct MonitorEventsHeaders int maxEventIdSize; int maxEventTimeSize; int maxEventNodeNameSize; + int maxEventRulePosSize; int maxEventDescSize; } MonitorEventsHeaders; diff --git a/src/bin/pg_autoctl/watch_colspecs.h b/src/bin/pg_autoctl/watch_colspecs.h index f3726581f..7b411e108 100644 --- a/src/bin/pg_autoctl/watch_colspecs.h +++ b/src/bin/pg_autoctl/watch_colspecs.h @@ -226,6 +226,7 @@ typedef enum EVENT_COLUMN_TYPE_LSN, EVENT_COLUMN_TYPE_CANDIDATE_PRIORITY, EVENT_COLUMN_TYPE_REPLICATION_QUORUM, + EVENT_COLUMN_TYPE_RULE_POS, EVENT_COLUMN_TYPE_DESCRIPTION, EVENT_COLUMN_TYPE_LAST @@ -291,6 +292,7 @@ EventColPolicy EventColumnPolicies[] = { { EVENT_COLUMN_TYPE_ID, "Id", 0 }, { EVENT_COLUMN_TYPE_TIME, "Event Time", 0 }, { EVENT_COLUMN_TYPE_NODE_NAME, "Name", 0 }, + { EVENT_COLUMN_TYPE_RULE_POS, "Rule", 0 }, { EVENT_COLUMN_TYPE_DESCRIPTION, "Description", 0 }, { EVENT_COLUMN_TYPE_LAST, "", 0 } } From a18595d72323ba1fcfb0fe2972dfeaf2c694565a Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 29 Jul 2026 18:09:14 +0200 Subject: [PATCH 11/52] Terminate every FSM-relevant regress test with a last_events() summary Adds an "event summary" query at the tail of every regress test that actually registers nodes and drives state transitions: workers, node_active_protocol (both formations it uses), guard_data_loss, fast_forward, drop_node, stale_primary_report, lock_and_fetch_migration, timeline_fork_detection (both formations), and failover_candidate_leaves_secondary. Skipped: create_extension, fsm, check_fsm_reachability, keeper_fsm_edges (no formation/node activity at all), dummy_update/drop_extension/upgrade (extension-version lifecycle tests, not FSM scenarios), and cluster_init_failover_rule_attribution (already has its own, more detailed rule-attribution query at its own tail). Each summary is: SELECT reportedstate, goalstate, rule_pos, rule_section, description FROM pgautofailover.last_events('', count => 100); Filtering by the test's own uniquely-named formation (gdl_test, ff_test, dn_test, etc.) isolates each summary from every other test sharing the same pgautofailover.event table across this schedule's serial run -- verified safe regardless of position, since nodeid/event rows carry no FK to pgautofailover.node (dropping a node doesn't cascade-delete its history). eventid and eventtime are deliberately never selected: eventid is a database-wide sequence shared by the whole schedule (see regress_schedule's own comment) and eventtime is a live timestamp -- neither is a value this file's own expected output could ever pin stably. This gives pgautofailover.last_events() -- broken for a long time until the previous two commits, entirely unexercised until this one -- real coverage against genuine multi-step scenarios (17-30 events each), not just the synthetic smoke test in monitor.sql. Reviewed every generated summary for sanity (correct node names, no cross-test bleed, sensible rule_pos/rule_section per event) before promoting. pg19-specific handling: expected/pg19/expected/{workers, node_active_protocol,guard_data_loss}.out are symlinks to the main expected file, so needed no separate edit. fast_forward.out and timeline_fork_detection.out are real, separate files (pg_lsn display format differs on PG19) -- appended the identical new tail content to both, since none of the summary's own columns touch lsn/timestamp fields. Verified: full regress+isolation Docker installcheck clean (18/18 regress including all nine modified tests, 6/6 isolation). --- src/monitor/expected/drop_node.out | 96 ++++++ .../failover_candidate_leaves_secondary.out | 78 +++++ src/monitor/expected/fast_forward.out | 114 +++++++ src/monitor/expected/guard_data_loss.out | 114 +++++++ .../expected/lock_and_fetch_migration.out | 192 ++++++++++++ src/monitor/expected/node_active_protocol.out | 288 ++++++++++++++++++ .../expected/pg19/expected/fast_forward.out | 114 +++++++ .../pg19/expected/timeline_fork_detection.out | 141 +++++++++ src/monitor/expected/stale_primary_report.out | 120 ++++++++ .../expected/timeline_fork_detection.out | 141 +++++++++ src/monitor/expected/workers.out | 30 ++ src/monitor/sql/drop_node.sql | 12 + .../failover_candidate_leaves_secondary.sql | 12 + src/monitor/sql/fast_forward.sql | 12 + src/monitor/sql/guard_data_loss.sql | 12 + src/monitor/sql/lock_and_fetch_migration.sql | 12 + src/monitor/sql/node_active_protocol.sql | 18 ++ src/monitor/sql/stale_primary_report.sql | 12 + src/monitor/sql/timeline_fork_detection.sql | 15 + src/monitor/sql/workers.sql | 12 + 20 files changed, 1545 insertions(+) diff --git a/src/monitor/expected/drop_node.out b/src/monitor/expected/drop_node.out index 5bf508602..38eaeda90 100644 --- a/src/monitor/expected/drop_node.out +++ b/src/monitor/expected/drop_node.out @@ -238,3 +238,99 @@ nodename | dn_p goalstate | single reportedstate | primary +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('dn_test', count => 100); +-[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 16 "dn_p" (dn_p:5432): "single" +-[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 17 "dn_s" (dn_s:5432): "wait_standby" +-[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 16 "dn_p" (dn_p:5432): "wait_primary" +-[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 17 "dn_s" (dn_s:5432): "catchingup" +-[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 17 "dn_s" (dn_s:5432): "secondary" +-[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 16 "dn_p" (dn_p:5432): "primary" +-[ RECORD 12 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | dropped +rule_pos | 103 +rule_section | api_triggered +description | remove_node, removed node cannot take writes -> dropped +-[ RECORD 13 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 14 ]+-------------------------------------------------------------------------------------------- +reportedstate | dropped +goalstate | dropped +rule_pos | +rule_section | +description | New state is reported by node 17 "dn_s" (dn_s:5432): "dropped" + diff --git a/src/monitor/expected/failover_candidate_leaves_secondary.out b/src/monitor/expected/failover_candidate_leaves_secondary.out index 10798ed36..1020251b0 100644 --- a/src/monitor/expected/failover_candidate_leaves_secondary.out +++ b/src/monitor/expected/failover_candidate_leaves_secondary.out @@ -182,3 +182,81 @@ nodename | fclma_s reportedstate | prepare_promotion goalstate | prepare_promotion +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('fclma_test', count => 100); +-[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 31 "fclma_p" (fclma_p:5432): "single" +-[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 32 "fclma_s" (fclma_s:5432): "wait_standby" +-[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 31 "fclma_p" (fclma_p:5432): "wait_primary" +-[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 32 "fclma_s" (fclma_s:5432): "catchingup" +-[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 32 "fclma_s" (fclma_s:5432): "secondary" +-[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 31 "fclma_p" (fclma_p:5432): "primary" + diff --git a/src/monitor/expected/fast_forward.out b/src/monitor/expected/fast_forward.out index e39f554ed..6b6bd878b 100644 --- a/src/monitor/expected/fast_forward.out +++ b/src/monitor/expected/fast_forward.out @@ -385,3 +385,117 @@ SELECT node_name, node_lsn, node_is_primary -- ── cleanup ─────────────────────────────────────────────────────────────────── RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('ff_test', count => 100); +-[ RECORD 1 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "single" +-[ RECORD 3 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "wait_standby" +-[ RECORD 4 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "wait_primary" +-[ RECORD 6 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "catchingup" +-[ RECORD 8 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "secondary" +-[ RECORD 10 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "primary" +-[ RECORD 12 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "wait_standby" +-[ RECORD 13 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "catchingup" +-[ RECORD 16 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "secondary" +-[ RECORD 17 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | report_lsn +goalstate | report_lsn +rule_pos | 363 +rule_section | reporting_node +description | MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, guard_data_loss=true -> report_lsn (retry once a source recovers) + diff --git a/src/monitor/expected/guard_data_loss.out b/src/monitor/expected/guard_data_loss.out index 98b07b2f7..903a66666 100644 --- a/src/monitor/expected/guard_data_loss.out +++ b/src/monitor/expected/guard_data_loss.out @@ -308,3 +308,117 @@ reportedstate | secondary RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('gdl_test', count => 100); +-[ RECORD 1 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 10 "p" (p:5432): "single" +-[ RECORD 3 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 11 "s1" (s1:5432): "wait_standby" +-[ RECORD 4 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 10 "p" (p:5432): "wait_primary" +-[ RECORD 6 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 11 "s1" (s1:5432): "catchingup" +-[ RECORD 8 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 11 "s1" (s1:5432): "secondary" +-[ RECORD 10 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 10 "p" (p:5432): "primary" +-[ RECORD 12 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 12 "s2" (s2:5432): "wait_standby" +-[ RECORD 13 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 12 "s2" (s2:5432): "catchingup" +-[ RECORD 16 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 12 "s2" (s2:5432): "secondary" +-[ RECORD 17 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | report_lsn +goalstate | prepare_promotion +rule_pos | 375 +rule_section | reporting_node +description | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate already has all WAL -> prepare_promotion (1 of 2 -- see this row's own comment on why both are listed) + diff --git a/src/monitor/expected/lock_and_fetch_migration.out b/src/monitor/expected/lock_and_fetch_migration.out index c48cad9f1..4156d16d9 100644 --- a/src/monitor/expected/lock_and_fetch_migration.out +++ b/src/monitor/expected/lock_and_fetch_migration.out @@ -316,3 +316,195 @@ nodename | lafm_s2_renamed goalstate | catchingup reportedstate | secondary +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('lafm_test', count => 100); +-[ RECORD 1 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "single" +-[ RECORD 3 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 22 "lafm_s1" (lafm_s1:5432): "wait_standby" +-[ RECORD 4 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "wait_primary" +-[ RECORD 6 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 22 "lafm_s1" (lafm_s1:5432): "catchingup" +-[ RECORD 8 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 22 "lafm_s1" (lafm_s1:5432): "secondary" +-[ RECORD 10 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "primary" +-[ RECORD 12 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 23 "lafm_s2" (lafm_s2:5432): "wait_standby" +-[ RECORD 13 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 23 "lafm_s2" (lafm_s2:5432): "catchingup" +-[ RECORD 16 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 23 "lafm_s2" (lafm_s2:5432): "secondary" +-[ RECORD 17 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | apply_settings +goalstate | apply_settings +rule_pos | +rule_section | +description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "apply_settings" +-[ RECORD 18 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | apply_settings +goalstate | primary +rule_pos | 415 +rule_section | primary_node +description | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 19 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "primary" +-[ RECORD 20 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 125 +rule_section | api_triggered +description | set_node_candidate_priority, primary not already apply_settings -> apply_settings +-[ RECORD 21 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 125 +rule_section | api_triggered +description | set_node_candidate_priority, primary not already apply_settings -> apply_settings +-[ RECORD 22 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 125 +rule_section | api_triggered +description | set_node_candidate_priority, primary not already apply_settings -> apply_settings +-[ RECORD 23 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 125 +rule_section | api_triggered +description | set_node_candidate_priority, primary not already apply_settings -> apply_settings +-[ RECORD 24 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 127 +rule_section | api_triggered +description | set_node_replication_quorum, primary not already apply_settings -> apply_settings +-[ RECORD 25 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | apply_settings +goalstate | apply_settings +rule_pos | +rule_section | +description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "apply_settings" +-[ RECORD 26 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | apply_settings +goalstate | primary +rule_pos | 415 +rule_section | primary_node +description | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 27 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "primary" +-[ RECORD 28 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | maintenance +rule_pos | 115 +rule_section | api_triggered +description | start_maintenance, secondary, ordinary case -> maintenance +-[ RECORD 29 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | maintenance +goalstate | maintenance +rule_pos | +rule_section | +description | New state is reported by node 22 "lafm_s1" (lafm_s1:5432): "maintenance" +-[ RECORD 30 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | maintenance +goalstate | catchingup +rule_pos | 123 +rule_section | api_triggered +description | stop_maintenance, ordinary case -> catchingup + diff --git a/src/monitor/expected/node_active_protocol.out b/src/monitor/expected/node_active_protocol.out index 1fb5b73c6..40ec2c8d9 100644 --- a/src/monitor/expected/node_active_protocol.out +++ b/src/monitor/expected/node_active_protocol.out @@ -670,3 +670,291 @@ SELECT pgautofailover.report_postgres_version(-1, 170003); -[ RECORD 1 ]-----------+- report_postgres_version | +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events, for each of the two formations used +-- above. Exercises pgautofailover.last_events() against a real scenario -- +-- its own SELECT list didn't match pgautofailover.event's column set for a +-- long time, breaking it outright, and nothing in this suite ever called +-- it to notice (see monitor.sql's own minimal-repro coverage). Filtering +-- by formationid isolates each summary from the other, and from every +-- other test in this schedule sharing the same event table -- safe +-- regardless of where in this file (or the whole schedule) it runs. +-- eventid/eventtime omitted: eventid is a database-wide sequence shared by +-- every test in this schedule (see regress_schedule's own comment) and +-- eventtime is a live timestamp -- neither is a stable value to pin here. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('fsm_test', count => 100); +-[ RECORD 1 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "single" +-[ RECORD 3 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "wait_standby" +-[ RECORD 4 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "wait_primary" +-[ RECORD 6 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "catchingup" +-[ RECORD 8 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "secondary" +-[ RECORD 10 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "primary" +-[ RECORD 12 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | prepare_promotion +rule_pos | 325 +rule_section | reporting_node +description | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) +-[ RECORD 13 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | draining +rule_pos | 325 +rule_section | reporting_node +description | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) +-[ RECORD 14 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | prepare_promotion +goalstate | prepare_promotion +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "prepare_promotion" +-[ RECORD 15 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | prepare_promotion +goalstate | stop_replication +rule_pos | 339 +rule_section | reporting_node +description | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) +-[ RECORD 16 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | demote_timeout +rule_pos | 339 +rule_section | reporting_node +description | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) +-[ RECORD 17 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | demote_timeout +goalstate | demote_timeout +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "demote_timeout" +-[ RECORD 18 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | stop_replication +goalstate | stop_replication +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "stop_replication" +-[ RECORD 19 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | stop_replication +goalstate | wait_primary +rule_pos | 345 +rule_section | reporting_node +description | stop_replication, primary converged demote_timeout -> wait_primary + demoted (1 of 3) +-[ RECORD 20 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | demote_timeout +goalstate | demoted +rule_pos | 345 +rule_section | reporting_node +description | stop_replication, primary converged demote_timeout -> wait_primary + demoted (1 of 3) +-[ RECORD 21 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "wait_primary" +-[ RECORD 22 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | demoted +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "primary" +-[ RECORD 23 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | demoted +goalstate | demoted +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "demoted" +-[ RECORD 24 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | demoted +goalstate | catchingup +rule_pos | 357 +rule_section | reporting_node +description | demoted, primary converged wait/join_primary/primary, healthy -> catchingup +-[ RECORD 25 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "catchingup" +-[ RECORD 26 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 27 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 6 "node1" (node1:5432): "secondary" +-[ RECORD 28 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 29 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 7 "node2" (node2:5432): "primary" +-[ RECORD 30 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | prepare_maintenance +rule_pos | 109 +rule_section | api_triggered +description | start_maintenance, primary, 2-node group -> prepare_maintenance (standby separately assigned prepare_promotion) +-[ RECORD 31 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | prepare_promotion +rule_pos | +rule_section | +description | Setting goal state of node 6 "node1" (node1:5432) to prepare_promotion after a user-initiated start_maintenance call. +-[ RECORD 32 ]+---------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | prepare_promotion +rule_pos | +rule_section | +description | Updating region to "dc2" for node 6 "node1" (node1:5432) + +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('killed_test', count => 100); +-[ RECORD 1 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 8 "ka" (ka:5432): "single" +-[ RECORD 3 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 9 "kb" (kb:5432): "wait_standby" +-[ RECORD 4 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 8 "ka" (ka:5432): "wait_primary" +-[ RECORD 6 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 9 "kb" (kb:5432): "catchingup" +-[ RECORD 8 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+-------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 9 "kb" (kb:5432): "secondary" +-[ RECORD 10 ]+-------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+-------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 8 "ka" (ka:5432): "primary" +-[ RECORD 12 ]+-------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | prepare_promotion +rule_pos | 325 +rule_section | reporting_node +description | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) +-[ RECORD 13 ]+-------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | draining +rule_pos | 325 +rule_section | reporting_node +description | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + diff --git a/src/monitor/expected/pg19/expected/fast_forward.out b/src/monitor/expected/pg19/expected/fast_forward.out index be1069382..e96755c8b 100644 --- a/src/monitor/expected/pg19/expected/fast_forward.out +++ b/src/monitor/expected/pg19/expected/fast_forward.out @@ -385,3 +385,117 @@ SELECT node_name, node_lsn, node_is_primary -- ── cleanup ─────────────────────────────────────────────────────────────────── RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('ff_test', count => 100); +-[ RECORD 1 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "single" +-[ RECORD 3 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "wait_standby" +-[ RECORD 4 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "wait_primary" +-[ RECORD 6 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "catchingup" +-[ RECORD 8 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 14 "ff_s1" (ff_s1:5432): "secondary" +-[ RECORD 10 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 13 "ff_p" (ff_p:5432): "primary" +-[ RECORD 12 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "wait_standby" +-[ RECORD 13 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "catchingup" +-[ RECORD 16 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 15 "ff_s2" (ff_s2:5432): "secondary" +-[ RECORD 17 ]+------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | report_lsn +goalstate | report_lsn +rule_pos | 363 +rule_section | reporting_node +description | MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, guard_data_loss=true -> report_lsn (retry once a source recovers) + diff --git a/src/monitor/expected/pg19/expected/timeline_fork_detection.out b/src/monitor/expected/pg19/expected/timeline_fork_detection.out index 23aac6f12..c68ad529e 100644 --- a/src/monitor/expected/pg19/expected/timeline_fork_detection.out +++ b/src/monitor/expected/pg19/expected/timeline_fork_detection.out @@ -540,3 +540,144 @@ resolved | t RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events, for each of the two formations used +-- above. Exercises pgautofailover.last_events() against a real scenario -- +-- its own SELECT list didn't match pgautofailover.event's column set for a +-- long time, breaking it outright, and nothing in this suite ever called +-- it to notice (see monitor.sql's own minimal-repro coverage). eventid/ +-- eventtime omitted: eventid is a database-wide sequence shared by every +-- test in this schedule (see regress_schedule's own comment) and +-- eventtime is a live timestamp -- neither is a stable value to pin here. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_unit', count => 100); +-[ RECORD 1 ]-+--------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single + +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_election', count => 100); +-[ RECORD 1 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 27 "p" (tlfe-p:5432): "single" +-[ RECORD 3 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 28 "s1" (tlfe-s1:5432): "wait_standby" +-[ RECORD 4 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 27 "p" (tlfe-p:5432): "wait_primary" +-[ RECORD 6 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 28 "s1" (tlfe-s1:5432): "catchingup" +-[ RECORD 8 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 28 "s1" (tlfe-s1:5432): "secondary" +-[ RECORD 10 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 27 "p" (tlfe-p:5432): "primary" +-[ RECORD 12 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 29 "s2" (tlfe-s2:5432): "wait_standby" +-[ RECORD 13 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 29 "s2" (tlfe-s2:5432): "catchingup" +-[ RECORD 16 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 29 "s2" (tlfe-s2:5432): "secondary" +-[ RECORD 17 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 30 "s3" (tlfe-s3:5432): "wait_standby" +-[ RECORD 18 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 30 "s3" (tlfe-s3:5432): "catchingup" +-[ RECORD 19 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 30 "s3" (tlfe-s3:5432): "secondary" +-[ RECORD 20 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | report_lsn +goalstate | prepare_promotion +rule_pos | 375 +rule_section | reporting_node +description | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate already has all WAL -> prepare_promotion (1 of 2 -- see this row's own comment on why both are listed) + diff --git a/src/monitor/expected/stale_primary_report.out b/src/monitor/expected/stale_primary_report.out index f6e216d4f..cc40f4d87 100644 --- a/src/monitor/expected/stale_primary_report.out +++ b/src/monitor/expected/stale_primary_report.out @@ -308,3 +308,123 @@ nodename | spr_s2 goalstate | report_lsn reportedstate | secondary +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('spr_test', count => 100); +-[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 18 "spr_p" (spr_p:5432): "single" +-[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "wait_standby" +-[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 18 "spr_p" (spr_p:5432): "wait_primary" +-[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "catchingup" +-[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "secondary" +-[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 18 "spr_p" (spr_p:5432): "primary" +-[ RECORD 12 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "wait_standby" +-[ RECORD 13 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "catchingup" +-[ RECORD 16 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "secondary" +-[ RECORD 17 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "primary" +-[ RECORD 18 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | report_lsn +rule_pos | 367 +rule_section | reporting_node +description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) + diff --git a/src/monitor/expected/timeline_fork_detection.out b/src/monitor/expected/timeline_fork_detection.out index 0cb535d8a..d777791c2 100644 --- a/src/monitor/expected/timeline_fork_detection.out +++ b/src/monitor/expected/timeline_fork_detection.out @@ -540,3 +540,144 @@ resolved | t RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events, for each of the two formations used +-- above. Exercises pgautofailover.last_events() against a real scenario -- +-- its own SELECT list didn't match pgautofailover.event's column set for a +-- long time, breaking it outright, and nothing in this suite ever called +-- it to notice (see monitor.sql's own minimal-repro coverage). eventid/ +-- eventtime omitted: eventid is a database-wide sequence shared by every +-- test in this schedule (see regress_schedule's own comment) and +-- eventtime is a live timestamp -- neither is a stable value to pin here. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_unit', count => 100); +-[ RECORD 1 ]-+--------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single + +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_election', count => 100); +-[ RECORD 1 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 27 "p" (tlfe-p:5432): "single" +-[ RECORD 3 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 28 "s1" (tlfe-s1:5432): "wait_standby" +-[ RECORD 4 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 27 "p" (tlfe-p:5432): "wait_primary" +-[ RECORD 6 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 28 "s1" (tlfe-s1:5432): "catchingup" +-[ RECORD 8 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 28 "s1" (tlfe-s1:5432): "secondary" +-[ RECORD 10 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 27 "p" (tlfe-p:5432): "primary" +-[ RECORD 12 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 29 "s2" (tlfe-s2:5432): "wait_standby" +-[ RECORD 13 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 29 "s2" (tlfe-s2:5432): "catchingup" +-[ RECORD 16 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 29 "s2" (tlfe-s2:5432): "secondary" +-[ RECORD 17 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 30 "s3" (tlfe-s3:5432): "wait_standby" +-[ RECORD 18 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 30 "s3" (tlfe-s3:5432): "catchingup" +-[ RECORD 19 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 30 "s3" (tlfe-s3:5432): "secondary" +-[ RECORD 20 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +reportedstate | report_lsn +goalstate | prepare_promotion +rule_pos | 375 +rule_section | reporting_node +description | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate already has all WAL -> prepare_promotion (1 of 2 -- see this row's own comment on why both are listed) + diff --git a/src/monitor/expected/workers.out b/src/monitor/expected/workers.out index 0cd3a6ef9..49dea7e3f 100644 --- a/src/monitor/expected/workers.out +++ b/src/monitor/expected/workers.out @@ -59,3 +59,33 @@ assigned_candidate_priority | 100 assigned_replication_quorum | t assigned_node_name | worker1a +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +select reportedstate, goalstate, rule_pos, rule_section, description + from pgautofailover.last_events('citus', count => 100); +-[ RECORD 1 ]-+--------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+--------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 4 "coord0a" (localhost:9876): "single" +-[ RECORD 3 ]-+--------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single + diff --git a/src/monitor/sql/drop_node.sql b/src/monitor/sql/drop_node.sql index 79c35754c..edc8195ad 100644 --- a/src/monitor/sql/drop_node.sql +++ b/src/monitor/sql/drop_node.sql @@ -171,3 +171,15 @@ SELECT nodename, goalstate, reportedstate FROM pgautofailover.node WHERE formationid = 'dn_test' ORDER BY nodename; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('dn_test', count => 100); diff --git a/src/monitor/sql/failover_candidate_leaves_secondary.sql b/src/monitor/sql/failover_candidate_leaves_secondary.sql index 8a587e4cd..9287d59f2 100644 --- a/src/monitor/sql/failover_candidate_leaves_secondary.sql +++ b/src/monitor/sql/failover_candidate_leaves_secondary.sql @@ -127,3 +127,15 @@ SELECT nodename, reportedstate, goalstate FROM pgautofailover.node WHERE formationid = 'fclma_test' ORDER BY nodename; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('fclma_test', count => 100); diff --git a/src/monitor/sql/fast_forward.sql b/src/monitor/sql/fast_forward.sql index cc3bda9c9..7c1ad5832 100644 --- a/src/monitor/sql/fast_forward.sql +++ b/src/monitor/sql/fast_forward.sql @@ -278,3 +278,15 @@ SELECT node_name, node_lsn, node_is_primary RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('ff_test', count => 100); diff --git a/src/monitor/sql/guard_data_loss.sql b/src/monitor/sql/guard_data_loss.sql index 09066b844..e75ff0f8a 100644 --- a/src/monitor/sql/guard_data_loss.sql +++ b/src/monitor/sql/guard_data_loss.sql @@ -220,3 +220,15 @@ SELECT nodename, goalstate, reportedstate RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('gdl_test', count => 100); diff --git a/src/monitor/sql/lock_and_fetch_migration.sql b/src/monitor/sql/lock_and_fetch_migration.sql index d0d8242d5..e9dccb5cc 100644 --- a/src/monitor/sql/lock_and_fetch_migration.sql +++ b/src/monitor/sql/lock_and_fetch_migration.sql @@ -172,3 +172,15 @@ SELECT nodename, goalstate, reportedstate FROM pgautofailover.node WHERE formationid = 'lafm_test' ORDER BY nodename; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('lafm_test', count => 100); diff --git a/src/monitor/sql/node_active_protocol.sql b/src/monitor/sql/node_active_protocol.sql index 355435fda..3f7fa3c95 100644 --- a/src/monitor/sql/node_active_protocol.sql +++ b/src/monitor/sql/node_active_protocol.sql @@ -440,3 +440,21 @@ SELECT pgautofailover.report_postgres_version(NULL, 170003); -- unknown node_id: silent no-op, not an error SELECT pgautofailover.report_postgres_version(-1, 170003); + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events, for each of the two formations used +-- above. Exercises pgautofailover.last_events() against a real scenario -- +-- its own SELECT list didn't match pgautofailover.event's column set for a +-- long time, breaking it outright, and nothing in this suite ever called +-- it to notice (see monitor.sql's own minimal-repro coverage). Filtering +-- by formationid isolates each summary from the other, and from every +-- other test in this schedule sharing the same event table -- safe +-- regardless of where in this file (or the whole schedule) it runs. +-- eventid/eventtime omitted: eventid is a database-wide sequence shared by +-- every test in this schedule (see regress_schedule's own comment) and +-- eventtime is a live timestamp -- neither is a stable value to pin here. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('fsm_test', count => 100); + +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('killed_test', count => 100); diff --git a/src/monitor/sql/stale_primary_report.sql b/src/monitor/sql/stale_primary_report.sql index 6db9732c4..45183caa6 100644 --- a/src/monitor/sql/stale_primary_report.sql +++ b/src/monitor/sql/stale_primary_report.sql @@ -208,3 +208,15 @@ SELECT nodename, goalstate, reportedstate FROM pgautofailover.node WHERE formationid = 'spr_test' ORDER BY nodename; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('spr_test', count => 100); diff --git a/src/monitor/sql/timeline_fork_detection.sql b/src/monitor/sql/timeline_fork_detection.sql index 67b987789..c605b98de 100644 --- a/src/monitor/sql/timeline_fork_detection.sql +++ b/src/monitor/sql/timeline_fork_detection.sql @@ -317,3 +317,18 @@ SELECT formationid, groupid, accepted_tli, resolved_at IS NOT NULL AS resolved RESET pgautofailover.guard_data_loss; RESET pgautofailover.startup_grace_period; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events, for each of the two formations used +-- above. Exercises pgautofailover.last_events() against a real scenario -- +-- its own SELECT list didn't match pgautofailover.event's column set for a +-- long time, breaking it outright, and nothing in this suite ever called +-- it to notice (see monitor.sql's own minimal-repro coverage). eventid/ +-- eventtime omitted: eventid is a database-wide sequence shared by every +-- test in this schedule (see regress_schedule's own comment) and +-- eventtime is a live timestamp -- neither is a stable value to pin here. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_unit', count => 100); + +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('tlf_election', count => 100); diff --git a/src/monitor/sql/workers.sql b/src/monitor/sql/workers.sql index d251168fc..5d44aa667 100644 --- a/src/monitor/sql/workers.sql +++ b/src/monitor/sql/workers.sql @@ -30,3 +30,15 @@ select * dbname => 'citus', desired_group_id => 1, node_kind => 'worker'); + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +select reportedstate, goalstate, rule_pos, rule_section, description + from pgautofailover.last_events('citus', count => 100); From 81e6e90c9acbed3e5445da45affbf24df9683e67 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 29 Jul 2026 19:31:29 +0200 Subject: [PATCH 12/52] Add otherNode as a role distinct from primaryNode in MonitorFSM[] otherNodeAssignedState's real target was always nac->primaryNode.node, hardcoded directly in DispatchMonitorFSMRule -- meaning "the other node in this transition" and "the group's primary" were the same concept by construction, with no way for a row to say otherwise. Decouples them: - NodeActiveContext gains otherNode (a NodeStatus, same shape as activeNode/primaryNode/candidateNode). BuildFromContextNodeActiveContext and BuildApiTriggerNodeActiveContext -- the only two builders that ever populate a real primaryNode -- now also set nac->otherNode = nac->primaryNode right after, so otherNode.node == primaryNode.node everywhere, exactly as before. (BuildForPrimaryNode NodeActiveContext and BuildMSFailoverNodeActiveContext leave .primaryNode at its memset-zero default already, same as before.) - MonitorFSMTransition gains a matching otherNode NodeStatusPattern field, wired into RuleMatches. Every existing row omits it, which NodeStatusPattern's own "omitted means don't-care" default already makes a no-op -- zero risk to any of the 74 existing rows, same guarantee this array's other additive fields (apiFunction, inMSFailoverCluster, etc.) already established when each was added. - DispatchMonitorFSMRule now assigns otherNodeAssignedState to nac->otherNode.node instead of nac->primaryNode.node directly -- behaviorally identical today (same pointer), but the dispatch mechanism's own vocabulary is now genuinely activeNode/otherNode, with primaryNode/candidateNode as separate, monitor-domain-specific roles a row can still constrain independently. This is purely additive groundwork, not yet used by any row: it decouples "who otherNodeAssignedState targets" from "the primary" at the type level, which is the prerequisite for a future otherNodesFn mechanism (see the design doc's own proposal) to resolve otherNode to something else entirely -- e.g. a dynamically selected failover candidate in the MS-failover cluster -- without having to rename or restructure every existing primaryNode-shaped row's own conditions. dump_fsm_edges()'s own edge resolution (NodeStatePatternResolveFromStates called against rule->primaryNode.statePattern) deliberately still reads primaryNode, not otherNode: every existing row's real reachability constraint lives on .primaryNode (.otherNode defaults to NODE_STATE_ANY, unset everywhere), so switching that reference now would make every resolved edge look reachable from any reported state -- a real regression. Flagged in-place as a follow-up for whenever a row first sets .otherNode.statePattern to something .primaryNode doesn't already say. Verified: clean compile, full regress+isolation Docker installcheck (18/18 regress, 6/6 isolation) with zero expected-output changes -- confirms this is fully behavior-neutral, as designed. --- src/monitor/group_state_machine.c | 54 +++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 8b4167053..9da6da2bd 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -552,6 +552,28 @@ typedef struct NodeActiveContext NodeStatus activeNode; NodeStatus primaryNode; + /* + * otherNode is who otherNodeAssignedState actually assigns to (see + * DispatchMonitorFSMRule). Set to a plain copy of primaryNode by every + * builder that populates a real primaryNode (BuildFromContextNodeActive + * Context, BuildApiTriggerNodeActiveContext) -- today, "the other node + * in this transition" and "the group's primary" are always the same + * node, so otherNode.node == primaryNode.node everywhere. Kept as its + * own field rather than reusing .primaryNode directly so that role + * stays conceptually activeNode/otherNode, matching the design doc's + * own framing and this array's own dispatch semantics ("assign + * activeNodeAssignedState to activeNode, otherNodeAssignedState to + * otherNode") independently of whichever node the monitor's own + * domain concepts (primary, candidate, ...) say it happens to be. A + * future otherNodesFn-resolved row (see the design doc's "MS-failover + * / candidate-selection cluster" section) could populate this from + * some other resolution entirely -- e.g. a dynamically selected + * failover candidate, not the primary -- without disturbing every + * existing row's own primaryNode-shaped conditions, which keep reading + * .primaryNode exactly as before. + */ + NodeStatus otherNode; + /* * candidateNode is only populated by BuildMSFailoverNodeActiveContext * (the MS-failover cluster's own nested-dispatch context builder, @@ -765,13 +787,38 @@ typedef struct MonitorFSMTransition NodeStatusPattern activeNode; NodeStatusPattern primaryNode; + + /* + * otherNode is the role otherNodeAssignedState actually targets (see + * that field's own comment): a genuinely distinct role from primaryNode, + * even though every row today has nac->otherNode.node == + * nac->primaryNode.node (see NodeActiveContext's own comment on + * .otherNode for why). Kept separate from primaryNode in this struct -- + * rather than just reusing .primaryNode wherever a row wants to + * constrain otherNodeAssignedState's target -- so a future row whose + * target is resolved some other way (e.g. a candidate an otherNodesFn + * selects, not simply "the primary") has a role to write conditions + * against without a name that falsely implies it's always the primary. + * Every row written before this field existed omits it, which matches + * NodeStatusPattern's own "omitted means don't-care" default -- exactly + * the same zero-risk-to-existing-rows guarantee apiFunction/inMSFailover + * Cluster/etc. already established when each was added. + */ + NodeStatusPattern otherNode; + NodeStatusPattern candidateNode; /* MS-failover sub-section rows only; see * NodeActiveContext's own comment on * .candidateNode */ NodeActiveContextPattern conditions; GoalStateAssignment activeNodeAssignedState; - GoalStateAssignment otherNodeAssignedState; /* target: nac->primaryNode.node */ + + /* + * otherNodeAssignedState targets nac->otherNode.node (see + * NodeActiveContext's own comment) -- not nac->primaryNode.node + * directly, even though today the two are always the same pointer. + */ + GoalStateAssignment otherNodeAssignedState; MonitorExtraActionFunction extraAction; @@ -915,6 +962,7 @@ RuleMatches(const NodeActiveContext *nac, const MonitorFSMTransition *rule) NodeMatchesPattern(&nac->activeNode, &rule->activeNode) && NodeMatchesPattern(&nac->primaryNode, &rule->primaryNode) && + NodeMatchesPattern(&nac->otherNode, &rule->otherNode) && NodeMatchesPattern(&nac->candidateNode, &rule->candidateNode) && BoolMatchesPattern(nac->groupHasExactlyOneNode, @@ -1039,7 +1087,7 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, if (rule->otherNodeAssignedState.kind == GOAL_STATE_SET) { - AssignDeclaredGoalState(rule, nac->primaryNode.node, + AssignDeclaredGoalState(rule, nac->otherNode.node, rule->otherNodeAssignedState.state, message); } @@ -1372,6 +1420,7 @@ BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *prim BuildNodeStatus(ctx, activeNode, &nac->activeNode); BuildNodeStatus(ctx, primaryNode, &nac->primaryNode); + nac->otherNode = nac->primaryNode; /* see NodeActiveContext's own comment on .otherNode */ /* isComparableToReferenceTli defaults to true (row :328 doesn't fire) -- a node that hasn't * reported a timeline yet (reportedTLI == 0) has nothing to check, same as the original. */ @@ -1579,6 +1628,7 @@ BuildApiTriggerNodeActiveContext(GroupStateContext *ctx, MonitorApiFunction apiF BuildNodeStatus(ctx, activeNode, &nac->activeNode); BuildNodeStatus(ctx, primaryNode, &nac->primaryNode); + nac->otherNode = nac->primaryNode; /* see NodeActiveContext's own comment on .otherNode */ nac->groupHasExactlyOneNode = (ctx->groupNodeCount == 1); nac->groupHasExactlyTwoNodes = (ctx->groupNodeCount == 2); From dd82a265a02338aec52e784ff70087339e18ad7f Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 01:44:22 +0200 Subject: [PATCH 13/52] monitor: replace MonitorFSM[] index-range boundaries with section paths Replace the 6 hand-maintained array-index boundary constants (MonitorFSM_EarlyChecksStart, MonitorFSM_FromContextStart, etc.) with a hierarchical MonitorFSMSectionPath carried on each row itself, matched via prefix containment (SectionPathIsUnderPrefix) instead of index ranges -- a row's membership no longer depends on where it happens to sit in the array. - Expand MonitorFSMSection in place with a NONE sentinel and fine-grained leaf values (reporting_node.ms_failover.* sub-sections); only sectionPath[0] is ever one of the original 4 SQL-visible values, so pgautofailover.fsm_section and rule_section are completely unaffected. - Add IntPattern (EXACTLY/AT_LEAST/AT_MOST) alongside the existing BoolPattern, and make ProceedGroupStateForMSFailover's three counting gates (missingNodesCount/candidateCount/quorumCandidateCount) declarative rows in MonitorFSM[], dispatched via a dedicated BuildMSFailoverCandidateGateNodeActiveContext and a new inMSFailoverCandidateGate guard (keeps them from being reachable through the existing MS-failover cluster's own broader scans). The decline-vs-continue control flow itself stays hand-written C; only the message text and conditions move into the table. - Add pgautofailover.fsm's new section_path column (ltree, cast at the view level only -- dump_fsm() itself just builds the dotted text) and the ltree control-file dependency. - Add candidate_count_gate.sql: dedicated regression coverage for the candidateCount == 0 gate, the one counting gate no existing test named explicitly. Full regress (19) + isolation (6) suites pass in Docker (PG17). --- src/monitor/Makefile | 2 +- src/monitor/expected/candidate_count_gate.out | 390 ++++++ ...cluster_init_failover_rule_attribution.out | 30 +- src/monitor/expected/create_extension.out | 1 + .../failover_candidate_leaves_secondary.out | 16 +- src/monitor/expected/fsm.out | 181 ++- src/monitor/expected/keeper_fsm_edges.out | 32 +- .../expected/lock_and_fetch_migration.out | 36 +- .../expected/timeline_fork_detection.out | 48 +- src/monitor/group_state_machine.c | 1174 ++++++++++++----- src/monitor/group_state_machine.h | 49 +- src/monitor/pgautofailover.control | 2 +- src/monitor/pgautofailover.sql | 24 +- src/monitor/regress_schedule | 1 + src/monitor/sql/candidate_count_gate.sql | 202 +++ src/monitor/sql/fsm.sql | 2 +- 16 files changed, 1763 insertions(+), 427 deletions(-) create mode 100644 src/monitor/expected/candidate_count_gate.out create mode 100644 src/monitor/sql/candidate_count_gate.sql diff --git a/src/monitor/Makefile b/src/monitor/Makefile index 43f1f85c8..a86986656 100644 --- a/src/monitor/Makefile +++ b/src/monitor/Makefile @@ -14,7 +14,7 @@ MODULE_big = $(EXTENSION) OBJS = $(patsubst ${SRC_DIR}%.c,%.o,$(wildcard ${SRC_DIR}*.c)) PG_CPPFLAGS = -Wall -Werror -Wno-unused-parameter -Iinclude -I$(libpq_srcdir) -g SHLIB_LINK = $(libpq) -REGRESS = create_extension monitor workers node_active_protocol guard_data_loss fast_forward drop_node stale_primary_report lock_and_fetch_migration timeline_fork_detection dummy_update drop_extension upgrade +REGRESS = create_extension monitor workers node_active_protocol guard_data_loss fast_forward drop_node stale_primary_report candidate_count_gate lock_and_fetch_migration timeline_fork_detection dummy_update drop_extension upgrade ISOLATION = concurrent_remove_node concurrent_remove_standby_and_primary_report concurrent_remove_standby_and_standby_report concurrent_second_primary_death_report concurrent_health_check_and_report concurrent_candidate_priority_and_quorum PG_CONFIG ?= pg_config diff --git a/src/monitor/expected/candidate_count_gate.out b/src/monitor/expected/candidate_count_gate.out new file mode 100644 index 000000000..9cfa786d7 --- /dev/null +++ b/src/monitor/expected/candidate_count_gate.out @@ -0,0 +1,390 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Regression test for ProceedGroupStateForMSFailover's candidateCount == 0 +-- gate (reporting_node.ms_failover.promotion_outcome.candidate_count_gate in +-- MonitorFSM[]): the window where the primary has gone unhealthy but NOT A +-- SINGLE standby has yet reported reaching report_lsn. +-- +-- Unlike the other two counting gates -- missingNodesCount (guard_data_loss.sql) +-- and quorumCandidateCount (stale_primary_report.sql), both named explicitly +-- in those files' own header comments -- no existing test exercises this one +-- by name. It has no dedicated log message either (the original hand-written +-- code silently `return`s false here, and the declarative row that now +-- matches this same condition carries no extraAction, matching that exactly) +-- so there is no pgautofailover.event row to check for it; the only +-- observable effect is what does NOT happen: neither standby should reach +-- prepare_promotion/fast_forward this round. +-- +-- guard_data_loss is set to false: with the default (true), the +-- missingNodesCount > 0 gate above this one in ProceedGroupStateForMSFailover +-- would itself decline and return before ever reaching this gate, since both +-- standbys are still SECONDARY/CATCHINGUP (each counted as missing, per +-- BuildCandidateList's own fan-out branch) at the moment this test polls +-- them. +-- +-- startup_grace_period is also lowered to 1, same as guard_data_loss.sql/ +-- fast_forward.sql/stale_primary_report.sql: NodeIsUnhealthy() only honors a +-- BAD health reading once at least this many seconds have passed since the +-- monitor process itself started (PgStartTime), to avoid spurious failovers +-- right after the monitor restarts. The default (10s) is longer than this +-- whole schedule takes to reach this test file when run automated, which +-- would silently make p never register as unhealthy and this test's own +-- ProceedGroupStateForMSFailover call never even fire. +\x on +-- ── formation and node registration ───────────────────────────────────────── +SELECT pgautofailover.create_formation('ccg_test', 'pgsql', 'postgres', true, 1); +-[ RECORD 1 ]----+------------------------------ +create_formation | (ccg_test,pgsql,postgres,t,1) + +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_p', 5432, + 'postgres', 'ccg_p', 1); +-[ RECORD 1 ]---------------+------- +assigned_node_id | 21 +assigned_group_id | 0 +assigned_group_state | single +assigned_candidate_priority | 100 +assigned_replication_quorum | t +assigned_node_name | ccg_p + +SELECT nodeid AS np FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p' \gset +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_s1', 5432, + 'postgres', 'ccg_s1', 1); +-[ RECORD 1 ]---------------+------------- +assigned_node_id | 22 +assigned_group_id | 0 +assigned_group_state | wait_standby +assigned_candidate_priority | 100 +assigned_replication_quorum | t +assigned_node_name | ccg_s1 + +SELECT nodeid AS ns1 FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_s1' \gset +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_s2', 5432, + 'postgres', 'ccg_s2', 1); +-[ RECORD 1 ]---------------+------------- +assigned_node_id | 23 +assigned_group_id | 0 +assigned_group_state | wait_standby +assigned_candidate_priority | 100 +assigned_replication_quorum | t +assigned_node_name | ccg_s2 + +SELECT nodeid AS ns2 FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_s2' \gset +-- ── bootstrap: drive the FSM to primary + secondary + secondary ───────────── +-- Same sequence as guard_data_loss.sql's own bootstrap. +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'single'); +-[ RECORD 1 ]--------+------- +assigned_group_state | single + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'wait_standby'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_standby + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'single', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+------------- +assigned_group_state | wait_primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'wait_standby'); +-[ RECORD 1 ]--------+----------- +assigned_group_state | catchingup + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+-------- +assigned_group_state | primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+-------- +assigned_group_state | primary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'wait_standby'); +-[ RECORD 1 ]--------+----------- +assigned_group_state | catchingup + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+----------- +assigned_group_state | catchingup + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+----------- +assigned_group_state | catchingup + +-- p: primary (refresh to pick up second secondary) +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+--------------- +assigned_group_state | apply_settings + +-- Verify bootstrap: p=primary, s1=secondary, s2=secondary. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; +-[ RECORD 1 ]-+--------------- +nodename | ccg_p +goalstate | apply_settings +reportedstate | primary +-[ RECORD 2 ]-+--------------- +nodename | ccg_s1 +goalstate | secondary +reportedstate | secondary +-[ RECORD 3 ]-+--------------- +nodename | ccg_s2 +goalstate | catchingup +reportedstate | secondary + +-- ── manufacture: primary unhealthy, NEITHER standby has reported yet ──────── +-- +-- p goes unhealthy and is demoted to draining/draining (same manufactured +-- shape as guard_data_loss.sql/stale_primary_report.sql). s1/s2 are left +-- exactly as bootstrap left them -- secondary/secondary, neither has been +-- assigned report_lsn yet -- so candidateCount is 0 for both of them: no +-- standby has reached report_lsn, the exact window this gate covers. +SET pgautofailover.startup_grace_period = 1; +SET pgautofailover.guard_data_loss TO false; +UPDATE pgautofailover.node + SET health = 0, + healthchecktime = now(), + reporttime = now() - interval '60 seconds' + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p'; +UPDATE pgautofailover.node + SET goalstate = 'draining', reportedstate = 'draining' + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p'; +-- Verify the manufactured state before the test call. +SELECT nodename, goalstate, reportedstate, health + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; +-[ RECORD 1 ]-+----------- +nodename | ccg_p +goalstate | draining +reportedstate | draining +health | 0 +-[ RECORD 2 ]-+----------- +nodename | ccg_s1 +goalstate | secondary +reportedstate | secondary +health | -1 +-[ RECORD 3 ]-+----------- +nodename | ccg_s2 +goalstate | catchingup +reportedstate | secondary +health | -1 + +-- ── test: poll exactly one secondary, candidateCount == 0 for both ────────── +-- +-- s1 reports secondary/0-5000 again (nothing new from its own point of +-- view). This drives ProceedGroupState(s1) -> ActionRunMultiStandbyFailover +-- Cascade -> ProceedGroupStateForMSFailover, which runs BuildCandidateList +-- over the WHOLE group (not just s1): both s1 and s2 are still SECONDARY/ +-- CATCHINGUP, so BOTH get fanned out to report_lsn in this same call (the +-- fan-out rows, already covered elsewhere) -- but candidateCount is 0 (no +-- node's reportedState is report_lsn yet), so the candidate_count_gate row +-- matches and the function returns false: no candidate is selected. +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); +-[ RECORD 1 ]--------+---------- +assigned_group_state | secondary + +-- s1 and s2 must both have been fanned out to report_lsn (goalstate), but +-- NEITHER may have reached prepare_promotion/fast_forward: the +-- candidate_count_gate declined before any candidate could be selected. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; +-[ RECORD 1 ]-+----------- +nodename | ccg_p +goalstate | draining +reportedstate | draining +-[ RECORD 2 ]-+----------- +nodename | ccg_s1 +goalstate | report_lsn +reportedstate | secondary +-[ RECORD 3 ]-+----------- +nodename | ccg_s2 +goalstate | report_lsn +reportedstate | secondary + +RESET pgautofailover.guard_data_loss; +RESET pgautofailover.startup_grace_period; +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('ccg_test', count => 100); +-[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +reportedstate | init +goalstate | single +rule_pos | 209 +rule_section | early_checks +description | alone in group, candidate-eligible -> single +-[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | single +rule_pos | +rule_section | +description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "single" +-[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "wait_standby" +-[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +reportedstate | single +goalstate | wait_primary +rule_pos | 401 +rule_section | primary_node +description | primary alone, another node reached wait_standby -> wait_primary +-[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | wait_primary +rule_pos | +rule_section | +description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "wait_primary" +-[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 315 +rule_section | reporting_node +description | wait_standby, primary converged wait/join_primary -> catchingup +-[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "catchingup" +-[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | secondary +rule_pos | 321 +rule_section | reporting_node +description | caught up, same TLI as primary, within sync threshold -> secondary +-[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | secondary +rule_pos | +rule_section | +description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "secondary" +-[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_primary +goalstate | primary +rule_pos | 411 +rule_section | primary_node +description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) +-[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | primary +rule_pos | +rule_section | +description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "primary" +-[ RECORD 12 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | wait_standby +rule_pos | +rule_section | +description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "wait_standby" +-[ RECORD 13 ]+-------------------------------------------------------------------------------------------- +reportedstate | wait_standby +goalstate | catchingup +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 14 ]+-------------------------------------------------------------------------------------------- +reportedstate | primary +goalstate | apply_settings +rule_pos | 317 +rule_section | reporting_node +description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings +-[ RECORD 15 ]+-------------------------------------------------------------------------------------------- +reportedstate | catchingup +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "catchingup" +-[ RECORD 16 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | catchingup +rule_pos | +rule_section | +description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "secondary" +-[ RECORD 17 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | report_lsn +rule_pos | 367 +rule_section | reporting_node +description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) +-[ RECORD 18 ]+-------------------------------------------------------------------------------------------- +reportedstate | secondary +goalstate | report_lsn +rule_pos | 367 +rule_section | reporting_node +description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) + diff --git a/src/monitor/expected/cluster_init_failover_rule_attribution.out b/src/monitor/expected/cluster_init_failover_rule_attribution.out index 7fd593626..95c2c3e11 100644 --- a/src/monitor/expected/cluster_init_failover_rule_attribution.out +++ b/src/monitor/expected/cluster_init_failover_rule_attribution.out @@ -22,7 +22,7 @@ SELECT * FROM pgautofailover.register_node('cifra_test', 'cifra_p', 5432, 'postgres', 'cifra_p', 1); -[ RECORD 1 ]---------------+-------- -assigned_node_id | 33 +assigned_node_id | 36 assigned_group_id | 0 assigned_group_state | single assigned_candidate_priority | 100 @@ -35,7 +35,7 @@ SELECT * FROM pgautofailover.register_node('cifra_test', 'cifra_s', 5432, 'postgres', 'cifra_s', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 34 +assigned_node_id | 37 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -171,7 +171,7 @@ SELECT e.eventid, e.nodename, e.reportedstate, e.goalstate, WHERE e.formationid = 'cifra_test' ORDER BY e.eventid; -[ RECORD 1 ]-+------------------------------------------------------------------------------------------------------------- -eventid | 186 +eventid | 204 nodename | cifra_p reportedstate | init goalstate | single @@ -179,7 +179,7 @@ rule_pos | 209 rule_section | early_checks rule_comment | alone in group, candidate-eligible -> single -[ RECORD 2 ]-+------------------------------------------------------------------------------------------------------------- -eventid | 187 +eventid | 205 nodename | cifra_p reportedstate | single goalstate | single @@ -187,7 +187,7 @@ rule_pos | rule_section | rule_comment | -[ RECORD 3 ]-+------------------------------------------------------------------------------------------------------------- -eventid | 188 +eventid | 206 nodename | cifra_s reportedstate | wait_standby goalstate | wait_standby @@ -195,7 +195,7 @@ rule_pos | rule_section | rule_comment | -[ RECORD 4 ]-+------------------------------------------------------------------------------------------------------------- -eventid | 189 +eventid | 207 nodename | cifra_p reportedstate | single goalstate | wait_primary @@ -203,7 +203,7 @@ rule_pos | 401 rule_section | primary_node rule_comment | primary alone, another node reached wait_standby -> wait_primary -[ RECORD 5 ]-+------------------------------------------------------------------------------------------------------------- -eventid | 190 +eventid | 208 nodename | cifra_p reportedstate | wait_primary goalstate | wait_primary @@ -211,7 +211,7 @@ rule_pos | rule_section | rule_comment | -[ RECORD 6 ]-+------------------------------------------------------------------------------------------------------------- -eventid | 191 +eventid | 209 nodename | cifra_s reportedstate | wait_standby goalstate | catchingup @@ -219,7 +219,7 @@ rule_pos | 315 rule_section | reporting_node rule_comment | wait_standby, primary converged wait/join_primary -> catchingup -[ RECORD 7 ]-+------------------------------------------------------------------------------------------------------------- -eventid | 192 +eventid | 210 nodename | cifra_s reportedstate | catchingup goalstate | catchingup @@ -227,7 +227,7 @@ rule_pos | rule_section | rule_comment | -[ RECORD 8 ]-+------------------------------------------------------------------------------------------------------------- -eventid | 193 +eventid | 211 nodename | cifra_s reportedstate | catchingup goalstate | secondary @@ -235,7 +235,7 @@ rule_pos | 321 rule_section | reporting_node rule_comment | caught up, same TLI as primary, within sync threshold -> secondary -[ RECORD 9 ]-+------------------------------------------------------------------------------------------------------------- -eventid | 194 +eventid | 212 nodename | cifra_s reportedstate | secondary goalstate | secondary @@ -243,7 +243,7 @@ rule_pos | rule_section | rule_comment | -[ RECORD 10 ]+------------------------------------------------------------------------------------------------------------- -eventid | 195 +eventid | 213 nodename | cifra_p reportedstate | wait_primary goalstate | primary @@ -251,7 +251,7 @@ rule_pos | 411 rule_section | primary_node rule_comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 11 ]+------------------------------------------------------------------------------------------------------------- -eventid | 196 +eventid | 214 nodename | cifra_p reportedstate | primary goalstate | primary @@ -259,7 +259,7 @@ rule_pos | rule_section | rule_comment | -[ RECORD 12 ]+------------------------------------------------------------------------------------------------------------- -eventid | 197 +eventid | 215 nodename | cifra_s reportedstate | secondary goalstate | prepare_promotion @@ -267,7 +267,7 @@ rule_pos | 105 rule_section | api_triggered rule_comment | manual failover, 2-node group, primary+standby both converged -> standby prepare_promotion, primary draining -[ RECORD 13 ]+------------------------------------------------------------------------------------------------------------- -eventid | 198 +eventid | 216 nodename | cifra_p reportedstate | primary goalstate | draining diff --git a/src/monitor/expected/create_extension.out b/src/monitor/expected/create_extension.out index 083b3077a..63e2e4d17 100644 --- a/src/monitor/expected/create_extension.out +++ b/src/monitor/expected/create_extension.out @@ -2,3 +2,4 @@ -- Licensed under the PostgreSQL License. create extension pgautofailover cascade; NOTICE: installing required extension "btree_gist" +NOTICE: installing required extension "ltree" diff --git a/src/monitor/expected/failover_candidate_leaves_secondary.out b/src/monitor/expected/failover_candidate_leaves_secondary.out index 1020251b0..3d7895ca2 100644 --- a/src/monitor/expected/failover_candidate_leaves_secondary.out +++ b/src/monitor/expected/failover_candidate_leaves_secondary.out @@ -52,7 +52,7 @@ SELECT * FROM pgautofailover.register_node('fclma_test', 'fclma_p', 5432, 'postgres', 'fclma_p', 1); -[ RECORD 1 ]---------------+-------- -assigned_node_id | 31 +assigned_node_id | 34 assigned_group_id | 0 assigned_group_state | single assigned_candidate_priority | 100 @@ -65,7 +65,7 @@ SELECT * FROM pgautofailover.register_node('fclma_test', 'fclma_s', 5432, 'postgres', 'fclma_s', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 32 +assigned_node_id | 35 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -204,13 +204,13 @@ reportedstate | single goalstate | single rule_pos | rule_section | -description | New state is reported by node 31 "fclma_p" (fclma_p:5432): "single" +description | New state is reported by node 34 "fclma_p" (fclma_p:5432): "single" -[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 32 "fclma_s" (fclma_s:5432): "wait_standby" +description | New state is reported by node 35 "fclma_s" (fclma_s:5432): "wait_standby" -[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- reportedstate | single goalstate | wait_primary @@ -222,7 +222,7 @@ reportedstate | wait_primary goalstate | wait_primary rule_pos | rule_section | -description | New state is reported by node 31 "fclma_p" (fclma_p:5432): "wait_primary" +description | New state is reported by node 34 "fclma_p" (fclma_p:5432): "wait_primary" -[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup @@ -234,7 +234,7 @@ reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | -description | New state is reported by node 32 "fclma_s" (fclma_s:5432): "catchingup" +description | New state is reported by node 35 "fclma_s" (fclma_s:5432): "catchingup" -[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | secondary @@ -246,7 +246,7 @@ reportedstate | secondary goalstate | secondary rule_pos | rule_section | -description | New state is reported by node 32 "fclma_s" (fclma_s:5432): "secondary" +description | New state is reported by node 35 "fclma_s" (fclma_s:5432): "secondary" -[ RECORD 10 ]+-------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | primary @@ -258,5 +258,5 @@ reportedstate | primary goalstate | primary rule_pos | rule_section | -description | New state is reported by node 31 "fclma_p" (fclma_p:5432): "primary" +description | New state is reported by node 34 "fclma_p" (fclma_p:5432): "primary" diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index 6aa9e8cc5..1441ad451 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -12,7 +12,7 @@ -- wider than a terminal (or this file's own diff-ability), and reads far -- worse than one field-per-line. \x on -SELECT pos, section, +SELECT pos, section, section_path, active_node_current_state, other_node_current_state, candidate_node_current_state, active_node_conditions, other_node_conditions, candidate_node_conditions, group_conditions, @@ -23,6 +23,7 @@ SELECT pos, section, -[ RECORD 1 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 101 section | api_triggered: remove_node +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -37,6 +38,7 @@ comment | remove_node, removed node can take writes -> drop -[ RECORD 2 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 103 section | api_triggered: remove_node +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -51,6 +53,7 @@ comment | remove_node, removed node cannot take writes -> d -[ RECORD 3 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 105 section | api_triggered: perform_failover +section_path | api_triggered active_node_current_state | secondary other_node_current_state | primary candidate_node_current_state | @@ -65,6 +68,7 @@ comment | manual failover, 2-node group, primary+standby bo -[ RECORD 4 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 107 section | api_triggered: perform_failover +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -79,6 +83,7 @@ comment | manual failover, >2-node group -> primary drains, -[ RECORD 5 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 109 section | api_triggered: start_maintenance +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -93,6 +98,7 @@ comment | start_maintenance, primary, 2-node group -> prepa -[ RECORD 6 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 111 section | api_triggered: start_maintenance +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -107,6 +113,7 @@ comment | start_maintenance, primary, >2-node group -> prep -[ RECORD 7 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 113 section | api_triggered: start_maintenance +section_path | api_triggered active_node_current_state | secondary, catchingup other_node_current_state | primary candidate_node_current_state | @@ -121,6 +128,7 @@ comment | start_maintenance, secondary, last healthy sync s -[ RECORD 8 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 115 section | api_triggered: start_maintenance +section_path | api_triggered active_node_current_state | secondary, catchingup other_node_current_state | primary candidate_node_current_state | @@ -135,6 +143,7 @@ comment | start_maintenance, secondary, ordinary case -> ma -[ RECORD 9 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 117 section | api_triggered: stop_maintenance +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -149,6 +158,7 @@ comment | stop_maintenance, no primary -> report_lsn -[ RECORD 10 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 119 section | api_triggered: stop_maintenance +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -163,6 +173,7 @@ comment | stop_maintenance, primary demoted -> report_lsn -[ RECORD 11 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 121 section | api_triggered: stop_maintenance +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -177,6 +188,7 @@ comment | stop_maintenance, failover in progress -> report_ -[ RECORD 12 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 123 section | api_triggered: stop_maintenance +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -191,6 +203,7 @@ comment | stop_maintenance, ordinary case -> catchingup -[ RECORD 13 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 125 section | api_triggered: set_node_candidate_priority +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -205,6 +218,7 @@ comment | set_node_candidate_priority, primary not already -[ RECORD 14 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 127 section | api_triggered: set_node_replication_quorum +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -219,6 +233,7 @@ comment | set_node_replication_quorum, primary not already -[ RECORD 15 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 129 section | api_triggered: set_formation_number_sync_standbys +section_path | api_triggered active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -233,6 +248,7 @@ comment | set_formation_number_sync_standbys, primary in pr -[ RECORD 16 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 201 section | early_checks +section_path | early_checks active_node_current_state | dropped other_node_current_state | candidate_node_current_state | @@ -247,6 +263,7 @@ comment | converged to dropped -> remove the node from the -[ RECORD 17 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 203 section | early_checks +section_path | early_checks active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -261,6 +278,7 @@ comment | goal already dropped -> no-op -[ RECORD 18 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 205 section | early_checks +section_path | early_checks active_node_current_state | maintenance other_node_current_state | candidate_node_current_state | @@ -275,6 +293,7 @@ comment | converged to maintenance -> no-op, frozen until s -[ RECORD 19 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 207 section | early_checks +section_path | early_checks active_node_current_state | demote_timeout other_node_current_state | candidate_node_current_state | @@ -289,6 +308,7 @@ comment | reported demote_timeout, assigned goal can't reac -[ RECORD 20 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 209 section | early_checks +section_path | early_checks active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -303,6 +323,7 @@ comment | alone in group, candidate-eligible -> single -[ RECORD 21 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 211 section | early_checks +section_path | early_checks active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -317,6 +338,7 @@ comment | alone in group, candidatePriority zero -> report_ -[ RECORD 22 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 301 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | secondary other_node_current_state | candidate_node_current_state | @@ -331,6 +353,7 @@ comment | converged secondary, reportedTLI not an ancestor -[ RECORD 23 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 303 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -345,6 +368,7 @@ comment | primary healthy, no standby past replication_stal -[ RECORD 24 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 305 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -359,6 +383,7 @@ comment | nodesCount>2, primary unhealthy -> draining/maint -[ RECORD 25 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 307 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | report_lsn other_node_current_state | wait_primary, join_primary candidate_node_current_state | @@ -373,6 +398,7 @@ comment | report_lsn, primary converged wait/join_primary, -[ RECORD 26 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 309 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | report_lsn other_node_current_state | primary candidate_node_current_state | @@ -387,6 +413,7 @@ comment | report_lsn, primary converged primary, healthy -> -[ RECORD 27 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 311 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | fast_forward other_node_current_state | candidate_node_current_state | @@ -401,6 +428,7 @@ comment | fast_forward done -> prepare_promotion -[ RECORD 28 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 313 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | report_lsn, fast_forward other_node_current_state | candidate_node_current_state | @@ -415,6 +443,7 @@ comment | report_lsn or fast_forward, continuing an already -[ RECORD 29 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 315 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | wait_standby other_node_current_state | wait_primary, join_primary candidate_node_current_state | @@ -429,6 +458,7 @@ comment | wait_standby, primary converged wait/join_primary -[ RECORD 30 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 317 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | wait_standby other_node_current_state | primary candidate_node_current_state | @@ -443,6 +473,7 @@ comment | wait_standby (quorum member), primary converged p -[ RECORD 31 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 319 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | wait_standby other_node_current_state | primary candidate_node_current_state | @@ -457,6 +488,7 @@ comment | wait_standby (not a quorum member), primary conve -[ RECORD 32 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 321 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | catchingup other_node_current_state | wait_primary, join_primary, primary candidate_node_current_state | @@ -471,6 +503,7 @@ comment | caught up, same TLI as primary, within sync thres -[ RECORD 33 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 323 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | secondary other_node_current_state | wait_primary candidate_node_current_state | @@ -485,6 +518,7 @@ comment | primary fails, already converged wait_primary (is -[ RECORD 34 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 325 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | secondary other_node_current_state | candidate_node_current_state | @@ -499,6 +533,7 @@ comment | primary fails, not already wait_primary -> second -[ RECORD 35 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 327 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | wait_maintenance other_node_current_state | wait_primary candidate_node_current_state | @@ -513,6 +548,7 @@ comment | wait_maintenance, primary converged wait_primary -[ RECORD 36 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 329 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | wait_maintenance other_node_current_state | candidate_node_current_state | @@ -527,6 +563,7 @@ comment | wait_maintenance, primary's goal no longer wait_p -[ RECORD 37 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 331 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | prepare_promotion other_node_current_state | prepare_maintenance candidate_node_current_state | @@ -541,6 +578,7 @@ comment | prepare_promotion, primary converged prepare_main -[ RECORD 38 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 333 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | prepare_promotion other_node_current_state | candidate_node_current_state | @@ -555,6 +593,7 @@ comment | Citus worker prepare_promotion, primary present - -[ RECORD 39 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 335 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | prepare_promotion other_node_current_state | candidate_node_current_state | @@ -569,6 +608,7 @@ comment | Citus worker prepare_promotion, primary removed - -[ RECORD 40 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 337 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | prepare_promotion other_node_current_state | wait_primary candidate_node_current_state | @@ -583,6 +623,7 @@ comment | prepare_promotion, primary already converged wait -[ RECORD 41 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 339 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | prepare_promotion other_node_current_state | candidate_node_current_state | @@ -597,6 +638,7 @@ comment | prepare_promotion, primary present, not in mainte -[ RECORD 42 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 341 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | prepare_promotion other_node_current_state | candidate_node_current_state | @@ -611,6 +653,7 @@ comment | prepare_promotion, primary removed -> wait_primar -[ RECORD 43 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 343 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | stop_replication other_node_current_state | prepare_maintenance candidate_node_current_state | @@ -625,6 +668,7 @@ comment | stop_replication, primary converged prepare_maint -[ RECORD 44 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 345 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | stop_replication other_node_current_state | demote_timeout candidate_node_current_state | @@ -639,6 +683,7 @@ comment | stop_replication, primary converged demote_timeou -[ RECORD 45 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 347 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | stop_replication other_node_current_state | candidate_node_current_state | @@ -653,6 +698,7 @@ comment | stop_replication, primary's drain time expired -> -[ RECORD 46 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 349 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | stop_replication other_node_current_state | candidate_node_current_state | @@ -667,6 +713,7 @@ comment | stop_replication, primary's goal wait_primary but -[ RECORD 47 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 351 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | stop_replication other_node_current_state | candidate_node_current_state | @@ -681,6 +728,7 @@ comment | Citus worker stop_replication, primary present -> -[ RECORD 48 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 353 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | stop_replication other_node_current_state | candidate_node_current_state | @@ -695,6 +743,7 @@ comment | Citus worker stop_replication, primary removed -> -[ RECORD 49 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 355 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | demoted other_node_current_state | wait_primary, join_primary candidate_node_current_state | @@ -709,6 +758,7 @@ comment | demoted, primary reported wait/join_primary with -[ RECORD 50 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 357 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | demoted other_node_current_state | wait_primary, join_primary, primary candidate_node_current_state | @@ -723,6 +773,7 @@ comment | demoted, primary converged wait/join_primary/prim -[ RECORD 51 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 359 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | join_secondary other_node_current_state | wait_primary candidate_node_current_state | @@ -737,6 +788,7 @@ comment | join_secondary, primary reported wait_primary wit -[ RECORD 52 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 361 section | reporting_node +section_path | reporting_node.from_context active_node_current_state | join_secondary other_node_current_state | primary candidate_node_current_state | @@ -751,6 +803,7 @@ comment | join_secondary, primary converged primary -> seco -[ RECORD 53 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 363 section | reporting_node +section_path | reporting_node.ms_failover.retry_reset active_node_current_state | report_lsn other_node_current_state | candidate_node_current_state | @@ -765,6 +818,7 @@ comment | MS-failover: candidate stuck in fast_forward, all -[ RECORD 54 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 365 section | reporting_node +section_path | reporting_node.ms_failover.candidate_join active_node_current_state | report_lsn other_node_current_state | candidate_node_current_state | @@ -779,6 +833,7 @@ comment | MS-failover: activeNode in report_lsn, failover c -[ RECORD 55 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 367 section | reporting_node +section_path | reporting_node.ms_failover.candidate_fanout active_node_current_state | secondary, catchingup other_node_current_state | candidate_node_current_state | @@ -793,6 +848,7 @@ comment | MS-failover fan-out: secondary/catchingup, not ye -[ RECORD 56 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 369 section | reporting_node +section_path | reporting_node.ms_failover.candidate_fanout active_node_current_state | maintenance other_node_current_state | candidate_node_current_state | @@ -807,6 +863,7 @@ comment | MS-failover fan-out: rejoining from maintenance - -[ RECORD 57 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 371 section | reporting_node +section_path | reporting_node.ms_failover.candidate_fanout active_node_current_state | draining, demoted other_node_current_state | candidate_node_current_state | @@ -821,6 +878,7 @@ comment | MS-failover fan-out: old primary converged draini -[ RECORD 58 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 373 section | reporting_node +section_path | reporting_node.ms_failover.candidate_fanout active_node_current_state | demoted other_node_current_state | candidate_node_current_state | @@ -835,6 +893,7 @@ comment | MS-failover fan-out: old primary demoted, was rej -[ RECORD 59 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 375 section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome active_node_current_state | report_lsn other_node_current_state | candidate_node_current_state | @@ -849,6 +908,7 @@ comment | MS-failover: no promotion in flight, most-advance -[ RECORD 60 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 377 section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome active_node_current_state | report_lsn other_node_current_state | candidate_node_current_state | @@ -863,6 +923,82 @@ comment | MS-failover: no promotion in flight, most-advance -[ RECORD 61 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 379 section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.missing_nodes_gate +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | guardDataLossEnabled=true, inMSFailoverCluster=true, inMSFailoverCandidateGate=true, missingNodesCount>=1 +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=true -> decline, wait for more reports (1 of 2) +-[ RECORD 62 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 381 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.missing_nodes_gate +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | guardDataLossEnabled=false, inMSFailoverCluster=true, inMSFailoverCandidateGate=true, missingNodesCount>=1 +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=false -> proceed despite possible data loss (2 of 2) +-[ RECORD 63 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 383 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.candidate_count_gate +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | inMSFailoverCluster=true, inMSFailoverCandidateGate=true, candidateCount=0 +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | f +comment | MS-failover: zero candidates have reported their LSN yet -> silent decline +-[ RECORD 64 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 385 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.quorum_candidate_gate +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | guardDataLossEnabled=true, inMSFailoverCluster=true, inMSFailoverCandidateGate=true, sufficientQuorumCandidates=false +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | MS-failover: not enough quorum candidates reported yet, guard_data_loss=true -> decline (1 of 2) +-[ RECORD 65 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 387 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.quorum_candidate_gate +active_node_current_state | +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | guardDataLossEnabled=false, inMSFailoverCluster=true, inMSFailoverCandidateGate=true, sufficientQuorumCandidates=false +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | t +comment | MS-failover: not enough quorum candidates reported yet, guard_data_loss=false -> proceed with fewer than required (2 of 2) +-[ RECORD 66 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 389 +section | reporting_node +section_path | reporting_node.ms_failover.promotion_outcome.no_candidate_yet active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -874,9 +1010,10 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | f comment | MS-failover: no promotion in flight, not enough (or not safe enough) candidates yet -> no-op besides the fan-out above --[ RECORD 62 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -pos | 381 +-[ RECORD 67 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 391 section | reporting_node +section_path | reporting_node.ms_failover.draining_or_maintenance active_node_current_state | other_node_current_state | candidate_node_current_state | @@ -888,9 +1025,10 @@ active_node_assigned_state | other_node_assigned_state | draining has_extra_action | f comment | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining --[ RECORD 63 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -pos | 383 +-[ RECORD 68 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 393 section | reporting_node +section_path | reporting_node.ms_failover.draining_or_maintenance active_node_current_state | other_node_current_state | prepare_maintenance candidate_node_current_state | @@ -902,9 +1040,10 @@ active_node_assigned_state | other_node_assigned_state | maintenance has_extra_action | f comment | nodesCount>2, primary unhealthy, converged prepare_maintenance -> primary maintenance --[ RECORD 64 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 69 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 401 section | primary_node +section_path | primary_node active_node_current_state | single other_node_current_state | candidate_node_current_state | @@ -916,9 +1055,10 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | primary alone, another node reached wait_standby -> wait_primary --[ RECORD 65 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 70 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 403 section | primary_node +section_path | primary_node active_node_current_state | primary, wait_primary, apply_settings other_node_current_state | candidate_node_current_state | @@ -930,9 +1070,10 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | t comment | all nodes async, zero secondaries -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 66 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 405 section | primary_node +section_path | primary_node active_node_current_state | primary, wait_primary, apply_settings other_node_current_state | candidate_node_current_state | @@ -944,9 +1085,10 @@ active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t comment | all nodes async, >=1 secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 67 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 407 section | primary_node +section_path | primary_node active_node_current_state | primary, apply_settings other_node_current_state | candidate_node_current_state | @@ -958,9 +1100,10 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | t comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 68 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 409 section | primary_node +section_path | primary_node active_node_current_state | primary, apply_settings other_node_current_state | candidate_node_current_state | @@ -972,9 +1115,10 @@ active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 69 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 411 section | primary_node +section_path | primary_node active_node_current_state | wait_primary other_node_current_state | candidate_node_current_state | @@ -986,9 +1130,10 @@ active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 70 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 75 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 413 section | primary_node +section_path | primary_node active_node_current_state | apply_settings other_node_current_state | candidate_node_current_state | @@ -1000,9 +1145,10 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | t comment | apply_settings, both zero -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 415 section | primary_node +section_path | primary_node active_node_current_state | apply_settings other_node_current_state | candidate_node_current_state | @@ -1014,9 +1160,10 @@ active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 417 section | primary_node +section_path | primary_node active_node_current_state | apply_settings other_node_current_state | candidate_node_current_state | @@ -1028,9 +1175,10 @@ active_node_assigned_state | primary other_node_assigned_state | has_extra_action | t comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 419 section | primary_node +section_path | primary_node active_node_current_state | primary, wait_primary, apply_settings other_node_current_state | candidate_node_current_state | @@ -1042,9 +1190,10 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | converged primary/wait_primary/apply_settings, no other condition applies -> no-op besides the unhealthy-secondary fan-out --[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 421 section | primary_node +section_path | primary_node active_node_current_state | join_primary other_node_current_state | candidate_node_current_state | diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 571c5aaa6..7a97c31ef 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -270,22 +270,22 @@ SELECT e.pos, e.current_state, e.assigned_state, f.comment 351 | fast_forward | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | join_secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 381 | init | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | demote_timeout | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | demoted | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | catchingup | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | prepare_promotion | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | stop_replication | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | wait_standby | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | prepare_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | wait_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | report_lsn | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | fast_forward | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | join_secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 381 | dropped | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | init | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | demote_timeout | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | demoted | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | catchingup | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | prepare_promotion | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | stop_replication | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | wait_standby | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | prepare_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | wait_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | report_lsn | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | fast_forward | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | join_secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | dropped | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining (134 rows) DROP TABLE keeper_fsm_edges; diff --git a/src/monitor/expected/lock_and_fetch_migration.out b/src/monitor/expected/lock_and_fetch_migration.out index 4156d16d9..5ab44389a 100644 --- a/src/monitor/expected/lock_and_fetch_migration.out +++ b/src/monitor/expected/lock_and_fetch_migration.out @@ -32,15 +32,15 @@ create_formation | (lafm_test,pgsql,postgres,t,1) SELECT pgautofailover.register_node('lafm_test', 'lafm_p', 5432, 'postgres', 'lafm_p', 1); -[ RECORD 1 ]-+--------------------------- -register_node | (21,0,single,100,t,lafm_p) +register_node | (24,0,single,100,t,lafm_p) SELECT pgautofailover.register_node('lafm_test', 'lafm_s1', 5432, 'postgres', 'lafm_s1', 1); -[ RECORD 1 ]-+---------------------------------- -register_node | (22,0,wait_standby,100,t,lafm_s1) +register_node | (25,0,wait_standby,100,t,lafm_s1) SELECT pgautofailover.register_node('lafm_test', 'lafm_s2', 5432, 'postgres', 'lafm_s2', 1); -[ RECORD 1 ]-+---------------------------------- -register_node | (23,0,wait_standby,100,t,lafm_s2) +register_node | (26,0,wait_standby,100,t,lafm_s2) SELECT nodeid AS np FROM pgautofailover.node WHERE formationid = 'lafm_test' AND nodename = 'lafm_p' \gset @@ -267,7 +267,7 @@ SELECT assigned_group_state FROM pgautofailover.node_active('lafm_test', :np, 0, assigned_group_state | primary SELECT pgautofailover.start_maintenance(:ns1); -WARNING: Starting maintenance on node 22 "lafm_s1" (lafm_s1:5432) will block writes on the primary node 21 "lafm_p" (lafm_p:5432) +WARNING: Starting maintenance on node 25 "lafm_s1" (lafm_s1:5432) will block writes on the primary node 24 "lafm_p" (lafm_p:5432) DETAIL: we now have 0 healthy node(s) left in the "secondary" state and formation "lafm_test" number-sync-standbys requires 1 sync standbys -[ RECORD 1 ]-----+-- start_maintenance | t @@ -338,13 +338,13 @@ reportedstate | single goalstate | single rule_pos | rule_section | -description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "single" +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "single" -[ RECORD 3 ]-+---------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 22 "lafm_s1" (lafm_s1:5432): "wait_standby" +description | New state is reported by node 25 "lafm_s1" (lafm_s1:5432): "wait_standby" -[ RECORD 4 ]-+---------------------------------------------------------------------------------------------------------------------- reportedstate | single goalstate | wait_primary @@ -356,7 +356,7 @@ reportedstate | wait_primary goalstate | wait_primary rule_pos | rule_section | -description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "wait_primary" +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "wait_primary" -[ RECORD 6 ]-+---------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup @@ -368,7 +368,7 @@ reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | -description | New state is reported by node 22 "lafm_s1" (lafm_s1:5432): "catchingup" +description | New state is reported by node 25 "lafm_s1" (lafm_s1:5432): "catchingup" -[ RECORD 8 ]-+---------------------------------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | secondary @@ -380,7 +380,7 @@ reportedstate | secondary goalstate | secondary rule_pos | rule_section | -description | New state is reported by node 22 "lafm_s1" (lafm_s1:5432): "secondary" +description | New state is reported by node 25 "lafm_s1" (lafm_s1:5432): "secondary" -[ RECORD 10 ]+---------------------------------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | primary @@ -392,13 +392,13 @@ reportedstate | primary goalstate | primary rule_pos | rule_section | -description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "primary" +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "primary" -[ RECORD 12 ]+---------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 23 "lafm_s2" (lafm_s2:5432): "wait_standby" +description | New state is reported by node 26 "lafm_s2" (lafm_s2:5432): "wait_standby" -[ RECORD 13 ]+---------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup @@ -416,19 +416,19 @@ reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | -description | New state is reported by node 23 "lafm_s2" (lafm_s2:5432): "catchingup" +description | New state is reported by node 26 "lafm_s2" (lafm_s2:5432): "catchingup" -[ RECORD 16 ]+---------------------------------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | catchingup rule_pos | rule_section | -description | New state is reported by node 23 "lafm_s2" (lafm_s2:5432): "secondary" +description | New state is reported by node 26 "lafm_s2" (lafm_s2:5432): "secondary" -[ RECORD 17 ]+---------------------------------------------------------------------------------------------------------------------- reportedstate | apply_settings goalstate | apply_settings rule_pos | rule_section | -description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "apply_settings" +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "apply_settings" -[ RECORD 18 ]+---------------------------------------------------------------------------------------------------------------------- reportedstate | apply_settings goalstate | primary @@ -440,7 +440,7 @@ reportedstate | primary goalstate | primary rule_pos | rule_section | -description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "primary" +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "primary" -[ RECORD 20 ]+---------------------------------------------------------------------------------------------------------------------- reportedstate | primary goalstate | apply_settings @@ -476,7 +476,7 @@ reportedstate | apply_settings goalstate | apply_settings rule_pos | rule_section | -description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "apply_settings" +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "apply_settings" -[ RECORD 26 ]+---------------------------------------------------------------------------------------------------------------------- reportedstate | apply_settings goalstate | primary @@ -488,7 +488,7 @@ reportedstate | primary goalstate | primary rule_pos | rule_section | -description | New state is reported by node 21 "lafm_p" (lafm_p:5432): "primary" +description | New state is reported by node 24 "lafm_p" (lafm_p:5432): "primary" -[ RECORD 28 ]+---------------------------------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | maintenance @@ -500,7 +500,7 @@ reportedstate | maintenance goalstate | maintenance rule_pos | rule_section | -description | New state is reported by node 22 "lafm_s1" (lafm_s1:5432): "maintenance" +description | New state is reported by node 25 "lafm_s1" (lafm_s1:5432): "maintenance" -[ RECORD 30 ]+---------------------------------------------------------------------------------------------------------------------- reportedstate | maintenance goalstate | catchingup diff --git a/src/monitor/expected/timeline_fork_detection.out b/src/monitor/expected/timeline_fork_detection.out index d777791c2..d8fbce08f 100644 --- a/src/monitor/expected/timeline_fork_detection.out +++ b/src/monitor/expected/timeline_fork_detection.out @@ -21,7 +21,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-p', 5432, 'postgres', 'p', 1); -[ RECORD 1 ]---------------+------- -assigned_node_id | 24 +assigned_node_id | 27 assigned_group_id | 0 assigned_group_state | single assigned_candidate_priority | 100 @@ -34,7 +34,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-s1', 5432, 'postgres', 's1', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 25 +assigned_node_id | 28 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -47,7 +47,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-s2', 5432, 'postgres', 's2', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 26 +assigned_node_id | 29 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -88,27 +88,27 @@ SELECT nodeid, tli, parenttli, switchpoint_lsn FROM pgautofailover.node_timeline_history ORDER BY nodeid, tli; -[ RECORD 1 ]---+------- -nodeid | 24 +nodeid | 27 tli | 1 parenttli | 0 switchpoint_lsn | 0/0 -[ RECORD 2 ]---+------- -nodeid | 25 +nodeid | 28 tli | 1 parenttli | 0 switchpoint_lsn | 0/0 -[ RECORD 3 ]---+------- -nodeid | 25 +nodeid | 28 tli | 2 parenttli | 1 switchpoint_lsn | 0/6000 -[ RECORD 4 ]---+------- -nodeid | 26 +nodeid | 29 tli | 1 parenttli | 0 switchpoint_lsn | 0/0 -[ RECORD 5 ]---+------- -nodeid | 26 +nodeid | 29 tli | 3 parenttli | 1 switchpoint_lsn | 0/6000 @@ -227,7 +227,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-p', 5432, 'postgres', 'p', 1); -[ RECORD 1 ]---------------+------- -assigned_node_id | 27 +assigned_node_id | 30 assigned_group_id | 0 assigned_group_state | single assigned_candidate_priority | 100 @@ -240,7 +240,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s1', 5432, 'postgres', 's1', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 28 +assigned_node_id | 31 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -253,7 +253,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s2', 5432, 'postgres', 's2', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 29 +assigned_node_id | 32 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -266,7 +266,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s3', 5432, 'postgres', 's3', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 30 +assigned_node_id | 33 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -571,13 +571,13 @@ reportedstate | single goalstate | single rule_pos | rule_section | -description | New state is reported by node 27 "p" (tlfe-p:5432): "single" +description | New state is reported by node 30 "p" (tlfe-p:5432): "single" -[ RECORD 3 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 28 "s1" (tlfe-s1:5432): "wait_standby" +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "wait_standby" -[ RECORD 4 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | single goalstate | wait_primary @@ -589,7 +589,7 @@ reportedstate | wait_primary goalstate | wait_primary rule_pos | rule_section | -description | New state is reported by node 27 "p" (tlfe-p:5432): "wait_primary" +description | New state is reported by node 30 "p" (tlfe-p:5432): "wait_primary" -[ RECORD 6 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup @@ -601,7 +601,7 @@ reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | -description | New state is reported by node 28 "s1" (tlfe-s1:5432): "catchingup" +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "catchingup" -[ RECORD 8 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | secondary @@ -613,7 +613,7 @@ reportedstate | secondary goalstate | secondary rule_pos | rule_section | -description | New state is reported by node 28 "s1" (tlfe-s1:5432): "secondary" +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "secondary" -[ RECORD 10 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | primary @@ -625,13 +625,13 @@ reportedstate | primary goalstate | primary rule_pos | rule_section | -description | New state is reported by node 27 "p" (tlfe-p:5432): "primary" +description | New state is reported by node 30 "p" (tlfe-p:5432): "primary" -[ RECORD 12 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 29 "s2" (tlfe-s2:5432): "wait_standby" +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "wait_standby" -[ RECORD 13 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup @@ -649,31 +649,31 @@ reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | -description | New state is reported by node 29 "s2" (tlfe-s2:5432): "catchingup" +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "catchingup" -[ RECORD 16 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | catchingup rule_pos | rule_section | -description | New state is reported by node 29 "s2" (tlfe-s2:5432): "secondary" +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "secondary" -[ RECORD 17 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 30 "s3" (tlfe-s3:5432): "wait_standby" +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "wait_standby" -[ RECORD 18 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 30 "s3" (tlfe-s3:5432): "catchingup" +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "catchingup" -[ RECORD 19 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 30 "s3" (tlfe-s3:5432): "secondary" +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "secondary" -[ RECORD 20 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | report_lsn goalstate | prepare_promotion diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 9da6da2bd..e03e64db7 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -133,6 +133,69 @@ BoolMatchesPattern(bool actual, BoolPattern pattern) } +/* + * IntPattern: the integer-valued analogue of BoolPattern, for facts that are + * counts rather than booleans -- BuildCandidateList's own missingNodesCount/ + * candidateCount/quorumCandidateCount (see NodeActiveContext's own fields of + * the same names), matched here instead of pre-flattened into one hand-named + * boolean per threshold the way atLeastOneHealthyCandidate etc. already are. + * INT_PATTERN_ANY is the same "omitted means don't-care" default every other + * pattern kind in this file uses. + * + * No array member (unlike ReplicationStateSet/NodeStatePattern -- see + * STATES()'s own comment on why those stay bare-brace): structurally + * identical to ApiTriggerPattern, so EXACTLY/AT_LEAST/AT_MOST's + * compound-literal cast below is exactly as safe as API_TRIGGER(fn)'s own + * cast (gcc's "initializer element is not constant" rejection is + * specifically about a nested array member, which neither struct has). + */ +typedef enum IntPatternKind +{ + INT_PATTERN_ANY = 0, + INT_PATTERN_EXACTLY, + INT_PATTERN_AT_LEAST, + INT_PATTERN_AT_MOST +} IntPatternKind; + +typedef struct IntPattern +{ + IntPatternKind kind; + int value; /* meaningful only when kind != INT_PATTERN_ANY */ +} IntPattern; + +#define EXACTLY(n) ((IntPattern) { .kind = INT_PATTERN_EXACTLY, .value = (n) }) +#define AT_LEAST(n) ((IntPattern) { .kind = INT_PATTERN_AT_LEAST, .value = (n) }) +#define AT_MOST(n) ((IntPattern) { .kind = INT_PATTERN_AT_MOST, .value = (n) }) + +static bool +IntMatchesPattern(int actual, IntPattern pattern) +{ + switch (pattern.kind) + { + case INT_PATTERN_EXACTLY: + { + return actual == pattern.value; + } + + case INT_PATTERN_AT_LEAST: + { + return actual >= pattern.value; + } + + case INT_PATTERN_AT_MOST: + { + return actual <= pattern.value; + } + + case INT_PATTERN_ANY: + default: + { + return true; + } + } +} + + typedef enum NodeStatePatternKind { NODE_STATE_ANY = 0, @@ -674,6 +737,45 @@ typedef struct NodeActiveContext * etc. regressing exactly this way before this field existed. */ bool inMSFailoverCluster; + + /* + * Set unconditionally true by BuildMSFailoverCandidateGateNodeActiveContext, + * false (the memset default) everywhere else -- narrows the 5 counting-gate + * rows (reporting_node.ms_failover.promotion_outcome.*_gate) to only ever + * match when dispatched from ProceedGroupStateForMSFailover's own gate + * checks. Without this, TryFanOutReportLsnRow's own candidateNode=NULL call + * (BuildMSFailoverNodeActiveContext) leaves candidateCount/missingNodesCount + * at their memset-0 default and candidatePromotionInProgress false -- the + * exact values the missing-nodes and candidate-count gate rows themselves + * match on -- so its own SectionMSFailover-wide scan could spuriously hit + * one of these gate rows instead of correctly falling through to its own + * plain-AssignGoalState fallback whenever pos 367-373 somehow didn't match. + */ + bool inMSFailoverCandidateGate; + + /* + * candidateCount/quorumCandidateCount/missingNodesCount mirror + * CandidateList's own fields of the same names (BuildCandidateList) -- + * set only by BuildMSFailoverCandidateGateNodeActiveContext, left at + * their memset-0 default everywhere else. Matched via IntPattern + * (EXACTLY/AT_LEAST/AT_MOST) instead of being pre-flattened into a + * one-off boolean per threshold, since these are the same reusable + * counts BuildCandidateList already computes once, not new facts. + */ + int candidateCount; + int quorumCandidateCount; + int missingNodesCount; + + /* + * quorumCandidateCount >= (ctx->formation->number_sync_standbys + 1), + * computed once by the same builder as the three counts above. A + * per-formation runtime threshold, not a literal -- IntPattern can only + * compare against a fixed value written into a row, so this one + * comparison is precomputed into a plain bool instead, exactly like + * mostAdvancedCandidateWithinPromoteThreshold already does for its own + * GUC-relative comparison (PromoteXlogThreshold) above. + */ + bool sufficientQuorumCandidates; } NodeActiveContext; typedef struct NodeActiveContextPattern @@ -706,6 +808,12 @@ typedef struct NodeActiveContextPattern BoolPattern mostAdvancedCandidateWithinPromoteThreshold; BoolPattern guardDataLossEnabled; BoolPattern inMSFailoverCluster; + BoolPattern inMSFailoverCandidateGate; + + IntPattern candidateCount; + IntPattern quorumCandidateCount; + IntPattern missingNodesCount; + BoolPattern sufficientQuorumCandidates; } NodeActiveContextPattern; @@ -750,40 +858,78 @@ typedef void (*MonitorExtraActionFunction) (GroupStateContext *ctx, * MonitorFSMSection itself is declared in group_state_machine.h, not here: * it needs to be visible to pg_auto_failover.c (to register * pgautofailover.fsm_section and wire up dump_fsm()) the same way - * ReplicationState is. The comment on the MonitorFSM_* index constants - * below explains what each of its three values corresponds to in the - * original if-chain. Every row is tagged with its own section explicitly - * (rather than leaving section membership implicit in array position - * alone) so a reader scanning the table sees which region a row belongs to - * without cross-referencing an index against a separate comment block, and - * so AssertMonitorFSMWellFormed() can verify the MonitorFSM_* index - * constants actually agree with where each section's rows really are. + * ReplicationState is. See its own comment there for the top-level-vs-leaf + * distinction and why expanding it with fine-grained values changed nothing + * SQL-visible. + * + * MonitorFSMSectionPath is a small, fixed-depth array of MonitorFSMSection + * values -- a row's own place in the section hierarchy (e.g. { REPORTING_NODE, + * MS_FAILOVER, MS_FAILOVER_RETRY_RESET }), replacing the hand-maintained + * array-index-range boundary constants this design used to rely on (see + * the design doc's own "Open items": those "need to stay in sync with the + * table by hand as rows are added, removed, or reordered"). Trailing unused + * slots default to MONITOR_FSM_SECTION_NONE via ordinary aggregate + * initialization -- a plain array member needs no compound-literal cast any + * more than ReplicationStateSet's own states[4] does (see STATES()'s own + * comment on why that stays bare-brace too). */ +#define MONITOR_FSM_SECTION_PATH_MAX_DEPTH 4 +typedef MonitorFSMSection MonitorFSMSectionPath[MONITOR_FSM_SECTION_PATH_MAX_DEPTH]; + +/* + * SectionPathIsUnderPrefix returns whether path is prefix (an ancestor of it, + * or itself) -- pure C, no parsing, no cache, no dependency on anything + * outside this file: comparing two small fixed-size enum arrays element by + * element. This is the replacement for FindMatchingMonitorFSMRuleIndexFrom's + * old [startIndex, endIndex) bound (see FindMatchingMonitorFSMRuleIndexUnderPath + * below): a search is now "every row whose sectionPath is under this + * prefix", not "every row between these two array indices". + */ +static bool +SectionPathIsUnderPrefix(const MonitorFSMSectionPath path, const MonitorFSMSectionPath prefix) +{ + for (int i = 0; i < MONITOR_FSM_SECTION_PATH_MAX_DEPTH; i++) + { + if (prefix[i] == MONITOR_FSM_SECTION_NONE) + { + return true; + } + + if (path[i] != prefix[i]) + { + return false; + } + } + + return true; +} typedef struct MonitorFSMTransition { /* - * pos and section are metadata, not match inputs: RuleMatches() never + * pos and sectionPath are metadata, not match inputs: RuleMatches() never * reads either. pos is a human-facing row number -- purely so a comment, * a bug report, or a SQL query against dump_fsm() can say "row 203" and * a reader can find it without counting braces up from the top of the - * array. Each MonitorFSMSection gets its own hundred-block, starting at - * *01 rather than *00 so the position within the block reads as an - * ordinary 1-based count (101 is the section's 1st row, 103 its 2nd, - * ...) instead of an off-by-one 0th/1st/2nd. Rows within a section are - * numbered every 2 (101, 103, 105, ...) rather than consecutively -- so - * a new row can be inserted between two existing ones (e.g. 102 between - * 101 and 103) without renumbering anything else in the file. - * section records which of the three real control-flow regions (see - * MonitorFSMSection above) this row belongs to; AssertMonitorFSMWellFormed() - * uses both to confirm the MonitorFSM_* index constants below are still - * correct, so a boundary drifting out of sync with the rows it's meant to - * bound fails at first use, not by accident months later. Both are also - * exposed to SQL via dump_fsm() and attributed to the pgautofailover.event - * row a matched rule produces (see rule_pos/rule_section below). + * array. Each top-level MonitorFSMSection gets its own hundred-block, + * starting at *01 rather than *00 so the position within the block reads + * as an ordinary 1-based count (101 is the section's 1st row, 103 its + * 2nd, ...) instead of an off-by-one 0th/1st/2nd. Rows within a section + * are numbered every 2 (101, 103, 105, ...) rather than consecutively -- + * so a new row can be inserted between two existing ones (e.g. 102 + * between 101 and 103) without renumbering anything else in the file. + * sectionPath records this row's own place in the section hierarchy (see + * MonitorFSMSectionPath above); AssertMonitorFSMWellFormed() uses both to + * confirm each row's sectionPath[0] agrees with what its pos's 100-block + * implies, so a mismatch fails at first use, not by accident months + * later. Both are also exposed to SQL via dump_fsm() (pos, section, and + * the new section_path) and attributed to the pgautofailover.event row a + * matched rule produces (see rule_pos/rule_section below) -- rule_section + * is always derived from sectionPath[0] alone, so it stays exactly the + * same 4-value enum it always was. */ int pos; - MonitorFSMSection section; + MonitorFSMSectionPath sectionPath; NodeStatusPattern activeNode; NodeStatusPattern primaryNode; @@ -830,7 +976,7 @@ typedef struct MonitorFSMTransition * for why ("One array, not three" in the design doc this table implements). * Forward-declared here so the extraActions defined above it (which each * perform one bounded, named nested search over it) can reference it by - * name; the four index constants below are forward-declared the same way, + * name; the section-path constants below are forward-declared the same way, * for the same reason -- both those actions and the top-level driver need * them to bound their searches. * @@ -841,109 +987,156 @@ typedef struct MonitorFSMTransition * row came from (whether .activeNode means "the reporting node" or "the * primary node substituted in"), and one specific fallback (the MS-failover * cascade declining) needs to resume scanning from a specific *later* point, - * not from the top. Each constant below is the index where one of those - * real regions starts, so a search can be bounded to exactly the region - * that's semantically valid for the situation at hand: + * not from the top. Each named MonitorFSMSectionPath constant below is the + * section a search should be bounded to, so it only ever considers rows + * semantically valid for the situation at hand -- replacing what used to be + * six hand-maintained array-index constants (recomputed by hand whenever a + * row was added, removed, or moved across a boundary; the design doc's own + * "Open items" flagged exactly this as fragile). A row's membership is now + * a fact carried on the row itself (.sectionPath, see MonitorFSMTransition + * above) rather than an implication of where it happens to sit in the + * array, so inserting, removing, or reordering rows within a section no + * longer requires touching any constant at all: * - * MonitorFSM_EarlyChecksStart = 15 - * Where MONITOR_FSM_SECTION_API_TRIGGERED ends and MONITOR_FSM_SECTION_ - * EARLY_CHECKS begins. Rows [0, 15) are the operator-triggered rows - * (perform_failover, remove_node, start/stop_maintenance, - * set_node_candidate_priority, set_node_replication_quorum, - * set_formation_number_sync_standbys) -- reached only via - * ProceedGroupStateForApiTrigger(), never via the ordinary node_active() - * heartbeat path (nac->apiFunction stays API_FUNCTION_NONE there, and - * every one of these 15 rows' .conditions.apiTrigger requires a specific - * non-NONE value -- see MatchApiTrigger). Used to: (a) start the - * api-triggered lookup at 0, bounded to end here, (b) start the - * early-checks lookup here instead of at 0, now that the heartbeat - * sections no longer begin at the very top of the array. + * SectionApiTriggered = { MONITOR_FSM_SECTION_API_TRIGGERED } + * Rows whose sectionPath[0] is API_TRIGGERED (pos 101-1xx) -- the + * operator-triggered rows (perform_failover, remove_node, + * start/stop_maintenance, set_node_candidate_priority, + * set_node_replication_quorum, set_formation_number_sync_standbys) -- + * reached only via ProceedGroupStateForApiTrigger(), never via the + * ordinary node_active() heartbeat path (nac->apiFunction stays + * API_FUNCTION_NONE there, and every one of these 15 rows' + * .conditions.apiTrigger requires a specific non-NONE value -- see + * MatchApiTrigger). * - * MonitorFSM_FromContextStart = 21 - * Where MONITOR_FSM_SECTION_REPORTING_NODE begins. Rows [15, 21) are the - * six checks (DROPPED, goal-DROPPED, MAINTENANCE, the demote_timeout + * SectionEarlyChecks = { MONITOR_FSM_SECTION_EARLY_CHECKS } + * Rows whose sectionPath[0] is EARLY_CHECKS (pos 201-211) -- the six + * checks (DROPPED, goal-DROPPED, MAINTENANCE, the demote_timeout * self-fence, both "alone in group" rows) that * ProceedGroupStateFromContext() runs unconditionally, before its one * real branch point (IsInPrimaryState(activeNode)) -- so they're tried - * first regardless of which way that branch goes, and everything from - * index 21 on is reached only when activeNode is confirmed NOT currently - * primary-role. Used to: (a) bound the early-checks lookup to [15, 21) - * in the top-level driver, (b) start the ordinary FromContext lookup at - * 21 once activeNode is confirmed non-primary. + * first regardless of which way that branch goes. + * + * SectionReportingNode = { MONITOR_FSM_SECTION_REPORTING_NODE } + * Rows whose sectionPath[0] is REPORTING_NODE (pos 301-38x) -- reached + * only when activeNode is confirmed NOT currently primary-role. Covers + * both the ordinary FromContext rows (sectionPath[1] == + * MONITOR_FSM_SECTION_FROM_CONTEXT) and the MS-failover cluster + * (sectionPath[1] == MONITOR_FSM_SECTION_MS_FAILOVER) as one prefix, + * matching the old MonitorFSM_FromContextStart..MonitorFSM_ + * PrimaryNodeSectionStart span exactly. * - * MonitorFSM_FromContextResumeStart = 24 - * A resume point *inside* MONITOR_FSM_SECTION_REPORTING_NODE, not a - * section boundary of its own -- the row right after the merged - * nodesCount>2-unhealthy-primary row (pos 209, "nodesCount>2, primary - * unhealthy -> draining/maintenance + MS-failover cascade") -- only used - * by ActionRunMultiStandbyFailoverCascade: when - * ProceedGroupStateForMSFailover() declines, the real source falls + * MonitorFSM_MultiStandbyCascadeResumeAfterPos = 305 + * Not a section at all: a resume point *inside* SectionReportingNode, + * used only by ActionRunMultiStandbyFailoverCascade -- the pos of the + * merged nodesCount>2-unhealthy-primary row itself ("nodesCount>2, + * primary unhealthy -> draining/maintenance + MS-failover cascade"). + * When ProceedGroupStateForMSFailover() declines, the real source falls * through to whatever if-statement is textually next, and this is where - * that "next" starts in this table. Named similarly to (but NOT the - * same concept as) the design doc's MonitorFSM_MSFailoverClusterStart -- - * see ActionRunMultiStandbyFailoverCascade's own comment for the + * that "next" starts in this table. Section-path containment alone + * can't express "resume after this specific row" (it answers "is this + * row under X?", not "in array order, after row Y") -- pos is already + * the row's own stable, human-facing identity, and + * AssertMonitorFSMWellFormed() confirms pos is strictly increasing == + * array order, so "pos > afterPos" is exactly the resume semantics + * needed. Named similarly to (but NOT the same concept as) the design + * doc's MonitorFSM_MSFailoverClusterStart -- see + * ActionRunMultiStandbyFailoverCascade's own comment for the * distinction. * - * MonitorFSM_MSFailoverStart = 52 - * A resume point *inside* MONITOR_FSM_SECTION_REPORTING_NODE, like - * MonitorFSM_FromContextResumeStart above, not a section boundary of its - * own: nine rows, appended at the end of the section rather than - * renumbered into it, for the MS-failover / candidate-selection - * cluster's own declarative transitions (see that section's own - * comment below) -- two (retry-reset, join_secondary) reached through + * SectionMSFailover = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER } + * Rows under the MS-failover / candidate-selection cluster's own + * declarative transitions (see that section's own comment below) -- + * two (retry-reset, join_secondary) reached through * TryMSFailoverDeclarativeRow's bounded nested dispatch, gated by the * exact same hand-written C condition ProceedGroupStateForMSFailover/ * ProceedWithMSFailover already evaluate; four (BuildCandidateList's * own fan-out) through TryFanOutReportLsnRow's bounded nested dispatch, * one per node the fan-out loop touches; two (PromoteSelectedNode's own * two outcomes) through DispatchMonitorFSMRuleByPos, since first-match- - * wins can't distinguish between them (see their own comment); one - * (the "still gathering candidates" case) purely for dump_fsm() - * completeness, never itself dispatched. None reached through the - * ordinary top-level driver. + * wins can't distinguish between them (see their own comment); three + * (the counting gates ProceedGroupStateForMSFailover's own hand-written + * ifs used to be, see BuildMSFailoverCandidateGateNodeActiveContext) and + * one more (the "still gathering candidates" catch-all) purely for + * dump_fsm() completeness/fallback, not reached through any other + * bounded search. None reached through the ordinary top-level driver. * - * MonitorFSM_PrimaryNodeSectionStart = 63 - * Where MONITOR_FSM_SECTION_PRIMARY_NODE begins: the declarative - * replacement for ProceedGroupStateForPrimaryNode()'s own if-chain, in - * which .activeNode means the *primary* node, not the reporting node. - * Used to: (a) bound the ordinary FromContext lookup to end here (so it - * can never wander into primary-role rows built under a different - * NodeActiveContext), (b) start the primary-role lookup here, both from - * the top-level driver (activeNode already primary-role) and from - * ActionRunPrimaryNodeTransition's nested pass on primaryNode - * (join_secondary's cascade row). Two rows short of MonitorFSM_MSFailoverStart's - * own eleven-row span (363-383, not 363-379): ActionRunMultiStandbyFailoverCascade's - * own DRAINING/MAINTENANCE outcomes (pos 381/383) share this same bound, - * appended at the end of the MS-failover cluster rather than renumbered - * into the ordinary REPORTING_NODE rows above -- see their own comment. + * SectionPrimaryNode = { MONITOR_FSM_SECTION_PRIMARY_NODE } + * Rows whose sectionPath[0] is PRIMARY_NODE (pos 401-421): the + * declarative replacement for ProceedGroupStateForPrimaryNode()'s own + * if-chain, in which .activeNode means the *primary* node, not the + * reporting node. Used both from the top-level driver (activeNode + * already primary-role) and from ActionRunPrimaryNodeTransition's + * nested pass on primaryNode (join_secondary's cascade row). * - * MonitorFSM_SIZE = 74 - * Total row count -- the end bound for the primary-role lookup (nothing - * comes after MONITOR_FSM_SECTION_PRIMARY_NODE), and the size every - * other bounded search is checked against. + * MonitorFSM_SIZE + * Total row count -- the size every bounded search's own linear scan + * runs over (see FindMatchingMonitorFSMRuleIndexUnderPath), and the end + * bound dump_fsm()/dump_fsm_edges()/AssertMonitorFSMWellFormed() each + * iterate to. * - * Kept as plain hardcoded integers, exactly as the design doc's own - * placeholders are -- recomputed by hand whenever a row is added, removed, - * or moved across a boundary. What makes a wrong value here safe to keep as - * a hand-maintained integer, rather than a foot-gun: every row also carries - * its own .pos and .section fields (see MonitorFSMTransition above), and - * AssertMonitorFSMWellFormed() (below the array) walks the whole table once - * and asserts these six constants agree with what the rows themselves say - * -- .pos matches array position, and .section actually changes from - * API_TRIGGERED to EARLY_CHECKS to REPORTING_NODE to PRIMARY_NODE exactly at - * these four indices and nowhere else. A boundary that drifts out of sync - * with a row added, removed, or moved fails loudly at first use (an - * assertion), not silently as a row from the wrong section matching - * unexpectedly or a search that scans zero rows and never matches. + * What makes a row's own sectionPath safe to hand-write (rather than a + * foot-gun): every row also carries its own .pos (see MonitorFSMTransition + * above), and AssertMonitorFSMWellFormed() (below the array) walks the + * whole table once and asserts .pos is strictly increasing and that each + * row's sectionPath[0] agrees with what its pos's own 100-block implies + * (API_TRIGGERED for 1xx, EARLY_CHECKS for 2xx, REPORTING_NODE for 3xx, + * PRIMARY_NODE for 4xx) -- a row whose sectionPath drifts out of sync with + * its own pos fails loudly at first use (an assertion), not silently as a + * row from the wrong section matching unexpectedly or a search that scans + * zero rows and never matches. */ static const MonitorFSMTransition MonitorFSM[]; -#define MonitorFSM_EarlyChecksStart 15 -#define MonitorFSM_FromContextStart 21 -#define MonitorFSM_FromContextResumeStart 24 -#define MonitorFSM_MSFailoverStart 52 -#define MonitorFSM_PrimaryNodeSectionStart 63 -#define MonitorFSM_SIZE 74 +/* + * Forward-declared for the same reason as MonitorFSM[] just above: these are + * defined far below (near BuildMSFailoverNodeActiveContext), but the + * counting-gate rows inside MonitorFSM[] itself reference them by name as + * their own .extraAction. + */ +static void ActionLogMSFailoverMissingNodesDecline(GroupStateContext *ctx, + NodeActiveContext *nac, + char *message); +static void ActionLogMSFailoverMissingNodesContinue(GroupStateContext *ctx, + NodeActiveContext *nac, + char *message); +static void ActionLogMSFailoverQuorumDecline(GroupStateContext *ctx, + NodeActiveContext *nac, + char *message); +static void ActionLogMSFailoverQuorumContinue(GroupStateContext *ctx, + NodeActiveContext *nac, + char *message); + +#define MonitorFSM_SIZE 79 +#define MonitorFSM_MultiStandbyCascadeResumeAfterPos 305 + +static const MonitorFSMSectionPath SectionApiTriggered = + { MONITOR_FSM_SECTION_API_TRIGGERED }; +static const MonitorFSMSectionPath SectionEarlyChecks = + { MONITOR_FSM_SECTION_EARLY_CHECKS }; +static const MonitorFSMSectionPath SectionReportingNode = + { MONITOR_FSM_SECTION_REPORTING_NODE }; +static const MonitorFSMSectionPath SectionPrimaryNode = + { MONITOR_FSM_SECTION_PRIMARY_NODE }; +static const MonitorFSMSectionPath SectionMSFailover = + { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER }; + +/* + * Leaf prefixes for the 3 MS-failover counting gates (missingNodesCount/ + * candidateCount/quorumCandidateCount, see ProceedGroupStateForMSFailover): + * exact 4-deep paths, each used only by that function's own dedicated + * dispatch call at the matching hand-written `if`, never via the broader + * SectionMSFailover scan other callers use (see inMSFailoverCandidateGate's + * own comment on NodeActiveContext for why that distinction matters). + */ +static const MonitorFSMSectionPath SectionMSFailoverMissingNodesGate = + { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE }; +static const MonitorFSMSectionPath SectionMSFailoverQuorumCandidateGate = + { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE }; /* Forward-declared for the same reason as MonitorFSM[] above: used by * extraActions (ActionRunPrimaryNodeTransition) defined before its real @@ -1003,16 +1196,47 @@ RuleMatches(const NodeActiveContext *nac, const MonitorFSMTransition *rule) BoolMatchesPattern(nac->mostAdvancedCandidateWithinPromoteThreshold, cond->mostAdvancedCandidateWithinPromoteThreshold) && BoolMatchesPattern(nac->guardDataLossEnabled, cond->guardDataLossEnabled) && - BoolMatchesPattern(nac->inMSFailoverCluster, cond->inMSFailoverCluster); + BoolMatchesPattern(nac->inMSFailoverCluster, cond->inMSFailoverCluster) && + BoolMatchesPattern(nac->inMSFailoverCandidateGate, + cond->inMSFailoverCandidateGate) && + + IntMatchesPattern(nac->missingNodesCount, cond->missingNodesCount) && + IntMatchesPattern(nac->candidateCount, cond->candidateCount) && + IntMatchesPattern(nac->quorumCandidateCount, cond->quorumCandidateCount) && + BoolMatchesPattern(nac->sufficientQuorumCandidates, + cond->sufficientQuorumCandidates); } +/* + * FindMatchingMonitorFSMRuleIndexUnderPath scans the whole table in array + * order, considering only rows whose own sectionPath is under prefix (see + * SectionPathIsUnderPrefix) and whose pos is greater than afterPos -- + * replacing the old [startIndex, endIndex) index-range bound with a + * section-membership one. pos is strictly increasing == array order (see + * AssertMonitorFSMWellFormed), so "pos > afterPos" is exactly "resume + * scanning after this row" -- afterPos = 0 (no real row has pos <= 0) means + * "from the very start of whichever rows are under prefix", the ordinary + * case; a specific pos is only ever passed for the one genuine mid-section + * resume point this table has (see MonitorFSM_MultiStandbyCascadeResumeAfterPos). + */ static int -FindMatchingMonitorFSMRuleIndexFrom(const MonitorFSMTransition table[], int tableSize, - int startIndex, const NodeActiveContext *nac) +FindMatchingMonitorFSMRuleIndexUnderPath(const MonitorFSMTransition table[], int tableSize, + const MonitorFSMSectionPath prefix, int afterPos, + const NodeActiveContext *nac) { - for (int i = startIndex; i < tableSize; i++) + for (int i = 0; i < tableSize; i++) { + if (table[i].pos <= afterPos) + { + continue; + } + + if (!SectionPathIsUnderPrefix(table[i].sectionPath, prefix)) + { + continue; + } + if (RuleMatches(nac, &table[i])) { return i; @@ -1067,7 +1291,7 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, int savedRuleSection = CurrentMonitorFSMRuleSection; CurrentMonitorFSMRulePos = rule->pos; - CurrentMonitorFSMRuleSection = (int) rule->section; + CurrentMonitorFSMRuleSection = (int) rule->sectionPath[0]; if (rule->comment != NULL) { @@ -1097,24 +1321,25 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, /* - * FindAndDispatchMonitorFSMRule bounds a search over MonitorFSM[] to - * [startIndex, endIndex) and dispatches the first match, if any -- the one - * building block every call site in this file needs: the top-level driver's - * own two straight-line lookups (early checks, then either the primary-role - * section or the rest of ProceedGroupStateFromContext()'s rows -- see its - * comment for why two, not the design doc's one), plus the two extraActions - * that each perform one further, separate bounded nested search of their own - * when their row's own cascade declines: ActionRunMultiStandbyFailoverCascade - * and ActionRunPrimaryNodeTransition below. Returns whether a row matched, so - * callers that need to distinguish "matched and handled" from "nothing in - * this range applied" can. + * FindAndDispatchMonitorFSMRuleUnderPath bounds a search over MonitorFSM[] to + * every row under prefix with pos > afterPos, and dispatches the first + * match, if any -- the one building block every call site in this file + * needs: the top-level driver's own two straight-line lookups (early checks, + * then either the primary-role section or the rest of + * ProceedGroupStateFromContext()'s rows -- see its comment for why two, not + * the design doc's one), plus the two extraActions that each perform one + * further, separate bounded nested search of their own when their row's own + * cascade declines: ActionRunMultiStandbyFailoverCascade and + * ActionRunPrimaryNodeTransition below. Returns whether a row matched, so + * callers that need to distinguish "matched and handled" from "nothing under + * this prefix applied" can. */ static bool -FindAndDispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, - int startIndex, int endIndex) +FindAndDispatchMonitorFSMRuleUnderPath(GroupStateContext *ctx, NodeActiveContext *nac, + const MonitorFSMSectionPath prefix, int afterPos) { - int index = FindMatchingMonitorFSMRuleIndexFrom(MonitorFSM, endIndex, startIndex, - nac); + int index = FindMatchingMonitorFSMRuleIndexUnderPath(MonitorFSM, MonitorFSM_SIZE, + prefix, afterPos, nac); if (index < 0) { @@ -1294,8 +1519,7 @@ ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext * { AutoFailoverNode *primaryNode = nac->primaryNode.node; - if (!FindAndDispatchMonitorFSMRule(ctx, nac, MonitorFSM_MSFailoverStart, - MonitorFSM_PrimaryNodeSectionStart)) + if (!FindAndDispatchMonitorFSMRuleUnderPath(ctx, nac, SectionMSFailover, 0)) { /* * Neither pos 381 nor pos 383 matched -- reproduce the original @@ -1334,8 +1558,8 @@ ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext * if (!ProceedGroupStateForMSFailover(ctx, primaryNode)) { - (void) FindAndDispatchMonitorFSMRule(ctx, nac, MonitorFSM_FromContextResumeStart, - MonitorFSM_PrimaryNodeSectionStart); + (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, nac, SectionReportingNode, + MonitorFSM_MultiStandbyCascadeResumeAfterPos); } } @@ -1371,9 +1595,7 @@ ActionRunPrimaryNodeTransition(GroupStateContext *ctx, NodeActiveContext *nac, BuildForPrimaryNodeNodeActiveContext(ctx, nac->primaryNode.node, &primaryNac); - (void) FindAndDispatchMonitorFSMRule(ctx, &primaryNac, - MonitorFSM_PrimaryNodeSectionStart, - MonitorFSM_SIZE); + (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, &primaryNac, SectionPrimaryNode, 0); } @@ -1700,9 +1922,8 @@ ProceedGroupStateForApiTrigger(MonitorApiFunction apiFunction, BuildGroupStateContext(&ctx, activeNode); BuildApiTriggerNodeActiveContext(&ctx, apiFunction, activeNode, primaryNode, &nac); - int index = FindMatchingMonitorFSMRuleIndexFrom(MonitorFSM, - MonitorFSM_EarlyChecksStart, - 0, &nac); + int index = FindMatchingMonitorFSMRuleIndexUnderPath(MonitorFSM, MonitorFSM_SIZE, + SectionApiTriggered, 0, &nac); if (index < 0) { @@ -1765,7 +1986,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * dispatch means whichever row is tried first wins when both could * apply, and only a node that canTakeWrites should get the fan-out. */ - { .pos = 101, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 101, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_REMOVE_NODE) }, .activeNode = { .canTakeWrites = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_DROPPED), @@ -1788,7 +2009,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * above do both itself, and this row only needing to cover the * non-primary case that never matched the row above at all. */ - { .pos = 103, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 103, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_REMOVE_NODE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_DROPPED), .comment = "remove_node, removed node cannot take writes -> dropped" }, @@ -1801,7 +2022,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * ProceedGroupStateForApiTrigger's own comment on pre/post side * effects); by the time dispatch runs, activeNode is that standby. */ - { .pos = 105, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 105, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_PERFORM_FAILOVER), .groupHasExactlyTwoNodes = BOOL_TRUE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY) }, @@ -1823,7 +2044,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * about biasing an election already left to the heartbeat-driven * MS-failover cluster rows, not a goal-state assignment of their own. */ - { .pos = 107, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 107, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_PERFORM_FAILOVER), .groupHasMoreThanTwoNodes = BOOL_TRUE }, .activeNode = { .isInPrimaryState = BOOL_TRUE }, @@ -1842,7 +2063,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * this row's own assignment (neither reads the other's freshly-committed * state), so it doesn't need extraAction to sequence correctly. */ - { .pos = 109, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 109, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), .groupHasExactlyTwoNodes = BOOL_TRUE }, .activeNode = { .isInPrimaryState = BOOL_TRUE }, @@ -1858,7 +2079,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * prepare_maintenance assignment has committed (it re-fetches fresh * state, so ordering matters here, unlike the 2-node row above). */ - { .pos = 111, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 111, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), .groupHasMoreThanTwoNodes = BOOL_TRUE }, .activeNode = { .isInPrimaryState = BOOL_TRUE }, @@ -1877,7 +2098,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * primaryNode state pattern, and only the more specific condition should * win. */ - { .pos = 113, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 113, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), .lastHealthySyncStandbyGoingToMaintenance = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_REPORTED, @@ -1895,7 +2116,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * start_maintenance(), secondary, ordinary case -- * node_active_protocol.c:1987-1996. */ - { .pos = 115, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 115, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE) }, .activeNode = { .statePattern = { .kind = NODE_STATE_REPORTED, .reportedStates = STATES( @@ -1918,7 +2139,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * disjunct is redundant here since the 2-node&&NULL case never reaches * dispatch at all, per the guard above). */ - { .pos = 117, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 117, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, .primaryNode = { .exists = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), @@ -1932,7 +2153,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * with no node-count condition, is exactly their shared real * condition). */ - { .pos = 119, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 119, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, .primaryNode = { .isDemotedPrimary = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), @@ -1945,7 +2166,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * REPORT_LSN -- a real, pre-existing message/behavior mismatch in the * source, not a modeling error in this table. */ - { .pos = 121, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 121, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE), .failoverInProgress = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), @@ -1959,7 +2180,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * failover is in progress -- exactly the real source's own final * "else" branch. */ - { .pos = 123, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 123, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "stop_maintenance, ordinary case -> catchingup" }, @@ -1975,7 +2196,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * -- this row is only reached once the wrapper has confirmed a primary * exists and isn't already apply_settings. */ - { .pos = 125, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 125, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER( API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY) }, .activeNode = { .statePattern = { .kind = NODE_STATE_NOT_STABLE, @@ -1990,7 +2211,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * set_node_replication_quorum(), node_active_protocol.c:2427-2441. Same * shape as set_node_candidate_priority above. */ - { .pos = 127, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 127, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER( API_FUNCTION_SET_NODE_REPLICATION_QUORUM) }, .activeNode = { .statePattern = { .kind = NODE_STATE_NOT_STABLE, @@ -2009,7 +2230,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * what's actually valid, and a future drift between the two fails loudly * via this row's own no-match ERROR rather than silently). */ - { .pos = 129, .section = MONITOR_FSM_SECTION_API_TRIGGERED, + { .pos = 129, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER( API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS) }, .activeNode = { .statePattern = { .kind = NODE_STATE_ASSIGNED, @@ -2022,30 +2243,30 @@ static const MonitorFSMTransition MonitorFSM[] = { "-> apply_settings" }, /* converged to dropped -> remove the node from the catalog entirely */ - { .pos = 201, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + { .pos = 201, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DROPPED) }, .extraAction = ActionRemoveDroppedNode, .comment = "converged to dropped -> remove the node from the catalog" }, /* goal already dropped (mid-drop, row above hasn't converged yet) -> no-op */ - { .pos = 203, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + { .pos = 203, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_DROPPED_GOAL }, .comment = "goal already dropped -> no-op" }, /* converged to maintenance -> no-op, frozen until stop_maintenance() */ - { .pos = 205, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + { .pos = 205, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_MAINTENANCE) }, .comment = "converged to maintenance -> no-op, frozen until stop_maintenance()" }, /* demote_timeout self-fence re-target (issue #1025) */ - { .pos = 207, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + { .pos = 207, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_REPORTED_DEMOTE_TIMEOUT, .unreachableFromDemoteTimeout = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), .comment = "reported demote_timeout, assigned goal can't reach it -> demoted" }, /* alone in group, candidate-eligible */ - { .pos = 209, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + { .pos = 209, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, .candidateEligible = BOOL_TRUE }, .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, @@ -2053,7 +2274,7 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "alone in group, candidate-eligible -> single" }, /* alone in group, not candidate-eligible */ - { .pos = 211, .section = MONITOR_FSM_SECTION_EARLY_CHECKS, + { .pos = 211, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, .candidateEligible = BOOL_FALSE }, .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, @@ -2067,7 +2288,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * NOT currently primary-role. */ /* converged secondary, reportedTLI not an ancestor of the group's reference timeline */ - { .pos = 301, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 301, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isComparableToReferenceTli = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), @@ -2075,7 +2296,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "converged secondary, reportedTLI not an ancestor of reference -> catchingup" }, /* replication stall (#997): primary healthy, no standby past replication_stall_timeout */ - { .pos = 303, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 303, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .primaryNode = { .isInPrimaryState = BOOL_TRUE, .isHealthy = BOOL_TRUE }, .conditions = { .replicationStallExceeded = BOOL_TRUE }, @@ -2086,7 +2307,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* nodesCount>2, primary unhealthy -- draining/maintenance/nothing decided inside the * action, then the MS-failover cascade unconditionally; must be a single row, see * ActionRunMultiStandbyFailoverCascade's comment for why. */ - { .pos = 305, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 305, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .primaryNode = { .isUnhealthy = BOOL_TRUE }, .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE }, .extraAction = ActionRunMultiStandbyFailoverCascade, @@ -2094,7 +2315,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade" }, /* report_lsn, primary converged wait/join_primary, healthy */ - { .pos = 307, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 307, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY, .isHealthy = BOOL_TRUE }, @@ -2103,7 +2324,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "report_lsn, primary converged wait/join_primary, healthy -> secondary" }, /* report_lsn, primary converged primary, healthy */ - { .pos = 309, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 309, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY), .isHealthy = BOOL_TRUE }, @@ -2111,7 +2332,7 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "report_lsn, primary converged primary, healthy -> secondary" }, /* fast_forward done -> prepare_promotion */ - { .pos = 311, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 311, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_FAST_FORWARD) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), .comment = "fast_forward done -> prepare_promotion" }, @@ -2119,21 +2340,21 @@ static const MonitorFSMTransition MonitorFSM[] = { /* continue an already-started MS failover -- a direct `return` in the real source, and no * later row in this table matches activeNode in REPORT_LSN/FAST_FORWARD, so it doesn't * matter here whether the extraAction's bool stops dispatch or lets it keep scanning. */ - { .pos = 313, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 313, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_REPORT_LSN_OR_FAST_FORWARD }, .extraAction = ActionRunPlainMSFailoverCascade, .comment = "report_lsn or fast_forward, continuing an already-started failover -> " "MS-failover cascade" }, /* wait_standby, primary converged wait/join_primary */ - { .pos = 315, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 315, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY) }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "wait_standby, primary converged wait/join_primary -> catchingup" }, /* wait_standby (quorum member), primary converged primary */ - { .pos = 317, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 317, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), .replicationQuorum = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, @@ -2143,7 +2364,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "catchingup + apply_settings" }, /* wait_standby (not a quorum member), primary converged primary */ - { .pos = 319, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 319, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), .replicationQuorum = BOOL_FALSE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, @@ -2152,7 +2373,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_standby (not a quorum member), primary converged primary -> catchingup" }, /* caught up, same TLI as primary, within sync threshold */ - { .pos = 321, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 321, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_CATCHINGUP), .isHealthy = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN }, @@ -2162,7 +2383,7 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "caught up, same TLI as primary, within sync threshold -> secondary" }, /* primary fails, already converged wait_primary (no draining edge, issue #1168) */ - { .pos = 323, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 323, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isHealthy = BOOL_TRUE, .candidateEligible = BOOL_TRUE }, @@ -2174,7 +2395,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "secondary -> prepare_promotion only (1 of 2)" }, /* primary fails, not already wait_primary */ - { .pos = 325, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 325, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isHealthy = BOOL_TRUE, .candidateEligible = BOOL_TRUE }, @@ -2189,14 +2410,14 @@ static const MonitorFSMTransition MonitorFSM[] = { "primary -> draining (2 of 2)" }, /* wait_maintenance, primary converged wait_primary */ - { .pos = 327, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 327, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), .comment = "wait_maintenance, primary converged wait_primary -> maintenance" }, /* wait_maintenance, primary's goal no longer wait_primary */ - { .pos = 329, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 329, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, .primaryNode = { .statePattern = FSM_NOT_ASSIGNED_WAIT_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), @@ -2204,7 +2425,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_maintenance, primary's goal no longer wait_primary -> maintenance" }, /* prepare_promotion, primary converged prepare_maintenance */ - { .pos = 331, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 331, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), @@ -2212,7 +2433,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "prepare_promotion, primary converged prepare_maintenance -> stop_replication" }, /* Citus worker, primary present */ - { .pos = 333, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 333, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), .isCitusWorkerGroup = BOOL_TRUE }, .primaryNode = { .exists = BOOL_TRUE }, @@ -2222,7 +2443,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "Citus worker prepare_promotion, primary present -> wait_primary + demoted" }, /* Citus worker, primary removed */ - { .pos = 335, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 335, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), .isCitusWorkerGroup = BOOL_TRUE }, .primaryNode = { .exists = BOOL_FALSE }, @@ -2230,7 +2451,7 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "Citus worker prepare_promotion, primary removed -> wait_primary" }, /* prepare_promotion, primary present, already converged wait_primary (issue #1168) */ - { .pos = 337, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 337, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .exists = BOOL_TRUE, .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY), @@ -2241,7 +2462,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "stop_replication only (1 of 2)" }, /* prepare_promotion, primary present, not in maintenance, not already wait_primary */ - { .pos = 339, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 339, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .exists = BOOL_TRUE, .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, @@ -2252,14 +2473,14 @@ static const MonitorFSMTransition MonitorFSM[] = { "stop_replication + demote_timeout (2 of 2)" }, /* prepare_promotion, primary removed */ - { .pos = 341, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 341, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .exists = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), .comment = "prepare_promotion, primary removed -> wait_primary" }, /* stop_replication, primary converged prepare_maintenance */ - { .pos = 343, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 343, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), @@ -2268,7 +2489,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_primary + maintenance" }, /* stop_replication, primary converged demote_timeout (3-way OR, 1 of 3) */ - { .pos = 345, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 345, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTE_TIMEOUT) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), @@ -2277,7 +2498,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_primary + demoted (1 of 3)" }, /* stop_replication, primary's drain time expired (3-way OR, 2 of 3) */ - { .pos = 347, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 347, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .primaryNode = { .drainTimeExpired = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), @@ -2286,7 +2507,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_primary + demoted (2 of 3)" }, /* stop_replication, primary's goal is wait_primary but presumed dead (3-way OR, 3 of 3) */ - { .pos = 349, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 349, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .conditions = { .primaryIsWaitPrimaryPresumedDead = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), @@ -2295,7 +2516,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_primary + demoted (3 of 3)" }, /* Citus worker, primary present */ - { .pos = 351, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 351, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), .isCitusWorkerGroup = BOOL_TRUE }, .primaryNode = { .exists = BOOL_TRUE }, @@ -2305,7 +2526,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "Citus worker stop_replication, primary present -> wait_primary + demoted" }, /* Citus worker, primary removed */ - { .pos = 353, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 353, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), .isCitusWorkerGroup = BOOL_TRUE }, .primaryNode = { .exists = BOOL_FALSE }, @@ -2313,7 +2534,7 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "Citus worker stop_replication, primary removed -> wait_primary" }, /* demoted, primary reported wait/join_primary with goal primary */ - { .pos = 355, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 355, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY, .isHealthy = BOOL_TRUE }, @@ -2322,7 +2543,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "demoted, primary reported wait/join_primary with goal primary -> catchingup" }, /* demoted, primary converged wait/join_primary/primary, healthy */ - { .pos = 357, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 357, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN, .isHealthy = BOOL_TRUE }, @@ -2332,7 +2553,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* join_secondary, primary reported wait_primary with goal wait/primary -- cascades into a * nested pass on primaryNode */ - { .pos = 359, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 359, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, .primaryNode = { .statePattern = FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), @@ -2342,7 +2563,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "secondary" }, /* join_secondary, primary converged primary */ - { .pos = 361, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 361, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), @@ -2365,7 +2586,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ /* MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, retry */ - { .pos = 363, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 363, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET }, .conditions = { .activeNodeAllWalSourcesUnhealthy = BOOL_TRUE, .guardDataLossEnabled = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, @@ -2380,7 +2601,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "guard_data_loss=true -> report_lsn (retry once a source recovers)" }, /* MS-failover: candidate ready to stream WAL -> follower joins as secondary */ - { .pos = 365, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 365, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN }, .conditions = { .candidatePromotionInProgress = BOOL_TRUE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, .candidateNode = { .isReadyToStreamWAL = BOOL_TRUE }, @@ -2414,7 +2635,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * goal states happened to line up -- see inMSFailoverCluster's own * comment on NodeActiveContext. */ - { .pos = 367, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 367, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, .reportedStates = STATES( @@ -2429,7 +2650,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn " "(1 of 4)" }, - { .pos = 369, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 369, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, .reportedStates = STATES( @@ -2441,7 +2662,7 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "MS-failover fan-out: rejoining from maintenance -> report_lsn (2 of 4)" }, - { .pos = 371, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 371, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_STABLE, .reportedStates = STATES( @@ -2452,7 +2673,7 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "MS-failover fan-out: old primary converged draining or demoted -> " "report_lsn (3 of 4)" }, - { .pos = 373, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 373, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, .reportedStates = STATES( @@ -2482,7 +2703,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * ("kept as 2 rows anyway, for dump_fsm() edge visibility... this pair * genuinely isn't disambiguated by this table's own dispatch model"). */ - { .pos = 375, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 375, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .candidatePromotionInProgress = BOOL_FALSE, .mostAdvancedCandidateWithinPromoteThreshold = BOOL_TRUE }, @@ -2492,7 +2713,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "threshold, selected candidate already has all WAL -> prepare_promotion " "(1 of 2 -- see this row's own comment on why both are listed)" }, - { .pos = 377, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 377, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .candidatePromotionInProgress = BOOL_FALSE, .mostAdvancedCandidateWithinPromoteThreshold = BOOL_TRUE }, @@ -2501,6 +2722,65 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "MS-failover: no promotion in flight, most-advanced candidate within " "threshold, selected candidate is lagging -> fast_forward (2 of 2)" }, + /* + * The 3 MS-failover counting gates (missingNodesCount/candidateCount/ + * quorumCandidateCount), dispatched from ProceedGroupStateForMSFailover's + * own hand-written `if` blocks via a dedicated + * BuildMSFailoverCandidateGateNodeActiveContext-built nac (see that + * builder's own comment) -- not part of the ordinary fan-out/promotion + * rows above, and never reached by their own SectionMSFailover-wide + * scans thanks to inMSFailoverCandidateGate (see NodeActiveContext's own + * comment on that field). Each pair below shares its own gate's real + * condition, split only on guard_data_loss (true -> decline, false -> + * proceed and log the data-loss risk) -- the decline/continue control + * flow itself stays hand-written C, unchanged; only the message text + * this pair's own extraAction builds is delegated here. + */ + { .pos = 379, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .inMSFailoverCandidateGate = BOOL_TRUE, + .missingNodesCount = AT_LEAST(1), + .guardDataLossEnabled = BOOL_TRUE }, + .extraAction = ActionLogMSFailoverMissingNodesDecline, + .comment = "MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=true " + "-> decline, wait for more reports (1 of 2)" }, + + { .pos = 381, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .inMSFailoverCandidateGate = BOOL_TRUE, + .missingNodesCount = AT_LEAST(1), + .guardDataLossEnabled = BOOL_FALSE }, + .extraAction = ActionLogMSFailoverMissingNodesContinue, + .comment = "MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=false " + "-> proceed despite possible data loss (2 of 2)" }, + + /* MS-failover: zero candidates have reported their LSN yet -- a hard, silent decline + * (the original code never logged here either). Never itself dispatched, listed for + * dump_fsm() completeness only, same as the no_candidate_yet row below. */ + { .pos = 383, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_CANDIDATE_COUNT_GATE }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .inMSFailoverCandidateGate = BOOL_TRUE, + .candidateCount = EXACTLY(0) }, + .comment = "MS-failover: zero candidates have reported their LSN yet -> silent decline" }, + + { .pos = 385, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .inMSFailoverCandidateGate = BOOL_TRUE, + .sufficientQuorumCandidates = BOOL_FALSE, + .guardDataLossEnabled = BOOL_TRUE }, + .extraAction = ActionLogMSFailoverQuorumDecline, + .comment = "MS-failover: not enough quorum candidates reported yet, guard_data_loss=true " + "-> decline (1 of 2)" }, + + { .pos = 387, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE }, + .conditions = { .inMSFailoverCluster = BOOL_TRUE, + .inMSFailoverCandidateGate = BOOL_TRUE, + .sufficientQuorumCandidates = BOOL_FALSE, + .guardDataLossEnabled = BOOL_FALSE }, + .extraAction = ActionLogMSFailoverQuorumContinue, + .comment = "MS-failover: not enough quorum candidates reported yet, guard_data_loss=false " + "-> proceed with fewer than required (2 of 2)" }, + /* MS-failover: no promotion in flight, either not enough candidates have reported yet * or the most-advanced one is still too far behind the primary to safely promote -- * a pure no-op besides BuildCandidateList's own fan-out (the four rows above); listed @@ -2510,7 +2790,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * missing activeNode state pattern (matches ANY state) would let it swallow the * ordinary top-level lookup's own "BUG: no MonitorFSM row matches" detection for * any otherwise-unmatched heartbeat, silently no-op'ing instead of erroring. */ - { .pos = 379, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 389, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_NO_CANDIDATE_YET }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .candidatePromotionInProgress = BOOL_FALSE }, .comment = "MS-failover: no promotion in flight, not enough (or not safe enough) " @@ -2534,7 +2814,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * plain AssignGoalState calls these replace, each falling back to the * original hand-written condition on no match. */ - { .pos = 381, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 391, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE }, .primaryNode = { .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, .isInPrimaryState = BOOL_TRUE, .isUnhealthy = BOOL_TRUE }, @@ -2544,7 +2824,7 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "nodesCount>2, primary unhealthy, in primary role but not yet " "wait_primary, >=1 healthy candidate -> primary draining" }, - { .pos = 383, .section = MONITOR_FSM_SECTION_REPORTING_NODE, + { .pos = 393, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE), .isUnhealthy = BOOL_TRUE }, .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE }, @@ -2559,14 +2839,14 @@ static const MonitorFSMTransition MonitorFSM[] = { * nested pass on primaryNode (the join_secondary cascade row above). */ /* primary alone, another node reached wait_standby */ - { .pos = 401, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + { .pos = 401, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SINGLE) }, .conditions = { .anyOtherNodeWaitingStandby = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), .comment = "primary alone, another node reached wait_standby -> wait_primary" }, /* all nodes async, zero secondaries */ - { .pos = 403, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + { .pos = 403, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, .secondaryNodesCountIsZero = BOOL_TRUE }, @@ -2576,7 +2856,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "(+ unhealthy-secondary fan-out to catchingup)" }, /* all nodes async, >=1 secondary */ - { .pos = 405, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + { .pos = 405, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, .secondaryNodesCountIsZero = BOOL_FALSE }, @@ -2587,7 +2867,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* converged primary/apply_settings (not wait_primary), no quorum secondaries, * number_sync_standbys=0, no failover in progress (issue #774) */ - { .pos = 407, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + { .pos = 407, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, .failoverInProgress = BOOL_FALSE, @@ -2600,7 +2880,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "(+ unhealthy-secondary fan-out to catchingup)" }, /* same, but number_sync_standbys>0 -> block writes on primary */ - { .pos = 409, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + { .pos = 409, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, .failoverInProgress = BOOL_FALSE, @@ -2613,7 +2893,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "(+ unhealthy-secondary fan-out to catchingup)" }, /* wait_primary, >=1 quorum secondary */ - { .pos = 411, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + { .pos = 411, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), @@ -2622,7 +2902,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "(+ unhealthy-secondary fan-out to catchingup)" }, /* apply_settings, both zero */ - { .pos = 413, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + { .pos = 413, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, .secondaryQuorumNodesCountIsZero = BOOL_TRUE }, @@ -2632,7 +2912,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "(+ unhealthy-secondary fan-out to catchingup)" }, /* apply_settings, number_sync_standbys != 0 (1 of 2 disjuncts) */ - { .pos = 415, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + { .pos = 415, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, .conditions = { .numberSyncStandbysIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), @@ -2642,7 +2922,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "(+ unhealthy-secondary fan-out to catchingup)" }, /* apply_settings, sync_standbys=0 but >=1 quorum secondary (2 of 2) */ - { .pos = 417, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + { .pos = 417, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, @@ -2653,7 +2933,7 @@ static const MonitorFSMTransition MonitorFSM[] = { "(+ unhealthy-secondary fan-out to catchingup)" }, /* converged primary/wait_primary/apply_settings, no other condition applies */ - { .pos = 419, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + { .pos = 419, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, .extraAction = ActionCatchupUnhealthySecondaries, .comment = @@ -2661,72 +2941,83 @@ static const MonitorFSMTransition MonitorFSM[] = { "no-op besides the unhealthy-secondary fan-out" }, /* backwards-compat: join_primary -> primary */ - { .pos = 421, .section = MONITOR_FSM_SECTION_PRIMARY_NODE, + { .pos = 421, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), .comment = "backwards-compat: join_primary -> primary" }, }; /* - * AssertMonitorFSMWellFormed cross-checks the five MonitorFSM_* index - * constants above against what the rows themselves declare via .pos/ - * .section, so a boundary that's drifted out of sync with an added, - * removed, or reordered row is caught here -- loudly, at first use -- - * instead of silently, as a row from the wrong section matching - * unexpectedly or a bounded search that scans zero rows and never matches. - * A no-op build (USE_ASSERT_CHECKING off) skips this entirely, matching - * every other structural check in this file (see AssignDeclaredGoalState). + * AssertMonitorFSMWellFormed cross-checks each row's own .pos against the + * top-level section its .sectionPath declares, so a row whose pos and + * section have drifted out of sync (e.g. moved into the wrong hundred-block + * without updating its path, or vice versa) is caught here -- loudly, at + * first use -- instead of silently, as a row matching (or failing to match) + * under the wrong prefix. A no-op build (USE_ASSERT_CHECKING off) skips this + * entirely, matching every other structural check in this file (see + * AssignDeclaredGoalState). * * .pos is NOT the array index: each section's rows are numbered starting * at *01 within its own hundred-block (101/201/301/401, matching how many - * MONITOR_FSM_SECTION_* values exist -- *01 rather than *00 so the count - * within the block reads as ordinary 1-based), stepping by 2 within the - * section rather than by 1 -- so a new row can be inserted between two - * existing ones (e.g. 202 between 201 and 203) without renumbering - * anything else in the file. What's checked here is weaker than "exactly - * i*2 + sectionStart + 1" as a result: strictly increasing, and within the - * section's own hundred-block -- consistent with rows being added over - * time at whatever free position is nearest where they belong, not at a - * specific computed slot. + * top-level MONITOR_FSM_SECTION_* values exist -- *01 rather than *00 so + * the count within the block reads as ordinary 1-based), stepping by 2 + * within the section rather than by 1 -- so a new row can be inserted + * between two existing ones (e.g. 202 between 201 and 203) without + * renumbering anything else in the file. What's checked here is weaker + * than "exactly i*2 + sectionStart + 1" as a result: strictly increasing, + * and within the section's own hundred-block -- consistent with rows being + * added over time at whatever free position is nearest where they belong, + * not at a specific computed slot. + * + * Every row's .sectionPath[0] must be one of the original four top-level + * values -- this is what makes it safe for MonitorFSMSectionGetEnum/ + * MonitorFSMSectionGetName (see their own comments) to only ever be called + * with .sectionPath[0], never a deeper path element. */ static void AssertMonitorFSMWellFormed(void) { #ifdef USE_ASSERT_CHECKING int previousPos = 0; + bool foundResumeAnchor = false; for (int i = 0; i < MonitorFSM_SIZE; i++) { - Assert(MonitorFSM[i].pos > previousPos); - previousPos = MonitorFSM[i].pos; - } + int pos = MonitorFSM[i].pos; + MonitorFSMSection top = MonitorFSM[i].sectionPath[0]; - for (int i = 0; i < MonitorFSM_EarlyChecksStart; i++) - { - Assert(MonitorFSM[i].section == MONITOR_FSM_SECTION_API_TRIGGERED); - Assert(MonitorFSM[i].pos >= 100 && MonitorFSM[i].pos < 200); - } + Assert(pos > previousPos); + previousPos = pos; - for (int i = MonitorFSM_EarlyChecksStart; i < MonitorFSM_FromContextStart; i++) - { - Assert(MonitorFSM[i].section == MONITOR_FSM_SECTION_EARLY_CHECKS); - Assert(MonitorFSM[i].pos >= 200 && MonitorFSM[i].pos < 300); - } - - for (int i = MonitorFSM_FromContextStart; i < MonitorFSM_PrimaryNodeSectionStart; i++) - { - Assert(MonitorFSM[i].section == MONITOR_FSM_SECTION_REPORTING_NODE); - Assert(MonitorFSM[i].pos >= 300 && MonitorFSM[i].pos < 400); - } + if (pos >= 100 && pos < 200) + { + Assert(top == MONITOR_FSM_SECTION_API_TRIGGERED); + } + else if (pos >= 200 && pos < 300) + { + Assert(top == MONITOR_FSM_SECTION_EARLY_CHECKS); + } + else if (pos >= 300 && pos < 400) + { + Assert(top == MONITOR_FSM_SECTION_REPORTING_NODE); + } + else if (pos >= 400 && pos < 500) + { + Assert(top == MONITOR_FSM_SECTION_PRIMARY_NODE); + } + else + { + Assert(false); + } - for (int i = MonitorFSM_PrimaryNodeSectionStart; i < MonitorFSM_SIZE; i++) - { - Assert(MonitorFSM[i].section == MONITOR_FSM_SECTION_PRIMARY_NODE); - Assert(MonitorFSM[i].pos >= 400 && MonitorFSM[i].pos < 500); + if (pos == MonitorFSM_MultiStandbyCascadeResumeAfterPos) + { + foundResumeAnchor = true; + Assert(SectionPathIsUnderPrefix(MonitorFSM[i].sectionPath, SectionReportingNode)); + } } - Assert(MonitorFSM_FromContextResumeStart > MonitorFSM_FromContextStart); - Assert(MonitorFSM_FromContextResumeStart < MonitorFSM_PrimaryNodeSectionStart); + Assert(foundResumeAnchor); #endif } @@ -2851,7 +3142,7 @@ MonitorApiFunctionGetName(MonitorApiFunction apiFunction) static Datum MonitorFSMTransitionSectionText(const MonitorFSMTransition *rule) { - const char *sectionName = MonitorFSMSectionGetName(rule->section); + const char *sectionName = MonitorFSMSectionGetName(rule->sectionPath[0]); if (rule->conditions.apiTrigger.kind == API_TRIGGER_SPECIFIC) { @@ -2868,6 +3159,126 @@ MonitorFSMTransitionSectionText(const MonitorFSMTransition *rule) } +/* + * MonitorFSMLeafSectionName returns the (enum) name of any MonitorFSMSection + * value, including the fine-grained leaves appended after PRIMARY_NODE -- + * unlike MonitorFSMSectionGetName (SQL-facing, and only ever called with one + * of the original 4 top-level values, see that function's own comment), this + * is used purely to render a row's own full section_path text below, one + * path element at a time. + */ +static const char * +MonitorFSMLeafSectionName(MonitorFSMSection section) +{ + switch (section) + { + case MONITOR_FSM_SECTION_API_TRIGGERED: + case MONITOR_FSM_SECTION_EARLY_CHECKS: + case MONITOR_FSM_SECTION_REPORTING_NODE: + case MONITOR_FSM_SECTION_PRIMARY_NODE: + { + return MonitorFSMSectionGetName(section); + } + + case MONITOR_FSM_SECTION_FROM_CONTEXT: + { + return "from_context"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER: + { + return "ms_failover"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET: + { + return "retry_reset"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN: + { + return "candidate_join"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT: + { + return "candidate_fanout"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME: + { + return "promotion_outcome"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE: + { + return "missing_nodes_gate"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_CANDIDATE_COUNT_GATE: + { + return "candidate_count_gate"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE: + { + return "quorum_candidate_gate"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_NO_CANDIDATE_YET: + { + return "no_candidate_yet"; + } + + case MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE: + { + return "draining_or_maintenance"; + } + + default: + { + ereport(ERROR, + (errmsg("bug: unknown MonitorFSMSection (%d)", section))); + } + } +} + + +/* + * MonitorFSMTransitionSectionPathText renders a row's full .sectionPath as a + * dotted string (e.g. "reporting_node.ms_failover.retry_reset") for + * dump_fsm()/pgautofailover.fsm's own section_path column -- the only place + * this project ever produces an ltree-shaped string, and only as plain text: + * the cast to real ltree happens in the pgautofailover.fsm view definition + * (pgautofailover.sql), via Postgres's own ordinary type-casting machinery, + * not from this C code. + */ +static Datum +MonitorFSMTransitionSectionPathText(const MonitorFSMTransition *rule) +{ + StringInfoData buf; + + initStringInfo(&buf); + + for (int i = 0; i < MONITOR_FSM_SECTION_PATH_MAX_DEPTH; i++) + { + if (rule->sectionPath[i] == MONITOR_FSM_SECTION_NONE) + { + break; + } + + if (buf.len > 0) + { + appendStringInfoString(&buf, "."); + } + + appendStringInfoString(&buf, MonitorFSMLeafSectionName(rule->sectionPath[i])); + } + + return CStringGetTextDatum(buf.data); +} + + /* * MonitorFSMSectionTypeOid returns the OID of the pgautofailover.fsm_section * type. Mirrors ReplicationStateTypeOid exactly, see its own comment for why @@ -3032,6 +3443,23 @@ NodeStatePatternReportedStatesText(const NodeStatePattern *pattern, bool *isNull } \ } while (0) +/* Same idea as APPEND_BOOL_CONDITION, for an IntPattern field: skips + * INT_PATTERN_ANY, otherwise renders "name=n"/"name>=n"/"name<=n" matching + * the EXACTLY/AT_LEAST/AT_MOST macro that built the pattern. */ +#define APPEND_INT_CONDITION(buf, name, intPattern) \ + do { \ + if ((intPattern).kind != INT_PATTERN_ANY) \ + { \ + const char *op = (intPattern).kind == INT_PATTERN_AT_LEAST ? ">=" : \ + (intPattern).kind == INT_PATTERN_AT_MOST ? "<=" : "="; \ + if ((buf)->len > 0) \ + { \ + appendStringInfoString((buf), ", "); \ + } \ + appendStringInfo((buf), "%s%s%d", (name), op, (intPattern).value); \ + } \ + } while (0) + /* * AppendNodeStateGoalCondition appends a role's own goal-state precondition * to buf, when its statePattern is NODE_STATE_ASSIGNED/NOT_ASSIGNED: "the @@ -3183,6 +3611,14 @@ NodeActiveContextPatternConditionsText(const NodeActiveContextPattern *cond, boo cond->mostAdvancedCandidateWithinPromoteThreshold); APPEND_BOOL_CONDITION(&buf, "guardDataLossEnabled", cond->guardDataLossEnabled); APPEND_BOOL_CONDITION(&buf, "inMSFailoverCluster", cond->inMSFailoverCluster); + APPEND_BOOL_CONDITION(&buf, "inMSFailoverCandidateGate", + cond->inMSFailoverCandidateGate); + + APPEND_INT_CONDITION(&buf, "missingNodesCount", cond->missingNodesCount); + APPEND_INT_CONDITION(&buf, "candidateCount", cond->candidateCount); + APPEND_INT_CONDITION(&buf, "quorumCandidateCount", cond->quorumCandidateCount); + APPEND_BOOL_CONDITION(&buf, "sufficientQuorumCandidates", + cond->sufficientQuorumCandidates); if (buf.len == 0) { @@ -3267,8 +3703,8 @@ dump_fsm(PG_FUNCTION_ARGS) for (int i = 0; i < MonitorFSM_SIZE; i++) { const MonitorFSMTransition *rule = &MonitorFSM[i]; - Datum values[13]; - bool isNull[13] = { false }; + Datum values[14]; + bool isNull[14] = { false }; values[0] = Int32GetDatum(rule->pos); values[1] = MonitorFSMTransitionSectionText(rule); @@ -3315,6 +3751,7 @@ dump_fsm(PG_FUNCTION_ARGS) } values[12] = BoolGetDatum(rule->extraAction != NULL); + values[13] = MonitorFSMTransitionSectionPathText(rule); tuplestore_putvalues(tupstore, tupdesc, values, isNull); } @@ -3541,7 +3978,7 @@ dump_fsm_edges(PG_FUNCTION_ARGS) { const MonitorFSMTransition *rule = &MonitorFSM[i]; - if (rule->section == MONITOR_FSM_SECTION_API_TRIGGERED) + if (rule->sectionPath[0] == MONITOR_FSM_SECTION_API_TRIGGERED) { continue; } @@ -3645,8 +4082,7 @@ ProceedGroupStateFromContext(GroupStateContext *ctx) BuildFromContextNodeActiveContext(ctx, NULL, &earlyNac); - if (FindAndDispatchMonitorFSMRule(ctx, &earlyNac, MonitorFSM_EarlyChecksStart, - MonitorFSM_FromContextStart)) + if (FindAndDispatchMonitorFSMRuleUnderPath(ctx, &earlyNac, SectionEarlyChecks, 0)) { return true; } @@ -3668,9 +4104,7 @@ ProceedGroupStateFromContext(GroupStateContext *ctx) BuildForPrimaryNodeNodeActiveContext(ctx, activeNode, &primaryNac); - return FindAndDispatchMonitorFSMRule(ctx, &primaryNac, - MonitorFSM_PrimaryNodeSectionStart, - MonitorFSM_SIZE); + return FindAndDispatchMonitorFSMRuleUnderPath(ctx, &primaryNac, SectionPrimaryNode, 0); } /* @@ -3716,8 +4150,7 @@ ProceedGroupStateFromContext(GroupStateContext *ctx) BuildFromContextNodeActiveContext(ctx, primaryNode, &nac); - return FindAndDispatchMonitorFSMRule(ctx, &nac, MonitorFSM_FromContextStart, - MonitorFSM_PrimaryNodeSectionStart); + return FindAndDispatchMonitorFSMRuleUnderPath(ctx, &nac, SectionReportingNode, 0); } @@ -3795,6 +4228,142 @@ BuildMSFailoverNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *activ } +/* + * BuildMSFailoverCandidateGateNodeActiveContext computes the facts the 3 + * MS-failover counting gates (missingNodesCount/candidateCount/ + * quorumCandidateCount, see ProceedGroupStateForMSFailover's own gate + * checks) match on, from BuildCandidateList's own CandidateList output. + * Called once, right after BuildCandidateList itself, and reused across all + * 3 gate checks -- the counts don't change between them within the same + * node_active() call. candidatePromotionInProgress is unconditionally false + * here: by the time ProceedGroupStateForMSFailover reaches these gates, its + * own "nodeBeingPromoted != NULL" branch has already returned, so no + * candidate is currently being promoted. + */ +static void +BuildMSFailoverCandidateGateNodeActiveContext(GroupStateContext *ctx, + AutoFailoverNode *primaryNode, + CandidateList *candidateList, + NodeActiveContext *nac) +{ + memset(nac, 0, sizeof(NodeActiveContext)); + + BuildNodeStatus(ctx, ctx->activeNode, &nac->activeNode); + BuildNodeStatus(ctx, primaryNode, &nac->primaryNode); + nac->otherNode = nac->primaryNode; /* see NodeActiveContext's own comment on .otherNode */ + + nac->inMSFailoverCluster = true; + nac->inMSFailoverCandidateGate = true; + nac->guardDataLossEnabled = GuardDataLoss; + nac->candidatePromotionInProgress = false; + + nac->candidateCount = candidateList->candidateCount; + nac->quorumCandidateCount = candidateList->quorumCandidateCount; + nac->missingNodesCount = candidateList->missingNodesCount; + + nac->sufficientQuorumCandidates = + candidateList->quorumCandidateCount >= (ctx->formation->number_sync_standbys + 1); +} + + +/* + * ActionLogMSFailoverMissingNodesDecline/Continue and ActionLogMSFailover + * QuorumDecline/Continue reproduce, verbatim, the LogAndNotifyMessage text + * ProceedGroupStateForMSFailover used to build inline for its own + * missingNodesCount/quorumCandidateCount gates -- moved here so the + * declarative rows that now match these same conditions (see the + * missing_nodes_gate/quorum_candidate_gate rows in MonitorFSM[]) are the + * single source of truth for the message, not a hand-written duplicate of + * it. Neither gate assigns a goal state either way (the original code + * never called AssignGoalState in either branch), so none of these four + * actions do either -- the control-flow decision itself (decline vs. + * continue) stays exactly the hand-written `if (GuardDataLoss)` in + * ProceedGroupStateForMSFailover, unchanged; only the message text is + * delegated here. + */ +static void +ActionLogMSFailoverMissingNodesDecline(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + AutoFailoverNode *activeNode = nac->activeNode.node; + + LogAndNotifyMessage( + message, BUFSIZE, + "Failover still in progress after %d nodes reported their LSN " + "and we are waiting for %d nodes to report, " + "activeNode is " NODE_FORMAT + " and reported state \"%s\"", + nac->candidateCount, + nac->missingNodesCount, + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->reportedState)); +} + + +static void +ActionLogMSFailoverMissingNodesContinue(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + AutoFailoverNode *activeNode = nac->activeNode.node; + + LogAndNotifyMessage( + message, BUFSIZE, + "Proceeding with failover despite %d unreported quorum node(s): " + "pgautofailover.guard_data_loss is false. " + "Committed transactions on missing node(s) may be lost. " + "activeNode is " NODE_FORMAT " and reported state \"%s\"", + nac->missingNodesCount, + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->reportedState)); +} + + +static void +ActionLogMSFailoverQuorumDecline(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + AutoFailoverNode *activeNode = nac->activeNode.node; + int minCandidates = ctx->formation->number_sync_standbys + 1; + + LogAndNotifyMessage( + message, BUFSIZE, + "Failover still in progress with %d candidates that participate " + "in the quorum having reported their LSN: %d nodes are required " + "in the quorum to satisfy number_sync_standbys=%d in " + "formation \"%s\", activeNode is " NODE_FORMAT + " and reported state \"%s\"", + nac->quorumCandidateCount, + minCandidates, + ctx->formation->number_sync_standbys, + ctx->formation->formationId, + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->reportedState)); +} + + +static void +ActionLogMSFailoverQuorumContinue(GroupStateContext *ctx, NodeActiveContext *nac, + char *message) +{ + AutoFailoverNode *activeNode = nac->activeNode.node; + int minCandidates = ctx->formation->number_sync_standbys + 1; + + LogAndNotifyMessage( + message, BUFSIZE, + "Proceeding with failover with only %d quorum candidate(s) despite " + "number_sync_standbys=%d requiring %d: " + "pgautofailover.guard_data_loss is false. " + "The new primary may start in wait_primary state with fewer " + "sync standbys than required. " + "activeNode is " NODE_FORMAT " and reported state \"%s\"", + nac->quorumCandidateCount, + ctx->formation->number_sync_standbys, + minCandidates, + NODE_FORMAT_ARGS(activeNode), + ReplicationStateGetName(activeNode->reportedState)); +} + + /* * TryMSFailoverDeclarativeRow attempts the MS-failover cluster's own two * declarative rows (pos 363/365) for activeNode/candidateNode, returning @@ -3819,8 +4388,7 @@ TryMSFailoverDeclarativeRow(GroupStateContext *ctx, AutoFailoverNode *activeNode BuildMSFailoverNodeActiveContext(ctx, activeNode, candidateNode, &msNac); - return FindAndDispatchMonitorFSMRule(ctx, &msNac, MonitorFSM_MSFailoverStart, - MonitorFSM_PrimaryNodeSectionStart); + return FindAndDispatchMonitorFSMRuleUnderPath(ctx, &msNac, SectionMSFailover, 0); } @@ -4051,6 +4619,21 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, BuildCandidateList(ctx, comparableNodesGroupList, &candidateList); + /* + * gateNac carries the 3 counting gates' own facts (missingNodesCount/ + * candidateCount/quorumCandidateCount/sufficientQuorumCandidates) for + * MonitorFSM[]'s own declarative rows under reporting_node.ms_failover. + * promotion_outcome.*_gate -- see BuildMSFailoverCandidateGateNodeActive + * Context's own comment. Each gate below still makes its own decline-vs- + * continue decision in plain C, exactly as before this refactor; only + * the message text each branch logs is now delegated to the matching + * row's own extraAction, so the table stays the single source of truth + * for what gets logged and why. + */ + NodeActiveContext gateNac; + + BuildMSFailoverCandidateGateNodeActiveContext(ctx, primaryNode, &candidateList, &gateNac); + /* * Time to select a candidate? * @@ -4064,33 +4647,13 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, */ if (candidateList.missingNodesCount > 0) { - char message[BUFSIZE] = { 0 }; + (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, &gateNac, + SectionMSFailoverMissingNodesGate, 0); if (GuardDataLoss) { - LogAndNotifyMessage( - message, BUFSIZE, - "Failover still in progress after %d nodes reported their LSN " - "and we are waiting for %d nodes to report, " - "activeNode is " NODE_FORMAT - " and reported state \"%s\"", - candidateList.candidateCount, - candidateList.missingNodesCount, - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->reportedState)); - return false; } - - LogAndNotifyMessage( - message, BUFSIZE, - "Proceeding with failover despite %d unreported quorum node(s): " - "pgautofailover.guard_data_loss is false. " - "Committed transactions on missing node(s) may be lost. " - "activeNode is " NODE_FORMAT " and reported state \"%s\"", - candidateList.missingNodesCount, - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->reportedState)); } /* @@ -4106,7 +4669,9 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, */ int minCandidates = ctx->formation->number_sync_standbys + 1; - /* no candidates is a hard pass */ + /* no candidates is a hard pass -- see MonitorFSM[]'s own candidate_count_gate + * row for this same fact, matched declaratively but never itself dispatched + * (a silent decline, same as the original code: no log here either). */ if (candidateList.candidateCount == 0) { return false; @@ -4115,40 +4680,13 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, /* not enough candidates to promote and then accept writes, pass */ if (candidateList.quorumCandidateCount < minCandidates) { - char message[BUFSIZE] = { 0 }; + (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, &gateNac, + SectionMSFailoverQuorumCandidateGate, 0); if (GuardDataLoss) { - LogAndNotifyMessage( - message, BUFSIZE, - "Failover still in progress with %d candidates that participate " - "in the quorum having reported their LSN: %d nodes are required " - "in the quorum to satisfy number_sync_standbys=%d in " - "formation \"%s\", activeNode is " NODE_FORMAT - " and reported state \"%s\"", - candidateList.quorumCandidateCount, - minCandidates, - ctx->formation->number_sync_standbys, - ctx->formation->formationId, - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->reportedState)); - return false; } - - LogAndNotifyMessage( - message, BUFSIZE, - "Proceeding with failover with only %d quorum candidate(s) despite " - "number_sync_standbys=%d requiring %d: " - "pgautofailover.guard_data_loss is false. " - "The new primary may start in wait_primary state with fewer " - "sync standbys than required. " - "activeNode is " NODE_FORMAT " and reported state \"%s\"", - candidateList.quorumCandidateCount, - ctx->formation->number_sync_standbys, - minCandidates, - NODE_FORMAT_ARGS(activeNode), - ReplicationStateGetName(activeNode->reportedState)); } /* enough candidates to promote and then accept writes, let's do it! */ diff --git a/src/monitor/group_state_machine.h b/src/monitor/group_state_machine.h index adc9cbfef..c40276f95 100644 --- a/src/monitor/group_state_machine.h +++ b/src/monitor/group_state_machine.h @@ -20,12 +20,27 @@ #include "node_metadata.h" /* - * MonitorFSMSection identifies which of the four real control-flow regions - * of the monitor's declarative dispatch table (MonitorFSM[] in - * group_state_machine.c) a row belongs to -- see that array's own comment - * for what each region corresponds to in the original if-chain/call sites. - * Declared here (not just in the .c file) so it can be exposed to SQL as - * pgautofailover.fsm_section, the same way ReplicationState is exposed as + * MonitorFSMSection identifies which node of the section hierarchy a row's + * .sectionPath (see MonitorFSMTransition/MonitorFSMSectionPath in + * group_state_machine.c) belongs to at some depth -- a row's own path is a + * small array of these, e.g. { MONITOR_FSM_SECTION_REPORTING_NODE, + * MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET }, + * matched via prefix/ancestor containment (SectionPathIsUnderPrefix) instead + * of the hand-maintained array-index-range constants this replaces -- see + * that mechanism's own comment for why (the design doc's own "Open items" + * flagged the old index constants as needing "to stay in sync with the + * table by hand as rows are added, removed, or reordered"). + * + * Only the first four values below (API_TRIGGERED/EARLY_CHECKS/ + * REPORTING_NODE/PRIMARY_NODE, unchanged in name and meaning from before this + * mechanism existed) are ever legal as a row's .sectionPath[0] -- every row's + * path always starts with exactly one of these four, enforced by + * AssertMonitorFSMWellFormed(). Declared here (not just in the .c file) so + * this top-level tag can be exposed to SQL as pgautofailover.fsm_section + * (mapped by NAME, not ordinal -- see MonitorFSMSectionGetEnum/ + * EnumGetMonitorFSMSection -- so inserting MONITOR_FSM_SECTION_NONE ahead of + * them, or appending new fine-grained values after PRIMARY_NODE, changes no + * SQL-visible behavior at all), the same way ReplicationState is exposed as * pgautofailover.replication_state (see replication_state.h/.c for that * pattern, mirrored below). * @@ -39,13 +54,33 @@ * contiguous pos range, matching one real entry point" property the other * three sections already have, rather than wedging operator rows into gaps * within the heartbeat sections. + * + * Values from MONITOR_FSM_SECTION_FROM_CONTEXT onward are fine-grained leaf + * labels, only ever used at path depth 1+ (reporting_node.from_context, + * reporting_node.ms_failover, and its own sub-leaves) -- purely for + * dump_fsm()'s own section_path column readability today; no caller needs + * to bound a search this narrowly (see each leaf's own row comment in + * MonitorFSM[]). */ typedef enum MonitorFSMSection { - MONITOR_FSM_SECTION_API_TRIGGERED = 0, + MONITOR_FSM_SECTION_NONE = 0, /* terminator / unused trailing path slot */ + MONITOR_FSM_SECTION_API_TRIGGERED, MONITOR_FSM_SECTION_EARLY_CHECKS, MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_PRIMARY_NODE, + + MONITOR_FSM_SECTION_FROM_CONTEXT, /* reporting_node.from_context */ + MONITOR_FSM_SECTION_MS_FAILOVER, /* reporting_node.ms_failover */ + MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_CANDIDATE_COUNT_GATE, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_NO_CANDIDATE_YET, + MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE, } MonitorFSMSection; /* public function declarations, mirroring replication_state.h's pattern */ diff --git a/src/monitor/pgautofailover.control b/src/monitor/pgautofailover.control index 081bc691b..6fcd33149 100644 --- a/src/monitor/pgautofailover.control +++ b/src/monitor/pgautofailover.control @@ -2,4 +2,4 @@ comment = 'pg_auto_failover' default_version = '2.3' module_pathname = '$libdir/pgautofailover' relocatable = false -requires = 'btree_gist' +requires = 'btree_gist, ltree' diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index 6a53e7489..321772cab 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -292,15 +292,35 @@ RETURNS TABLE group_conditions text, active_node_assigned_state pgautofailover.replication_state, other_node_assigned_state pgautofailover.replication_state, - has_extra_action bool + has_extra_action bool, + section_path text ) LANGUAGE C SECURITY DEFINER AS 'MODULE_PATHNAME', $$dump_fsm$$; grant execute on function pgautofailover.dump_fsm() to autoctl_node; +-- section_path is cast to ltree here (a plain SQL cast, going through +-- Postgres's own ordinary type-casting machinery) rather than in the C +-- function itself: dump_fsm() only ever builds the dotted text, this view +-- is the sole place pgautofailover takes a dependency on ltree. CREATE VIEW pgautofailover.fsm AS - SELECT * FROM pgautofailover.dump_fsm() ORDER BY pos; + SELECT pos, + section, + comment, + active_node_current_state, + other_node_current_state, + candidate_node_current_state, + active_node_conditions, + other_node_conditions, + candidate_node_conditions, + group_conditions, + active_node_assigned_state, + other_node_assigned_state, + has_extra_action, + section_path::ltree AS section_path + FROM pgautofailover.dump_fsm() + ORDER BY pos; -- Flat, fully-resolved (pos, current_state, assigned_state) edges derived -- from MonitorFSM[] -- see dump_fsm_edges()'s own C-side comment. Never diff --git a/src/monitor/regress_schedule b/src/monitor/regress_schedule index 5cc5768ce..4d2ca7bdb 100644 --- a/src/monitor/regress_schedule +++ b/src/monitor/regress_schedule @@ -41,6 +41,7 @@ test: guard_data_loss test: fast_forward test: drop_node test: stale_primary_report +test: candidate_count_gate test: lock_and_fetch_migration test: timeline_fork_detection test: failover_candidate_leaves_secondary diff --git a/src/monitor/sql/candidate_count_gate.sql b/src/monitor/sql/candidate_count_gate.sql new file mode 100644 index 000000000..a08ab7703 --- /dev/null +++ b/src/monitor/sql/candidate_count_gate.sql @@ -0,0 +1,202 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Regression test for ProceedGroupStateForMSFailover's candidateCount == 0 +-- gate (reporting_node.ms_failover.promotion_outcome.candidate_count_gate in +-- MonitorFSM[]): the window where the primary has gone unhealthy but NOT A +-- SINGLE standby has yet reported reaching report_lsn. +-- +-- Unlike the other two counting gates -- missingNodesCount (guard_data_loss.sql) +-- and quorumCandidateCount (stale_primary_report.sql), both named explicitly +-- in those files' own header comments -- no existing test exercises this one +-- by name. It has no dedicated log message either (the original hand-written +-- code silently `return`s false here, and the declarative row that now +-- matches this same condition carries no extraAction, matching that exactly) +-- so there is no pgautofailover.event row to check for it; the only +-- observable effect is what does NOT happen: neither standby should reach +-- prepare_promotion/fast_forward this round. +-- +-- guard_data_loss is set to false: with the default (true), the +-- missingNodesCount > 0 gate above this one in ProceedGroupStateForMSFailover +-- would itself decline and return before ever reaching this gate, since both +-- standbys are still SECONDARY/CATCHINGUP (each counted as missing, per +-- BuildCandidateList's own fan-out branch) at the moment this test polls +-- them. +-- +-- startup_grace_period is also lowered to 1, same as guard_data_loss.sql/ +-- fast_forward.sql/stale_primary_report.sql: NodeIsUnhealthy() only honors a +-- BAD health reading once at least this many seconds have passed since the +-- monitor process itself started (PgStartTime), to avoid spurious failovers +-- right after the monitor restarts. The default (10s) is longer than this +-- whole schedule takes to reach this test file when run automated, which +-- would silently make p never register as unhealthy and this test's own +-- ProceedGroupStateForMSFailover call never even fire. + +\x on + +-- ── formation and node registration ───────────────────────────────────────── + +SELECT pgautofailover.create_formation('ccg_test', 'pgsql', 'postgres', true, 1); + +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_p', 5432, + 'postgres', 'ccg_p', 1); + +SELECT nodeid AS np FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p' \gset + +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_s1', 5432, + 'postgres', 'ccg_s1', 1); + +SELECT nodeid AS ns1 FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_s1' \gset + +SELECT * + FROM pgautofailover.register_node('ccg_test', 'ccg_s2', 5432, + 'postgres', 'ccg_s2', 1); + +SELECT nodeid AS ns2 FROM pgautofailover.node + WHERE formationid = 'ccg_test' AND nodename = 'ccg_s2' \gset + +-- ── bootstrap: drive the FSM to primary + secondary + secondary ───────────── +-- Same sequence as guard_data_loss.sql's own bootstrap. + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'single'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'wait_standby'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'single', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'wait_standby'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'wait_primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'wait_standby'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'catchingup', + current_lsn => '0/5000'); + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns2, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +-- p: primary (refresh to pick up second secondary) +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :np, 0, + current_group_role => 'primary', + current_lsn => '0/5000'); + +-- Verify bootstrap: p=primary, s1=secondary, s2=secondary. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; + +-- ── manufacture: primary unhealthy, NEITHER standby has reported yet ──────── +-- +-- p goes unhealthy and is demoted to draining/draining (same manufactured +-- shape as guard_data_loss.sql/stale_primary_report.sql). s1/s2 are left +-- exactly as bootstrap left them -- secondary/secondary, neither has been +-- assigned report_lsn yet -- so candidateCount is 0 for both of them: no +-- standby has reached report_lsn, the exact window this gate covers. + +SET pgautofailover.startup_grace_period = 1; +SET pgautofailover.guard_data_loss TO false; + +UPDATE pgautofailover.node + SET health = 0, + healthchecktime = now(), + reporttime = now() - interval '60 seconds' + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p'; + +UPDATE pgautofailover.node + SET goalstate = 'draining', reportedstate = 'draining' + WHERE formationid = 'ccg_test' AND nodename = 'ccg_p'; + +-- Verify the manufactured state before the test call. +SELECT nodename, goalstate, reportedstate, health + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; + +-- ── test: poll exactly one secondary, candidateCount == 0 for both ────────── +-- +-- s1 reports secondary/0-5000 again (nothing new from its own point of +-- view). This drives ProceedGroupState(s1) -> ActionRunMultiStandbyFailover +-- Cascade -> ProceedGroupStateForMSFailover, which runs BuildCandidateList +-- over the WHOLE group (not just s1): both s1 and s2 are still SECONDARY/ +-- CATCHINGUP, so BOTH get fanned out to report_lsn in this same call (the +-- fan-out rows, already covered elsewhere) -- but candidateCount is 0 (no +-- node's reportedState is report_lsn yet), so the candidate_count_gate row +-- matches and the function returns false: no candidate is selected. + +SELECT assigned_group_state + FROM pgautofailover.node_active('ccg_test', :ns1, 0, + current_group_role => 'secondary', + current_lsn => '0/5000'); + +-- s1 and s2 must both have been fanned out to report_lsn (goalstate), but +-- NEITHER may have reached prepare_promotion/fast_forward: the +-- candidate_count_gate declined before any candidate could be selected. +SELECT nodename, goalstate, reportedstate + FROM pgautofailover.node + WHERE formationid = 'ccg_test' + ORDER BY nodename; + +RESET pgautofailover.guard_data_loss; +RESET pgautofailover.startup_grace_period; + +-- event summary: which MonitorFSM[] rule (if any) produced each of this +-- test's own state-change events. Exercises pgautofailover.last_events() +-- against a real scenario -- its own SELECT list didn't match +-- pgautofailover.event's column set for a long time, breaking it outright, +-- and nothing in this suite ever called it to notice (see monitor.sql's +-- own minimal-repro coverage). eventid/eventtime omitted: eventid is a +-- database-wide sequence shared by every test in this schedule (see +-- regress_schedule's own comment) and eventtime is a live timestamp -- +-- neither is a stable value to pin in this file's own expected output. +SELECT reportedstate, goalstate, rule_pos, rule_section, description + FROM pgautofailover.last_events('ccg_test', count => 100); diff --git a/src/monitor/sql/fsm.sql b/src/monitor/sql/fsm.sql index 67b850adb..782944dce 100644 --- a/src/monitor/sql/fsm.sql +++ b/src/monitor/sql/fsm.sql @@ -14,7 +14,7 @@ \x on -SELECT pos, section, +SELECT pos, section, section_path, active_node_current_state, other_node_current_state, candidate_node_current_state, active_node_conditions, other_node_conditions, candidate_node_conditions, group_conditions, From 73478fe7c7b5c062871cbee67725d44a65f33767 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 02:41:51 +0200 Subject: [PATCH 14/52] monitor: sectionPath one-per-line formatting; keeper edge reverse-check; otherNodesFn; Assert fallbacks Three follow-ups from review of the section-path/IntPattern work: - Reformat every row's .sectionPath as one element per physical line (citus_indent-stable), instead of one long line per row -- easier to review diffs when a leaf changes. - keeper_fsm_edges.sql: add the reverse cross-check (step 2b) alongside the existing monitor-not-in-keeper one (step 2a) -- every keeper edge no MonitorFSM[] row can produce. - Implement the otherNodesFn mechanism the design doc's own comments gestured at but never built: MonitorFSMTransition gains an otherNodesFn field (GroupStateContext, NodeActiveContext) -> List *; when set, DispatchMonitorFSMRule loops the resolved list and calls AssignDeclaredGoalState per node instead of targeting the single nac->otherNode.node, giving every fanned-out node the same rule_pos/ rule_section attribution any other row's assignment already gets. Converts ActionCatchupUnhealthySecondaries (pos 403-419, 8 rows) and ActionFanOutReportLsnOnPrimaryRemoval (pos 101) from hand-written extraActions with zero dump_fsm() visibility into real otherNodeAssignedState-bearing rows. dump_fsm_edges() skips otherNodesFn rows explicitly: their target's current-state precondition isn't a NodeStatePattern, so resolving one would fabricate bogus edges from the NODE_STATE_ANY default. - Add Assert(false) to the 7 remaining raw AssignGoalState fallback sites (the "row should always match here" defensive branches): grepped every fallback's own message text against the full expected/*.out corpus and confirmed zero hits, so none has ever fired. Assertions are compiled out in production but catch a future row/call-site divergence immediately in any assert-enabled build instead of silently reverting to un-attributed pre-refactor behavior forever. Full regress (19) + isolation (6) suites pass in Docker (PG17). --- src/monitor/expected/fsm.out | 42 +- src/monitor/expected/keeper_fsm_edges.out | 61 +- src/monitor/group_state_machine.c | 681 +++++++++++++++++----- src/monitor/sql/keeper_fsm_edges.sql | 20 +- 4 files changed, 648 insertions(+), 156 deletions(-) diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index 1441ad451..ccf9bacf3 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -32,8 +32,8 @@ other_node_conditions | candidate_node_conditions | group_conditions | active_node_assigned_state | dropped -other_node_assigned_state | -has_extra_action | t +other_node_assigned_state | report_lsn +has_extra_action | f comment | remove_node, removed node can take writes -> dropped, every surviving non-maintenance standby joins report_lsn -[ RECORD 2 ]----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 103 @@ -1067,8 +1067,8 @@ other_node_conditions | candidate_node_conditions | group_conditions | replicationQuorumCountIsZero=true, secondaryNodesCountIsZero=true active_node_assigned_state | wait_primary -other_node_assigned_state | -has_extra_action | t +other_node_assigned_state | catchingup +has_extra_action | f comment | all nodes async, zero secondaries -> wait_primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 405 @@ -1082,8 +1082,8 @@ other_node_conditions | candidate_node_conditions | group_conditions | replicationQuorumCountIsZero=true, secondaryNodesCountIsZero=false active_node_assigned_state | primary -other_node_assigned_state | -has_extra_action | t +other_node_assigned_state | catchingup +has_extra_action | f comment | all nodes async, >=1 secondary -> primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 407 @@ -1097,8 +1097,8 @@ other_node_conditions | candidate_node_conditions | group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNodesCountIsZero=true, failoverInProgress=false active_node_assigned_state | wait_primary -other_node_assigned_state | -has_extra_action | t +other_node_assigned_state | catchingup +has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 409 @@ -1112,8 +1112,8 @@ other_node_conditions | candidate_node_conditions | group_conditions | numberSyncStandbysIsZero=false, secondaryQuorumNodesCountIsZero=true, failoverInProgress=false active_node_assigned_state | primary -other_node_assigned_state | -has_extra_action | t +other_node_assigned_state | catchingup +has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 411 @@ -1127,8 +1127,8 @@ other_node_conditions | candidate_node_conditions | group_conditions | secondaryQuorumNodesCountIsZero=false active_node_assigned_state | primary -other_node_assigned_state | -has_extra_action | t +other_node_assigned_state | catchingup +has_extra_action | f comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 75 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 413 @@ -1142,8 +1142,8 @@ other_node_conditions | candidate_node_conditions | group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNodesCountIsZero=true active_node_assigned_state | wait_primary -other_node_assigned_state | -has_extra_action | t +other_node_assigned_state | catchingup +has_extra_action | f comment | apply_settings, both zero -> wait_primary (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 415 @@ -1157,8 +1157,8 @@ other_node_conditions | candidate_node_conditions | group_conditions | numberSyncStandbysIsZero=false active_node_assigned_state | primary -other_node_assigned_state | -has_extra_action | t +other_node_assigned_state | catchingup +has_extra_action | f comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 417 @@ -1172,8 +1172,8 @@ other_node_conditions | candidate_node_conditions | group_conditions | numberSyncStandbysIsZero=true, secondaryQuorumNodesCountIsZero=false active_node_assigned_state | primary -other_node_assigned_state | -has_extra_action | t +other_node_assigned_state | catchingup +has_extra_action | f comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) (+ unhealthy-secondary fan-out to catchingup) -[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 419 @@ -1187,9 +1187,9 @@ other_node_conditions | candidate_node_conditions | group_conditions | active_node_assigned_state | -other_node_assigned_state | -has_extra_action | t -comment | converged primary/wait_primary/apply_settings, no other condition applies -> no-op besides the unhealthy-secondary fan-out +other_node_assigned_state | catchingup +has_extra_action | f +comment | converged primary/wait_primary/apply_settings, no other condition applies -> no-op besides the unhealthy-secondary fan-out to catchingup -[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 421 section | primary_node diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 7a97c31ef..0567e5be3 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -132,7 +132,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; dropped | dropped (97 rows) --- Step 2: the actual cross-check -- every pgautofailover.dump_fsm_edges() +-- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A -- non-empty result here is a real, actionable gap (unlike -- check_fsm_reachability.sql's own synthetic-input test, which only @@ -288,4 +288,63 @@ SELECT e.pos, e.current_state, e.assigned_state, f.comment 391 | dropped | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining (134 rows) +-- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper +-- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the +-- full, fully-resolved edge set the monitor's table can reach, see its own +-- comment). A non-empty result here means the keeper is prepared to +-- transition through a (current, assigned) pair the monitor itself would +-- never assign -- either genuinely dead keeper code, or a real coverage +-- gap on the monitor side, same "investigate before assuming which" caveat +-- as step 2a's own comment. +SELECT k.current_state, k.assigned_state + FROM keeper_fsm_edges k + WHERE NOT EXISTS ( + SELECT 1 + FROM pgautofailover.dump_fsm_edges() e + WHERE e.current_state = k.current_state + AND e.assigned_state = k.assigned_state + ) + ORDER BY k.current_state, k.assigned_state; + current_state | assigned_state +---------------------+--------------------- + init | wait_standby + init | dropped + single | dropped + wait_primary | join_primary + wait_primary | apply_settings + wait_primary | dropped + primary | maintenance + primary | join_primary + primary | prepare_maintenance + primary | dropped + draining | dropped + demote_timeout | primary + demote_timeout | dropped + demoted | dropped + catchingup | prepare_promotion + catchingup | maintenance + catchingup | wait_maintenance + catchingup | dropped + secondary | wait_standby + secondary | maintenance + secondary | wait_maintenance + secondary | dropped + prepare_promotion | dropped + stop_replication | dropped + wait_standby | dropped + maintenance | catchingup + maintenance | dropped + join_primary | dropped + apply_settings | join_primary + apply_settings | dropped + prepare_maintenance | catchingup + prepare_maintenance | dropped + wait_maintenance | dropped + report_lsn | dropped + fast_forward | dropped + join_secondary | dropped + dropped | wait_standby + dropped | dropped +(38 rows) + DROP TABLE keeper_fsm_edges; diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index e03e64db7..9ea34bb9d 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -833,6 +833,23 @@ typedef struct GoalStateAssignment * another static aggregate's designated initializer. */ #define GOAL(x) { .kind = GOAL_STATE_SET, .state = (x) } +/* + * A row's otherNodesFn, when set, resolves a dynamically-sized list of + * nodes -- rather than the single nac->otherNode.node target -- that its + * own otherNodeAssignedState gets assigned to. DispatchMonitorFSMRule loops + * over the resolved list and calls AssignDeclaredGoalState once per node, + * so each of those assignments gets exactly the same rule_pos/rule_section + * attribution any other row's own single-target assignment already gets -- + * closing the gap the design doc's own "otherNodesFn" concept was meant to + * close (see otherNode's own comment above, and + * ActionFanOutReportLsnOnPrimaryRemoval's -- both flagged this as a future + * mechanism before it existed). A row sets at most one of "otherNodesFn + * resolves the target list" (this) or "nac->otherNode.node is the single, + * already-resolved target" (omitted, the original mechanism) -- never both. + */ +typedef List *(*MonitorOtherNodesResolverFunction) (GroupStateContext *ctx, + NodeActiveContext *nac); + /* * A row's extraAction runs before its own activeNodeAssignedState/ * otherNodeAssignedState are applied (matching the original if-chain's @@ -936,16 +953,19 @@ typedef struct MonitorFSMTransition /* * otherNode is the role otherNodeAssignedState actually targets (see - * that field's own comment): a genuinely distinct role from primaryNode, - * even though every row today has nac->otherNode.node == + * that field's own comment) when the target is the single, already- + * resolved nac->otherNode.node -- a genuinely distinct role from + * primaryNode, even though every row using it has nac->otherNode.node == * nac->primaryNode.node (see NodeActiveContext's own comment on * .otherNode for why). Kept separate from primaryNode in this struct -- * rather than just reusing .primaryNode wherever a row wants to - * constrain otherNodeAssignedState's target -- so a future row whose - * target is resolved some other way (e.g. a candidate an otherNodesFn - * selects, not simply "the primary") has a role to write conditions - * against without a name that falsely implies it's always the primary. - * Every row written before this field existed omits it, which matches + * constrain otherNodeAssignedState's target -- so a role exists to write + * conditions against without a name that falsely implies it's always the + * primary. Doesn't apply to a row using otherNodesFn instead (see that + * field's own comment): such a row's target is a dynamically resolved + * list, not this single node, so .otherNode stays at its + * NodeStatusPattern default (omitted, don't-care) there too. Every row + * written before this field existed omits it, which matches * NodeStatusPattern's own "omitted means don't-care" default -- exactly * the same zero-risk-to-existing-rows guarantee apiFunction/inMSFailover * Cluster/etc. already established when each was added. @@ -962,10 +982,20 @@ typedef struct MonitorFSMTransition /* * otherNodeAssignedState targets nac->otherNode.node (see * NodeActiveContext's own comment) -- not nac->primaryNode.node - * directly, even though today the two are always the same pointer. + * directly, even though today the two are always the same pointer -- + * unless otherNodesFn (below) is set, in which case it targets every + * node otherNodesFn resolves instead. */ GoalStateAssignment otherNodeAssignedState; + /* + * When set, otherNodeAssignedState (above) is assigned to every node + * this resolves, not to the single nac->otherNode.node -- see + * MonitorOtherNodesResolverFunction's own comment. NULL (the default) + * for every row using the single-target otherNode mechanism instead. + */ + MonitorOtherNodesResolverFunction otherNodesFn; + MonitorExtraActionFunction extraAction; const char *comment; @@ -1311,8 +1341,24 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, if (rule->otherNodeAssignedState.kind == GOAL_STATE_SET) { - AssignDeclaredGoalState(rule, nac->otherNode.node, - rule->otherNodeAssignedState.state, message); + if (rule->otherNodesFn != NULL) + { + List *otherNodesList = rule->otherNodesFn(ctx, nac); + ListCell *otherNodeCell = NULL; + + foreach(otherNodeCell, otherNodesList) + { + AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(otherNodeCell); + + AssignDeclaredGoalState(rule, otherNode, + rule->otherNodeAssignedState.state, message); + } + } + else + { + AssignDeclaredGoalState(rule, nac->otherNode.node, + rule->otherNodeAssignedState.state, message); + } } CurrentMonitorFSMRulePos = savedRulePos; @@ -1522,14 +1568,29 @@ ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext * if (!FindAndDispatchMonitorFSMRuleUnderPath(ctx, nac, SectionMSFailover, 0)) { /* - * Neither pos 381 nor pos 383 matched -- reproduce the original + * Neither pos 391 nor pos 393 matched -- reproduce the original * hand-written condition exactly. nac->atLeastOneHealthyCandidate is * already the exact same fact those rows' own * atLeastOneHealthyCandidate condition checks (same * AutoFailoverOtherNodesListInState + CountHealthyCandidates * computation, see BuildFromContextNodeActiveContext), so it's reused * here rather than recomputed. + * + * Should never happen: verified by inspection that pos 391/393's own + * conditions are exactly this hand-written condition, transcribed + * 1:1, and confirmed by grepping every expected output file in the full + * regress+isolation suite for this branch's own message text -- zero + * matches, so nothing has ever exercised it. Kept as a hard + * assertion (not silently dropped) rather than removed outright: if + * a future edit to either the row's conditions or this hand-written + * one ever makes them diverge, USE_ASSERT_CHECKING builds (regress, + * isolation, any dev build) fail loudly and immediately instead of + * silently falling back to this branch forever, unnoticed. See + * AssignDeclaredGoalState's own Assert for the same idea applied to + * a single declared state instead of a whole condition. */ + Assert(false); + if (IsInPrimaryState(primaryNode) && !IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) && nac->atLeastOneHealthyCandidate) @@ -1599,12 +1660,22 @@ ActionRunPrimaryNodeTransition(GroupStateContext *ctx, NodeActiveContext *nac, } -static void -ActionCatchupUnhealthySecondaries(GroupStateContext *ctx, NodeActiveContext *nac, - char *message) +/* + * OtherNodesDueForCatchingUp is pos 419's own otherNodesFn: every other node + * in the group that OtherNodeIsDueForCatchingUp() says has come back + * healthy after being sent to catchingup earlier. Replaces the former + * ActionCatchupUnhealthySecondaries extraAction -- the loop-and- + * AssignGoalState body is unchanged, just returning the list instead of + * assigning inline, so DispatchMonitorFSMRule's own otherNodesFn loop (see + * its own comment) does the assignment and gets the same rule_pos/ + * rule_section attribution any other row's assignment gets. + */ +static List * +OtherNodesDueForCatchingUp(GroupStateContext *ctx, NodeActiveContext *nac) { AutoFailoverNode *primaryNode = nac->activeNode.node; List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); + List *dueNodesList = NIL; ListCell *nodeCell = NULL; foreach(nodeCell, otherNodesGroupList) @@ -1613,16 +1684,11 @@ ActionCatchupUnhealthySecondaries(GroupStateContext *ctx, NodeActiveContext *nac if (OtherNodeIsDueForCatchingUp(ctx, otherNode)) { - char otherMessage[BUFSIZE] = { 0 }; - - snprintf(otherMessage, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to catchingup after it became unhealthy.", - NODE_FORMAT_ARGS(otherNode)); - - AssignGoalState(otherNode, REPLICATION_STATE_CATCHINGUP, otherMessage); + dueNodesList = lappend(dueNodesList, otherNode); } } + + return dueNodesList; } @@ -1776,23 +1842,24 @@ BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, /* - * ActionFanOutReportLsnOnPrimaryRemoval implements RemoveNode's own + * OtherNodesNotInMaintenance is pos 101's own otherNodesFn: RemoveNode's own * primary-removal fan-out (node_active_protocol.c's RemoveNode(), * "if (currentNodeIsPrimary) { foreach other node not in maintenance -> - * report_lsn }"), reusing the exact otherNodesFn-style shape already - * established for the heartbeat side's own fan-out - * (ActionCatchupUnhealthySecondaries above) -- a dynamically-sized list of - * nodes can't be expressed as a single declared activeNodeAssignedState/ - * otherNodeAssignedState slot, operator-triggered or not. Runs before the - * row's own activeNodeAssignedState = DROPPED (see DispatchMonitorFSMRule), - * matching the real source's own order: fan out to the survivors first, - * then mark the removed node itself dropped. + * report_lsn }"). Assigned via this row's own otherNodeAssignedState = + * GOAL(REPORT_LSN) instead of an extraAction -- DispatchMonitorFSMRule's own + * otherNodesFn loop (see its own comment) now runs after this row's + * activeNodeAssignedState = DROPPED, same order as every other row using + * both slots together (e.g. the perform_failover/reporting_node rows that + * already assign both roles). The real source's own order (fan out to the + * survivors, then mark the removed node dropped) doesn't matter here: which + * nodes qualify for the fan-out, and what they're assigned, is entirely + * independent of the removed node's own goalState. */ -static void -ActionFanOutReportLsnOnPrimaryRemoval(GroupStateContext *ctx, NodeActiveContext *nac, - char *message) +static List * +OtherNodesNotInMaintenance(GroupStateContext *ctx, NodeActiveContext *nac) { List *otherNodesGroupList = AutoFailoverOtherNodesList(nac->activeNode.node); + List *eligibleNodesList = NIL; ListCell *nodeCell = NULL; foreach(nodeCell, otherNodesGroupList) @@ -1804,15 +1871,10 @@ ActionFanOutReportLsnOnPrimaryRemoval(GroupStateContext *ctx, NodeActiveContext continue; } - char otherMessage[BUFSIZE] = { 0 }; - - snprintf(otherMessage, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn after primary node removal.", - NODE_FORMAT_ARGS(otherNode)); - - AssignGoalState(otherNode, REPLICATION_STATE_REPORT_LSN, otherMessage); + eligibleNodesList = lappend(eligibleNodesList, otherNode); } + + return eligibleNodesList; } @@ -1986,11 +2048,15 @@ static const MonitorFSMTransition MonitorFSM[] = { * dispatch means whichever row is tried first wins when both could * apply, and only a node that canTakeWrites should get the fan-out. */ - { .pos = 101, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 101, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_REMOVE_NODE) }, .activeNode = { .canTakeWrites = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_DROPPED), - .extraAction = ActionFanOutReportLsnOnPrimaryRemoval, + .otherNodesFn = OtherNodesNotInMaintenance, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), .comment = "remove_node, removed node can take writes -> dropped, " "every surviving non-maintenance standby joins report_lsn" }, @@ -2009,7 +2075,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * above do both itself, and this row only needing to cover the * non-primary case that never matched the row above at all. */ - { .pos = 103, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 103, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_REMOVE_NODE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_DROPPED), .comment = "remove_node, removed node cannot take writes -> dropped" }, @@ -2022,7 +2091,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * ProceedGroupStateForApiTrigger's own comment on pre/post side * effects); by the time dispatch runs, activeNode is that standby. */ - { .pos = 105, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 105, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_PERFORM_FAILOVER), .groupHasExactlyTwoNodes = BOOL_TRUE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY) }, @@ -2044,7 +2116,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * about biasing an election already left to the heartbeat-driven * MS-failover cluster rows, not a goal-state assignment of their own. */ - { .pos = 107, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 107, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_PERFORM_FAILOVER), .groupHasMoreThanTwoNodes = BOOL_TRUE }, .activeNode = { .isInPrimaryState = BOOL_TRUE }, @@ -2063,7 +2138,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * this row's own assignment (neither reads the other's freshly-committed * state), so it doesn't need extraAction to sequence correctly. */ - { .pos = 109, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 109, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), .groupHasExactlyTwoNodes = BOOL_TRUE }, .activeNode = { .isInPrimaryState = BOOL_TRUE }, @@ -2079,7 +2157,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * prepare_maintenance assignment has committed (it re-fetches fresh * state, so ordering matters here, unlike the 2-node row above). */ - { .pos = 111, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 111, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), .groupHasMoreThanTwoNodes = BOOL_TRUE }, .activeNode = { .isInPrimaryState = BOOL_TRUE }, @@ -2098,7 +2179,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * primaryNode state pattern, and only the more specific condition should * win. */ - { .pos = 113, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 113, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), .lastHealthySyncStandbyGoingToMaintenance = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_REPORTED, @@ -2116,7 +2200,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * start_maintenance(), secondary, ordinary case -- * node_active_protocol.c:1987-1996. */ - { .pos = 115, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 115, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE) }, .activeNode = { .statePattern = { .kind = NODE_STATE_REPORTED, .reportedStates = STATES( @@ -2139,7 +2226,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * disjunct is redundant here since the 2-node&&NULL case never reaches * dispatch at all, per the guard above). */ - { .pos = 117, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 117, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, .primaryNode = { .exists = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), @@ -2153,7 +2243,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * with no node-count condition, is exactly their shared real * condition). */ - { .pos = 119, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 119, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, .primaryNode = { .isDemotedPrimary = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), @@ -2166,7 +2259,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * REPORT_LSN -- a real, pre-existing message/behavior mismatch in the * source, not a modeling error in this table. */ - { .pos = 121, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 121, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE), .failoverInProgress = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), @@ -2180,7 +2276,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * failover is in progress -- exactly the real source's own final * "else" branch. */ - { .pos = 123, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 123, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "stop_maintenance, ordinary case -> catchingup" }, @@ -2196,7 +2295,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * -- this row is only reached once the wrapper has confirmed a primary * exists and isn't already apply_settings. */ - { .pos = 125, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 125, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER( API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY) }, .activeNode = { .statePattern = { .kind = NODE_STATE_NOT_STABLE, @@ -2211,7 +2313,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * set_node_replication_quorum(), node_active_protocol.c:2427-2441. Same * shape as set_node_candidate_priority above. */ - { .pos = 127, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 127, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER( API_FUNCTION_SET_NODE_REPLICATION_QUORUM) }, .activeNode = { .statePattern = { .kind = NODE_STATE_NOT_STABLE, @@ -2230,7 +2335,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * what's actually valid, and a future drift between the two fails loudly * via this row's own no-match ERROR rather than silently). */ - { .pos = 129, .sectionPath = { MONITOR_FSM_SECTION_API_TRIGGERED }, + { .pos = 129, + .sectionPath = { + MONITOR_FSM_SECTION_API_TRIGGERED + }, .conditions = { .apiTrigger = API_TRIGGER( API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS) }, .activeNode = { .statePattern = { .kind = NODE_STATE_ASSIGNED, @@ -2243,30 +2351,45 @@ static const MonitorFSMTransition MonitorFSM[] = { "-> apply_settings" }, /* converged to dropped -> remove the node from the catalog entirely */ - { .pos = 201, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, + { .pos = 201, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DROPPED) }, .extraAction = ActionRemoveDroppedNode, .comment = "converged to dropped -> remove the node from the catalog" }, /* goal already dropped (mid-drop, row above hasn't converged yet) -> no-op */ - { .pos = 203, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, + { .pos = 203, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, .activeNode = { .statePattern = FSM_DROPPED_GOAL }, .comment = "goal already dropped -> no-op" }, /* converged to maintenance -> no-op, frozen until stop_maintenance() */ - { .pos = 205, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, + { .pos = 205, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_MAINTENANCE) }, .comment = "converged to maintenance -> no-op, frozen until stop_maintenance()" }, /* demote_timeout self-fence re-target (issue #1025) */ - { .pos = 207, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, + { .pos = 207, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, .activeNode = { .statePattern = FSM_REPORTED_DEMOTE_TIMEOUT, .unreachableFromDemoteTimeout = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), .comment = "reported demote_timeout, assigned goal can't reach it -> demoted" }, /* alone in group, candidate-eligible */ - { .pos = 209, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, + { .pos = 209, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, .candidateEligible = BOOL_TRUE }, .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, @@ -2274,7 +2397,10 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "alone in group, candidate-eligible -> single" }, /* alone in group, not candidate-eligible */ - { .pos = 211, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, + { .pos = 211, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, .candidateEligible = BOOL_FALSE }, .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, @@ -2288,7 +2414,11 @@ static const MonitorFSMTransition MonitorFSM[] = { * NOT currently primary-role. */ /* converged secondary, reportedTLI not an ancestor of the group's reference timeline */ - { .pos = 301, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 301, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isComparableToReferenceTli = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), @@ -2296,7 +2426,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "converged secondary, reportedTLI not an ancestor of reference -> catchingup" }, /* replication stall (#997): primary healthy, no standby past replication_stall_timeout */ - { .pos = 303, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 303, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .primaryNode = { .isInPrimaryState = BOOL_TRUE, .isHealthy = BOOL_TRUE }, .conditions = { .replicationStallExceeded = BOOL_TRUE }, @@ -2307,7 +2441,11 @@ static const MonitorFSMTransition MonitorFSM[] = { /* nodesCount>2, primary unhealthy -- draining/maintenance/nothing decided inside the * action, then the MS-failover cascade unconditionally; must be a single row, see * ActionRunMultiStandbyFailoverCascade's comment for why. */ - { .pos = 305, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 305, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .primaryNode = { .isUnhealthy = BOOL_TRUE }, .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE }, .extraAction = ActionRunMultiStandbyFailoverCascade, @@ -2315,7 +2453,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade" }, /* report_lsn, primary converged wait/join_primary, healthy */ - { .pos = 307, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 307, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY, .isHealthy = BOOL_TRUE }, @@ -2324,7 +2466,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "report_lsn, primary converged wait/join_primary, healthy -> secondary" }, /* report_lsn, primary converged primary, healthy */ - { .pos = 309, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 309, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY), .isHealthy = BOOL_TRUE }, @@ -2332,7 +2478,11 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "report_lsn, primary converged primary, healthy -> secondary" }, /* fast_forward done -> prepare_promotion */ - { .pos = 311, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 311, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_FAST_FORWARD) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), .comment = "fast_forward done -> prepare_promotion" }, @@ -2340,21 +2490,33 @@ static const MonitorFSMTransition MonitorFSM[] = { /* continue an already-started MS failover -- a direct `return` in the real source, and no * later row in this table matches activeNode in REPORT_LSN/FAST_FORWARD, so it doesn't * matter here whether the extraAction's bool stops dispatch or lets it keep scanning. */ - { .pos = 313, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 313, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_REPORT_LSN_OR_FAST_FORWARD }, .extraAction = ActionRunPlainMSFailoverCascade, .comment = "report_lsn or fast_forward, continuing an already-started failover -> " "MS-failover cascade" }, /* wait_standby, primary converged wait/join_primary */ - { .pos = 315, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 315, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY) }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "wait_standby, primary converged wait/join_primary -> catchingup" }, /* wait_standby (quorum member), primary converged primary */ - { .pos = 317, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 317, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), .replicationQuorum = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, @@ -2364,7 +2526,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "catchingup + apply_settings" }, /* wait_standby (not a quorum member), primary converged primary */ - { .pos = 319, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 319, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), .replicationQuorum = BOOL_FALSE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, @@ -2373,7 +2539,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_standby (not a quorum member), primary converged primary -> catchingup" }, /* caught up, same TLI as primary, within sync threshold */ - { .pos = 321, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 321, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_CATCHINGUP), .isHealthy = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN }, @@ -2383,7 +2553,11 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "caught up, same TLI as primary, within sync threshold -> secondary" }, /* primary fails, already converged wait_primary (no draining edge, issue #1168) */ - { .pos = 323, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 323, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isHealthy = BOOL_TRUE, .candidateEligible = BOOL_TRUE }, @@ -2395,7 +2569,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "secondary -> prepare_promotion only (1 of 2)" }, /* primary fails, not already wait_primary */ - { .pos = 325, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 325, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isHealthy = BOOL_TRUE, .candidateEligible = BOOL_TRUE }, @@ -2410,14 +2588,22 @@ static const MonitorFSMTransition MonitorFSM[] = { "primary -> draining (2 of 2)" }, /* wait_maintenance, primary converged wait_primary */ - { .pos = 327, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 327, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), .comment = "wait_maintenance, primary converged wait_primary -> maintenance" }, /* wait_maintenance, primary's goal no longer wait_primary */ - { .pos = 329, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 329, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, .primaryNode = { .statePattern = FSM_NOT_ASSIGNED_WAIT_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_MAINTENANCE), @@ -2425,7 +2611,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_maintenance, primary's goal no longer wait_primary -> maintenance" }, /* prepare_promotion, primary converged prepare_maintenance */ - { .pos = 331, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 331, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_STOP_REPLICATION), @@ -2433,7 +2623,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "prepare_promotion, primary converged prepare_maintenance -> stop_replication" }, /* Citus worker, primary present */ - { .pos = 333, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 333, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), .isCitusWorkerGroup = BOOL_TRUE }, .primaryNode = { .exists = BOOL_TRUE }, @@ -2443,7 +2637,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "Citus worker prepare_promotion, primary present -> wait_primary + demoted" }, /* Citus worker, primary removed */ - { .pos = 335, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 335, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), .isCitusWorkerGroup = BOOL_TRUE }, .primaryNode = { .exists = BOOL_FALSE }, @@ -2451,7 +2649,11 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "Citus worker prepare_promotion, primary removed -> wait_primary" }, /* prepare_promotion, primary present, already converged wait_primary (issue #1168) */ - { .pos = 337, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 337, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .exists = BOOL_TRUE, .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY), @@ -2462,7 +2664,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "stop_replication only (1 of 2)" }, /* prepare_promotion, primary present, not in maintenance, not already wait_primary */ - { .pos = 339, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 339, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .exists = BOOL_TRUE, .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, @@ -2473,14 +2679,22 @@ static const MonitorFSMTransition MonitorFSM[] = { "stop_replication + demote_timeout (2 of 2)" }, /* prepare_promotion, primary removed */ - { .pos = 341, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 341, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .exists = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), .comment = "prepare_promotion, primary removed -> wait_primary" }, /* stop_replication, primary converged prepare_maintenance */ - { .pos = 343, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 343, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), @@ -2489,7 +2703,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_primary + maintenance" }, /* stop_replication, primary converged demote_timeout (3-way OR, 1 of 3) */ - { .pos = 345, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 345, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTE_TIMEOUT) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), @@ -2498,7 +2716,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_primary + demoted (1 of 3)" }, /* stop_replication, primary's drain time expired (3-way OR, 2 of 3) */ - { .pos = 347, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 347, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .primaryNode = { .drainTimeExpired = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), @@ -2507,7 +2729,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_primary + demoted (2 of 3)" }, /* stop_replication, primary's goal is wait_primary but presumed dead (3-way OR, 3 of 3) */ - { .pos = 349, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 349, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .conditions = { .primaryIsWaitPrimaryPresumedDead = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), @@ -2516,7 +2742,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_primary + demoted (3 of 3)" }, /* Citus worker, primary present */ - { .pos = 351, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 351, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), .isCitusWorkerGroup = BOOL_TRUE }, .primaryNode = { .exists = BOOL_TRUE }, @@ -2526,7 +2756,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "Citus worker stop_replication, primary present -> wait_primary + demoted" }, /* Citus worker, primary removed */ - { .pos = 353, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 353, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), .isCitusWorkerGroup = BOOL_TRUE }, .primaryNode = { .exists = BOOL_FALSE }, @@ -2534,7 +2768,11 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "Citus worker stop_replication, primary removed -> wait_primary" }, /* demoted, primary reported wait/join_primary with goal primary */ - { .pos = 355, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 355, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY, .isHealthy = BOOL_TRUE }, @@ -2543,7 +2781,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "demoted, primary reported wait/join_primary with goal primary -> catchingup" }, /* demoted, primary converged wait/join_primary/primary, healthy */ - { .pos = 357, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 357, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN, .isHealthy = BOOL_TRUE }, @@ -2553,7 +2795,11 @@ static const MonitorFSMTransition MonitorFSM[] = { /* join_secondary, primary reported wait_primary with goal wait/primary -- cascades into a * nested pass on primaryNode */ - { .pos = 359, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 359, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, .primaryNode = { .statePattern = FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), @@ -2563,7 +2809,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "secondary" }, /* join_secondary, primary converged primary */ - { .pos = 361, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, + { .pos = 361, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), @@ -2586,7 +2836,12 @@ static const MonitorFSMTransition MonitorFSM[] = { */ /* MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, retry */ - { .pos = 363, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET }, + { .pos = 363, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET + }, .conditions = { .activeNodeAllWalSourcesUnhealthy = BOOL_TRUE, .guardDataLossEnabled = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, @@ -2601,7 +2856,12 @@ static const MonitorFSMTransition MonitorFSM[] = { "guard_data_loss=true -> report_lsn (retry once a source recovers)" }, /* MS-failover: candidate ready to stream WAL -> follower joins as secondary */ - { .pos = 365, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN }, + { .pos = 365, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN + }, .conditions = { .candidatePromotionInProgress = BOOL_TRUE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, .candidateNode = { .isReadyToStreamWAL = BOOL_TRUE }, @@ -2635,7 +2895,12 @@ static const MonitorFSMTransition MonitorFSM[] = { * goal states happened to line up -- see inMSFailoverCluster's own * comment on NodeActiveContext. */ - { .pos = 367, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, + { .pos = 367, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, .reportedStates = STATES( @@ -2650,7 +2915,12 @@ static const MonitorFSMTransition MonitorFSM[] = { "MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn " "(1 of 4)" }, - { .pos = 369, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, + { .pos = 369, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, .reportedStates = STATES( @@ -2662,7 +2932,12 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "MS-failover fan-out: rejoining from maintenance -> report_lsn (2 of 4)" }, - { .pos = 371, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, + { .pos = 371, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_STABLE, .reportedStates = STATES( @@ -2673,7 +2948,12 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "MS-failover fan-out: old primary converged draining or demoted -> " "report_lsn (3 of 4)" }, - { .pos = 373, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, + { .pos = 373, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, .reportedStates = STATES( @@ -2703,7 +2983,12 @@ static const MonitorFSMTransition MonitorFSM[] = { * ("kept as 2 rows anyway, for dump_fsm() edge visibility... this pair * genuinely isn't disambiguated by this table's own dispatch model"). */ - { .pos = 375, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME }, + { .pos = 375, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .candidatePromotionInProgress = BOOL_FALSE, .mostAdvancedCandidateWithinPromoteThreshold = BOOL_TRUE }, @@ -2713,7 +2998,12 @@ static const MonitorFSMTransition MonitorFSM[] = { "threshold, selected candidate already has all WAL -> prepare_promotion " "(1 of 2 -- see this row's own comment on why both are listed)" }, - { .pos = 377, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME }, + { .pos = 377, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .candidatePromotionInProgress = BOOL_FALSE, .mostAdvancedCandidateWithinPromoteThreshold = BOOL_TRUE }, @@ -2736,7 +3026,13 @@ static const MonitorFSMTransition MonitorFSM[] = { * flow itself stays hand-written C, unchanged; only the message text * this pair's own extraAction builds is delegated here. */ - { .pos = 379, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE }, + { .pos = 379, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .inMSFailoverCandidateGate = BOOL_TRUE, .missingNodesCount = AT_LEAST(1), @@ -2745,7 +3041,13 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=true " "-> decline, wait for more reports (1 of 2)" }, - { .pos = 381, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE }, + { .pos = 381, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .inMSFailoverCandidateGate = BOOL_TRUE, .missingNodesCount = AT_LEAST(1), @@ -2757,13 +3059,25 @@ static const MonitorFSMTransition MonitorFSM[] = { /* MS-failover: zero candidates have reported their LSN yet -- a hard, silent decline * (the original code never logged here either). Never itself dispatched, listed for * dump_fsm() completeness only, same as the no_candidate_yet row below. */ - { .pos = 383, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_CANDIDATE_COUNT_GATE }, + { .pos = 383, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_CANDIDATE_COUNT_GATE + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .inMSFailoverCandidateGate = BOOL_TRUE, .candidateCount = EXACTLY(0) }, .comment = "MS-failover: zero candidates have reported their LSN yet -> silent decline" }, - { .pos = 385, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE }, + { .pos = 385, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .inMSFailoverCandidateGate = BOOL_TRUE, .sufficientQuorumCandidates = BOOL_FALSE, @@ -2772,7 +3086,13 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "MS-failover: not enough quorum candidates reported yet, guard_data_loss=true " "-> decline (1 of 2)" }, - { .pos = 387, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE }, + { .pos = 387, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .inMSFailoverCandidateGate = BOOL_TRUE, .sufficientQuorumCandidates = BOOL_FALSE, @@ -2790,7 +3110,13 @@ static const MonitorFSMTransition MonitorFSM[] = { * missing activeNode state pattern (matches ANY state) would let it swallow the * ordinary top-level lookup's own "BUG: no MonitorFSM row matches" detection for * any otherwise-unmatched heartbeat, silently no-op'ing instead of erroring. */ - { .pos = 389, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_NO_CANDIDATE_YET }, + { .pos = 389, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_NO_CANDIDATE_YET + }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .candidatePromotionInProgress = BOOL_FALSE }, .comment = "MS-failover: no promotion in flight, not enough (or not safe enough) " @@ -2814,7 +3140,12 @@ static const MonitorFSMTransition MonitorFSM[] = { * plain AssignGoalState calls these replace, each falling back to the * original hand-written condition on no match. */ - { .pos = 391, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE }, + { .pos = 391, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE + }, .primaryNode = { .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, .isInPrimaryState = BOOL_TRUE, .isUnhealthy = BOOL_TRUE }, @@ -2824,7 +3155,12 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "nodesCount>2, primary unhealthy, in primary role but not yet " "wait_primary, >=1 healthy candidate -> primary draining" }, - { .pos = 393, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE }, + { .pos = 393, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE + }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE), .isUnhealthy = BOOL_TRUE }, .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE }, @@ -2839,109 +3175,151 @@ static const MonitorFSMTransition MonitorFSM[] = { * nested pass on primaryNode (the join_secondary cascade row above). */ /* primary alone, another node reached wait_standby */ - { .pos = 401, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, + { .pos = 401, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SINGLE) }, .conditions = { .anyOtherNodeWaitingStandby = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), .comment = "primary alone, another node reached wait_standby -> wait_primary" }, /* all nodes async, zero secondaries */ - { .pos = 403, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, + { .pos = 403, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, .secondaryNodesCountIsZero = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "all nodes async, zero secondaries -> wait_primary " "(+ unhealthy-secondary fan-out to catchingup)" }, /* all nodes async, >=1 secondary */ - { .pos = 405, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, + { .pos = 405, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, .secondaryNodesCountIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "all nodes async, >=1 secondary -> primary " "(+ unhealthy-secondary fan-out to catchingup)" }, /* converged primary/apply_settings (not wait_primary), no quorum secondaries, * number_sync_standbys=0, no failover in progress (issue #774) */ - { .pos = 407, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, + { .pos = 407, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, .failoverInProgress = BOOL_FALSE, .numberSyncStandbysIsZero = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "converged primary/apply_settings, no quorum secondaries, no failover in " "progress, number_sync_standbys=0 -> wait_primary " "(+ unhealthy-secondary fan-out to catchingup)" }, /* same, but number_sync_standbys>0 -> block writes on primary */ - { .pos = 409, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, + { .pos = 409, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, .failoverInProgress = BOOL_FALSE, .numberSyncStandbysIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "converged primary/apply_settings, no quorum secondaries, no failover in " "progress, number_sync_standbys>0 -> primary (block writes) " "(+ unhealthy-secondary fan-out to catchingup)" }, /* wait_primary, >=1 quorum secondary */ - { .pos = 411, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, + { .pos = 411, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "wait_primary, >=1 quorum secondary -> primary " "(+ unhealthy-secondary fan-out to catchingup)" }, /* apply_settings, both zero */ - { .pos = 413, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, + { .pos = 413, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, .secondaryQuorumNodesCountIsZero = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "apply_settings, both zero -> wait_primary " "(+ unhealthy-secondary fan-out to catchingup)" }, /* apply_settings, number_sync_standbys != 0 (1 of 2 disjuncts) */ - { .pos = 415, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, + { .pos = 415, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, .conditions = { .numberSyncStandbysIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) " "(+ unhealthy-secondary fan-out to catchingup)" }, /* apply_settings, sync_standbys=0 but >=1 quorum secondary (2 of 2) */ - { .pos = 417, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, + { .pos = 417, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), - .extraAction = ActionCatchupUnhealthySecondaries, + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) " "(+ unhealthy-secondary fan-out to catchingup)" }, /* converged primary/wait_primary/apply_settings, no other condition applies */ - { .pos = 419, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, + { .pos = 419, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, - .extraAction = ActionCatchupUnhealthySecondaries, + .otherNodesFn = OtherNodesDueForCatchingUp, + .otherNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "converged primary/wait_primary/apply_settings, no other condition applies -> " - "no-op besides the unhealthy-secondary fan-out" }, + "no-op besides the unhealthy-secondary fan-out to catchingup" }, /* backwards-compat: join_primary -> primary */ - { .pos = 421, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE }, + { .pos = 421, + .sectionPath = { + MONITOR_FSM_SECTION_PRIMARY_NODE + }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), .comment = "backwards-compat: join_primary -> primary" }, @@ -4008,7 +4386,19 @@ dump_fsm_edges(PG_FUNCTION_ARGS) } } - if (rule->otherNodeAssignedState.kind == GOAL_STATE_SET) + /* + * otherNodesFn rows are skipped here: their "other node" target's + * own current-state precondition isn't a NodeStatePattern at all + * (it's whatever filtering the resolver function itself does, e.g. + * OtherNodeIsDueForCatchingUp's own health/state checks) -- reading + * .primaryNode.statePattern for such a row would resolve its + * NODE_STATE_ANY default to all 21 states, fabricating 21 bogus + * edges no keeper FSM could ever have. Same "not every row's edges + * are representable this way" precedent as the api_triggered + * section's own exclusion above. + */ + if (rule->otherNodeAssignedState.kind == GOAL_STATE_SET && + rule->otherNodesFn == NULL) { int count; ReplicationState *states = @@ -4521,6 +4911,11 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, */ if (!TryMSFailoverDeclarativeRow(ctx, activeNode, activeNode)) { + /* should never happen -- see ActionRunMultiStandbyFailoverCascade's + * own Assert(false) comment for why this stays a hard assertion + * rather than a silently-kept fallback. */ + Assert(false); + LogAndNotifyMessage( message, BUFSIZE, "Failover candidate " NODE_FORMAT @@ -4910,6 +5305,11 @@ BuildCandidateList(GroupStateContext *ctx, List *nodesGroupList, if (!TryFanOutReportLsnRow(ctx, node)) { + /* should never happen -- see ActionRunMultiStandbyFailoverCascade's + * own Assert(false) comment for why this stays a hard assertion + * rather than a silently-kept fallback. */ + Assert(false); + LogAndNotifyMessage( message, BUFSIZE, "Setting goal state of " NODE_FORMAT @@ -4966,6 +5366,11 @@ ProceedWithMSFailover(GroupStateContext *ctx, AutoFailoverNode *activeNode, { char message[BUFSIZE]; + /* should never happen -- see ActionRunMultiStandbyFailoverCascade's + * own Assert(false) comment for why this stays a hard assertion + * rather than a silently-kept fallback. */ + Assert(false); + LogAndNotifyMessage( message, BUFSIZE, "Setting goal state of " NODE_FORMAT @@ -5301,6 +5706,11 @@ PromoteSelectedNode(GroupStateContext *ctx, if (!DispatchMonitorFSMRuleByPos(ctx, &promotionNac, 375)) { + /* should never happen -- see ActionRunMultiStandbyFailoverCascade's + * own Assert(false) comment for why this stays a hard assertion + * rather than a silently-kept fallback. */ + Assert(false); + AssignGoalState(selectedNode, REPLICATION_STATE_PREPARE_PROMOTION, message); @@ -5340,6 +5750,11 @@ PromoteSelectedNode(GroupStateContext *ctx, if (!DispatchMonitorFSMRuleByPos(ctx, &promotionNac, 377)) { + /* should never happen -- see ActionRunMultiStandbyFailoverCascade's + * own Assert(false) comment for why this stays a hard assertion + * rather than a silently-kept fallback. */ + Assert(false); + AssignGoalState(selectedNode, REPLICATION_STATE_FAST_FORWARD, message); } diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index b19239158..e202e5e68 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -34,7 +34,7 @@ SELECT DISTINCT SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; --- Step 2: the actual cross-check -- every pgautofailover.dump_fsm_edges() +-- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A -- non-empty result here is a real, actionable gap (unlike -- check_fsm_reachability.sql's own synthetic-input test, which only @@ -53,4 +53,22 @@ SELECT e.pos, e.current_state, e.assigned_state, f.comment ) ORDER BY e.pos, e.current_state; +-- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper +-- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the +-- full, fully-resolved edge set the monitor's table can reach, see its own +-- comment). A non-empty result here means the keeper is prepared to +-- transition through a (current, assigned) pair the monitor itself would +-- never assign -- either genuinely dead keeper code, or a real coverage +-- gap on the monitor side, same "investigate before assuming which" caveat +-- as step 2a's own comment. +SELECT k.current_state, k.assigned_state + FROM keeper_fsm_edges k + WHERE NOT EXISTS ( + SELECT 1 + FROM pgautofailover.dump_fsm_edges() e + WHERE e.current_state = k.current_state + AND e.assigned_state = k.assigned_state + ) + ORDER BY k.current_state, k.assigned_state; + DROP TABLE keeper_fsm_edges; From 67678af94e76c6dbc525833a3ff814dba3df1f07 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 04:15:00 +0200 Subject: [PATCH 15/52] monitor: remove dead AssignGoalState fallbacks, replace with hard errors Every raw AssignGoalState call outside AssignDeclaredGoalState is gone now (only the forward declaration, the one call inside AssignDeclaredGoalState itself, and the definition remain): - ActionRunMultiStandbyFailoverCascade's DRAINING/MAINTENANCE fallback: this one's "no match" case is a legitimate, expected no-op (the original hand-written if/else-if had no final else -- "neither applies" is a real, common outcome, not a bug), so last commit's Assert(false) here was wrong -- it would have tripped on ordinary heartbeats in an assert-enabled build. Simplified to a bare, return-value-ignored dispatch call, matching the file's own existing pattern for this exact situation. - The other 5 fallback sites (pos 363's retry-reset, the 367-373 fan-out, pos 365's join_secondary, pos 375/377's promotion outcome) are each reached only from inside a hand-written `if` whose own condition already guarantees a specific row must match -- genuinely "can't happen", not "shouldn't happen". Replaced with ereport(ERROR, "BUG: ...") instead of Assert(false) + a silently-kept AssignGoalState call: this fails loudly in every build, not just assert-enabled ones, and there's no more duplicate hand-written logic left to silently drift out of sync with the row it's supposed to mirror. Full regress (19) + isolation (6) suites pass in Docker (PG17), unchanged from before -- confirms every removed fallback really was dead code. --- src/monitor/group_state_machine.c | 165 +++++++++--------------------- 1 file changed, 51 insertions(+), 114 deletions(-) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 9ea34bb9d..00580eceb 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -1565,57 +1565,16 @@ ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext * { AutoFailoverNode *primaryNode = nac->primaryNode.node; - if (!FindAndDispatchMonitorFSMRuleUnderPath(ctx, nac, SectionMSFailover, 0)) - { - /* - * Neither pos 391 nor pos 393 matched -- reproduce the original - * hand-written condition exactly. nac->atLeastOneHealthyCandidate is - * already the exact same fact those rows' own - * atLeastOneHealthyCandidate condition checks (same - * AutoFailoverOtherNodesListInState + CountHealthyCandidates - * computation, see BuildFromContextNodeActiveContext), so it's reused - * here rather than recomputed. - * - * Should never happen: verified by inspection that pos 391/393's own - * conditions are exactly this hand-written condition, transcribed - * 1:1, and confirmed by grepping every expected output file in the full - * regress+isolation suite for this branch's own message text -- zero - * matches, so nothing has ever exercised it. Kept as a hard - * assertion (not silently dropped) rather than removed outright: if - * a future edit to either the row's conditions or this hand-written - * one ever makes them diverge, USE_ASSERT_CHECKING builds (regress, - * isolation, any dev build) fail loudly and immediately instead of - * silently falling back to this branch forever, unnoticed. See - * AssignDeclaredGoalState's own Assert for the same idea applied to - * a single declared state instead of a whole condition. - */ - Assert(false); - - if (IsInPrimaryState(primaryNode) && - !IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) && - nac->atLeastOneHealthyCandidate) - { - char drainingMessage[BUFSIZE] = { 0 }; - - snprintf(drainingMessage, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to draining after it became unhealthy.", - NODE_FORMAT_ARGS(primaryNode)); - - AssignGoalState(primaryNode, REPLICATION_STATE_DRAINING, drainingMessage); - } - else if (IsCurrentState(primaryNode, REPLICATION_STATE_PREPARE_MAINTENANCE)) - { - char maintenanceMessage[BUFSIZE] = { 0 }; - - snprintf(maintenanceMessage, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to maintenance after it converged to prepare_maintenance.", - NODE_FORMAT_ARGS(primaryNode)); - - AssignGoalState(primaryNode, REPLICATION_STATE_MAINTENANCE, maintenanceMessage); - } - } + /* + * pos 391/393 are the only two rows FindAndDispatchMonitorFSMRuleUnderPath + * can reach here (nac->inMSFailoverCluster is false for this nac, so + * every other row under SectionMSFailover is unreachable -- see + * inMSFailoverCluster's own comment). A false return means neither + * row's own condition held: a genuine, expected no-op (the primary is + * unhealthy but neither draining nor prepare_maintenance applies yet), + * not a bug -- nothing further to do this round. + */ + (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, nac, SectionMSFailover, 0); if (!ProceedGroupStateForMSFailover(ctx, primaryNode)) { @@ -4911,24 +4870,18 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, */ if (!TryMSFailoverDeclarativeRow(ctx, activeNode, activeNode)) { - /* should never happen -- see ActionRunMultiStandbyFailoverCascade's - * own Assert(false) comment for why this stays a hard assertion - * rather than a silently-kept fallback. */ - Assert(false); - - LogAndNotifyMessage( - message, BUFSIZE, - "Failover candidate " NODE_FORMAT - " is stuck in fast_forward: all WAL source nodes are " - "unhealthy and pgautofailover.guard_data_loss is true. " - "Resetting candidate to report_lsn to retry when a " - "source recovers. Use pg_autoctl perform failover " - "--allow-data-loss to promote with available WAL.", - NODE_FORMAT_ARGS(activeNode)); - - AssignGoalState(activeNode, - REPLICATION_STATE_REPORT_LSN, - message); + /* + * Can't happen: the if-condition just above already + * establishes exactly what pos 363's own conditions + * require (same activeNodeAllWalSourcesUnhealthy/ + * guardDataLossEnabled facts, see + * BuildMSFailoverNodeActiveContext). + */ + ereport(ERROR, + (errmsg("BUG: pos 363 didn't match " NODE_FORMAT + " although its own conditions should " + "always hold here", + NODE_FORMAT_ARGS(activeNode)))); } return true; @@ -5299,24 +5252,20 @@ BuildCandidateList(GroupStateContext *ctx, List *nodesGroupList, (node->reportedState == REPLICATION_STATE_DEMOTED && node->goalState == REPLICATION_STATE_CATCHINGUP)))) { - char message[BUFSIZE] = { 0 }; - ++(candidateList->missingNodesCount); if (!TryFanOutReportLsnRow(ctx, node)) { - /* should never happen -- see ActionRunMultiStandbyFailoverCascade's - * own Assert(false) comment for why this stays a hard assertion - * rather than a silently-kept fallback. */ - Assert(false); - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to report_lsn to find the failover candidate", - NODE_FORMAT_ARGS(node)); - - AssignGoalState(node, REPLICATION_STATE_REPORT_LSN, message); + /* + * Can't happen: the if-condition just above already + * establishes exactly what pos 367-373's own conditions + * require. + */ + ereport(ERROR, + (errmsg("BUG: no MS-failover fan-out row matched " + NODE_FORMAT " although its own conditions " + "should always hold here", + NODE_FORMAT_ARGS(node)))); } continue; @@ -5364,22 +5313,15 @@ ProceedWithMSFailover(GroupStateContext *ctx, AutoFailoverNode *activeNode, */ if (!TryMSFailoverDeclarativeRow(ctx, activeNode, candidateNode)) { - char message[BUFSIZE]; - - /* should never happen -- see ActionRunMultiStandbyFailoverCascade's - * own Assert(false) comment for why this stays a hard assertion - * rather than a silently-kept fallback. */ - Assert(false); - - LogAndNotifyMessage( - message, BUFSIZE, - "Setting goal state of " NODE_FORMAT - " to join_secondary after " NODE_FORMAT - " got selected as the failover candidate.", - NODE_FORMAT_ARGS(activeNode), - NODE_FORMAT_ARGS(candidateNode)); - - AssignGoalState(activeNode, REPLICATION_STATE_JOIN_SECONDARY, message); + /* + * Can't happen: the if-condition just above already establishes + * exactly what pos 365's own conditions require. + */ + ereport(ERROR, + (errmsg("BUG: pos 365 didn't match " NODE_FORMAT + " although its own conditions should always " + "hold here", + NODE_FORMAT_ARGS(activeNode)))); } return true; @@ -5706,14 +5648,11 @@ PromoteSelectedNode(GroupStateContext *ctx, if (!DispatchMonitorFSMRuleByPos(ctx, &promotionNac, 375)) { - /* should never happen -- see ActionRunMultiStandbyFailoverCascade's - * own Assert(false) comment for why this stays a hard assertion - * rather than a silently-kept fallback. */ - Assert(false); - - AssignGoalState(selectedNode, - REPLICATION_STATE_PREPARE_PROMOTION, - message); + /* can't happen: pos 375 is a fixed, always-present row (see + * AssertMonitorFSMWellFormed) -- DispatchMonitorFSMRuleByPos only + * fails to find a pos that doesn't exist in the table at all. */ + ereport(ERROR, + (errmsg("BUG: MonitorFSM[] has no row with pos = 375"))); } /* leave the other nodes in ReportLSN state for now */ @@ -5750,13 +5689,11 @@ PromoteSelectedNode(GroupStateContext *ctx, if (!DispatchMonitorFSMRuleByPos(ctx, &promotionNac, 377)) { - /* should never happen -- see ActionRunMultiStandbyFailoverCascade's - * own Assert(false) comment for why this stays a hard assertion - * rather than a silently-kept fallback. */ - Assert(false); - - AssignGoalState(selectedNode, - REPLICATION_STATE_FAST_FORWARD, message); + /* can't happen: pos 377 is a fixed, always-present row (see + * AssertMonitorFSMWellFormed) -- DispatchMonitorFSMRuleByPos only + * fails to find a pos that doesn't exist in the table at all. */ + ereport(ERROR, + (errmsg("BUG: MonitorFSM[] has no row with pos = 377"))); } return true; From 0d7fb46dd44c33ce04fc251e25c77f485bd54230 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 04:40:31 +0200 Subject: [PATCH 16/52] monitor: reflow group_state_machine.c's own comments to 80 columns Every /* */ comment (standalone single-line, multi-line, and multi-line with text on the opening line) now wraps at 80 rendered columns (tabs expanded to 4) instead of the ad hoc widths they'd accumulated -- mechanical paragraph reflow via a small script (kept out of the repo), preserving blank-line paragraph breaks, bullet items, and anything containing real code syntax (braces, named C identifiers, #define/typedef) verbatim/unwrapped rather than risking corrupting it. Investigated using citus_indent/uncrustify itself for this (code_width=80, cmt_width=80, cmt_reflow_mode=2 via a local config copied out of the citus/stylechecker:no-py image): it technically supports both settings, but produces objectively broken output on this file -- a missing space after the '*' continuation marker on many reflowed lines, and ugly mid-declaration splits (pointer type separated from its own variable name). Reverted that approach; comments were reflowed by hand/script instead. Confirmed the result is stable under the project's own standard style-checker invocation (zero changes on a second pass) and that citus-style.cfg's own code_width=90/ cmt_width=0 (reflow disabled) stay untouched -- this is a one-file, purely cosmetic change with no config changes needed anywhere else. Code lines (signatures, expressions, struct literals) are intentionally left at their existing width -- only prose comments were in scope here. Full regress (19) + isolation (6) suites pass in Docker (PG17), as expected for a comment-only change. --- src/monitor/group_state_machine.c | 666 +++++++++++++++++------------- 1 file changed, 390 insertions(+), 276 deletions(-) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 00580eceb..4485c93ff 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -264,13 +264,18 @@ MatchStateSet(ReplicationState actual, ReplicationStateSet declared) } -/* Same reasoning as STATES() above: no compound-literal cast, since every - * use is nested inside another static aggregate's designated initializer - * (e.g. ".statePattern = FSM_STATE(x)" inside a MonitorFSMTransition row). */ +/* + * Same reasoning as STATES() above: no compound-literal cast, since every use + * is nested inside another static aggregate's designated initializer (e.g. + * ".statePattern = FSM_STATE(x)" inside a MonitorFSMTransition row). + */ #define FSM_STATE(x) \ { .kind = NODE_STATE_STABLE, .reportedStates = STATES(x) } -/* group_state_machine.c:504-523/1059-1106 -- three IsCurrentState(primaryNode, X) ORed */ +/* + * group_state_machine.c:504-523/1059-1106 -- three IsCurrentState(primaryNode, + * X) ORed + */ static const NodeStatePattern FSM_PRIMARY_OR_WAIT_OR_JOIN = { .kind = NODE_STATE_STABLE, .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, @@ -278,17 +283,22 @@ static const NodeStatePattern FSM_PRIMARY_OR_WAIT_OR_JOIN = { REPLICATION_STATE_PRIMARY), }; -/* WAIT_PRIMARY/JOIN_PRIMARY only, not PRIMARY -- a distinct, narrower set from the one above */ +/* + * WAIT_PRIMARY/JOIN_PRIMARY only, not PRIMARY -- a distinct, narrower set from + * the one above + */ static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY = { .kind = NODE_STATE_STABLE, .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, REPLICATION_STATE_JOIN_PRIMARY), }; -/* the "primary role" states MONITOR_FSM_SECTION_PRIMARY_NODE's own rows match +/* + * the "primary role" states MONITOR_FSM_SECTION_PRIMARY_NODE's own rows match * against (the declarative replacement for the old, now-removed * ProceedGroupStateForPrimaryNode()) -- a different three-element set from - * FSM_PRIMARY_OR_WAIT_OR_JOIN above (no JOIN_PRIMARY, has APPLY_SETTINGS) */ + * FSM_PRIMARY_OR_WAIT_OR_JOIN above (no JOIN_PRIMARY, has APPLY_SETTINGS) + */ static const NodeStatePattern FSM_PRIMARY_ROLE_STATES = { .kind = NODE_STATE_STABLE, .reportedStates = STATES(REPLICATION_STATE_PRIMARY, @@ -296,21 +306,30 @@ static const NodeStatePattern FSM_PRIMARY_ROLE_STATES = { REPLICATION_STATE_APPLY_SETTINGS), }; -/* same "primary role" scope, minus WAIT_PRIMARY -- a narrower enumerated STABLE set */ +/* + * same "primary role" scope, minus WAIT_PRIMARY -- a narrower enumerated STABLE + * set + */ static const NodeStatePattern FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY = { .kind = NODE_STATE_STABLE, .reportedStates = STATES(REPLICATION_STATE_PRIMARY, REPLICATION_STATE_APPLY_SETTINGS), }; -/* reported WAIT_PRIMARY, goal in {WAIT_PRIMARY, PRIMARY} -- join_secondary's cascade row */ +/* + * reported WAIT_PRIMARY, goal in {WAIT_PRIMARY, PRIMARY} -- join_secondary's + * cascade row + */ static const NodeStatePattern FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY = { .kind = NODE_STATE_TRANSITIONING, .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY), .assignedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, REPLICATION_STATE_PRIMARY), }; -/* reported in {WAIT_PRIMARY,JOIN_PRIMARY}, goal PRIMARY -- demoted->catchingup, first disjunct */ +/* + * reported in {WAIT_PRIMARY,JOIN_PRIMARY}, goal PRIMARY -- demoted->catchingup, + * first disjunct + */ static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY = { .kind = NODE_STATE_TRANSITIONING, .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, @@ -318,13 +337,19 @@ static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY .assignedStates = STATES(REPLICATION_STATE_PRIMARY), }; -/* goalState != WAIT_PRIMARY, reportedState irrelevant -- wait_maintenance's second row */ +/* + * goalState != WAIT_PRIMARY, reportedState irrelevant -- wait_maintenance's + * second row + */ static const NodeStatePattern FSM_NOT_ASSIGNED_WAIT_PRIMARY = { .kind = NODE_STATE_NOT_ASSIGNED, .assignedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY), }; -/* !IsCurrentState(node, WAIT_PRIMARY): not converged to wait_primary, for any reason */ +/* + * !IsCurrentState(node, WAIT_PRIMARY): not converged to wait_primary, for any + * reason + */ static const NodeStatePattern FSM_NOT_STABLE_WAIT_PRIMARY = { .kind = NODE_STATE_NOT_STABLE, .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY), @@ -342,17 +367,23 @@ static const NodeStatePattern FSM_DROPPED_GOAL = { .assignedStates = STATES(REPLICATION_STATE_DROPPED), }; -/* reportedState == DEMOTE_TIMEOUT, goalState irrelevant. NOT FSM_STATE(DEMOTE_TIMEOUT), which - * would also require goalState == DEMOTE_TIMEOUT -- the opposite of what this self-fence guard - * needs: it must catch a node whose goalState is still whatever was assigned before the - * self-fence fired (see the real comment on this guard, preserved below). */ +/* + * reportedState == DEMOTE_TIMEOUT, goalState irrelevant. NOT + * FSM_STATE(DEMOTE_TIMEOUT), which would also require goalState == + * DEMOTE_TIMEOUT -- the opposite of what this self-fence guard needs: it must + * catch a node whose goalState is still whatever was assigned before the + * self-fence fired (see the real comment on this guard, preserved below). + */ static const NodeStatePattern FSM_REPORTED_DEMOTE_TIMEOUT = { .kind = NODE_STATE_REPORTED, .reportedStates = STATES(REPLICATION_STATE_DEMOTE_TIMEOUT), }; -/* reportedState in {REPORT_LSN, FAST_FORWARD}, converged -- "continue an already started - * failover" guard, the direct (non-cascading) entry into ProceedGroupStateForMSFailover */ +/* + * reportedState in {REPORT_LSN, FAST_FORWARD}, converged -- "continue an + * already started failover" guard, the direct (non-cascading) entry into + * ProceedGroupStateForMSFailover + */ static const NodeStatePattern FSM_REPORT_LSN_OR_FAST_FORWARD = { .kind = NODE_STATE_STABLE, .reportedStates = STATES(REPLICATION_STATE_REPORT_LSN, @@ -365,9 +396,12 @@ NodeStateMatchesPattern(const AutoFailoverNode *node, const NodeStatePattern *pa { if (node == NULL) { - /* No node this round (no primary). Only NODE_STATE_ANY can still match -- every other - * kind needs a real reportedState/goalState to compare, which a nonexistent node simply - * doesn't have. Existence itself is checked separately via NodeStatusPattern.exists. */ + /* + * No node this round (no primary). Only NODE_STATE_ANY can still match + * -- every other kind needs a real reportedState/goalState to compare, + * which a nonexistent node simply doesn't have. Existence itself is + * checked separately via NodeStatusPattern.exists. + */ return pattern->kind == NODE_STATE_ANY; } @@ -423,7 +457,8 @@ NodeStateMatchesPattern(const AutoFailoverNode *node, const NodeStatePattern *pa /* * NodeStatus: every per-node fact this table's conditions need, computed once - * per role (activeNode, primaryNode) at the top of dispatch by BuildNodeStatus(). + * per role (activeNode, primaryNode) at the top of dispatch by + * BuildNodeStatus(). */ typedef struct NodeStatus { @@ -465,8 +500,10 @@ BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *stat if (node == NULL) { - /* NodeIsUnhealthy(NULL, ctx) returns true -- a nonexistent node being "unhealthy" is - * exactly the semantics the original if-chain relies on. */ + /* + * NodeIsUnhealthy(NULL, ctx) returns true -- a nonexistent node being + * "unhealthy" is exactly the semantics the original if-chain relies on. + */ status->isUnhealthy = true; return; } @@ -692,30 +729,37 @@ typedef struct NodeActiveContext * default) for every other dispatch pass. */ - /* WalSourceNodesAreAllUnhealthy(ctx, ctx->groupNodeList, activeNode) -- - * activeNode-specific despite living at this level (not on NodeStatus): - * it depends on the whole group's other REPORT_LSN peers, not on - * activeNode's own state alone. */ + /* + * WalSourceNodesAreAllUnhealthy(ctx, ctx->groupNodeList, activeNode) -- + * activeNode-specific despite living at this level (not on NodeStatus): it + * depends on the whole group's other REPORT_LSN peers, not on activeNode's + * own state alone. + */ bool activeNodeAllWalSourcesUnhealthy; - /* mirrors whether ProceedGroupStateForMSFailover's own - * "nodeBeingPromoted != NULL" branch will actually drive that candidate - * via ProceedWithMSFailover this round, rather than falling through to + /* + * mirrors whether ProceedGroupStateForMSFailover's own "nodeBeingPromoted + * != NULL" branch will actually drive that candidate via + * ProceedWithMSFailover this round, rather than falling through to * BuildCandidateList/selection instead -- see this file's own * ActionRunMultiStandbyFailoverCascade comment and the design doc's - * identically-named fact for the full derivation. */ + * identically-named fact for the full derivation. + */ bool candidatePromotionInProgress; - /* WalDifferenceWithin(mostAdvancedNode, ctx->primaryNode, - * PromoteXlogThreshold) for the current MS-failover candidate pool's - * own most-advanced member -- SelectFailoverCandidateNode's own - * data-loss guard. */ + /* + * WalDifferenceWithin(mostAdvancedNode, ctx->primaryNode, + * PromoteXlogThreshold) for the current MS-failover candidate pool's own + * most-advanced member -- SelectFailoverCandidateNode's own data-loss + * guard. + */ bool mostAdvancedCandidateWithinPromoteThreshold; - /* GuardDataLoss (metadata.c), the pgautofailover.guard_data_loss GUC -- - * a single global process-wide bool, grouped here (not read directly - * inside an extraAction) so any row gated on it says so in its own - * conditions. */ + /* + * GuardDataLoss (metadata.c), the pgautofailover.guard_data_loss GUC -- a + * single global process-wide bool, grouped here (not read directly inside + * an extraAction) so any row gated on it says so in its own conditions. + */ bool guardDataLossEnabled; /* @@ -739,17 +783,19 @@ typedef struct NodeActiveContext bool inMSFailoverCluster; /* - * Set unconditionally true by BuildMSFailoverCandidateGateNodeActiveContext, - * false (the memset default) everywhere else -- narrows the 5 counting-gate - * rows (reporting_node.ms_failover.promotion_outcome.*_gate) to only ever - * match when dispatched from ProceedGroupStateForMSFailover's own gate - * checks. Without this, TryFanOutReportLsnRow's own candidateNode=NULL call - * (BuildMSFailoverNodeActiveContext) leaves candidateCount/missingNodesCount - * at their memset-0 default and candidatePromotionInProgress false -- the - * exact values the missing-nodes and candidate-count gate rows themselves - * match on -- so its own SectionMSFailover-wide scan could spuriously hit - * one of these gate rows instead of correctly falling through to its own - * plain-AssignGoalState fallback whenever pos 367-373 somehow didn't match. + * Set unconditionally true by + * BuildMSFailoverCandidateGateNodeActiveContext, false (the memset default) + * everywhere else -- narrows the 5 counting-gate rows + * (reporting_node.ms_failover.promotion_outcome.*_gate) to only ever match + * when dispatched from ProceedGroupStateForMSFailover's own gate checks. + * Without this, TryFanOutReportLsnRow's own candidateNode=NULL call + * (BuildMSFailoverNodeActiveContext) leaves + * candidateCount/missingNodesCount at their memset-0 default and + * candidatePromotionInProgress false -- the exact values the missing-nodes + * and candidate-count gate rows themselves match on -- so its own + * SectionMSFailover-wide scan could spuriously hit one of these gate rows + * instead of correctly falling through to its own plain-AssignGoalState + * fallback whenever pos 367-373 somehow didn't match. */ bool inMSFailoverCandidateGate; @@ -780,8 +826,9 @@ typedef struct NodeActiveContext typedef struct NodeActiveContextPattern { - ApiTriggerPattern apiTrigger; /* omitted -> {0} -> API_TRIGGER_NODE_ACTIVE, matching every - * row's existing meaning with no changes required elsewhere */ + ApiTriggerPattern apiTrigger; /* omitted -> {0} -> API_TRIGGER_NODE_ACTIVE, + * matching every row's existing meaning with + * no changes required elsewhere */ BoolPattern groupHasExactlyOneNode; BoolPattern groupHasExactlyTwoNodes; @@ -829,8 +876,10 @@ typedef struct GoalStateAssignment ReplicationState state; } GoalStateAssignment; -/* No compound-literal cast -- see STATES()'s comment: every use nests inside - * another static aggregate's designated initializer. */ +/* + * No compound-literal cast -- see STATES()'s comment: every use nests inside + * another static aggregate's designated initializer. + */ #define GOAL(x) { .kind = GOAL_STATE_SET, .state = (x) } /* @@ -895,12 +944,12 @@ typedef MonitorFSMSection MonitorFSMSectionPath[MONITOR_FSM_SECTION_PATH_MAX_DEP /* * SectionPathIsUnderPrefix returns whether path is prefix (an ancestor of it, - * or itself) -- pure C, no parsing, no cache, no dependency on anything - * outside this file: comparing two small fixed-size enum arrays element by - * element. This is the replacement for FindMatchingMonitorFSMRuleIndexFrom's - * old [startIndex, endIndex) bound (see FindMatchingMonitorFSMRuleIndexUnderPath - * below): a search is now "every row whose sectionPath is under this - * prefix", not "every row between these two array indices". + * or itself) -- pure C, no parsing, no cache, no dependency on anything outside + * this file: comparing two small fixed-size enum arrays element by element. + * This is the replacement for FindMatchingMonitorFSMRuleIndexFrom's old + * [startIndex, endIndex) bound (see FindMatchingMonitorFSMRuleIndexUnderPath + * below): a search is now "every row whose sectionPath is under this prefix", + * not "every row between these two array indices". */ static bool SectionPathIsUnderPrefix(const MonitorFSMSectionPath path, const MonitorFSMSectionPath prefix) @@ -1074,7 +1123,8 @@ typedef struct MonitorFSMTransition * ActionRunMultiStandbyFailoverCascade's own comment for the * distinction. * - * SectionMSFailover = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER } + * SectionMSFailover = { MONITOR_FSM_SECTION_REPORTING_NODE, + * MONITOR_FSM_SECTION_MS_FAILOVER } * Rows under the MS-failover / candidate-selection cluster's own * declarative transitions (see that section's own comment below) -- * two (retry-reset, join_secondary) reached through @@ -1168,9 +1218,11 @@ static const MonitorFSMSectionPath SectionMSFailoverQuorumCandidateGate = MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE }; -/* Forward-declared for the same reason as MonitorFSM[] above: used by +/* + * Forward-declared for the same reason as MonitorFSM[] above: used by * extraActions (ActionRunPrimaryNodeTransition) defined before its real - * definition further down. */ + * definition further down. + */ static void BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *primaryNode, NodeActiveContext *nac); @@ -1241,14 +1293,14 @@ RuleMatches(const NodeActiveContext *nac, const MonitorFSMTransition *rule) /* * FindMatchingMonitorFSMRuleIndexUnderPath scans the whole table in array * order, considering only rows whose own sectionPath is under prefix (see - * SectionPathIsUnderPrefix) and whose pos is greater than afterPos -- - * replacing the old [startIndex, endIndex) index-range bound with a - * section-membership one. pos is strictly increasing == array order (see - * AssertMonitorFSMWellFormed), so "pos > afterPos" is exactly "resume - * scanning after this row" -- afterPos = 0 (no real row has pos <= 0) means - * "from the very start of whichever rows are under prefix", the ordinary - * case; a specific pos is only ever passed for the one genuine mid-section - * resume point this table has (see MonitorFSM_MultiStandbyCascadeResumeAfterPos). + * SectionPathIsUnderPrefix) and whose pos is greater than afterPos -- replacing + * the old [startIndex, endIndex) index-range bound with a section-membership + * one. pos is strictly increasing == array order (see + * AssertMonitorFSMWellFormed), so "pos > afterPos" is exactly "resume scanning + * after this row" -- afterPos = 0 (no real row has pos <= 0) means "from the + * very start of whichever rows are under prefix", the ordinary case; a specific + * pos is only ever passed for the one genuine mid-section resume point this + * table has (see MonitorFSM_MultiStandbyCascadeResumeAfterPos). */ static int FindMatchingMonitorFSMRuleIndexUnderPath(const MonitorFSMTransition table[], int tableSize, @@ -1502,50 +1554,50 @@ ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *me * MAINTENANCE/nothing if/else-if decision, followed unconditionally by * ProceedGroupStateForMSFailover(). The DRAINING/MAINTENANCE decision itself * is dispatched through MonitorFSM[]'s own pos 381/383 rows first (see their - * own comment) -- the hand-written if/else-if below only runs as a fallback - * if neither row's own conditions somehow line up with the ones just - * checked (should never happen). The real source never `return`s after - * assigning DRAINING/MAINTENANCE to the primary -- it always falls through to - * try ProceedGroupStateForMSFailover next, in the SAME outer if-block, and if - * THAT declines (returns false), falls through further still to the rest of - * the original source's own if-chain inside ProceedGroupStateFromContext (now - * the report_lsn/prepare_promotion/stop_replication/... rows further down + * own comment) -- the hand-written if/else-if below only runs as a fallback if + * neither row's own conditions somehow line up with the ones just checked + * (should never happen). The real source never `return`s after assigning + * DRAINING/MAINTENANCE to the primary -- it always falls through to try + * ProceedGroupStateForMSFailover next, in the SAME outer if-block, and if THAT + * declines (returns false), falls through further still to the rest of the + * original source's own if-chain inside ProceedGroupStateFromContext (now the + * report_lsn/prepare_promotion/stop_replication/... rows further down * MonitorFSM[]'s REPORTING_NODE section, for this SAME activeNode). * * This has to be ONE row/action, not three separate declarative rows sharing * this action (as an earlier version of this file had it): once dispatch - * continues past a declined row, it keeps scanning forward and a later, - * broader row matching the same outer "nodesCount>2, primary unhealthy" - * condition (the catch-all "neither DRAINING nor MAINTENANCE applies" case) - * would match too and re-invoke ProceedGroupStateForMSFailover a *second* - * time in the same node_active() call -- something the original single-pass - * if/else-if structure never does. Confirmed by concurrent_second_primary_ - * death_report and concurrent_health_check_and_report, which got stuck (the - * former) or produced a spurious second cascade invocation changing the - * outcome (the latter) until this was folded into a single row/action pair. + * continues past a declined row, it keeps scanning forward and a later, broader + * row matching the same outer "nodesCount>2, primary unhealthy" condition (the + * catch-all "neither DRAINING nor MAINTENANCE applies" case) would match too + * and re-invoke ProceedGroupStateForMSFailover a *second* time in the same + * node_active() call -- something the original single-pass if/else-if structure + * never does. Confirmed by concurrent_second_primary_ death_report and + * concurrent_health_check_and_report, which got stuck (the former) or produced + * a spurious second cascade invocation changing the outcome (the latter) until + * this was folded into a single row/action pair. * - * When ProceedGroupStateForMSFailover() declines, the fallthrough to "the - * rest of ProceedGroupStateFromContext" is a single bounded nested search - * from MonitorFSM_FromContextResumeStart, not a flag back to the top-level - * driver: FindAndDispatchMonitorFSMRule's own internal loop already finds - * whichever row is the correct next match, however many rows down that is, - * in one call -- no repeated re-dispatch needed to walk past intervening - * non-matches. + * When ProceedGroupStateForMSFailover() declines, the fallthrough to "the rest + * of ProceedGroupStateFromContext" is a single bounded nested search from + * MonitorFSM_FromContextResumeStart, not a flag back to the top-level driver: + * FindAndDispatchMonitorFSMRule's own internal loop already finds whichever row + * is the correct next match, however many rows down that is, in one call -- no + * repeated re-dispatch needed to walk past intervening non-matches. * * NOTE on naming: MonitorFSM_FromContextResumeStart is NOT * MonitorFSM_MSFailoverStart, despite both marking a conceptually similar - * "resume point" -- they bound two different things. MonitorFSM_FromContextResumeStart - * (used here) just marks "resume scanning ordinary REPORTING_NODE rows after - * ProceedGroupStateForMSFailover declines" -- ProceedGroupStateForMSFailover() - * itself, and the BuildCandidateList/SelectFailoverCandidateNode/ - * PromoteSelectedNode functions it calls, stay hand-written C, called - * wholesale from here exactly as before this refactor (the candidate-selection - * algorithm itself doesn't reduce to declarative conditions any more cleanly - * than it did before -- see this file's own top-of-file design comment). - * MonitorFSM_MSFailoverStart, by contrast, bounds the *separate* eleven-row - * MS-failover cluster (pos 363-383, "MS-failover / candidate-selection - * cluster" section below) that those same hand-written functions now reach - * *into*, at their own tail end, via TryMSFailoverDeclarativeRow/ + * "resume point" -- they bound two different things. + * MonitorFSM_FromContextResumeStart (used here) just marks "resume scanning + * ordinary REPORTING_NODE rows after ProceedGroupStateForMSFailover declines" + * -- ProceedGroupStateForMSFailover() itself, and the + * BuildCandidateList/SelectFailoverCandidateNode/ PromoteSelectedNode functions + * it calls, stay hand-written C, called wholesale from here exactly as before + * this refactor (the candidate-selection algorithm itself doesn't reduce to + * declarative conditions any more cleanly than it did before -- see this file's + * own top-of-file design comment). MonitorFSM_MSFailoverStart, by contrast, + * bounds the *separate* eleven-row MS-failover cluster (pos 363-383, + * "MS-failover / candidate-selection cluster" section below) that those same + * hand-written functions now reach *into*, at their own tail end, via + * TryMSFailoverDeclarativeRow/ * TryFanOutReportLsnRow/DispatchMonitorFSMRuleByPos -- covering the plain * per-node goal assignments (retry-reset, join_secondary, BuildCandidateList's * own report_lsn fan-out, PromoteSelectedNode's prepare_promotion/fast_forward @@ -1554,10 +1606,10 @@ ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *me * last two rows in that same cluster (pos 381/383) are a fourth, unrelated * caller reusing the same bounded range: ActionRunMultiStandbyFailoverCascade's * own DRAINING/MAINTENANCE outcomes (see that function's own comment) -- not - * part of the candidate-selection machinery at all, just sharing the same - * "safe to scan with the ordinary nac" bound. The candidate-selection - * algorithm's own logic (priority sort, LSN comparison, WAL-fetch - * orchestration) is not part of either bounded range. + * part of the candidate-selection machinery at all, just sharing the same "safe + * to scan with the ordinary nac" bound. The candidate-selection algorithm's own + * logic (priority sort, LSN comparison, WAL-fetch orchestration) is not part of + * either bounded range. */ static void ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, @@ -1669,8 +1721,11 @@ BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *prim BuildNodeStatus(ctx, primaryNode, &nac->primaryNode); nac->otherNode = nac->primaryNode; /* see NodeActiveContext's own comment on .otherNode */ - /* isComparableToReferenceTli defaults to true (row :328 doesn't fire) -- a node that hasn't - * reported a timeline yet (reportedTLI == 0) has nothing to check, same as the original. */ + /* + * isComparableToReferenceTli defaults to true (row :328 doesn't fire) -- a + * node that hasn't reported a timeline yet (reportedTLI == 0) has nothing + * to check, same as the original. + */ nac->activeNode.isComparableToReferenceTli = true; if (activeNode->reportedTLI > 0) { @@ -1732,12 +1787,12 @@ BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *prim /* - * BuildForPrimaryNodeNodeActiveContext computes every fact the - * ForPrimaryNode section of MonitorFSM[] (from MonitorFSM_PrimaryNodeSectionStart - * onward) needs, mirroring the counting loop that used to be inline at the - * top of the old, now-folded-in ProceedGroupStateForPrimaryNode() (the same - * loop OtherNodeIsDueForCatchingUp's condition drives the fan-out assignment - * for, in ActionCatchupUnhealthySecondaries above). + * BuildForPrimaryNodeNodeActiveContext computes every fact the ForPrimaryNode + * section of MonitorFSM[] (from MonitorFSM_PrimaryNodeSectionStart onward) + * needs, mirroring the counting loop that used to be inline at the top of the + * old, now-folded-in ProceedGroupStateForPrimaryNode() (the same loop + * OtherNodeIsDueForCatchingUp's condition drives the fan-out assignment for, + * in ActionCatchupUnhealthySecondaries above). */ static void BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, @@ -1748,9 +1803,11 @@ BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, BuildNodeStatus(ctx, primaryNode, &nac->activeNode); - /* .primaryNode role is unused by every row in MonitorFSM[]'s ForPrimaryNode section -- - * primaryNode IS activeNode here, so every condition is expressed against .activeNode - * directly. */ + /* + * .primaryNode role is unused by every row in MonitorFSM[]'s + * ForPrimaryNode section -- primaryNode IS activeNode here, so every + * condition is expressed against .activeNode directly. + */ List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); int otherNodesCount = list_length(otherNodesGroupList); @@ -2129,14 +2186,14 @@ static const MonitorFSMTransition MonitorFSM[] = { /* * start_maintenance(), secondary, last healthy sync standby -- - * node_active_protocol.c:1973-1986. lastHealthySyncStandbyGoingToMaintenance - * is computed by BuildApiTriggerNodeActiveContext (see its own comment) - * only for this apiFunction, mirroring the real source's own - * number_sync_standbys==0 && secondaryNodesCount==1 && - * IsHealthySyncStandby(currentNode) check verbatim. Must come before the - * ordinary-secondary row below: both match the same activeNode/ - * primaryNode state pattern, and only the more specific condition should - * win. + * node_active_protocol.c:1973-1986. + * lastHealthySyncStandbyGoingToMaintenance is computed by + * BuildApiTriggerNodeActiveContext (see its own comment) only for this + * apiFunction, mirroring the real source's own number_sync_standbys==0 && + * secondaryNodesCount==1 && IsHealthySyncStandby(currentNode) check + * verbatim. Must come before the ordinary-secondary row below: both match + * the same activeNode/ primaryNode state pattern, and only the more + * specific condition should win. */ { .pos = 113, .sectionPath = { @@ -2245,14 +2302,14 @@ static const MonitorFSMTransition MonitorFSM[] = { /* * set_node_candidate_priority(), node_active_protocol.c:2282-2296. - * activeNode IS the primary here (mirrors BuildForPrimaryNodeNodeActiveContext's - * own "primaryNode IS activeNode" convention): the row's only real target - * is the primary's own apply_settings transition. The "primary is - * already apply_settings" ereport(ERROR) and the "no primary yet, just - * proceed" no-op both stay hand-written pre-dispatch in - * set_node_candidate_priority() itself, exactly where they already are - * -- this row is only reached once the wrapper has confirmed a primary - * exists and isn't already apply_settings. + * activeNode IS the primary here (mirrors + * BuildForPrimaryNodeNodeActiveContext's own "primaryNode IS activeNode" + * convention): the row's only real target is the primary's own + * apply_settings transition. The "primary is already apply_settings" + * ereport(ERROR) and the "no primary yet, just proceed" no-op both stay + * hand-written pre-dispatch in set_node_candidate_priority() itself, + * exactly where they already are -- this row is only reached once the + * wrapper has confirmed a primary exists and isn't already apply_settings. */ { .pos = 125, .sectionPath = { @@ -2318,7 +2375,9 @@ static const MonitorFSMTransition MonitorFSM[] = { .extraAction = ActionRemoveDroppedNode, .comment = "converged to dropped -> remove the node from the catalog" }, - /* goal already dropped (mid-drop, row above hasn't converged yet) -> no-op */ + /* + * goal already dropped (mid-drop, row above hasn't converged yet) -> no-op + */ { .pos = 203, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS @@ -2366,13 +2425,18 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), .comment = "alone in group, candidatePriority zero -> report_lsn" }, - /* --- [MonitorFSM_FromContextStart, MonitorFSM_PrimaryNodeSectionStart): the rest of - * ProceedGroupStateFromContext()'s own sequential if-chain -- everything from the - * timeline-fork check (group_state_machine.c:328, right after the - * IsInPrimaryState(activeNode) early return) onward. Reached only when activeNode is - * NOT currently primary-role. */ + /* + * --- [MonitorFSM_FromContextStart, MonitorFSM_PrimaryNodeSectionStart): + * the rest of ProceedGroupStateFromContext()'s own sequential if-chain -- + * everything from the timeline-fork check (group_state_machine.c:328, right + * after the IsInPrimaryState(activeNode) early return) onward. Reached only + * when activeNode is NOT currently primary-role. + */ - /* converged secondary, reportedTLI not an ancestor of the group's reference timeline */ + /* + * converged secondary, reportedTLI not an ancestor of the group's reference + * timeline + */ { .pos = 301, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2384,7 +2448,10 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "converged secondary, reportedTLI not an ancestor of reference -> catchingup" }, - /* replication stall (#997): primary healthy, no standby past replication_stall_timeout */ + /* + * replication stall (#997): primary healthy, no standby past + * replication_stall_timeout + */ { .pos = 303, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2397,9 +2464,11 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "primary healthy, no standby past replication_stall_timeout -> wait_primary" }, - /* nodesCount>2, primary unhealthy -- draining/maintenance/nothing decided inside the - * action, then the MS-failover cascade unconditionally; must be a single row, see - * ActionRunMultiStandbyFailoverCascade's comment for why. */ + /* + * nodesCount>2, primary unhealthy -- draining/maintenance/nothing decided + * inside the action, then the MS-failover cascade unconditionally; must be + * a single row, see ActionRunMultiStandbyFailoverCascade's comment for why. + */ { .pos = 305, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2446,9 +2515,12 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), .comment = "fast_forward done -> prepare_promotion" }, - /* continue an already-started MS failover -- a direct `return` in the real source, and no - * later row in this table matches activeNode in REPORT_LSN/FAST_FORWARD, so it doesn't - * matter here whether the extraAction's bool stops dispatch or lets it keep scanning. */ + /* + * continue an already-started MS failover -- a direct `return` in the real + * source, and no later row in this table matches activeNode in + * REPORT_LSN/FAST_FORWARD, so it doesn't matter here whether the + * extraAction's bool stops dispatch or lets it keep scanning. + */ { .pos = 313, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2511,7 +2583,10 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), .comment = "caught up, same TLI as primary, within sync threshold -> secondary" }, - /* primary fails, already converged wait_primary (no draining edge, issue #1168) */ + /* + * primary fails, already converged wait_primary (no draining edge, issue + * #1168) + */ { .pos = 323, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2607,7 +2682,10 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNodeAssignedState = GOAL(REPLICATION_STATE_WAIT_PRIMARY), .comment = "Citus worker prepare_promotion, primary removed -> wait_primary" }, - /* prepare_promotion, primary present, already converged wait_primary (issue #1168) */ + /* + * prepare_promotion, primary present, already converged wait_primary (issue + * #1168) + */ { .pos = 337, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2622,7 +2700,10 @@ static const MonitorFSMTransition MonitorFSM[] = { "prepare_promotion, primary already converged wait_primary (issue #1168) -> " "stop_replication only (1 of 2)" }, - /* prepare_promotion, primary present, not in maintenance, not already wait_primary */ + /* + * prepare_promotion, primary present, not in maintenance, not already + * wait_primary + */ { .pos = 339, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2687,7 +2768,10 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "stop_replication, primary's drain time expired -> " "wait_primary + demoted (2 of 3)" }, - /* stop_replication, primary's goal is wait_primary but presumed dead (3-way OR, 3 of 3) */ + /* + * stop_replication, primary's goal is wait_primary but presumed dead (3-way + * OR, 3 of 3) + */ { .pos = 349, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2752,8 +2836,10 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "demoted, primary converged wait/join_primary/primary, healthy -> catchingup" }, - /* join_secondary, primary reported wait_primary with goal wait/primary -- cascades into a - * nested pass on primaryNode */ + /* + * join_secondary, primary reported wait_primary with goal wait/primary -- + * cascades into a nested pass on primaryNode + */ { .pos = 359, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2794,7 +2880,10 @@ static const MonitorFSMTransition MonitorFSM[] = { * renumbered into them, so nothing else shifts. */ - /* MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, retry */ + /* + * MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, + * retry + */ { .pos = 363, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2814,7 +2903,9 @@ static const MonitorFSMTransition MonitorFSM[] = { "MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, " "guard_data_loss=true -> report_lsn (retry once a source recovers)" }, - /* MS-failover: candidate ready to stream WAL -> follower joins as secondary */ + /* + * MS-failover: candidate ready to stream WAL -> follower joins as secondary + */ { .pos = 365, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -3015,9 +3106,12 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=false " "-> proceed despite possible data loss (2 of 2)" }, - /* MS-failover: zero candidates have reported their LSN yet -- a hard, silent decline - * (the original code never logged here either). Never itself dispatched, listed for - * dump_fsm() completeness only, same as the no_candidate_yet row below. */ + /* + * MS-failover: zero candidates have reported their LSN yet -- a hard, + * silent decline (the original code never logged here either). Never itself + * dispatched, listed for dump_fsm() completeness only, same as the + * no_candidate_yet row below. + */ { .pos = 383, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -3060,15 +3154,19 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "MS-failover: not enough quorum candidates reported yet, guard_data_loss=false " "-> proceed with fewer than required (2 of 2)" }, - /* MS-failover: no promotion in flight, either not enough candidates have reported yet - * or the most-advanced one is still too far behind the primary to safely promote -- - * a pure no-op besides BuildCandidateList's own fan-out (the four rows above); listed - * for dump_fsm() completeness, never itself dispatched (no assignment to attribute). + /* + * MS-failover: no promotion in flight, either not enough candidates have + * reported yet or the most-advanced one is still too far behind the primary + * to safely promote -- a pure no-op besides BuildCandidateList's own + * fan-out (the four rows above); listed for dump_fsm() completeness, never + * itself dispatched (no assignment to attribute). * inMSFailoverCluster=BOOL_TRUE despite that: without it this row's bare - * candidatePromotionInProgress=BOOL_FALSE (true by default everywhere) plus its - * missing activeNode state pattern (matches ANY state) would let it swallow the - * ordinary top-level lookup's own "BUG: no MonitorFSM row matches" detection for - * any otherwise-unmatched heartbeat, silently no-op'ing instead of erroring. */ + * candidatePromotionInProgress=BOOL_FALSE (true by default everywhere) plus + * its missing activeNode state pattern (matches ANY state) would let it + * swallow the ordinary top-level lookup's own "BUG: no MonitorFSM row + * matches" detection for any otherwise-unmatched heartbeat, silently + * no-op'ing instead of erroring. + */ { .pos = 389, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -3083,21 +3181,21 @@ static const MonitorFSMTransition MonitorFSM[] = { /* * ActionRunMultiStandbyFailoverCascade's own two outcomes (pos 305's - * extraAction, group_state_machine.c). Both rows below share pos 305's - * own gating conditions (primaryNode.isUnhealthy, groupHasMoreThanTwoNodes) - * verbatim -- not a new marker field -- which is what keeps them safe - * from the ordinary top-level lookup: that lookup can only ever reach - * these two rows by first reaching pos 305 itself (earlier in the array, - * with no activeNode-state restriction of its own), which unconditionally - * dispatches through here via its own extraAction before the ordinary - * scan could ever resume past it in the same call. atLeastOneHealthyCandidate - * is the exact same fact (same AutoFailoverOtherNodesListInState + + * extraAction, group_state_machine.c). Both rows below share pos 305's own + * gating conditions (primaryNode.isUnhealthy, groupHasMoreThanTwoNodes) + * verbatim -- not a new marker field -- which is what keeps them safe from + * the ordinary top-level lookup: that lookup can only ever reach these two + * rows by first reaching pos 305 itself (earlier in the array, with no + * activeNode-state restriction of its own), which unconditionally + * dispatches through here via its own extraAction before the ordinary scan + * could ever resume past it in the same call. atLeastOneHealthyCandidate is + * the exact same fact (same AutoFailoverOtherNodesListInState + * CountHealthyCandidates computation, same isUnhealthy/groupNodeCount>2 * gate) ActionRunMultiStandbyFailoverCascade used to compute locally -- * reused here instead of duplicated. ResolveAcceptedTimeline-style side - * effects don't apply to either row (there are none here); only the - * plain AssignGoalState calls these replace, each falling back to the - * original hand-written condition on no match. + * effects don't apply to either row (there are none here); only the plain + * AssignGoalState calls these replace, each falling back to the original + * hand-written condition on no match. */ { .pos = 391, .sectionPath = { @@ -3127,11 +3225,14 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "nodesCount>2, primary unhealthy, converged prepare_maintenance -> " "primary maintenance" }, - /* --- [MonitorFSM_PrimaryNodeSectionStart, MonitorFSM_SIZE): the declarative replacement - * for ProceedGroupStateForPrimaryNode()'s own sequential if-chain. Here .activeNode maps - * to the primaryNode parameter, not a reporting node -- reached either directly by the - * top-level driver (activeNode already primary-role) or via ActionRunPrimaryNodeTransition's - * nested pass on primaryNode (the join_secondary cascade row above). */ + /* + * --- [MonitorFSM_PrimaryNodeSectionStart, MonitorFSM_SIZE): the + * declarative replacement for ProceedGroupStateForPrimaryNode()'s own + * sequential if-chain. Here .activeNode maps to the primaryNode parameter, + * not a reporting node -- reached either directly by the top-level driver + * (activeNode already primary-role) or via ActionRunPrimaryNodeTransition's + * nested pass on primaryNode (the join_secondary cascade row above). + */ /* primary alone, another node reached wait_standby */ { .pos = 401, @@ -3171,8 +3272,10 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "all nodes async, >=1 secondary -> primary " "(+ unhealthy-secondary fan-out to catchingup)" }, - /* converged primary/apply_settings (not wait_primary), no quorum secondaries, - * number_sync_standbys=0, no failover in progress (issue #774) */ + /* + * converged primary/apply_settings (not wait_primary), no quorum + * secondaries, number_sync_standbys=0, no failover in progress (issue #774) + */ { .pos = 407, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE @@ -3262,7 +3365,9 @@ static const MonitorFSMTransition MonitorFSM[] = { "apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) " "(+ unhealthy-secondary fan-out to catchingup)" }, - /* converged primary/wait_primary/apply_settings, no other condition applies */ + /* + * converged primary/wait_primary/apply_settings, no other condition applies + */ { .pos = 419, .sectionPath = { MONITOR_FSM_SECTION_PRIMARY_NODE @@ -3760,13 +3865,15 @@ NodeStatePatternReportedStatesText(const NodeStatePattern *pattern, bool *isNull } -/* Appends "name=true"/"name=false" to buf (with a ", " separator when buf - * isn't empty), skipping BOOL_ANY entirely -- BOOL_ANY means this row - * doesn't care about the fact at all, so it's noise, not a guard. Shared by +/* + * Appends "name=true"/"name=false" to buf (with a ", " separator when buf isn't + * empty), skipping BOOL_ANY entirely -- BOOL_ANY means this row doesn't care + * about the fact at all, so it's noise, not a guard. Shared by * NodeStatusPatternConditionsText and NodeActiveContextPatternConditionsText - * below, one line per BoolPattern field rather than a loop: there's no - * runtime array of (name, value) pairs to iterate, since the field names - * only exist at compile time. */ + * below, one line per BoolPattern field rather than a loop: there's no runtime + * array of (name, value) pairs to iterate, since the field names only exist at + * compile time. + */ #define APPEND_BOOL_CONDITION(buf, name, boolPattern) \ do { \ if ((boolPattern) != BOOL_ANY) \ @@ -3780,9 +3887,11 @@ NodeStatePatternReportedStatesText(const NodeStatePattern *pattern, bool *isNull } \ } while (0) -/* Same idea as APPEND_BOOL_CONDITION, for an IntPattern field: skips - * INT_PATTERN_ANY, otherwise renders "name=n"/"name>=n"/"name<=n" matching - * the EXACTLY/AT_LEAST/AT_MOST macro that built the pattern. */ +/* + * Same idea as APPEND_BOOL_CONDITION, for an IntPattern field: skips + * INT_PATTERN_ANY, otherwise renders "name=n"/"name>=n"/"name<=n" matching the + * EXACTLY/AT_LEAST/AT_MOST macro that built the pattern. + */ #define APPEND_INT_CONDITION(buf, name, intPattern) \ do { \ if ((intPattern).kind != INT_PATTERN_ANY) \ @@ -3975,22 +4084,22 @@ PG_FUNCTION_INFO_V1(dump_fsm); * (first-match-wins order -- the same order RuleMatches() itself scans in). * This is the cross-check surface the design doc's dump_fsm()/ * check_fsm_reachability() proposal calls for: enough to correlate a - * pgautofailover.event row's rule_pos/rule_section (see notifications.h) - * back to the exact rule that produced it, and enough for an operator or a - * future keeper-side reachability check to see every transition the - * monitor's table can produce, without reading the C source. The - * active_node_current_state/other_node_current_state/candidate_node_current_state - * columns (rendered via NodeStatePatternReportedStatesText, see its own + * pgautofailover.event row's rule_pos/rule_section (see notifications.h) back + * to the exact rule that produced it, and enough for an operator or a future + * keeper-side reachability check to see every transition the monitor's table + * can produce, without reading the C source. The active_node_current_state/ + * other_node_current_state/candidate_node_current_state columns (rendered via + * NodeStatePatternReportedStatesText, see its own * comment) plus the assigned-state columns already here give a keeper- * cross-check tool the full (fromState, toState) shape it needs to compare * against KeeperFSMTransition. The *_conditions columns (rendered via - * NodeStatusPatternConditionsText/NodeActiveContextPatternConditionsText, - * see their own comments) additionally surface every non-default BoolPattern - * guard on the row -- health foremost among them, but also maintenance, - * candidate eligibility, WAL thresholds, guard_data_loss, and the rest -- - * so a reader (or a future automated check) doesn't have to open the C - * source to see what else has to be true for a given row to fire. The - * section column (rendered via MonitorFSMTransitionSectionText) also names + * NodeStatusPatternConditionsText/NodeActiveContextPatternConditionsText, see + * their own comments) additionally surface every non-default BoolPattern guard + * on the row -- health foremost among them, but also maintenance, candidate + * eligibility, WAL thresholds, guard_data_loss, and the rest -- so a reader (or + * a future automated check) doesn't have to open the C source to see what else + * has to be true for a given row to fire. The section column (rendered via + * MonitorFSMTransitionSectionText) also names * which operator-triggered SQL entry point a MONITOR_FSM_SECTION_API_TRIGGERED * row requires, e.g. "api_triggered: remove_node" -- just the plain section * name for every other row, since only that section's rows carry a specific @@ -4211,64 +4320,63 @@ PG_FUNCTION_INFO_V1(dump_fsm_edges); /* * dump_fsm_edges exposes MonitorFSM[] as a flat set of concrete * (pos, current_state, assigned_state) edges -- one row per actual - * (reportedState, assignedState) pair a row can produce, fully resolved - * (never NULL, unlike dump_fsm()'s own pattern-summary columns, which stay - * unresolved for human display). This is the keeper-cross-check surface the - * design doc's check_fsm_reachability() proposal needs: pgautofailover. + * (reportedState, assignedState) pair a row can produce, fully resolved (never + * NULL, unlike dump_fsm()'s own pattern-summary columns, which stay unresolved + * for human display). This is the keeper-cross-check surface the design doc's + * check_fsm_reachability() proposal needs: pgautofailover. * check_fsm_reachability(jsonb) anti-joins this against a keeper's own - * KeeperFSM[] edges (see KeeperFSMToJSON(), src/bin/pg_autoctl/fsm.c) to - * find any monitor transition with no matching keeper edge -- exactly the - * bug class issue #774 was. + * KeeperFSM[] edges (see KeeperFSMToJSON(), src/bin/pg_autoctl/fsm.c) to find + * any monitor transition with no matching keeper edge -- exactly the bug class + * issue #774 was. * * Two potential edges per row, resolved independently: activeNode's own - * (statePattern -> activeNodeAssignedState) when the row actually assigns - * one, and primaryNode's own (statePattern -> otherNodeAssignedState) when - * it does. candidateNode never has an assignment slot of its own (see - * MonitorFSMTransition's own comment), so it never contributes an edge. - * pgKind is deliberately not modeled here -- see this function's own header - * comment in the design doc discussion; every Citus-specific KeeperFSM[] - * edge already has a NODE_KIND_ANY counterpart with the same - * (current, assigned) shape (fsm_mermaid.c's own comment establishes this - * as an invariant this codebase already relies on elsewhere), so a flat, - * pgKind-blind edge set on both sides is already correct. + * (statePattern -> activeNodeAssignedState) when the row actually assigns one, + * and primaryNode's own (statePattern -> otherNodeAssignedState) when it does. + * candidateNode never has an assignment slot of its own (see + * MonitorFSMTransition's own comment), so it never contributes an edge. pgKind + * is deliberately not modeled here -- see this function's own header comment in + * the design doc discussion; every Citus-specific KeeperFSM[] edge already has + * a NODE_KIND_ANY counterpart with the same (current, assigned) shape + * (fsm_mermaid.c's own comment establishes this as an invariant this codebase + * already relies on elsewhere), so a flat, pgKind-blind edge set on both sides + * is already correct. * * Two categories of edges are deliberately never emitted, both confirmed by * running pg_autoctl inspect fsm check for real and tracing every one of the * mismatches it reported back to its actual root cause rather than assuming: * * - Reflexive (current == assigned) edges. keeper_fsm_reach_assigned_state() - * (src/bin/pg_autoctl/fsm.c) returns true the moment - * current_role == assigned_role, before ever consulting KeeperFSM[] -- - * confirmed by reading that function directly. A self-loop can therefore - * never have (or need) a matching keeper edge; including one here would - * always report a false gap. This alone explained every mismatch on pos - * 363, 403, 405, 409 during that live run. + * (src/bin/pg_autoctl/fsm.c) returns true the moment current_role == + * assigned_role, before ever consulting KeeperFSM[] -- confirmed by reading + * that function directly. A self-loop can therefore never have (or need) a + * matching keeper edge; including one here would always report a false gap. + * This alone explained every mismatch on pos 363, 403, 405, 409 during that + * live run. * * - MONITOR_FSM_SECTION_API_TRIGGERED rows entirely. Every one of these is - * reached from an operator-facing SQL wrapper (remove_node, - * perform_failover, start_maintenance, ...) that resolves activeNode to a - * specific, already-validated role (almost always the primary) via - * hand-written C *before* dispatch ever runs -- ProceedGroupStateForApiTrigger's - * own comment, and several of these rows' own comments ("there's nothing - * else for activeNode to be here but the primary itself", "activeNode IS - * the primary here"), document this explicitly. The row's own - * NodeStatePattern for these (typically ANY, or a BoolPattern condition - * like isInPrimaryState with no accompanying state-set restriction) is - * consequently far broader than what's actually reachable: it was never - * meant to double as a full reachability precondition, because that - * precondition already lives in hand-written C outside this table, per - * this design's own "pre/post side effects stay hand-written, not modeled - * as a row" principle. Expanding it here (as this function otherwise - * correctly does for the ordinary heartbeat-driven sections) manufactures - * the exact same kind of false gap for every one of these ten rows, all - * confirmed by that same live run. + * reached from an operator-facing SQL wrapper (remove_node, perform_failover, + * start_maintenance, ...) that resolves activeNode to a specific, + * already-validated role (almost always the primary) via hand-written C + * *before* dispatch ever runs -- ProceedGroupStateForApiTrigger's own comment, + * and several of these rows' own comments ("there's nothing else for activeNode + * to be here but the primary itself", "activeNode IS the primary here"), + * document this explicitly. The row's own NodeStatePattern for these (typically + * ANY, or a BoolPattern condition like isInPrimaryState with no accompanying + * state-set restriction) is consequently far broader than what's actually + * reachable: it was never meant to double as a full reachability precondition, + * because that precondition already lives in hand-written C outside this table, + * per this design's own "pre/post side effects stay hand-written, not modeled + * as a row" principle. Expanding it here (as this function otherwise correctly + * does for the ordinary heartbeat-driven sections) manufactures the exact same + * kind of false gap for every one of these ten rows, all confirmed by that same + * live run. * * Every other mismatch that same run found (early_checks 209/211, * reporting_node 303/325/333/339/347/349/351) stayed reachable from a live - * node's own genuine reported state with no such structural excuse, so - * they're deliberately NOT filtered here -- each is a real candidate for - * individual investigation against the keeper's actual KeeperFSM[] rows, - * not a known-safe artifact of this function's own edge derivation. + * node's own genuine reported state with no such structural excuse, so they're + * deliberately NOT filtered here -- each is a real candidate for individual + * investigation against the keeper's actual KeeperFSM[] rows, not a known-safe + * artifact of this function's own edge derivation. */ Datum dump_fsm_edges(PG_FUNCTION_ARGS) @@ -4544,17 +4652,16 @@ WalSourceNodesAreAllUnhealthy(GroupStateContext *ctx, /* - * BuildMSFailoverNodeActiveContext computes the facts the MS-failover - * cluster's own declarative rows (MonitorFSM_MSFailoverStart onwards) need. - * candidateNode is NULL at exactly one call site (TryFanOutReportLsnRow, - * wrapping BuildCandidateList's own fan-out loop, which hasn't selected a - * candidate yet) -- everywhere else (both call sites TryMSFailoverDeclarativeRow - * wraps) it's non-NULL, called from within ProceedGroupStateForMSFailover's - * own "nodeBeingPromoted != NULL" branch. Either way - * candidatePromotionInProgress is exactly (candidateNode != NULL), and the - * activeNodeAllWalSourcesUnhealthy computation below is skipped whenever - * candidateNode is NULL, so passing NULL never risks matching it against - * the wrong node. + * BuildMSFailoverNodeActiveContext computes the facts the MS-failover cluster's + * own declarative rows (MonitorFSM_MSFailoverStart onwards) need. candidateNode + * is NULL at exactly one call site (TryFanOutReportLsnRow, wrapping + * BuildCandidateList's own fan-out loop, which hasn't selected a candidate yet) + * -- everywhere else (both call sites TryMSFailoverDeclarativeRow wraps) it's + * non-NULL, called from within ProceedGroupStateForMSFailover's own + * "nodeBeingPromoted != NULL" branch. Either way candidatePromotionInProgress + * is exactly (candidateNode != NULL), and the activeNodeAllWalSourcesUnhealthy + * computation below is skipped whenever candidateNode is NULL, so passing NULL + * never risks matching it against the wrong node. */ static void BuildMSFailoverNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *activeNode, @@ -5017,9 +5124,12 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, */ int minCandidates = ctx->formation->number_sync_standbys + 1; - /* no candidates is a hard pass -- see MonitorFSM[]'s own candidate_count_gate - * row for this same fact, matched declaratively but never itself dispatched - * (a silent decline, same as the original code: no log here either). */ + /* + * no candidates is a hard pass -- see MonitorFSM[]'s own + * candidate_count_gate row for this same fact, matched declaratively but + * never itself dispatched (a silent decline, same as the original code: + * no log here either). + */ if (candidateList.candidateCount == 0) { return false; @@ -5648,9 +5758,11 @@ PromoteSelectedNode(GroupStateContext *ctx, if (!DispatchMonitorFSMRuleByPos(ctx, &promotionNac, 375)) { - /* can't happen: pos 375 is a fixed, always-present row (see + /* + * can't happen: pos 375 is a fixed, always-present row (see * AssertMonitorFSMWellFormed) -- DispatchMonitorFSMRuleByPos only - * fails to find a pos that doesn't exist in the table at all. */ + * fails to find a pos that doesn't exist in the table at all. + */ ereport(ERROR, (errmsg("BUG: MonitorFSM[] has no row with pos = 375"))); } @@ -5689,9 +5801,11 @@ PromoteSelectedNode(GroupStateContext *ctx, if (!DispatchMonitorFSMRuleByPos(ctx, &promotionNac, 377)) { - /* can't happen: pos 377 is a fixed, always-present row (see + /* + * can't happen: pos 377 is a fixed, always-present row (see * AssertMonitorFSMWellFormed) -- DispatchMonitorFSMRuleByPos only - * fails to find a pos that doesn't exist in the table at all. */ + * fails to find a pos that doesn't exist in the table at all. + */ ereport(ERROR, (errmsg("BUG: MonitorFSM[] has no row with pos = 377"))); } From 8a405c57afd0ce400835310fea0795701e009e5e Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 15:14:38 +0200 Subject: [PATCH 17/52] monitor: exclude api_triggered-only dropped edges from keeper_fsm_edges step 2b Step 2b (keeper -> monitor reverse gap) was flagging 21 of its 38 rows as unreachable purely because dump_fsm_edges() excludes the entire api_triggered section, where remove_node()'s own DROPPED assignment lives. KeeperFSM[]'s two ANY_STATE -> DROPPED rows expand to one edge per concrete current_state (fsm.c's KeeperFSMToJSON()), so every one of them was a guaranteed false positive, not a real gap. Filter them out explicitly, and note in the comment that the remaining rows still need the same per-row api_triggered-aware judgment as step 2a. --- src/monitor/expected/keeper_fsm_edges.out | 47 ++++++++++++----------- src/monitor/sql/keeper_fsm_edges.sql | 24 +++++++++++- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 0567e5be3..2d78c863b 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -296,9 +296,31 @@ SELECT e.pos, e.current_state, e.assigned_state, f.comment -- never assign -- either genuinely dead keeper code, or a real coverage -- gap on the monitor side, same "investigate before assuming which" caveat -- as step 2a's own comment. +-- +-- assigned_state <> 'dropped' excludes a known, 100%-explained artifact, +-- not a real gap: KeeperFSM[]'s two ANY_STATE -> DROPPED rows (fsm.c) get +-- expanded by KeeperFSMToJSON() into one edge per concrete current_state +-- (21 of them), but the monitor's own equivalent (remove_node(), pos +-- 101/103) lives entirely in the api_triggered section, which +-- dump_fsm_edges() deliberately excludes (see its own comment) since those +-- rows resolve their target via hand-written C, not a NodeStatePattern -- +-- so dump_fsm_edges() can never produce a single edge assigning 'dropped', +-- by construction, regardless of current_state. Filtering these out here +-- avoids drowning the rows below in guaranteed noise. +-- +-- The rows that remain still need the same per-row judgment as step 2a: +-- some of their target states (e.g. maintenance, prepare_maintenance, +-- wait_standby, join_primary, wait_maintenance) are ALSO only reachable +-- through api_triggered rows and so are equally artifacts of that same +-- exclusion; others (e.g. primary, catchingup, prepare_promotion) do have +-- some non-api_triggered coverage in dump_fsm_edges(), so a gap against +-- one of those is more likely a genuine reachability question worth +-- investigating -- don't assume either way without checking the specific +-- (current, assigned) pair against dump_fsm_edges() and pgautofailover.fsm. SELECT k.current_state, k.assigned_state FROM keeper_fsm_edges k - WHERE NOT EXISTS ( + WHERE k.assigned_state <> 'dropped' + AND NOT EXISTS ( SELECT 1 FROM pgautofailover.dump_fsm_edges() e WHERE e.current_state = k.current_state @@ -308,43 +330,22 @@ SELECT k.current_state, k.assigned_state current_state | assigned_state ---------------------+--------------------- init | wait_standby - init | dropped - single | dropped wait_primary | join_primary wait_primary | apply_settings - wait_primary | dropped primary | maintenance primary | join_primary primary | prepare_maintenance - primary | dropped - draining | dropped demote_timeout | primary - demote_timeout | dropped - demoted | dropped catchingup | prepare_promotion catchingup | maintenance catchingup | wait_maintenance - catchingup | dropped secondary | wait_standby secondary | maintenance secondary | wait_maintenance - secondary | dropped - prepare_promotion | dropped - stop_replication | dropped - wait_standby | dropped maintenance | catchingup - maintenance | dropped - join_primary | dropped apply_settings | join_primary - apply_settings | dropped prepare_maintenance | catchingup - prepare_maintenance | dropped - wait_maintenance | dropped - report_lsn | dropped - fast_forward | dropped - join_secondary | dropped dropped | wait_standby - dropped | dropped -(38 rows) +(17 rows) DROP TABLE keeper_fsm_edges; diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index e202e5e68..b7aafc634 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -61,9 +61,31 @@ SELECT e.pos, e.current_state, e.assigned_state, f.comment -- never assign -- either genuinely dead keeper code, or a real coverage -- gap on the monitor side, same "investigate before assuming which" caveat -- as step 2a's own comment. +-- +-- assigned_state <> 'dropped' excludes a known, 100%-explained artifact, +-- not a real gap: KeeperFSM[]'s two ANY_STATE -> DROPPED rows (fsm.c) get +-- expanded by KeeperFSMToJSON() into one edge per concrete current_state +-- (21 of them), but the monitor's own equivalent (remove_node(), pos +-- 101/103) lives entirely in the api_triggered section, which +-- dump_fsm_edges() deliberately excludes (see its own comment) since those +-- rows resolve their target via hand-written C, not a NodeStatePattern -- +-- so dump_fsm_edges() can never produce a single edge assigning 'dropped', +-- by construction, regardless of current_state. Filtering these out here +-- avoids drowning the rows below in guaranteed noise. +-- +-- The rows that remain still need the same per-row judgment as step 2a: +-- some of their target states (e.g. maintenance, prepare_maintenance, +-- wait_standby, join_primary, wait_maintenance) are ALSO only reachable +-- through api_triggered rows and so are equally artifacts of that same +-- exclusion; others (e.g. primary, catchingup, prepare_promotion) do have +-- some non-api_triggered coverage in dump_fsm_edges(), so a gap against +-- one of those is more likely a genuine reachability question worth +-- investigating -- don't assume either way without checking the specific +-- (current, assigned) pair against dump_fsm_edges() and pgautofailover.fsm. SELECT k.current_state, k.assigned_state FROM keeper_fsm_edges k - WHERE NOT EXISTS ( + WHERE k.assigned_state <> 'dropped' + AND NOT EXISTS ( SELECT 1 FROM pgautofailover.dump_fsm_edges() e WHERE e.current_state = k.current_state From 7b6b096e2f77de1a1c4de90c5003522b85c996ac Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 15:40:20 +0200 Subject: [PATCH 18/52] monitor+keeper: keep ANY_STATE as a literal 'any' sentinel, match it in SQL KeeperFSMToJSON() previously expanded a KeeperFSM[] row's ANY_STATE wildcard into one concrete edge per entry of a hand-maintained AllRealNodeStates C list. That has two problems: it silently under-covers the real wildcard semantics the moment the list drifts out of sync with the NodeState enum (a false negative nothing would ever catch), and it explodes a single keeper rule into 21 separate rows for every downstream comparison, which is exactly what forced keeper_fsm_edges.sql's previous 'assigned_state <> dropped' filter. Emit the literal string 'any' instead, and do the wildcard matching in SQL: - check_fsm_reachability(): CASE WHEN k.current = 'any' THEN true ELSE k.current::replication_state = e.current_state END -- CASE reliably short-circuits so 'any' never hits the enum cast, while every other value (including real typos) still fails loudly. - keeper_fsm_edges.sql: current_state becomes text (not the enum, since 'any' isn't a legal value); Step 1 still round-trips every non-'any' value through the enum for the same fail-loudly guarantee. Step 2a matches 'any' against every current_state. Step 2b matches it existentially, which collapses the old 21-row 'any -> dropped' explosion into a single row and removes the need for the blanket dropped filter entirely. Regenerated keeper_fsm_edges.json from the rebuilt binary and the matching expected output. Full regress (19) + isolation (6) suites pass; check_fsm_reachability's 'unrecognized state name fails loudly' case still holds. --- src/bin/pg_autoctl/fsm.c | 98 +++------ src/monitor/expected/keeper_fsm_edges.out | 234 +++++++++++----------- src/monitor/keeper_fsm_edges.json | 164 +-------------- src/monitor/pgautofailover.sql | 19 +- src/monitor/sql/keeper_fsm_edges.sql | 79 +++++--- 5 files changed, 205 insertions(+), 389 deletions(-) diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index 9d74ceafe..8ffe189fd 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -1318,46 +1318,13 @@ print_fsm_for_graphviz(void) } -/* - * AllRealNodeStates is every "real" (non-sentinel) NodeState a keeper can - * genuinely report or be assigned, used only to expand a KeeperFSMTransition - * row's ANY_STATE wildcard (see KeeperFSMToJSON's own comment) into concrete - * states -- NO_STATE (the array terminator) and ANY_STATE itself excluded. - */ -static const NodeState AllRealNodeStates[] = { - INIT_STATE, - SINGLE_STATE, - PRIMARY_STATE, - WAIT_PRIMARY_STATE, - WAIT_STANDBY_STATE, - DEMOTED_STATE, - DEMOTE_TIMEOUT_STATE, - DRAINING_STATE, - SECONDARY_STATE, - CATCHINGUP_STATE, - PREP_PROMOTION_STATE, - STOP_REPLICATION_STATE, - MAINTENANCE_STATE, - JOIN_PRIMARY_STATE, - APPLY_SETTINGS_STATE, - PREPARE_MAINTENANCE_STATE, - WAIT_MAINTENANCE_STATE, - REPORT_LSN_STATE, - FAST_FORWARD_STATE, - JOIN_SECONDARY_STATE, - DROPPED_STATE -}; - -#define ALL_REAL_NODE_STATES_COUNT \ - ((int) (sizeof(AllRealNodeStates) / sizeof(AllRealNodeStates[0]))) - - /* * KeeperFSMToJSONAppendEdge appends one {"current": ..., "assigned": ...} - * object to array for a single, already-concrete (current, assigned) pair. - * Factored out of KeeperFSMToJSON so its own ANY_STATE-expansion loop (see - * that function's comment) can call it once per expanded state instead of - * duplicating the object-building code. + * object to array for a single KeeperFSMTransition row. current is rendered + * as the literal string "any" for ANY_STATE (state_matches()'s wildcard, + * e.g. fsm.c's "drop node from any state" rows) instead of + * NodeStateToString(ANY_STATE)'s own "#any state#" -- see KeeperFSMToJSON's + * own comment for why this sentinel exists and how the SQL side matches it. */ static void KeeperFSMToJSONAppendEdge(JSON_Array *array, NodeState current, NodeState assigned) @@ -1365,7 +1332,8 @@ KeeperFSMToJSONAppendEdge(JSON_Array *array, NodeState current, NodeState assign JSON_Value *jsEntry = json_value_init_object(); JSON_Object *jsObj = json_value_get_object(jsEntry); - json_object_set_string(jsObj, "current", NodeStateToString(current)); + json_object_set_string(jsObj, "current", + current == ANY_STATE ? "any" : NodeStateToString(current)); json_object_set_string(jsObj, "assigned", NodeStateToString(assigned)); json_array_append_value(array, jsEntry); @@ -1374,25 +1342,28 @@ KeeperFSMToJSONAppendEdge(JSON_Array *array, NodeState current, NodeState assign /* * KeeperFSMToJSON serializes KeeperFSM[] into a JSON array of - * {"current": ..., "assigned": ...} objects -- one per concrete - * (current, assigned) edge a KeeperFSMTransition row produces, walked the - * same way print_fsm_for_graphviz/print_reachable_states already do. This - * is the keeper-side half of the monitor/keeper FSM reachability - * cross-check: "pg_autoctl inspect fsm check" sends this verbatim to the - * monitor's pgautofailover.check_fsm_reachability(jsonb), which anti-joins - * it against every edge the monitor's own MonitorFSM[] table - * (group_state_machine.c) can produce (see dump_fsm_edges()'s own comment - * there) and reports any monitor transition with no matching entry here. + * {"current": ..., "assigned": ...} objects -- one per KeeperFSMTransition + * row, walked the same way print_fsm_for_graphviz/print_reachable_states + * already do. This is the keeper-side half of the monitor/keeper FSM + * reachability cross-check: "pg_autoctl inspect fsm check" sends this + * verbatim to the monitor's pgautofailover.check_fsm_reachability(jsonb), + * which anti-joins it against every edge the monitor's own MonitorFSM[] + * table (group_state_machine.c) can produce (see dump_fsm_edges()'s own + * comment there) and reports any monitor transition with no matching entry + * here. * - * A row's own .current can be ANY_STATE (state_matches()'s wildcard, e.g. - * fsm.c's "drop node from any state" rows) -- NodeStateToString(ANY_STATE) - * returns the literal string "#any state#", which is not a valid - * pgautofailover.replication_state and would fail check_fsm_reachability()'s - * own cast loudly (confirmed: this is exactly what happened the first time - * this function ran for real, against a live monitor+keeper pair). Expanded - * here into one concrete edge per AllRealNodeStates entry instead, mirroring - * NodeStatePatternResolveFromStates' own ANY-kind handling on the monitor - * side. .assigned is never ANY_STATE in any current KeeperFSM[] row (an + * A row's own .current can be ANY_STATE -- rendered here as the literal + * sentinel "any" (see KeeperFSMToJSONAppendEdge), not expanded into one + * edge per concrete state: an earlier version of this function expanded it + * against a hand-maintained list of "every real NodeState", which silently + * under-covers the real ANY_STATE semantics the moment that list drifts out + * of sync with the actual NodeState enum -- a false negative baked + * permanently into the fixture, with nothing to ever catch it. Emitting the + * wildcard literally instead lets the SQL comparison (check_fsm_reachability(), + * and this project's own keeper_fsm_edges.sql regress test) match it against + * whatever current_state values pgautofailover.dump_fsm_edges() actually + * produces, so it can never drift out of sync with the real state universe. + * .assigned is never ANY_STATE in any current KeeperFSM[] row (an * assignment target wildcard has no sensible meaning), so it's passed * through as-is. * @@ -1416,18 +1387,7 @@ KeeperFSMToJSON(void) while (transition.current != NO_STATE) { - if (transition.current == ANY_STATE) - { - for (int i = 0; i < ALL_REAL_NODE_STATES_COUNT; i++) - { - KeeperFSMToJSONAppendEdge(array, AllRealNodeStates[i], - transition.assigned); - } - } - else - { - KeeperFSMToJSONAppendEdge(array, transition.current, transition.assigned); - } + KeeperFSMToJSONAppendEdge(array, transition.current, transition.assigned); transition = KeeperFSM[++transitionIndex]; } diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 2d78c863b..388f9fee7 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -18,33 +18,80 @@ -- distinct keeper edge, so this test's own expected/keeper_fsm_edges.out -- shows the whole keeper FSM line by line, human-reviewable, with a diff on -- every change to KeeperFSM[] -- the same discipline fsm.sql's own dump --- already gives MonitorFSM[]. DISTINCT: KeeperFSMToJSON()'s ANY_STATE --- expansion (see its own comment, fsm.c) can make two different --- KeeperFSM[] rows resolve to the exact same (current, assigned) pair -- --- the JSON has no per-row provenance to tell such duplicates apart by, so --- they carry no extra information here and would only clutter the --- reviewable list. +-- already gives MonitorFSM[]. +-- +-- current_state is plain text, not pgautofailover.replication_state: a row +-- whose real KeeperFSM[] .current is ANY_STATE (state_matches()'s wildcard) +-- is serialized by KeeperFSMToJSON() as the literal string "any" (see its +-- own comment, fsm.c), which is not a legal enum value by design -- it's a +-- sentinel Step 2a/2b below match structurally, not a real reported state. +-- Every other value is still round-tripped through the enum (CASE ... ELSE +-- ... ::pgautofailover.replication_state ... END) so a typo'd or unrecognized +-- state name in the fixture still fails loudly here, same as +-- check_fsm_reachability()'s own cast does for the live-cluster path. +-- +-- DISTINCT: nothing in KeeperFSM[] guarantees two different rows can't +-- resolve to the exact same (current, assigned) pair, and the JSON has no +-- per-row provenance to tell such duplicates apart by, so they carry no +-- extra information here and would only clutter the reviewable list. \set keeper_json `cat keeper_fsm_edges.json` CREATE TABLE keeper_fsm_edges AS SELECT DISTINCT - (edge ->> 'current')::pgautofailover.replication_state AS current_state, + CASE WHEN (edge ->> 'current') = 'any' + THEN 'any' + ELSE ((edge ->> 'current')::pgautofailover.replication_state)::text + END AS current_state, (edge ->> 'assigned')::pgautofailover.replication_state AS assigned_state FROM jsonb_array_elements(:'keeper_json'::jsonb) AS edge; SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; current_state | assigned_state ---------------------+--------------------- + any | dropped + apply_settings | single + apply_settings | wait_primary + apply_settings | primary + apply_settings | draining + apply_settings | demote_timeout + apply_settings | demoted + apply_settings | join_primary + catchingup | single + catchingup | secondary + catchingup | prepare_promotion + catchingup | maintenance + catchingup | wait_maintenance + catchingup | report_lsn + demote_timeout | single + demote_timeout | primary + demote_timeout | demoted + demoted | single + demoted | catchingup + demoted | report_lsn + draining | single + draining | demote_timeout + draining | demoted + draining | report_lsn + dropped | single + dropped | wait_standby + dropped | report_lsn + fast_forward | prepare_promotion init | single init | wait_standby init | report_lsn - init | dropped - single | wait_primary - single | dropped - wait_primary | single - wait_primary | primary - wait_primary | demoted - wait_primary | join_primary - wait_primary | apply_settings - wait_primary | dropped + join_primary | single + join_primary | wait_primary + join_primary | primary + join_primary | draining + join_primary | demote_timeout + join_primary | demoted + join_secondary | secondary + maintenance | catchingup + maintenance | report_lsn + prepare_maintenance | catchingup + prepare_maintenance | maintenance + prepare_maintenance | report_lsn + prepare_promotion | single + prepare_promotion | wait_primary + prepare_promotion | stop_replication primary | single primary | wait_primary primary | draining @@ -54,27 +101,11 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; primary | join_primary primary | apply_settings primary | prepare_maintenance - primary | dropped - draining | single - draining | demote_timeout - draining | demoted - draining | report_lsn - draining | dropped - demote_timeout | single - demote_timeout | primary - demote_timeout | demoted - demote_timeout | dropped - demoted | single - demoted | catchingup - demoted | report_lsn - demoted | dropped - catchingup | single - catchingup | secondary - catchingup | prepare_promotion - catchingup | maintenance - catchingup | wait_maintenance - catchingup | report_lsn - catchingup | dropped + report_lsn | single + report_lsn | secondary + report_lsn | prepare_promotion + report_lsn | fast_forward + report_lsn | join_secondary secondary | single secondary | catchingup secondary | prepare_promotion @@ -82,55 +113,17 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; secondary | maintenance secondary | wait_maintenance secondary | report_lsn - secondary | dropped - prepare_promotion | single - prepare_promotion | wait_primary - prepare_promotion | stop_replication - prepare_promotion | dropped + single | wait_primary stop_replication | single stop_replication | wait_primary - stop_replication | dropped - wait_standby | catchingup - wait_standby | dropped - maintenance | catchingup - maintenance | report_lsn - maintenance | dropped - join_primary | single - join_primary | wait_primary - join_primary | primary - join_primary | draining - join_primary | demote_timeout - join_primary | demoted - join_primary | dropped - apply_settings | single - apply_settings | wait_primary - apply_settings | primary - apply_settings | draining - apply_settings | demote_timeout - apply_settings | demoted - apply_settings | join_primary - apply_settings | dropped - prepare_maintenance | catchingup - prepare_maintenance | maintenance - prepare_maintenance | report_lsn - prepare_maintenance | dropped wait_maintenance | maintenance - wait_maintenance | dropped - report_lsn | single - report_lsn | secondary - report_lsn | prepare_promotion - report_lsn | fast_forward - report_lsn | join_secondary - report_lsn | dropped - fast_forward | prepare_promotion - fast_forward | dropped - join_secondary | secondary - join_secondary | dropped - dropped | single - dropped | wait_standby - dropped | report_lsn - dropped | dropped -(97 rows) + wait_primary | single + wait_primary | primary + wait_primary | demoted + wait_primary | join_primary + wait_primary | apply_settings + wait_standby | catchingup +(77 rows) -- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A @@ -140,14 +133,19 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- project's own investigation of these mismatches (dump_fsm_edges()'s own -- comment, group_state_machine.c, and the design doc) for which of them -- are genuine keeper gaps versus artifacts already excluded upstream. +-- +-- k.current_state = 'any' matches every e.current_state -- a keeper row +-- covering every current state also covers this specific one, so it counts +-- as a match here exactly like a literal (e.current_state, e.assigned_state) +-- row would. SELECT e.pos, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos WHERE NOT EXISTS ( SELECT 1 FROM keeper_fsm_edges k - WHERE k.current_state = e.current_state - AND k.assigned_state = e.assigned_state + WHERE k.assigned_state = e.assigned_state + AND (k.current_state = 'any' OR k.current_state = e.current_state::text) ) ORDER BY e.pos, e.current_state; pos | current_state | assigned_state | comment @@ -297,55 +295,51 @@ SELECT e.pos, e.current_state, e.assigned_state, f.comment -- gap on the monitor side, same "investigate before assuming which" caveat -- as step 2a's own comment. -- --- assigned_state <> 'dropped' excludes a known, 100%-explained artifact, --- not a real gap: KeeperFSM[]'s two ANY_STATE -> DROPPED rows (fsm.c) get --- expanded by KeeperFSMToJSON() into one edge per concrete current_state --- (21 of them), but the monitor's own equivalent (remove_node(), pos --- 101/103) lives entirely in the api_triggered section, which --- dump_fsm_edges() deliberately excludes (see its own comment) since those --- rows resolve their target via hand-written C, not a NodeStatePattern -- --- so dump_fsm_edges() can never produce a single edge assigning 'dropped', --- by construction, regardless of current_state. Filtering these out here --- avoids drowning the rows below in guaranteed noise. --- --- The rows that remain still need the same per-row judgment as step 2a: --- some of their target states (e.g. maintenance, prepare_maintenance, --- wait_standby, join_primary, wait_maintenance) are ALSO only reachable --- through api_triggered rows and so are equally artifacts of that same --- exclusion; others (e.g. primary, catchingup, prepare_promotion) do have --- some non-api_triggered coverage in dump_fsm_edges(), so a gap against --- one of those is more likely a genuine reachability question worth --- investigating -- don't assume either way without checking the specific --- (current, assigned) pair against dump_fsm_edges() and pgautofailover.fsm. +-- For a k.current_state = 'any' row, "matches" is existential: at least one +-- current_state for which the monitor assigns this same target is enough +-- to say the target is implementable at all, so a gap here means the +-- monitor can NEVER produce this assigned_state from ANY current state -- +-- e.g. "any -> dropped": KeeperFSM[]'s two ANY_STATE -> DROPPED rows +-- collapse to this single row (see Step 1's own comment on why "any" isn't +-- expanded per-state anymore -- the previous per-state expansion turned +-- this one keeper rule into 21 separate flagged rows here, a lot of +-- redundant noise for a single underlying fact), and it stays flagged +-- because dump_fsm_edges() really can never produce a 'dropped' edge at +-- all: the monitor's own equivalent (remove_node(), pos 101/103) lives +-- entirely in the api_triggered section, which dump_fsm_edges() +-- deliberately excludes (see its own comment) since those rows resolve +-- their target via hand-written C, not a NodeStatePattern. Same +-- "investigate before assuming which" caveat as the rest of this file +-- applies to every row below, "any" or not. SELECT k.current_state, k.assigned_state FROM keeper_fsm_edges k - WHERE k.assigned_state <> 'dropped' - AND NOT EXISTS ( + WHERE NOT EXISTS ( SELECT 1 FROM pgautofailover.dump_fsm_edges() e - WHERE e.current_state = k.current_state - AND e.assigned_state = k.assigned_state + WHERE e.assigned_state = k.assigned_state + AND (k.current_state = 'any' OR k.current_state = e.current_state::text) ) ORDER BY k.current_state, k.assigned_state; current_state | assigned_state ---------------------+--------------------- + any | dropped + apply_settings | join_primary + catchingup | prepare_promotion + catchingup | maintenance + catchingup | wait_maintenance + demote_timeout | primary + dropped | wait_standby init | wait_standby - wait_primary | join_primary - wait_primary | apply_settings + maintenance | catchingup + prepare_maintenance | catchingup primary | maintenance primary | join_primary primary | prepare_maintenance - demote_timeout | primary - catchingup | prepare_promotion - catchingup | maintenance - catchingup | wait_maintenance secondary | wait_standby secondary | maintenance secondary | wait_maintenance - maintenance | catchingup - apply_settings | join_primary - prepare_maintenance | catchingup - dropped | wait_standby -(17 rows) + wait_primary | join_primary + wait_primary | apply_settings +(18 rows) DROP TABLE keeper_fsm_edges; diff --git a/src/monitor/keeper_fsm_edges.json b/src/monitor/keeper_fsm_edges.json index 58d3c2a35..2eca675ce 100644 --- a/src/monitor/keeper_fsm_edges.json +++ b/src/monitor/keeper_fsm_edges.json @@ -392,171 +392,11 @@ "assigned": "report_lsn" }, { - "current": "init", - "assigned": "dropped" - }, - { - "current": "single", - "assigned": "dropped" - }, - { - "current": "primary", - "assigned": "dropped" - }, - { - "current": "wait_primary", - "assigned": "dropped" - }, - { - "current": "wait_standby", - "assigned": "dropped" - }, - { - "current": "demoted", - "assigned": "dropped" - }, - { - "current": "demote_timeout", - "assigned": "dropped" - }, - { - "current": "draining", - "assigned": "dropped" - }, - { - "current": "secondary", - "assigned": "dropped" - }, - { - "current": "catchingup", - "assigned": "dropped" - }, - { - "current": "prepare_promotion", - "assigned": "dropped" - }, - { - "current": "stop_replication", - "assigned": "dropped" - }, - { - "current": "maintenance", - "assigned": "dropped" - }, - { - "current": "join_primary", - "assigned": "dropped" - }, - { - "current": "apply_settings", - "assigned": "dropped" - }, - { - "current": "prepare_maintenance", - "assigned": "dropped" - }, - { - "current": "wait_maintenance", - "assigned": "dropped" - }, - { - "current": "report_lsn", - "assigned": "dropped" - }, - { - "current": "fast_forward", - "assigned": "dropped" - }, - { - "current": "join_secondary", - "assigned": "dropped" - }, - { - "current": "dropped", + "current": "any", "assigned": "dropped" }, { - "current": "init", - "assigned": "dropped" - }, - { - "current": "single", - "assigned": "dropped" - }, - { - "current": "primary", - "assigned": "dropped" - }, - { - "current": "wait_primary", - "assigned": "dropped" - }, - { - "current": "wait_standby", - "assigned": "dropped" - }, - { - "current": "demoted", - "assigned": "dropped" - }, - { - "current": "demote_timeout", - "assigned": "dropped" - }, - { - "current": "draining", - "assigned": "dropped" - }, - { - "current": "secondary", - "assigned": "dropped" - }, - { - "current": "catchingup", - "assigned": "dropped" - }, - { - "current": "prepare_promotion", - "assigned": "dropped" - }, - { - "current": "stop_replication", - "assigned": "dropped" - }, - { - "current": "maintenance", - "assigned": "dropped" - }, - { - "current": "join_primary", - "assigned": "dropped" - }, - { - "current": "apply_settings", - "assigned": "dropped" - }, - { - "current": "prepare_maintenance", - "assigned": "dropped" - }, - { - "current": "wait_maintenance", - "assigned": "dropped" - }, - { - "current": "report_lsn", - "assigned": "dropped" - }, - { - "current": "fast_forward", - "assigned": "dropped" - }, - { - "current": "join_secondary", - "assigned": "dropped" - }, - { - "current": "dropped", + "current": "any", "assigned": "dropped" } ] diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index 321772cab..8545bf38c 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -345,9 +345,16 @@ grant execute on function pgautofailover.dump_fsm_edges() to autoctl_node; -- result means every transition the monitor can ever assign has somewhere -- for the keeper to go. keeper_edges is expected to be a jsonb array of -- {"current": ..., "assigned": ...} objects, one per KeeperFSMTransition --- row; casting both fields to pgautofailover.replication_state means a --- keeper reporting a state name this enum doesn't recognize fails loudly, --- with a real cast error, rather than silently never matching. +-- row. "current" can be the literal string "any" (KeeperFSMToJSON()'s own +-- sentinel for a row whose real .current is ANY_STATE): matched against +-- every e.current_state without ever casting it to +-- pgautofailover.replication_state, via a CASE (not "k.current = 'any' OR +-- k.current::...= e.current_state", which does not reliably guarantee the +-- cast is skipped once the left side matches -- CASE WHEN/THEN is the only +-- construct Postgres guarantees short-circuits). Any other current value, +-- and "assigned" always, still go through the enum cast unconditionally -- +-- a keeper reporting a state name this enum doesn't recognize still fails +-- loudly, with a real cast error, rather than silently never matching. CREATE FUNCTION pgautofailover.check_fsm_reachability(keeper_edges jsonb) RETURNS TABLE ( @@ -364,8 +371,10 @@ AS $$ WHERE NOT EXISTS ( SELECT 1 FROM jsonb_to_recordset(keeper_edges) AS k(current text, assigned text) - WHERE k.current::pgautofailover.replication_state = e.current_state - AND k.assigned::pgautofailover.replication_state = e.assigned_state) + WHERE k.assigned::pgautofailover.replication_state = e.assigned_state + AND CASE WHEN k.current = 'any' THEN true + ELSE k.current::pgautofailover.replication_state = e.current_state + END) ORDER BY e.pos; $$; diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index b7aafc634..b9f10937a 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -18,17 +18,30 @@ -- distinct keeper edge, so this test's own expected/keeper_fsm_edges.out -- shows the whole keeper FSM line by line, human-reviewable, with a diff on -- every change to KeeperFSM[] -- the same discipline fsm.sql's own dump --- already gives MonitorFSM[]. DISTINCT: KeeperFSMToJSON()'s ANY_STATE --- expansion (see its own comment, fsm.c) can make two different --- KeeperFSM[] rows resolve to the exact same (current, assigned) pair -- --- the JSON has no per-row provenance to tell such duplicates apart by, so --- they carry no extra information here and would only clutter the --- reviewable list. +-- already gives MonitorFSM[]. +-- +-- current_state is plain text, not pgautofailover.replication_state: a row +-- whose real KeeperFSM[] .current is ANY_STATE (state_matches()'s wildcard) +-- is serialized by KeeperFSMToJSON() as the literal string "any" (see its +-- own comment, fsm.c), which is not a legal enum value by design -- it's a +-- sentinel Step 2a/2b below match structurally, not a real reported state. +-- Every other value is still round-tripped through the enum (CASE ... ELSE +-- ... ::pgautofailover.replication_state ... END) so a typo'd or unrecognized +-- state name in the fixture still fails loudly here, same as +-- check_fsm_reachability()'s own cast does for the live-cluster path. +-- +-- DISTINCT: nothing in KeeperFSM[] guarantees two different rows can't +-- resolve to the exact same (current, assigned) pair, and the JSON has no +-- per-row provenance to tell such duplicates apart by, so they carry no +-- extra information here and would only clutter the reviewable list. \set keeper_json `cat keeper_fsm_edges.json` CREATE TABLE keeper_fsm_edges AS SELECT DISTINCT - (edge ->> 'current')::pgautofailover.replication_state AS current_state, + CASE WHEN (edge ->> 'current') = 'any' + THEN 'any' + ELSE ((edge ->> 'current')::pgautofailover.replication_state)::text + END AS current_state, (edge ->> 'assigned')::pgautofailover.replication_state AS assigned_state FROM jsonb_array_elements(:'keeper_json'::jsonb) AS edge; @@ -42,14 +55,19 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- project's own investigation of these mismatches (dump_fsm_edges()'s own -- comment, group_state_machine.c, and the design doc) for which of them -- are genuine keeper gaps versus artifacts already excluded upstream. +-- +-- k.current_state = 'any' matches every e.current_state -- a keeper row +-- covering every current state also covers this specific one, so it counts +-- as a match here exactly like a literal (e.current_state, e.assigned_state) +-- row would. SELECT e.pos, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos WHERE NOT EXISTS ( SELECT 1 FROM keeper_fsm_edges k - WHERE k.current_state = e.current_state - AND k.assigned_state = e.assigned_state + WHERE k.assigned_state = e.assigned_state + AND (k.current_state = 'any' OR k.current_state = e.current_state::text) ) ORDER BY e.pos, e.current_state; @@ -62,34 +80,29 @@ SELECT e.pos, e.current_state, e.assigned_state, f.comment -- gap on the monitor side, same "investigate before assuming which" caveat -- as step 2a's own comment. -- --- assigned_state <> 'dropped' excludes a known, 100%-explained artifact, --- not a real gap: KeeperFSM[]'s two ANY_STATE -> DROPPED rows (fsm.c) get --- expanded by KeeperFSMToJSON() into one edge per concrete current_state --- (21 of them), but the monitor's own equivalent (remove_node(), pos --- 101/103) lives entirely in the api_triggered section, which --- dump_fsm_edges() deliberately excludes (see its own comment) since those --- rows resolve their target via hand-written C, not a NodeStatePattern -- --- so dump_fsm_edges() can never produce a single edge assigning 'dropped', --- by construction, regardless of current_state. Filtering these out here --- avoids drowning the rows below in guaranteed noise. --- --- The rows that remain still need the same per-row judgment as step 2a: --- some of their target states (e.g. maintenance, prepare_maintenance, --- wait_standby, join_primary, wait_maintenance) are ALSO only reachable --- through api_triggered rows and so are equally artifacts of that same --- exclusion; others (e.g. primary, catchingup, prepare_promotion) do have --- some non-api_triggered coverage in dump_fsm_edges(), so a gap against --- one of those is more likely a genuine reachability question worth --- investigating -- don't assume either way without checking the specific --- (current, assigned) pair against dump_fsm_edges() and pgautofailover.fsm. +-- For a k.current_state = 'any' row, "matches" is existential: at least one +-- current_state for which the monitor assigns this same target is enough +-- to say the target is implementable at all, so a gap here means the +-- monitor can NEVER produce this assigned_state from ANY current state -- +-- e.g. "any -> dropped": KeeperFSM[]'s two ANY_STATE -> DROPPED rows +-- collapse to this single row (see Step 1's own comment on why "any" isn't +-- expanded per-state anymore -- the previous per-state expansion turned +-- this one keeper rule into 21 separate flagged rows here, a lot of +-- redundant noise for a single underlying fact), and it stays flagged +-- because dump_fsm_edges() really can never produce a 'dropped' edge at +-- all: the monitor's own equivalent (remove_node(), pos 101/103) lives +-- entirely in the api_triggered section, which dump_fsm_edges() +-- deliberately excludes (see its own comment) since those rows resolve +-- their target via hand-written C, not a NodeStatePattern. Same +-- "investigate before assuming which" caveat as the rest of this file +-- applies to every row below, "any" or not. SELECT k.current_state, k.assigned_state FROM keeper_fsm_edges k - WHERE k.assigned_state <> 'dropped' - AND NOT EXISTS ( + WHERE NOT EXISTS ( SELECT 1 FROM pgautofailover.dump_fsm_edges() e - WHERE e.current_state = k.current_state - AND e.assigned_state = k.assigned_state + WHERE e.assigned_state = k.assigned_state + AND (k.current_state = 'any' OR k.current_state = e.current_state::text) ) ORDER BY k.current_state, k.assigned_state; From 3b126342e8be980f5f77a4d16a39e93d9f6b2de6 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 15:52:35 +0200 Subject: [PATCH 19/52] monitor: add per-rule subtotal rows to keeper_fsm_edges.sql Step 2a Step 2a's 134-row gap list collapses to just 10 distinct MonitorFSM[] rules (renamed pos -> rule in the select list), each fanning out across many current_states via a broad NodeStatePattern. Add a GROUPING SETS summary row per (rule, assigned_state, comment) showing the fan-out count (n), ordered first via NULLS FIRST so each rule's own detail rows follow its own header. Makes the shape of the gap list ('10 broad rules, not 134 independent problems') visible without counting detail rows by hand. --- src/monitor/expected/keeper_fsm_edges.out | 300 ++++++++++++---------- src/monitor/sql/keeper_fsm_edges.sql | 16 +- 2 files changed, 175 insertions(+), 141 deletions(-) diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 388f9fee7..e03dd0469 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -138,7 +138,15 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- covering every current state also covers this specific one, so it counts -- as a match here exactly like a literal (e.current_state, e.assigned_state) -- row would. -SELECT e.pos, e.current_state, e.assigned_state, f.comment +-- +-- GROUPING SETS adds one summary row per rule (pos, assigned_state, comment) +-- -- current_state NULL, n = how many current_states that single +-- MonitorFSM[] rule fans out to -- alongside the ordinary per-current_state +-- detail rows, so a rule using a broad NodeStatePattern (matching many +-- states at once) is immediately visible as one big number instead of +-- having to count its own detail rows by hand. NULLS FIRST puts each rule's +-- summary row right before its own detail rows, as a header. +SELECT e.pos AS rule, e.current_state, e.assigned_state, f.comment, count(*) AS n FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos WHERE NOT EXISTS ( @@ -147,144 +155,158 @@ SELECT e.pos, e.current_state, e.assigned_state, f.comment WHERE k.assigned_state = e.assigned_state AND (k.current_state = 'any' OR k.current_state = e.current_state::text) ) - ORDER BY e.pos, e.current_state; - pos | current_state | assigned_state | comment ------+---------------------+----------------+---------------------------------------------------------------------------------------------------------------------- - 209 | wait_standby | single | alone in group, candidate-eligible -> single - 209 | maintenance | single | alone in group, candidate-eligible -> single - 209 | prepare_maintenance | single | alone in group, candidate-eligible -> single - 209 | wait_maintenance | single | alone in group, candidate-eligible -> single - 209 | fast_forward | single | alone in group, candidate-eligible -> single - 209 | join_secondary | single | alone in group, candidate-eligible -> single - 211 | wait_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | primary | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | demote_timeout | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | prepare_promotion | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | wait_standby | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | join_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | apply_settings | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | wait_maintenance | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | fast_forward | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | join_secondary | report_lsn | alone in group, candidatePriority zero -> report_lsn - 303 | init | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | draining | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | demote_timeout | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | demoted | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | catchingup | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | wait_standby | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | prepare_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | wait_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | report_lsn | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | fast_forward | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | join_secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | dropped | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 325 | init | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | single | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | demote_timeout | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | demoted | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | catchingup | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | prepare_promotion | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | stop_replication | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | wait_standby | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | prepare_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | wait_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | report_lsn | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | fast_forward | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | join_secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | dropped | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 333 | init | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | single | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | catchingup | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | prepare_promotion | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | stop_replication | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | wait_standby | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | prepare_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | wait_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | report_lsn | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | fast_forward | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | join_secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | dropped | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 339 | init | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | single | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | demoted | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | catchingup | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | prepare_promotion | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | stop_replication | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | wait_standby | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | prepare_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | wait_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | report_lsn | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | fast_forward | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | join_secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | dropped | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 347 | init | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | single | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | catchingup | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | prepare_promotion | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | stop_replication | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | wait_standby | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | prepare_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | wait_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | report_lsn | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | fast_forward | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | join_secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | dropped | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 349 | init | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | single | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | catchingup | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | prepare_promotion | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | stop_replication | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | wait_standby | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | prepare_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | wait_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | report_lsn | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | fast_forward | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | join_secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | dropped | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 351 | init | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | single | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | catchingup | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | prepare_promotion | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | stop_replication | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | wait_standby | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | prepare_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | wait_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | report_lsn | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | fast_forward | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | join_secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 391 | init | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | demote_timeout | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | demoted | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | catchingup | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | prepare_promotion | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | stop_replication | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | wait_standby | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | prepare_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | wait_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | report_lsn | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | fast_forward | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | join_secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | dropped | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining -(134 rows) + GROUP BY GROUPING SETS ( + (e.pos, e.assigned_state, f.comment, e.current_state), + (e.pos, e.assigned_state, f.comment) + ) + ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; + rule | current_state | assigned_state | comment | n +------+---------------------+----------------+----------------------------------------------------------------------------------------------------------------------+---- + 209 | | single | alone in group, candidate-eligible -> single | 6 + 209 | wait_standby | single | alone in group, candidate-eligible -> single | 1 + 209 | maintenance | single | alone in group, candidate-eligible -> single | 1 + 209 | prepare_maintenance | single | alone in group, candidate-eligible -> single | 1 + 209 | wait_maintenance | single | alone in group, candidate-eligible -> single | 1 + 209 | fast_forward | single | alone in group, candidate-eligible -> single | 1 + 209 | join_secondary | single | alone in group, candidate-eligible -> single | 1 + 211 | | report_lsn | alone in group, candidatePriority zero -> report_lsn | 11 + 211 | wait_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 + 211 | primary | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 + 211 | demote_timeout | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 + 211 | prepare_promotion | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 + 211 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 + 211 | wait_standby | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 + 211 | join_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 + 211 | apply_settings | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 + 211 | wait_maintenance | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 + 211 | fast_forward | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 + 211 | join_secondary | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 + 303 | | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 14 + 303 | init | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | draining | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | demote_timeout | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | demoted | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | catchingup | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | wait_standby | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | prepare_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | wait_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | report_lsn | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | fast_forward | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | join_secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 303 | dropped | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 + 325 | | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 16 + 325 | init | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | single | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | demote_timeout | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | demoted | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | catchingup | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | prepare_promotion | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | stop_replication | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | wait_standby | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | prepare_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | wait_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | report_lsn | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | fast_forward | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | join_secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 325 | dropped | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 + 333 | | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 14 + 333 | init | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | single | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | catchingup | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | prepare_promotion | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | stop_replication | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | wait_standby | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | prepare_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | wait_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | report_lsn | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | fast_forward | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | join_secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 333 | dropped | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 + 339 | | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 15 + 339 | init | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | single | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | demoted | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | catchingup | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | prepare_promotion | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | stop_replication | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | wait_standby | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | prepare_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | wait_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | report_lsn | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | fast_forward | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | join_secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 339 | dropped | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 + 347 | | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 14 + 347 | init | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | single | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | catchingup | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | prepare_promotion | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | stop_replication | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | wait_standby | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | prepare_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | wait_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | report_lsn | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | fast_forward | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | join_secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 347 | dropped | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 + 349 | | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 14 + 349 | init | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | single | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | catchingup | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | prepare_promotion | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | stop_replication | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | wait_standby | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | prepare_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | wait_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | report_lsn | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | fast_forward | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | join_secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 349 | dropped | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 + 351 | | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 14 + 351 | init | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | single | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | catchingup | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | prepare_promotion | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | stop_replication | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | wait_standby | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | prepare_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | wait_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | report_lsn | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | fast_forward | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | join_secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 351 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 + 391 | | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 16 + 391 | init | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | demote_timeout | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | demoted | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | catchingup | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | prepare_promotion | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | stop_replication | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | wait_standby | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | prepare_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | wait_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | report_lsn | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | fast_forward | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | join_secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + 391 | dropped | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 +(144 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index b9f10937a..67575401c 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -60,7 +60,15 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- covering every current state also covers this specific one, so it counts -- as a match here exactly like a literal (e.current_state, e.assigned_state) -- row would. -SELECT e.pos, e.current_state, e.assigned_state, f.comment +-- +-- GROUPING SETS adds one summary row per rule (pos, assigned_state, comment) +-- -- current_state NULL, n = how many current_states that single +-- MonitorFSM[] rule fans out to -- alongside the ordinary per-current_state +-- detail rows, so a rule using a broad NodeStatePattern (matching many +-- states at once) is immediately visible as one big number instead of +-- having to count its own detail rows by hand. NULLS FIRST puts each rule's +-- summary row right before its own detail rows, as a header. +SELECT e.pos AS rule, e.current_state, e.assigned_state, f.comment, count(*) AS n FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos WHERE NOT EXISTS ( @@ -69,7 +77,11 @@ SELECT e.pos, e.current_state, e.assigned_state, f.comment WHERE k.assigned_state = e.assigned_state AND (k.current_state = 'any' OR k.current_state = e.current_state::text) ) - ORDER BY e.pos, e.current_state; + GROUP BY GROUPING SETS ( + (e.pos, e.assigned_state, f.comment, e.current_state), + (e.pos, e.assigned_state, f.comment) + ) + ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the From be4d1630dc2044b42f4690957d66d58d4f95b1df Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 15:57:12 +0200 Subject: [PATCH 20/52] monitor: document Step 2a's 10-rule historical/precedent investigation Investigated each of the 10 rules behind Step 2a's 134-row gap list against the pre-refactor hand-written code (commit 9c9c9b9^): in every case the old code's own condition was already broad (a role/negation check or opaque NodeIsXxx() helper, never an enumerated state list), so the new NodeStatePattern's fan-out is a faithful translation, not a refactor-introduced widening. Real regression/tap-spec precedent (issues #997, #1168) exists for the 'obvious' current_state each rule targets, but none of the 10 is tested from one of the other fanned-out states -- a structural artifact of this check, not 10 separate bugs. Recorded so this doesn't need re-investigating next time the gap list is reviewed. --- src/monitor/sql/keeper_fsm_edges.sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 67575401c..9874d5507 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -68,6 +68,23 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- states at once) is immediately visible as one big number instead of -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. +-- +-- As of this writing the gap list is exactly 10 rules this way (134 detail +-- rows total), each an "alone in group"/failover/Citus-worker rule whose +-- NodeStatePattern is a role or predicate check (e.g. !IsCurrentState(...), +-- an opaque NodeIsXxx() helper, or no state restriction at all) rather than +-- an enumerated state list -- investigated rule by rule against the +-- pre-refactor hand-written code (commit 9c9c9b9^): in every one of the 10, +-- that breadth already existed before this refactor (this is a faithful, +-- behavior-preserving translation, not a widening introduced here). Real +-- regression/tap-spec precedent exists for the "obvious" current_state each +-- rule is clearly meant for (e.g. issue #997 for pos 303, issue #1168 for +-- pos 325/347/349's sibling branches), but none of the 10 has a test +-- exercising the transition from one of the other, more exotic fanned-out +-- current_states (dropped, fast_forward, join_secondary, and similar) -- +-- this is Step 2a's own structural artifact of enumerating a role/predicate +-- gate across every syntactically possible current_state, not a sign of 10 +-- separate functional bugs. SELECT e.pos AS rule, e.current_state, e.assigned_state, f.comment, count(*) AS n FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos From 660b00a67db6ddd2bdbfc0fd7f113e30e2d67c92 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 16:02:19 +0200 Subject: [PATCH 21/52] monitor: move keeper_fsm_edges Step 2a's n column right after rule --- src/monitor/expected/keeper_fsm_edges.out | 311 ++++++++++++---------- src/monitor/sql/keeper_fsm_edges.sql | 2 +- 2 files changed, 165 insertions(+), 148 deletions(-) diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index e03dd0469..4eb6fddc7 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -146,7 +146,24 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- states at once) is immediately visible as one big number instead of -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. -SELECT e.pos AS rule, e.current_state, e.assigned_state, f.comment, count(*) AS n +-- +-- As of this writing the gap list is exactly 10 rules this way (134 detail +-- rows total), each an "alone in group"/failover/Citus-worker rule whose +-- NodeStatePattern is a role or predicate check (e.g. !IsCurrentState(...), +-- an opaque NodeIsXxx() helper, or no state restriction at all) rather than +-- an enumerated state list -- investigated rule by rule against the +-- pre-refactor hand-written code (commit 9c9c9b9^): in every one of the 10, +-- that breadth already existed before this refactor (this is a faithful, +-- behavior-preserving translation, not a widening introduced here). Real +-- regression/tap-spec precedent exists for the "obvious" current_state each +-- rule is clearly meant for (e.g. issue #997 for pos 303, issue #1168 for +-- pos 325/347/349's sibling branches), but none of the 10 has a test +-- exercising the transition from one of the other, more exotic fanned-out +-- current_states (dropped, fast_forward, join_secondary, and similar) -- +-- this is Step 2a's own structural artifact of enumerating a role/predicate +-- gate across every syntactically possible current_state, not a sign of 10 +-- separate functional bugs. +SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos WHERE NOT EXISTS ( @@ -160,152 +177,152 @@ SELECT e.pos AS rule, e.current_state, e.assigned_state, f.comment, count(*) AS (e.pos, e.assigned_state, f.comment) ) ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; - rule | current_state | assigned_state | comment | n -------+---------------------+----------------+----------------------------------------------------------------------------------------------------------------------+---- - 209 | | single | alone in group, candidate-eligible -> single | 6 - 209 | wait_standby | single | alone in group, candidate-eligible -> single | 1 - 209 | maintenance | single | alone in group, candidate-eligible -> single | 1 - 209 | prepare_maintenance | single | alone in group, candidate-eligible -> single | 1 - 209 | wait_maintenance | single | alone in group, candidate-eligible -> single | 1 - 209 | fast_forward | single | alone in group, candidate-eligible -> single | 1 - 209 | join_secondary | single | alone in group, candidate-eligible -> single | 1 - 211 | | report_lsn | alone in group, candidatePriority zero -> report_lsn | 11 - 211 | wait_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 - 211 | primary | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 - 211 | demote_timeout | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 - 211 | prepare_promotion | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 - 211 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 - 211 | wait_standby | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 - 211 | join_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 - 211 | apply_settings | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 - 211 | wait_maintenance | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 - 211 | fast_forward | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 - 211 | join_secondary | report_lsn | alone in group, candidatePriority zero -> report_lsn | 1 - 303 | | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 14 - 303 | init | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | draining | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | demote_timeout | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | demoted | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | catchingup | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | wait_standby | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | prepare_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | wait_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | report_lsn | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | fast_forward | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | join_secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 303 | dropped | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary | 1 - 325 | | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 16 - 325 | init | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | single | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | demote_timeout | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | demoted | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | catchingup | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | prepare_promotion | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | stop_replication | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | wait_standby | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | prepare_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | wait_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | report_lsn | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | fast_forward | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | join_secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 325 | dropped | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) | 1 - 333 | | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 14 - 333 | init | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | single | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | catchingup | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | prepare_promotion | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | stop_replication | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | wait_standby | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | prepare_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | wait_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | report_lsn | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | fast_forward | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | join_secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 333 | dropped | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted | 1 - 339 | | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 15 - 339 | init | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | single | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | demoted | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | catchingup | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | prepare_promotion | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | stop_replication | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | wait_standby | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | prepare_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | wait_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | report_lsn | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | fast_forward | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | join_secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 339 | dropped | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) | 1 - 347 | | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 14 - 347 | init | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | single | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | catchingup | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | prepare_promotion | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | stop_replication | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | wait_standby | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | prepare_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | wait_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | report_lsn | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | fast_forward | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | join_secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 347 | dropped | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) | 1 - 349 | | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 14 - 349 | init | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | single | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | catchingup | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | prepare_promotion | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | stop_replication | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | wait_standby | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | prepare_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | wait_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | report_lsn | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | fast_forward | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | join_secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 349 | dropped | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) | 1 - 351 | | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 14 - 351 | init | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | single | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | catchingup | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | prepare_promotion | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | stop_replication | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | wait_standby | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | prepare_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | wait_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | report_lsn | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | fast_forward | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | join_secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 351 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted | 1 - 391 | | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 16 - 391 | init | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | demote_timeout | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | demoted | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | catchingup | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | prepare_promotion | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | stop_replication | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | wait_standby | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | prepare_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | wait_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | report_lsn | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | fast_forward | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | join_secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 - 391 | dropped | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining | 1 + rule | n | current_state | assigned_state | comment +------+----+---------------------+----------------+---------------------------------------------------------------------------------------------------------------------- + 209 | 6 | | single | alone in group, candidate-eligible -> single + 209 | 1 | wait_standby | single | alone in group, candidate-eligible -> single + 209 | 1 | maintenance | single | alone in group, candidate-eligible -> single + 209 | 1 | prepare_maintenance | single | alone in group, candidate-eligible -> single + 209 | 1 | wait_maintenance | single | alone in group, candidate-eligible -> single + 209 | 1 | fast_forward | single | alone in group, candidate-eligible -> single + 209 | 1 | join_secondary | single | alone in group, candidate-eligible -> single + 211 | 11 | | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | wait_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | primary | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | demote_timeout | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | prepare_promotion | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | wait_standby | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | join_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | apply_settings | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | wait_maintenance | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | fast_forward | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | join_secondary | report_lsn | alone in group, candidatePriority zero -> report_lsn + 303 | 14 | | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | init | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | draining | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | demote_timeout | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | demoted | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | catchingup | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | wait_standby | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | prepare_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | wait_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | report_lsn | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | fast_forward | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | join_secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 303 | 1 | dropped | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary + 325 | 16 | | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | init | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | single | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | demote_timeout | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | demoted | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | catchingup | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | prepare_promotion | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | stop_replication | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | wait_standby | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | prepare_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | wait_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | report_lsn | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | fast_forward | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | join_secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | dropped | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 333 | 14 | | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | init | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | single | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | catchingup | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | prepare_promotion | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | stop_replication | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | wait_standby | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | prepare_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | wait_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | report_lsn | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | fast_forward | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | join_secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 1 | dropped | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 339 | 15 | | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | init | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | single | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | demoted | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | catchingup | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | prepare_promotion | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | stop_replication | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | wait_standby | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | prepare_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | wait_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | report_lsn | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | fast_forward | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | join_secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 1 | dropped | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 347 | 14 | | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | init | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | single | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | catchingup | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | prepare_promotion | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | stop_replication | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | wait_standby | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | prepare_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | wait_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | report_lsn | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | fast_forward | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | join_secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 1 | dropped | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 349 | 14 | | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | init | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | single | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | catchingup | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | prepare_promotion | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | stop_replication | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | wait_standby | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | prepare_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | wait_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | report_lsn | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | fast_forward | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | join_secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 1 | dropped | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 351 | 14 | | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | init | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | single | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | catchingup | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | prepare_promotion | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | stop_replication | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | wait_standby | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | prepare_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | wait_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | report_lsn | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | fast_forward | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | join_secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 1 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 391 | 16 | | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | init | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | demote_timeout | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | demoted | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | catchingup | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | prepare_promotion | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | stop_replication | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | wait_standby | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | prepare_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | wait_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | report_lsn | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | fast_forward | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | join_secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | dropped | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining (144 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 9874d5507..a3f1c8bb1 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -85,7 +85,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- this is Step 2a's own structural artifact of enumerating a role/predicate -- gate across every syntactically possible current_state, not a sign of 10 -- separate functional bugs. -SELECT e.pos AS rule, e.current_state, e.assigned_state, f.comment, count(*) AS n +SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos WHERE NOT EXISTS ( From 147f216e3391f9d0efaf8a4d61aa5dc296d19d61 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 16:52:23 +0200 Subject: [PATCH 22/52] monitor: add pgaftest coverage for pos 211's keeper gap, document shadowing caveat tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf reproduces pos 211's real gap live: a lone priority-zero primary (node2, after node1 is dropped) gets assigned 'report_lsn' by the monitor but has no KeeperFSM[] transition from PRIMARY_STATE to reach it, confirmed via the exact fsm.c log_fatal firing every retry ('pg_autoctl does not know how to reach state "report_lsn" from "primary"'). Not added to any schedule (same convention as debug_failover_pg19.pgaf) since it's expected to fail once the missing transition is implemented. Attempting the equivalent for pos 209's 'maintenance' fanout state disproved it instead: pos 205 ('converged to maintenance -> no-op', unconditional on MAINTENANCE_STATE) intercepts every node_active() call from a maintenance node first, so pos 209 can never actually fire for that state -- confirmed live (monitor never re-assigned 'single'). dump_fsm_edges() resolves each row's edges independently and does not model this first-match-wins shadowing between rows, so it can report edges that are structurally unreachable in practice. Documented as a caveat in dump_fsm_edges()'s own comment for whoever investigates the remaining pos 209/211 fanned-out states next. --- src/monitor/group_state_machine.c | 31 +++++- ...per_fsm_gap_211_primary_priority_zero.pgaf | 101 ++++++++++++++++++ 2 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 4485c93ff..3a5387736 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -4372,11 +4372,32 @@ PG_FUNCTION_INFO_V1(dump_fsm_edges); * live run. * * Every other mismatch that same run found (early_checks 209/211, - * reporting_node 303/325/333/339/347/349/351) stayed reachable from a live - * node's own genuine reported state with no such structural excuse, so they're - * deliberately NOT filtered here -- each is a real candidate for individual - * investigation against the keeper's actual KeeperFSM[] rows, not a known-safe - * artifact of this function's own edge derivation. + * reporting_node 303/325/333/339/347/349/351) is deliberately NOT filtered + * here -- each is a real candidate for individual investigation against the + * keeper's actual KeeperFSM[] rows, not a known-safe artifact of this + * function's own edge derivation in the same structural sense as the two + * categories above. + * + * That said, this function resolves each row's own edges independently (the + * for-loop above never looks at any OTHER row), so it does NOT model + * first-match-wins shadowing between rows: a row can report an edge for a + * current_state that an EARLIER row (lower array index, matched + * unconditionally or under a broader condition) would actually intercept + * first in real dispatch, making that edge practically unreachable even + * though this function still emits it. Confirmed concretely for one such + * case via a live pgaftest run (tests/tap/specs/ + * keeper_fsm_gap_211_primary_priority_zero.pgaf's own investigation): pos + * 209's "maintenance" edge looked like a real gap here, but pos 205 + * ("converged to maintenance -> no-op, frozen until stop_maintenance()", + * unconditional on MAINTENANCE_STATE, no groupHasExactlyOneNode check) comes + * first in array order and intercepts every node_active() call from a node + * in MAINTENANCE_STATE regardless of group size -- pos 209 can never + * actually fire for that state, single-node group or not. Anyone + * individually investigating one of the mismatches named above should rule + * this out first (does an earlier row in MonitorFSM[] match the same + * current_state unconditionally, or under a condition the one being + * investigated doesn't also exclude?) before concluding a row's own + * fanned-out current_state is a real, reachable gap. */ Datum dump_fsm_edges(PG_FUNCTION_ARGS) diff --git a/tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf b/tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf new file mode 100644 index 000000000..ac0b40e4d --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf @@ -0,0 +1,101 @@ +# Diagnostic spec for MonitorFSM[] pos 211 ("alone in group, +# candidatePriority zero -> report_lsn", src/monitor/group_state_machine.c) +# -- NOT in the CI schedule, same convention as debug_failover_pg19.pgaf, +# since it is expected to FAIL until the missing KeeperFSM[] transition +# documented below is implemented. +# +# src/monitor/sql/keeper_fsm_edges.sql's Step 2a found that pos 211's +# NodeStatePattern is willing to assign REPORT_LSN_STATE to a lone remaining +# node reporting PRIMARY_STATE as its current_state (among ten other +# fanned-out states), but KeeperFSM[] (src/bin/pg_autoctl/fsm.c) has no +# PRIMARY_STATE -> REPORT_LSN_STATE row at all. +# +# This spec manufactures that exact (current, assigned) pair deterministically +# (no timing race: candidate-priority and node role are both explicit, +# durable node properties, not transient states): +# +# 1. node2 is promoted to primary, then its own candidate-priority is set +# to 0. +# 2. node1 (now the only secondary) is dropped, leaving node2 alone in the +# group -- still reporting PRIMARY_STATE, since nothing prompts it to +# change role on its own. +# 3. The monitor recomputes: groupHasExactlyOneNode is now true, node2's +# own reported current_state is "primary", and its candidate priority +# is 0 (not eligible), so pos 211 fires and the monitor assigns +# REPORT_LSN. +# 4. node2's keeper receives that assigned state, but +# keeper_fsm_reach_assigned_state() (fsm.c) finds no matching +# KeeperFSM[] row for (primary -> report_lsn) and logs +# "pg_autoctl does not know how to reach state \"report_lsn\" from +# \"primary\"" (fsm.c's own log_fatal call) -- node2 never converges, +# staying stuck reporting "primary" forever (still accepting writes, +# which the monitor no longer expects once "report_lsn" is assigned). +# +# Expected fix: add a PRIMARY_STATE -> REPORT_LSN_STATE row to KeeperFSM[], +# see this file's own header comment in fsm.c for the reusable-code question +# (a lone priority-zero primary demoting itself to report_lsn is a novel +# transition shape; check whether any existing transition function already +# does "stop being a writable primary without a promotion counterpart" that +# could be reused, e.g. the DEMOTED_STATE-family transitions). + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_001_promote_node2_and_zero_its_priority { + promote node2 + wait until node2 state is primary + and node1 state is secondary + timeout 90s + exec node2 pg_autoctl set node candidate-priority 0 +} + +step test_002_drop_secondary_leaves_lone_priority_zero_primary { + exec node1 pg_autoctl drop node --no-wait + wait until node1 stopped timeout 60s + # The monitor now sees exactly one node (node2) in the group, still + # reporting "primary", with candidate-priority 0 -- pos 211 must fire + # and assign "report_lsn". + wait until node2 assigned-state = report_lsn timeout 60s +} + +step test_003_node2_cannot_reach_report_lsn_from_primary { + # This is the failure this spec exists to demonstrate: node2 is assigned + # "report_lsn" (confirmed above) but has no KeeperFSM[] transition to get + # there from "primary", so it never converges -- it keeps reporting + # "primary" indefinitely instead, retrying the failed transition on every + # loop iteration. Queried directly against pgautofailover.node rather + # than via "assert ... state is ..."/"assert ... stays ... while { }": + # neither of those constructs fit here, since node2's *reported* state + # (reportedstate) and its *goal* state (goalstate) have genuinely and + # permanently diverged -- there's no single moment to "wait until", and + # no unchanged baseline to assert "stays" against. + # + # Once the missing transition is implemented, this query's second column + # is expected to change to "report_lsn" (node2 actually converges), at + # which point this spec should be rewritten to assert that convergence + # instead. + sleep 5s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { primary } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { report_lsn } + logs node2 contains "does not know how to reach state" +} From dcb1e466236e4c0077bc3cfc2fde4fdc6f2d1a3e Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 17:16:47 +0200 Subject: [PATCH 23/52] monitor: make dump_fsm_edges() detect first-match-wins shadowing between rows dump_fsm_edges() previously resolved each MonitorFSM[] row's edges independently, so it could report an edge for a current_state that an EARLIER, unconditional row (lower array index, same top-level section) would actually intercept first in real dispatch -- confirmed live via pgaftest for pos 209's 'maintenance' fanout state, shadowed by pos 205's unconditional 'converged to maintenance -> no-op' row. Add RuleUnconditionallyMatchesActiveNodeState/...PrimaryNodeState: true iff a row's RuleMatches() would hold for EVERY possible dispatch context whose activeNode/primaryNode reports a given state -- every other NodeStatus role fully unconstrained, the role under test has no condition beyond its own state, and every .conditions field is at its 'don't care' default (mirrors RuleMatches()'s own field list one-for-one; a comment ties them together so future fields get added to both). EdgeIsShadowedByEarlierRule scans earlier same-section rows for one that qualifies, and both edge-emission loops in dump_fsm_edges() now skip a candidate edge when this is true. Caught and fixed one real bug while building this: a first draft treated NodeStatePatternResolveFromStates' ASSIGNED/NOT_ASSIGNED/ TRANSITIONING handling (which resolves to 'the full state universe' or 'reportedStates alone, ignoring assignedStates' -- correct for that function's own edge-SOURCE purpose) as if it meant those rows matched unconditionally on reportedState too. It doesn't: ASSIGNED/NOT_ASSIGNED key off goalState only, a separate runtime fact. Without excluding those three kinds, pos 203 ('goalState == DROPPED, reportedState irrelevant') looked unconditional for every state and wrongly suppressed 5 of pos 209's other genuinely-reachable fanned states (wait_standby, prepare_maintenance, wait_maintenance, fast_forward, join_secondary) -- caught by re-running the pos 211 pgaftest spec and noticing pos 209/211 lost far more than the one confirmed case. Net effect after the fix: pos 209 and 211 each correctly lose exactly two fanned states (maintenance, shadowed by pos 205; dropped, shadowed by pos 201's own unconditional 'converged to dropped' row) and keep every other one. total_edge_count: 251 -> 244. Deliberately scoped to same-top-level-section shadowing only (documented as a sound under-approximation, not exhaustive: EARLY_CHECKS unconditionally precedes REPORTING_NODE in the same dispatch chain, so cross-section shadowing can also happen in principle, just not detected here). Full regress (19) + isolation (6) suites pass; the pos 211 pgaftest spec (tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf) still correctly reproduces its real, unshadowed gap after the fix. --- .../expected/check_fsm_reachability.out | 4 +- src/monitor/expected/keeper_fsm_edges.out | 12 +- src/monitor/group_state_machine.c | 333 ++++++++++++++++-- 3 files changed, 317 insertions(+), 32 deletions(-) diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index ca403706b..9c7cb65ce 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 251 + 244 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 251 + 244 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 4eb6fddc7..f4924691d 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -179,9 +179,8 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; rule | n | current_state | assigned_state | comment ------+----+---------------------+----------------+---------------------------------------------------------------------------------------------------------------------- - 209 | 6 | | single | alone in group, candidate-eligible -> single + 209 | 5 | | single | alone in group, candidate-eligible -> single 209 | 1 | wait_standby | single | alone in group, candidate-eligible -> single - 209 | 1 | maintenance | single | alone in group, candidate-eligible -> single 209 | 1 | prepare_maintenance | single | alone in group, candidate-eligible -> single 209 | 1 | wait_maintenance | single | alone in group, candidate-eligible -> single 209 | 1 | fast_forward | single | alone in group, candidate-eligible -> single @@ -323,7 +322,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 391 | 1 | fast_forward | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining 391 | 1 | join_secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining 391 | 1 | dropped | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining -(144 rows) +(143 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the @@ -367,18 +366,23 @@ SELECT k.current_state, k.assigned_state catchingup | maintenance catchingup | wait_maintenance demote_timeout | primary + dropped | single dropped | wait_standby + dropped | report_lsn init | wait_standby maintenance | catchingup prepare_maintenance | catchingup primary | maintenance primary | join_primary primary | prepare_maintenance + report_lsn | prepare_promotion + report_lsn | fast_forward + report_lsn | join_secondary secondary | wait_standby secondary | maintenance secondary | wait_maintenance wait_primary | join_primary wait_primary | apply_settings -(18 rows) +(23 rows) DROP TABLE keeper_fsm_edges; diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 3a5387736..5fbb1484a 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -4315,6 +4315,269 @@ NodeStatePatternResolveFromStates(const NodeStatePattern *pattern, int *outCount } +/* + * NodeStatePatternIncludesState is a single-state membership query built on + * top of NodeStatePatternResolveFromStates -- used by the shadowing check + * below, which needs to ask "does this OTHER row's own pattern also match + * this one concrete state" without caring about the rest of that row's + * resolved set. + */ +static bool +NodeStatePatternIncludesState(const NodeStatePattern *pattern, ReplicationState state) +{ + int count; + ReplicationState *states = NodeStatePatternResolveFromStates(pattern, &count); + + for (int i = 0; i < count; i++) + { + if (states[i] == state) + { + return true; + } + } + + return false; +} + + +/* + * NodeStatePatternKindIsReportedStateOnly: true for the pattern kinds whose + * match genuinely depends only on reportedState (ignoring, for STABLE, its + * own additional "reported == goal" requirement -- see this function's own + * comment for why that specific simplification is deliberate). ASSIGNED, + * NOT_ASSIGNED, and TRANSITIONING all have a real, separate dependency on + * goalState (NodeStateMatchesPattern's own switch: ASSIGNED/NOT_ASSIGNED + * check goalState exclusively, TRANSITIONING checks both), which + * NodeStatePatternResolveFromStates papers over for edge-SOURCE purposes by + * resolving them to either the literal reportedStates list (TRANSITIONING, + * silently dropping its own assignedStates half) or the full state universe + * (ASSIGNED/NOT_ASSIGNED, since goalState alone decides those, independently + * of reportedState) -- both correct over-approximations for "what could this + * row's reported-state source legitimately be", but wrong for this function's + * different question, "does this row match unconditionally whenever reported + * state equals state". A row like pos 203 ("goalState == DROPPED, + * reportedState irrelevant", ASSIGNED kind) would otherwise look like it + * resolves to (and therefore unconditionally matches) every one of the 21 + * states, when it actually still requires a completely separate, real fact + * (goalState == DROPPED) that has nothing to do with reportedState at all -- + * confirmed by dump_fsm_edges()'s own regression: an early, buggy version of + * this shadowing check treated pos 203 as unconditional and wrongly + * suppressed several of pos 209's genuinely reachable fanned-out states + * (wait_standby, prepare_maintenance, wait_maintenance, fast_forward, + * join_secondary) that have nothing to do with the node's goal being + * DROPPED. + * + * STABLE is kept eligible despite its own "reported == goal" wrinkle: for + * every row actually written using it (a bare FSM_STATE(x), no other + * condition), the codebase's own edge-source resolution + * (NodeStatePatternResolveFromStates) already treats STABLE identically to + * REPORTED, and confirmed live (this file's own comment on pos 205/pos 209) + * that treatment is what makes real shadowing detection work at all -- + * requiring the stricter, fully rigorous "and goal really does equal + * reported in every possible calling context" would need modeling whether + * some OTHER row's otherNodeAssignedState could have changed this exact + * node's own goalState moments earlier, out of scope for a static, + * per-table check like this one. + */ +static bool +NodeStatePatternKindIsReportedStateOnly(NodeStatePatternKind kind) +{ + switch (kind) + { + case NODE_STATE_ANY: + case NODE_STATE_STABLE: + case NODE_STATE_REPORTED: + case NODE_STATE_NOT_STABLE: + { + return true; + } + + case NODE_STATE_ASSIGNED: + case NODE_STATE_NOT_ASSIGNED: + case NODE_STATE_TRANSITIONING: + default: + { + return false; + } + } +} + + +/* + * NodeStatusPatternOtherFieldsAreAny/NodeStatusPatternIsFullyAny/ + * NodeActiveContextPatternIsAny mirror RuleMatches()'s own field list, + * field-for-field, checking each is at its BOOL_ANY/INT_PATTERN_ANY/ + * API_TRIGGER_NODE_ACTIVE "don't care" default -- if MonitorFSMTransition's + * pattern structs ever gain a new field, RuleMatches() needs to grow a new + * conjunct for it, and these three functions need the matching ANY-check + * added right alongside, or the shadowing detection below silently starts + * ignoring that new field (treating a row as unconditional when it no + * longer is). + */ +static bool +NodeStatusPatternOtherFieldsAreAny(const NodeStatusPattern *pattern) +{ + return pattern->exists == BOOL_ANY && + pattern->isHealthy == BOOL_ANY && + pattern->isUnhealthy == BOOL_ANY && + pattern->candidateEligible == BOOL_ANY && + pattern->isInPrimaryState == BOOL_ANY && + pattern->isInMaintenance == BOOL_ANY && + pattern->isDemotedPrimary == BOOL_ANY && + pattern->canTakeWrites == BOOL_ANY && + pattern->isReadyToStreamWAL == BOOL_ANY && + pattern->drainTimeExpired == BOOL_ANY && + pattern->isCitusWorkerGroup == BOOL_ANY && + pattern->replicationQuorum == BOOL_ANY && + pattern->isComparableToReferenceTli == BOOL_ANY && + pattern->unreachableFromDemoteTimeout == BOOL_ANY; +} + + +static bool +NodeStatusPatternIsFullyAny(const NodeStatusPattern *pattern) +{ + return pattern->statePattern.kind == NODE_STATE_ANY && + NodeStatusPatternOtherFieldsAreAny(pattern); +} + + +static bool +NodeActiveContextPatternIsAny(const NodeActiveContextPattern *cond) +{ + return cond->apiTrigger.kind == API_TRIGGER_NODE_ACTIVE && + + cond->groupHasExactlyOneNode == BOOL_ANY && + cond->groupHasExactlyTwoNodes == BOOL_ANY && + cond->groupHasMoreThanTwoNodes == BOOL_ANY && + cond->anyOtherNodeWaitingStandby == BOOL_ANY && + cond->numberSyncStandbysIsZero == BOOL_ANY && + cond->replicationQuorumCountIsZero == BOOL_ANY && + cond->secondaryNodesCountIsZero == BOOL_ANY && + cond->secondaryQuorumNodesCountIsZero == BOOL_ANY && + cond->atLeastOneHealthyCandidate == BOOL_ANY && + cond->walWithinPromoteThreshold == BOOL_ANY && + cond->walWithinSyncThreshold == BOOL_ANY && + cond->activeAndPrimaryTliMatch == BOOL_ANY && + cond->primaryIsWaitPrimaryPresumedDead == BOOL_ANY && + cond->failoverInProgress == BOOL_ANY && + cond->replicationStallExceeded == BOOL_ANY && + cond->lastHealthySyncStandbyGoingToMaintenance == BOOL_ANY && + cond->activeNodeAllWalSourcesUnhealthy == BOOL_ANY && + cond->candidatePromotionInProgress == BOOL_ANY && + cond->mostAdvancedCandidateWithinPromoteThreshold == BOOL_ANY && + cond->guardDataLossEnabled == BOOL_ANY && + cond->inMSFailoverCluster == BOOL_ANY && + cond->inMSFailoverCandidateGate == BOOL_ANY && + + cond->candidateCount.kind == INT_PATTERN_ANY && + cond->quorumCandidateCount.kind == INT_PATTERN_ANY && + cond->missingNodesCount.kind == INT_PATTERN_ANY && + cond->sufficientQuorumCandidates == BOOL_ANY; +} + + +/* + * RuleUnconditionallyMatchesActiveNodeState/RuleUnconditionallyMatchesPrimary + * NodeState: true iff rule's own RuleMatches() would return true for EVERY + * possible NodeActiveContext whose activeNode (resp. primaryNode) reports + * state -- i.e. rule's activeNode.statePattern (resp. primaryNode. + * statePattern) accepts state, and nothing else about the row narrows it any + * further: every other NodeStatus role is entirely unconstrained, the role + * being tested has no OTHER constraint beyond its own state, and every + * .conditions field is at its own "don't care" default. A row like this is + * a pure, unconditional catch-all for that one reported state -- exactly pos + * 205's "converged to maintenance -> no-op" row (see dump_fsm_edges()'s own + * comment on the confirmed pos 209/maintenance case this was built to catch). + * + * Deliberately does NOT require rule->otherNodesFn == NULL: otherNodesFn only + * changes who a matched row's otherNodeAssignedState is assigned to, not + * whether the row matches in the first place, so it has no bearing on + * whether this row shadows another one. + */ +static bool +RuleUnconditionallyMatchesActiveNodeState(const MonitorFSMTransition *rule, + ReplicationState state) +{ + return NodeStatePatternKindIsReportedStateOnly(rule->activeNode.statePattern.kind) && + NodeStatePatternIncludesState(&rule->activeNode.statePattern, state) && + NodeStatusPatternOtherFieldsAreAny(&rule->activeNode) && + NodeStatusPatternIsFullyAny(&rule->primaryNode) && + NodeStatusPatternIsFullyAny(&rule->otherNode) && + NodeStatusPatternIsFullyAny(&rule->candidateNode) && + NodeActiveContextPatternIsAny(&rule->conditions); +} + + +static bool +RuleUnconditionallyMatchesPrimaryNodeState(const MonitorFSMTransition *rule, + ReplicationState state) +{ + return NodeStatePatternKindIsReportedStateOnly(rule->primaryNode.statePattern.kind) && + NodeStatePatternIncludesState(&rule->primaryNode.statePattern, state) && + NodeStatusPatternOtherFieldsAreAny(&rule->primaryNode) && + NodeStatusPatternIsFullyAny(&rule->activeNode) && + NodeStatusPatternIsFullyAny(&rule->otherNode) && + NodeStatusPatternIsFullyAny(&rule->candidateNode) && + NodeActiveContextPatternIsAny(&rule->conditions); +} + + +/* + * EdgeIsShadowedByEarlierRule scans MonitorFSM[0 .. beforeIndex) for a row + * that would unconditionally intercept state before dispatch ever reaches + * beforeIndex -- i.e. a real, always-invoked scan of this section (the + * default one, starting from array position 0, which every top-level + * section has independently of any narrower resume-point scan some specific + * caller might also use) would never actually reach beforeIndex for a node + * reporting state, making an edge reported for it purely an artifact of this + * function resolving each row independently (see dump_fsm_edges()'s own + * comment). + * + * Deliberately bounded to rows sharing beforeIndex's own top-level section, + * not the whole array: every section IS reachable via its own independent + * top-level scan (FindAndDispatchMonitorFSMRuleUnderPath's callers), so + * shadowing within one section is straightforward to prove -- the section's + * own default scan, starting from array position 0, is a real call that + * genuinely exists and always runs in that order. Note this is a + * conservative under-approximation, not a claim that cross-section + * shadowing is impossible: EARLY_CHECKS always runs before REPORTING_NODE in + * the same overall dispatch chain (ProceedGroupStateFromContext), so an + * unconditional EARLY_CHECKS row (pos 201's "converged to dropped", say) + * could in principle also shadow a REPORTING_NODE row's own "dropped" + * fanned-out state -- this function does not attempt to detect that, since + * doing so soundly requires modeling the fixed section-to-section call + * order across the whole dispatch entry point graph, not just one section's + * own internal array order. Left as a known limitation rather than a false + * suppression risk: this scoping can only ever under-detect shadowing + * (leaving a real-but-unreachable edge in the output), never wrongly hide a + * genuinely reachable one. + */ +static bool +EdgeIsShadowedByEarlierRule(int beforeIndex, ReplicationState state, bool primaryNodeSide, + MonitorFSMSection topLevelSection) +{ + for (int j = 0; j < beforeIndex; j++) + { + const MonitorFSMTransition *earlier = &MonitorFSM[j]; + + if (earlier->sectionPath[0] != topLevelSection) + { + continue; + } + + if (primaryNodeSide + ? RuleUnconditionallyMatchesPrimaryNodeState(earlier, state) + : RuleUnconditionallyMatchesActiveNodeState(earlier, state)) + { + return true; + } + } + + return false; +} + + PG_FUNCTION_INFO_V1(dump_fsm_edges); /* @@ -4371,33 +4634,41 @@ PG_FUNCTION_INFO_V1(dump_fsm_edges); * kind of false gap for every one of these ten rows, all confirmed by that same * live run. * - * Every other mismatch that same run found (early_checks 209/211, - * reporting_node 303/325/333/339/347/349/351) is deliberately NOT filtered - * here -- each is a real candidate for individual investigation against the - * keeper's actual KeeperFSM[] rows, not a known-safe artifact of this - * function's own edge derivation in the same structural sense as the two - * categories above. + * A third category is filtered the same way, for a different reason: this + * function used to resolve each row's own edges entirely independently, + * never considering any OTHER row, so it could report an edge for a + * current_state that an EARLIER row (lower array index, matching + * unconditionally -- e.g. a bare no-op like pos 205's "converged to + * maintenance -> no-op, frozen until stop_maintenance()") would actually + * intercept first in real first-match-wins dispatch, making that edge + * practically unreachable. Confirmed concretely via a live pgaftest run + * (tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf's own + * investigation): pos 209's "maintenance" edge looked like a real gap here, + * but pos 205 comes first in array order and intercepts every node_active() + * call from a node in MAINTENANCE_STATE regardless of group size, so pos + * 209 can never actually fire for that state at all. * - * That said, this function resolves each row's own edges independently (the - * for-loop above never looks at any OTHER row), so it does NOT model - * first-match-wins shadowing between rows: a row can report an edge for a - * current_state that an EARLIER row (lower array index, matched - * unconditionally or under a broader condition) would actually intercept - * first in real dispatch, making that edge practically unreachable even - * though this function still emits it. Confirmed concretely for one such - * case via a live pgaftest run (tests/tap/specs/ - * keeper_fsm_gap_211_primary_priority_zero.pgaf's own investigation): pos - * 209's "maintenance" edge looked like a real gap here, but pos 205 - * ("converged to maintenance -> no-op, frozen until stop_maintenance()", - * unconditional on MAINTENANCE_STATE, no groupHasExactlyOneNode check) comes - * first in array order and intercepts every node_active() call from a node - * in MAINTENANCE_STATE regardless of group size -- pos 209 can never - * actually fire for that state, single-node group or not. Anyone - * individually investigating one of the mismatches named above should rule - * this out first (does an earlier row in MonitorFSM[] match the same - * current_state unconditionally, or under a condition the one being - * investigated doesn't also exclude?) before concluding a row's own - * fanned-out current_state is a real, reachable gap. + * EdgeIsShadowedByEarlierRule (below) now detects exactly this: for each + * candidate edge, it scans every earlier row sharing the same top-level + * section (every section is its own independent top-level scan, so a row + * outside it is never reachable in the same dispatch pass regardless of + * array order -- see FindAndDispatchMonitorFSMRuleUnderPath's callers) for + * one that would match unconditionally whenever the same NodeStatus role + * reports that same state, regardless of anything else in the dispatch + * context. This is a sound, deliberately conservative check: it only + * suppresses an edge when an earlier row is PROVABLY unconditional for that + * state (every other pattern field at its own "don't care" default -- see + * RuleUnconditionallyMatchesActiveNodeState/...PrimaryNodeState's own + * comment), never merely "plausibly likely to match first" -- a row with + * even one real extra condition is left alone, since whether it actually + * fires first still depends on runtime facts this function can't know + * statically. Every mismatch that live pgaftest run found beyond pos 209's + * (early_checks 211, reporting_node 303/325/333/339/347/349/351) survived + * this check and remains a real, reachable gap -- pos 211's specifically was + * independently confirmed live in that same investigation (a lone + * priority-zero primary really does get assigned report_lsn from PRIMARY_ + * STATE with no shadowing row in front of it, and the keeper really has no + * transition for it). */ Datum dump_fsm_edges(PG_FUNCTION_ARGS) @@ -4465,6 +4736,11 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + if (EdgeIsShadowedByEarlierRule(i, states[j], false, rule->sectionPath[0])) + { + continue; + } + values[0] = Int32GetDatum(rule->pos); values[1] = ObjectIdGetDatum(ReplicationStateGetEnum(states[j])); values[2] = ObjectIdGetDatum( @@ -4503,6 +4779,11 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + if (EdgeIsShadowedByEarlierRule(i, states[j], true, rule->sectionPath[0])) + { + continue; + } + values[0] = Int32GetDatum(rule->pos); values[1] = ObjectIdGetDatum(ReplicationStateGetEnum(states[j])); values[2] = ObjectIdGetDatum( From 39a771c8d252005ba9270a0dba6ea74f81143875 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 17:29:39 +0200 Subject: [PATCH 24/52] monitor: document why cross-section shadowing detection was tried and rejected Attempted extending EdgeIsShadowedByEarlierRule to also treat MONITOR_FSM_SECTION_EARLY_CHECKS as unconditionally preceding every other section (justified by ProceedGroupStateFromContext() always trying SectionEarlyChecks first). Found a concrete counter-example before committing it: pos 205's STABLE-kind pattern requires reportedState == goalState == maintenance, an equality NOT guaranteed just because reportedState == maintenance -- stop_maintenance() on a multi-node group dispatches through the separate api_triggered path and can advance goalState independently of the node's own next heartbeat. The cross-section version would have wrongly treated pos 205 as shadowing pos 369 ("MS-failover fan-out: rejoining from maintenance -> report_lsn"), which specifically requires reportedState == maintenance AND goalState == catchingup, deleting a real, reachable edge. Caught by hand-tracing before committing, not by any test failure. Reverted to the already-committed same-section-only behavior (verified byte-identical regress+isolation output), with the failed attempt and its concrete counter-example documented in EdgeIsShadowedByEarlierRule's own comment so it isn't re-attempted without this context. --- src/monitor/group_state_machine.c | 41 +++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 5fbb1484a..624fc3499 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -4539,19 +4539,34 @@ RuleUnconditionallyMatchesPrimaryNodeState(const MonitorFSMTransition *rule, * top-level scan (FindAndDispatchMonitorFSMRuleUnderPath's callers), so * shadowing within one section is straightforward to prove -- the section's * own default scan, starting from array position 0, is a real call that - * genuinely exists and always runs in that order. Note this is a - * conservative under-approximation, not a claim that cross-section - * shadowing is impossible: EARLY_CHECKS always runs before REPORTING_NODE in - * the same overall dispatch chain (ProceedGroupStateFromContext), so an - * unconditional EARLY_CHECKS row (pos 201's "converged to dropped", say) - * could in principle also shadow a REPORTING_NODE row's own "dropped" - * fanned-out state -- this function does not attempt to detect that, since - * doing so soundly requires modeling the fixed section-to-section call - * order across the whole dispatch entry point graph, not just one section's - * own internal array order. Left as a known limitation rather than a false - * suppression risk: this scoping can only ever under-detect shadowing - * (leaving a real-but-unreachable edge in the output), never wrongly hide a - * genuinely reachable one. + * genuinely exists and always runs in that order. + * + * A cross-section version of this check was tried and REJECTED, not merely + * left as a TODO: ProceedGroupStateFromContext() does always try + * SectionEarlyChecks first, before SectionPrimaryNode/SectionReportingNode + * (see its own comment), which looks like it should let an unconditional + * early_checks row (pos 205's "converged to maintenance -> no-op") shadow a + * same-state edge in a later section too. It doesn't, in general: pos 205's + * own STABLE-kind pattern requires reportedState == goalState == maintenance, + * and that equality is NOT guaranteed just because reportedState == + * maintenance -- stop_maintenance() on a multi-node group dispatches through + * the *separate* MONITOR_FSM_SECTION_API_TRIGGERED path (ProceedGroupStateFor + * ApiTrigger, not ProceedGroupStateFromContext at all) and assigns a new goal + * directly, independently of the target node's own next heartbeat -- so a + * node can genuinely present reportedState == maintenance with goalState + * already advanced past it. A first attempt at this cross-section extension + * treated pos 205 as shadowing pos 369 ("MS-failover fan-out: rejoining from + * maintenance -> report_lsn", a TRANSITIONING-kind row requiring exactly + * reportedState == maintenance AND goalState == catchingup) this way, and + * would have wrongly deleted a real, reachable edge -- caught before + * committing by checking the field-level trace by hand, not by any test. + * Soundly generalizing this would require modeling every place a node's own + * goalState can be written independently of its own next reportedState + * update (every apiTrigger row, not just stop_maintenance), which is a much + * larger undertaking than this function's own scope; same-section-only + * stays the safe, committed behavior. This under-approximates real + * shadowing (some cross-section cases go undetected), never over-approximates + * (no risk of wrongly hiding a genuinely reachable edge). */ static bool EdgeIsShadowedByEarlierRule(int beforeIndex, ReplicationState state, bool primaryNodeSide, From abb47d424a4bea2834e3bfb077c6f93c0ae99fab Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 18:58:22 +0200 Subject: [PATCH 25/52] monitor: filter dump_fsm_edges() candidates by isInPrimaryState too; add pos 209 wait_maintenance spec dump_fsm_edges() resolved a row's candidate current_state purely from .statePattern, never looking at that same role's other BoolPattern conditions -- isInPrimaryState in particular. Several primaryNode-side rows (pos 303, 325, 391) set .primaryNode.statePattern to ANY (or a broad NOT_STABLE set) alongside .isInPrimaryState = BOOL_TRUE, so dump_fsm_edges() enumerated all 21 (or ~20) states as candidate sources even though IsInPrimaryState() (node_metadata.c) can only ever be true for a small, fixed set: CanTakeWritesInState's own {single, primary, wait_primary, join_primary, apply_settings}, plus primary/apply_settings via its own second disjunct (already a subset). Every other state (catchingup, secondary, dropped, maintenance, ...) is structurally impossible for a node IsInPrimaryState() accepts, regardless of goalState. Added StateCanSatisfyIsInPrimaryState (a direct, provably-correct satisfiability check against IsInPrimaryState's own two-disjunct definition) and NodeStatusPatternSurvivesIsInPrimaryState, wired into both edge-emission loops in dump_fsm_edges(). Deliberately narrow in scope to only this one field -- sibling state-dependent fields (isInMaintenance, canTakeWrites, drainTimeExpired, unreachableFromDemoteTimeout) are NOT touched here, since EdgeIsShadowedByEarlierRule's own comment already documents a concrete case (pos 369) where a seemingly-safe assumption about one of these turned out wrong once a separate write path was accounted for -- extending further needs the same due diligence per field, not a blanket generalization. Effect, confirmed via full regress+isolation: total_edge_count 244 -> 198, Step 2a's gap list 143 -> 98 rows. Pos 303 is now fully closed (zero remaining gap states -- every state IsInPrimaryState can ever admit is already covered by a real KeeperFSM[] transition). Pos 325 and 391 drop from 16 states each to 1 ("single"). Pos 333/339/347/349/ 351 are unaffected (verified: none of them actually set .isInPrimaryState -- 333/351 use bare .exists, 339 uses NOT_STABLE + isInMaintenance, 347 uses drainTimeExpired, 349 uses a .conditions flag with no .primaryNode restriction at all), confirming the fix only touches the rows it's actually justified for. Also adds tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf, the second fully-validated pgaftest reproduction (after pos 211's): network-disconnecting the primary durably strands a lone last-quorum- member secondary at WAIT_MAINTENANCE_STATE (not a narrow timing window -- it can never receive the primary's ack while disconnected), then dropping the primary directly via pgautofailover.remove_node() leaves it alone in the group, confirming live that the monitor assigns SINGLE and the keeper has no WAIT_MAINTENANCE_STATE -> SINGLE_STATE transition (log_fatal captured). Re-verified both this spec and the pos 211 one still reproduce their real gaps after the isInPrimaryState fix. --- .../expected/check_fsm_reachability.out | 4 +- src/monitor/expected/keeper_fsm_edges.out | 51 +--------- src/monitor/group_state_machine.c | 96 +++++++++++++++++++ .../keeper_fsm_gap_209_wait_maintenance.pgaf | 87 +++++++++++++++++ 4 files changed, 188 insertions(+), 50 deletions(-) create mode 100644 tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index 9c7cb65ce..d2cc76df2 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 244 + 198 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 244 + 198 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index f4924691d..a95ebbbcd 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -197,38 +197,8 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 211 | 1 | wait_maintenance | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | fast_forward | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | join_secondary | report_lsn | alone in group, candidatePriority zero -> report_lsn - 303 | 14 | | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | init | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | draining | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | demote_timeout | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | demoted | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | catchingup | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | wait_standby | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | prepare_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | wait_maintenance | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | report_lsn | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | fast_forward | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | join_secondary | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 303 | 1 | dropped | wait_primary | primary healthy, no standby past replication_stall_timeout -> wait_primary - 325 | 16 | | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | init | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) + 325 | 1 | | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) 325 | 1 | single | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | demote_timeout | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | demoted | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | catchingup | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | prepare_promotion | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | stop_replication | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | wait_standby | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | prepare_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | wait_maintenance | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | report_lsn | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | fast_forward | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | join_secondary | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | dropped | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) 333 | 14 | | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | init | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | single | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted @@ -305,24 +275,9 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 351 | 1 | fast_forward | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | join_secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 391 | 16 | | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | init | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining + 391 | 1 | | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining 391 | 1 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | demote_timeout | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | demoted | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | catchingup | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | prepare_promotion | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | stop_replication | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | wait_standby | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | prepare_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | wait_maintenance | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | report_lsn | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | fast_forward | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | join_secondary | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | dropped | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining -(143 rows) +(98 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 624fc3499..7fe510982 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -4340,6 +4340,92 @@ NodeStatePatternIncludesState(const NodeStatePattern *pattern, ReplicationState } +/* + * StateCanSatisfyIsInPrimaryState answers, for a hypothetical reportedState + * of state, whether SOME goalState exists making IsInPrimaryState() + * (node_metadata.c) evaluate to required -- i.e. whether state is a genuinely + * possible edge SOURCE for a row whose own .isInPrimaryState field demands + * required, as opposed to merely a state dump_fsm_edges()'s own + * NodeStatePatternResolveFromStates() would enumerate without knowing + * anything about IsInPrimaryState() at all (that function only ever reads + * .statePattern; every other NodeStatusPattern field, isInPrimaryState + * included, is invisible to it by construction, see its own comment). + * + * IsInPrimaryState(node) is exactly: + * (goal == reported && CanTakeWritesInState(goal)) + * || ((goal in {APPLY_SETTINGS, PRIMARY}) && (reported in {PRIMARY, APPLY_SETTINGS})) + * + * For required == true: choosing goal == state makes the first disjunct + * CanTakeWritesInState(state) -- always achievable when true. The second + * disjunct additionally admits state == PRIMARY or APPLY_SETTINGS via an + * appropriate goal choice even when CanTakeWritesInState(state) doesn't + * already cover it (it does, today -- CanTakeWritesInState already includes + * both -- but this is spelled out explicitly since the two conditions are + * independently maintained code and could diverge later). + * + * For required == false: some goal can always be chosen making both + * disjuncts fail (goal different from state, and not the specific + * APPLY_SETTINGS/PRIMARY pairing) -- IsInPrimaryState is never forced true + * for every possible goal by reportedState alone, so "false" is always + * satisfiable regardless of state. + * + * Confirmed real-world impact: pos 303's own .primaryNode.statePattern is + * NODE_STATE_ANY (no restriction at all) alongside .isInPrimaryState = + * BOOL_TRUE -- so dump_fsm_edges() would otherwise enumerate all 21 states + * as candidate primaryNode "current_state" sources for it, most of which + * (catchingup, secondary, dropped, maintenance, ...) IsInPrimaryState() + * could never actually accept regardless of goalState. This function + * narrows that down to the 5 states IsInPrimaryState can ever admit: + * single, primary, wait_primary, join_primary, apply_settings (from + * CanTakeWritesInState's own set). + * + * Deliberately narrow in scope: only .isInPrimaryState is modeled this way. + * Several sibling fields (isInMaintenance, canTakeWrites, drainTimeExpired, + * unreachableFromDemoteTimeout) are ALSO state-dependent in the same sense + * and could in principle be filtered the same way, but each needs its own + * careful satisfiability proof first -- isInMaintenance in particular looks + * deceptively similar to isInPrimaryState but is NOT safe to treat the same + * way without one: EdgeIsShadowedByEarlierRule's own comment documents a + * concrete case (pos 369) where a reportedState commonly assumed + * incompatible with a goal-dependent condition turned out to be reachable + * anyway, once a real, separate write path (stop_maintenance()'s + * api_triggered dispatch) was accounted for. Extending this file naively to + * every state-dependent field without doing that same due diligence for + * each risks reintroducing exactly that class of bug. + */ +static bool +StateCanSatisfyIsInPrimaryState(ReplicationState state, bool required) +{ + if (!required) + { + return true; + } + + return CanTakeWritesInState(state) || + state == REPLICATION_STATE_PRIMARY || + state == REPLICATION_STATE_APPLY_SETTINGS; +} + + +/* + * NodeStatusPatternSurvivesIsInPrimaryState filters a candidate edge-source + * state against pattern's own .isInPrimaryState field (BOOL_ANY -- the vast + * majority of rows -- always survives; see StateCanSatisfyIsInPrimaryState's + * own comment for BOOL_TRUE/BOOL_FALSE). + */ +static bool +NodeStatusPatternSurvivesIsInPrimaryState(const NodeStatusPattern *pattern, + ReplicationState state) +{ + if (pattern->isInPrimaryState == BOOL_ANY) + { + return true; + } + + return StateCanSatisfyIsInPrimaryState(state, pattern->isInPrimaryState == BOOL_TRUE); +} + + /* * NodeStatePatternKindIsReportedStateOnly: true for the pattern kinds whose * match genuinely depends only on reportedState (ignoring, for STABLE, its @@ -4751,6 +4837,11 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->activeNode, states[j])) + { + continue; + } + if (EdgeIsShadowedByEarlierRule(i, states[j], false, rule->sectionPath[0])) { continue; @@ -4794,6 +4885,11 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->primaryNode, states[j])) + { + continue; + } + if (EdgeIsShadowedByEarlierRule(i, states[j], true, rule->sectionPath[0])) { continue; diff --git a/tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf b/tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf new file mode 100644 index 000000000..8d77fcffc --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf @@ -0,0 +1,87 @@ +# Diagnostic spec for MonitorFSM[] pos 209 ("alone in group, +# candidate-eligible -> single", src/monitor/group_state_machine.c) -- NOT in +# the CI schedule, same convention as debug_failover_pg19.pgaf, since it is +# expected to FAIL until the missing KeeperFSM[] transition documented below +# is implemented. +# +# src/monitor/sql/keeper_fsm_edges.sql's Step 2a found that pos 209's +# NodeStatePattern is willing to assign SINGLE_STATE to a lone remaining +# node reporting WAIT_MAINTENANCE_STATE as its current_state, but KeeperFSM[] +# (src/bin/pg_autoctl/fsm.c) has no WAIT_MAINTENANCE_STATE -> SINGLE_STATE +# row at all. +# +# This spec manufactures that exact (current, assigned) pair: +# +# 1. node1 (primary) is network-disconnected, so it can never acknowledge +# a replication-quorum change. +# 2. node2 (the only secondary, hence the last quorum member) is told to +# enter maintenance. Per service_keeper.c's own comment, a secondary +# that is the last quorum member goes SECONDARY -> WAIT_MAINTENANCE +# first, waiting for the primary to drop it from synchronous_standby_ +# names before proceeding to MAINTENANCE -- since node1 can never see +# or acknowledge that request, node2 is durably stuck at +# WAIT_MAINTENANCE_STATE, not a narrow timing window. +# 3. node1 is dropped directly via pgautofailover.remove_node(), bypassing +# node1 entirely (it doesn't need to cooperate for the monitor's own +# catalog to drop it) -- leaving node2 alone in the group, still +# reporting wait_maintenance. +# 4. The monitor recomputes: groupHasExactlyOneNode is now true, node2's +# own reported current_state is "wait_maintenance", and its candidate +# priority is > 0, so pos 209 fires and the monitor assigns SINGLE. +# 5. node2's keeper receives that assigned state, but +# keeper_fsm_reach_assigned_state() (fsm.c) finds no matching +# KeeperFSM[] row for (wait_maintenance -> single) and logs +# "pg_autoctl does not know how to reach state \"single\" from +# \"wait_maintenance\"" -- node2 never converges, staying stuck +# reporting wait_maintenance forever. +# +# Expected fix: add a WAIT_MAINTENANCE_STATE -> SINGLE_STATE row to +# KeeperFSM[]. Check for reuse first: WAIT_MAINTENANCE_STATE -> +# MAINTENANCE_STATE already exists (fsm.c) and its transition function may +# be adaptable, since both targets ultimately mean "stop waiting for a peer +# that isn't coming." + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_001_disconnect_primary { + network disconnect node1 +} + +step test_002_secondary_stuck_at_wait_maintenance { + # pg_autoctl enable maintenance itself polls for convergence and exits + # non-zero when it can't reach "maintenance" quickly (the primary is + # unreachable, exactly the point of this scenario) -- the real state + # transition still happens regardless of this command's own exit code, + # confirmed by the wait below. + exec-fails node2 pg_autoctl enable maintenance + wait until node2 state is wait_maintenance timeout 60s +} + +step test_003_drop_primary_bypass_and_check_stuck { + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + wait until node2 assigned-state = single timeout 60s + sleep 5s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { wait_maintenance } + logs node2 contains "does not know how to reach state" +} From 9f75fbfa406774eb2864b16d6070cc564d66c899 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 22:19:27 +0200 Subject: [PATCH 26/52] monitor FSM: fix lone priority-zero primary self-demotion, remove MonitorFSM_SIZE pos 210 ("alone in group, already primary despite candidatePriority zero -> single") was gated on isInPrimaryState=true, which also requires goalState to already agree with reportedState. A live pgaftest run (keeper_fsm_gap_211_primary_priority_zero.pgaf) caught a real self-undermining oscillation: the row's own extraAction reassigns goalState to SINGLE, which makes isInPrimaryState() evaluate false on the very next dispatch (goalState=single no longer agrees with reportedState=primary), so the row stopped matching one dispatch after firing and pos 211 (no such requirement) fired right behind it, overwriting the assignment back to REPORT_LSN -- silently undoing the fix in production. Replace the isInPrimaryState condition with a new NODE_STATE_REPORTED pattern (FSM_REPORTED_PRIMARY_ROLE_STATES) that matches on reportedState alone, ignoring goalState entirely. Since pos 210's own action never touches reportedState, the match stays stable across dispatches until the keeper itself actually converges to single. Rewrote keeper_fsm_gap_211_primary_priority_zero.pgaf to assert the fixed convergence-to-single behavior instead of the old stuck-at-report_lsn bug it was written to demonstrate. Also remove MonitorFSM_SIZE, a hand-maintained #define that a prior commit had already caught silently dropping the array's last row (pos 421) from every bounded loop when a new row was added without also bumping it, and had patched with an Assert(size == sizeof(...)/sizeof(...)) safeguard. That safeguard only works after the array's full definition is lexically visible, but two call sites need the count before that point (only an incomplete forward declaration of MonitorFSM[] is visible there), so it couldn't close the gap for every caller. Replace the whole mechanism with a terminator row (.pos left at its zero default) at the end of MonitorFSM[]: every loop now walks until it sees pos == 0 instead of a separately maintained count, so a row added before the terminator is automatically in scope everywhere, and the terminator can't itself drift out of sync since it's part of the array's own literal initializer. AssertMonitorFSMWellFormed() keeps a generous iteration-count sanity check as a backstop against the terminator being removed or a row being added after it by mistake. Verified: full regress (19/19) + isolation (6/6) suites pass, fsm.out and check_fsm_reachability.out regenerated (zero change to keeper_fsm_edges.out -- same set of assigned edges, just a more precise match condition), both keeper_fsm_gap_209/211 pgaftest specs pass live against a rebuilt Docker image. --- .../expected/check_fsm_reachability.out | 4 +- src/monitor/expected/fsm.out | 131 +++++++++------- src/monitor/group_state_machine.c | 144 +++++++++++++++--- ...per_fsm_gap_211_primary_priority_zero.pgaf | 95 +++++------- 4 files changed, 241 insertions(+), 133 deletions(-) diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index d2cc76df2..e7a569f7f 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 198 + 202 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 198 + 202 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index ccf9bacf3..926454ff0 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -321,6 +321,21 @@ other_node_assigned_state | has_extra_action | f comment | alone in group, candidate-eligible -> single -[ RECORD 21 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 210 +section | early_checks +section_path | early_checks +active_node_current_state | primary, wait_primary, join_primary, apply_settings +other_node_current_state | +candidate_node_current_state | +active_node_conditions | candidateEligible=false +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyOneNode=true +active_node_assigned_state | single +other_node_assigned_state | +has_extra_action | f +comment | alone in group, already primary despite candidatePriority zero -> single +-[ RECORD 22 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 211 section | early_checks section_path | early_checks @@ -335,7 +350,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | alone in group, candidatePriority zero -> report_lsn --[ RECORD 22 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 23 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 301 section | reporting_node section_path | reporting_node.from_context @@ -350,7 +365,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | has_extra_action | f comment | converged secondary, reportedTLI not an ancestor of reference -> catchingup --[ RECORD 23 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 24 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 303 section | reporting_node section_path | reporting_node.from_context @@ -365,7 +380,7 @@ active_node_assigned_state | other_node_assigned_state | wait_primary has_extra_action | f comment | primary healthy, no standby past replication_stall_timeout -> wait_primary --[ RECORD 24 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 25 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 305 section | reporting_node section_path | reporting_node.from_context @@ -380,7 +395,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade --[ RECORD 25 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 26 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 307 section | reporting_node section_path | reporting_node.from_context @@ -395,7 +410,7 @@ active_node_assigned_state | secondary other_node_assigned_state | has_extra_action | f comment | report_lsn, primary converged wait/join_primary, healthy -> secondary --[ RECORD 26 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 27 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 309 section | reporting_node section_path | reporting_node.from_context @@ -410,7 +425,7 @@ active_node_assigned_state | secondary other_node_assigned_state | has_extra_action | f comment | report_lsn, primary converged primary, healthy -> secondary --[ RECORD 27 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 28 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 311 section | reporting_node section_path | reporting_node.from_context @@ -425,7 +440,7 @@ active_node_assigned_state | prepare_promotion other_node_assigned_state | has_extra_action | f comment | fast_forward done -> prepare_promotion --[ RECORD 28 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 29 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 313 section | reporting_node section_path | reporting_node.from_context @@ -440,7 +455,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | report_lsn or fast_forward, continuing an already-started failover -> MS-failover cascade --[ RECORD 29 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 30 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 315 section | reporting_node section_path | reporting_node.from_context @@ -455,7 +470,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | has_extra_action | f comment | wait_standby, primary converged wait/join_primary -> catchingup --[ RECORD 30 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 31 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 317 section | reporting_node section_path | reporting_node.from_context @@ -470,7 +485,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | apply_settings has_extra_action | f comment | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings --[ RECORD 31 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 32 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 319 section | reporting_node section_path | reporting_node.from_context @@ -485,7 +500,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | has_extra_action | f comment | wait_standby (not a quorum member), primary converged primary -> catchingup --[ RECORD 32 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 33 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 321 section | reporting_node section_path | reporting_node.from_context @@ -500,7 +515,7 @@ active_node_assigned_state | secondary other_node_assigned_state | has_extra_action | f comment | caught up, same TLI as primary, within sync threshold -> secondary --[ RECORD 33 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 34 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 323 section | reporting_node section_path | reporting_node.from_context @@ -515,7 +530,7 @@ active_node_assigned_state | prepare_promotion other_node_assigned_state | has_extra_action | f comment | primary fails, already converged wait_primary (issue #1168) -> secondary -> prepare_promotion only (1 of 2) --[ RECORD 34 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 35 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 325 section | reporting_node section_path | reporting_node.from_context @@ -530,7 +545,7 @@ active_node_assigned_state | prepare_promotion other_node_assigned_state | draining has_extra_action | f comment | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) --[ RECORD 35 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 36 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 327 section | reporting_node section_path | reporting_node.from_context @@ -545,7 +560,7 @@ active_node_assigned_state | maintenance other_node_assigned_state | has_extra_action | f comment | wait_maintenance, primary converged wait_primary -> maintenance --[ RECORD 36 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 37 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 329 section | reporting_node section_path | reporting_node.from_context @@ -560,7 +575,7 @@ active_node_assigned_state | maintenance other_node_assigned_state | has_extra_action | f comment | wait_maintenance, primary's goal no longer wait_primary -> maintenance --[ RECORD 37 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 38 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 331 section | reporting_node section_path | reporting_node.from_context @@ -575,7 +590,7 @@ active_node_assigned_state | stop_replication other_node_assigned_state | has_extra_action | f comment | prepare_promotion, primary converged prepare_maintenance -> stop_replication --[ RECORD 38 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 39 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 333 section | reporting_node section_path | reporting_node.from_context @@ -590,7 +605,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | demoted has_extra_action | f comment | Citus worker prepare_promotion, primary present -> wait_primary + demoted --[ RECORD 39 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 40 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 335 section | reporting_node section_path | reporting_node.from_context @@ -605,7 +620,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | Citus worker prepare_promotion, primary removed -> wait_primary --[ RECORD 40 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 41 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 337 section | reporting_node section_path | reporting_node.from_context @@ -620,7 +635,7 @@ active_node_assigned_state | stop_replication other_node_assigned_state | has_extra_action | f comment | prepare_promotion, primary already converged wait_primary (issue #1168) -> stop_replication only (1 of 2) --[ RECORD 41 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 42 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 339 section | reporting_node section_path | reporting_node.from_context @@ -635,7 +650,7 @@ active_node_assigned_state | stop_replication other_node_assigned_state | demote_timeout has_extra_action | f comment | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) --[ RECORD 42 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 43 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 341 section | reporting_node section_path | reporting_node.from_context @@ -650,7 +665,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | prepare_promotion, primary removed -> wait_primary --[ RECORD 43 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 44 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 343 section | reporting_node section_path | reporting_node.from_context @@ -665,7 +680,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | maintenance has_extra_action | f comment | stop_replication, primary converged prepare_maintenance -> wait_primary + maintenance --[ RECORD 44 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 45 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 345 section | reporting_node section_path | reporting_node.from_context @@ -680,7 +695,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | demoted has_extra_action | f comment | stop_replication, primary converged demote_timeout -> wait_primary + demoted (1 of 3) --[ RECORD 45 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 46 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 347 section | reporting_node section_path | reporting_node.from_context @@ -695,7 +710,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | demoted has_extra_action | f comment | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) --[ RECORD 46 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 47 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 349 section | reporting_node section_path | reporting_node.from_context @@ -710,7 +725,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | demoted has_extra_action | f comment | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) --[ RECORD 47 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 48 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 351 section | reporting_node section_path | reporting_node.from_context @@ -725,7 +740,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | demoted has_extra_action | f comment | Citus worker stop_replication, primary present -> wait_primary + demoted --[ RECORD 48 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 49 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 353 section | reporting_node section_path | reporting_node.from_context @@ -740,7 +755,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | Citus worker stop_replication, primary removed -> wait_primary --[ RECORD 49 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 50 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 355 section | reporting_node section_path | reporting_node.from_context @@ -755,7 +770,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | has_extra_action | f comment | demoted, primary reported wait/join_primary with goal primary -> catchingup --[ RECORD 50 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 51 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 357 section | reporting_node section_path | reporting_node.from_context @@ -770,7 +785,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | has_extra_action | f comment | demoted, primary converged wait/join_primary/primary, healthy -> catchingup --[ RECORD 51 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 52 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 359 section | reporting_node section_path | reporting_node.from_context @@ -785,7 +800,7 @@ active_node_assigned_state | secondary other_node_assigned_state | has_extra_action | t comment | join_secondary, primary reported wait_primary with goal wait/primary -> secondary --[ RECORD 52 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 53 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 361 section | reporting_node section_path | reporting_node.from_context @@ -800,7 +815,7 @@ active_node_assigned_state | secondary other_node_assigned_state | has_extra_action | f comment | join_secondary, primary converged primary -> secondary --[ RECORD 53 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 54 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 363 section | reporting_node section_path | reporting_node.ms_failover.retry_reset @@ -815,7 +830,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, guard_data_loss=true -> report_lsn (retry once a source recovers) --[ RECORD 54 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 55 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 365 section | reporting_node section_path | reporting_node.ms_failover.candidate_join @@ -830,7 +845,7 @@ active_node_assigned_state | join_secondary other_node_assigned_state | has_extra_action | f comment | MS-failover: activeNode in report_lsn, failover candidate ready to stream WAL -> join_secondary --[ RECORD 55 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 56 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 367 section | reporting_node section_path | reporting_node.ms_failover.candidate_fanout @@ -845,7 +860,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) --[ RECORD 56 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 57 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 369 section | reporting_node section_path | reporting_node.ms_failover.candidate_fanout @@ -860,7 +875,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | MS-failover fan-out: rejoining from maintenance -> report_lsn (2 of 4) --[ RECORD 57 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 58 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 371 section | reporting_node section_path | reporting_node.ms_failover.candidate_fanout @@ -875,7 +890,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | MS-failover fan-out: old primary converged draining or demoted -> report_lsn (3 of 4) --[ RECORD 58 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 59 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 373 section | reporting_node section_path | reporting_node.ms_failover.candidate_fanout @@ -890,7 +905,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | MS-failover fan-out: old primary demoted, was rejoining a now-failed primary -> report_lsn (4 of 4) --[ RECORD 59 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 60 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 375 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome @@ -905,7 +920,7 @@ active_node_assigned_state | prepare_promotion other_node_assigned_state | has_extra_action | f comment | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate already has all WAL -> prepare_promotion (1 of 2 -- see this row's own comment on why both are listed) --[ RECORD 60 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 61 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 377 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome @@ -920,7 +935,7 @@ active_node_assigned_state | fast_forward other_node_assigned_state | has_extra_action | f comment | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate is lagging -> fast_forward (2 of 2) --[ RECORD 61 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 62 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 379 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.missing_nodes_gate @@ -935,7 +950,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=true -> decline, wait for more reports (1 of 2) --[ RECORD 62 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 63 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 381 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.missing_nodes_gate @@ -950,7 +965,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=false -> proceed despite possible data loss (2 of 2) --[ RECORD 63 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 64 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 383 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.candidate_count_gate @@ -965,7 +980,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | f comment | MS-failover: zero candidates have reported their LSN yet -> silent decline --[ RECORD 64 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 65 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 385 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.quorum_candidate_gate @@ -980,7 +995,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | MS-failover: not enough quorum candidates reported yet, guard_data_loss=true -> decline (1 of 2) --[ RECORD 65 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 66 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 387 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.quorum_candidate_gate @@ -995,7 +1010,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | MS-failover: not enough quorum candidates reported yet, guard_data_loss=false -> proceed with fewer than required (2 of 2) --[ RECORD 66 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 67 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 389 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.no_candidate_yet @@ -1010,7 +1025,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | f comment | MS-failover: no promotion in flight, not enough (or not safe enough) candidates yet -> no-op besides the fan-out above --[ RECORD 67 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 68 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 391 section | reporting_node section_path | reporting_node.ms_failover.draining_or_maintenance @@ -1025,7 +1040,7 @@ active_node_assigned_state | other_node_assigned_state | draining has_extra_action | f comment | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining --[ RECORD 68 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 69 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 393 section | reporting_node section_path | reporting_node.ms_failover.draining_or_maintenance @@ -1040,7 +1055,7 @@ active_node_assigned_state | other_node_assigned_state | maintenance has_extra_action | f comment | nodesCount>2, primary unhealthy, converged prepare_maintenance -> primary maintenance --[ RECORD 69 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 70 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 401 section | primary_node section_path | primary_node @@ -1055,7 +1070,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | primary alone, another node reached wait_standby -> wait_primary --[ RECORD 70 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 403 section | primary_node section_path | primary_node @@ -1070,7 +1085,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | all nodes async, zero secondaries -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 405 section | primary_node section_path | primary_node @@ -1085,7 +1100,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | all nodes async, >=1 secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 407 section | primary_node section_path | primary_node @@ -1100,7 +1115,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 409 section | primary_node section_path | primary_node @@ -1115,7 +1130,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 75 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 411 section | primary_node section_path | primary_node @@ -1130,7 +1145,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 75 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 413 section | primary_node section_path | primary_node @@ -1145,7 +1160,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, both zero -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 415 section | primary_node section_path | primary_node @@ -1160,7 +1175,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 417 section | primary_node section_path | primary_node @@ -1175,7 +1190,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 419 section | primary_node section_path | primary_node @@ -1190,7 +1205,7 @@ active_node_assigned_state | other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/wait_primary/apply_settings, no other condition applies -> no-op besides the unhealthy-secondary fan-out to catchingup --[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 80 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 421 section | primary_node section_path | primary_node diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 7fe510982..839d3cb71 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -316,6 +316,25 @@ static const NodeStatePattern FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY = { REPLICATION_STATE_APPLY_SETTINGS), }; +/* + * reportedState alone (goalState irrelevant) is one of the roles + * CanTakeWritesInState() recognizes as "serving as primary", minus SINGLE -- + * NODE_STATE_REPORTED, deliberately not NODE_STATE_STABLE: a STABLE pattern + * requires reportedState == goalState, which breaks the moment a row using + * this pattern reassigns goalState itself (see pos 210's own comment for the + * real oscillation this caused when it instead relied on IsInPrimaryState(), + * which has the same reported == goal requirement built in). Matching on + * reportedState only means the match survives across the row's own + * extraAction, since nothing here ever touches reportedState directly. + */ +static const NodeStatePattern FSM_REPORTED_PRIMARY_ROLE_STATES = { + .kind = NODE_STATE_REPORTED, + .reportedStates = STATES(REPLICATION_STATE_PRIMARY, + REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_JOIN_PRIMARY, + REPLICATION_STATE_APPLY_SETTINGS), +}; + /* * reported WAIT_PRIMARY, goal in {WAIT_PRIMARY, PRIMARY} -- join_secondary's * cascade row @@ -1149,11 +1168,13 @@ typedef struct MonitorFSMTransition * already primary-role) and from ActionRunPrimaryNodeTransition's * nested pass on primaryNode (join_secondary's cascade row). * - * MonitorFSM_SIZE - * Total row count -- the size every bounded search's own linear scan - * runs over (see FindMatchingMonitorFSMRuleIndexUnderPath), and the end - * bound dump_fsm()/dump_fsm_edges()/AssertMonitorFSMWellFormed() each - * iterate to. + * Table end + * MonitorFSM[] ends with a terminator row (.pos left at its zero + * default, which no real row ever has) rather than a separately + * maintained count -- every bounded search's own linear scan (see + * FindMatchingMonitorFSMRuleIndexUnderPath) and every full-table walk + * (dump_fsm()/dump_fsm_edges()/AssertMonitorFSMWellFormed()) stops there + * instead. * * What makes a row's own sectionPath safe to hand-write (rather than a * foot-gun): every row also carries its own .pos (see MonitorFSMTransition @@ -1187,7 +1208,6 @@ static void ActionLogMSFailoverQuorumContinue(GroupStateContext *ctx, NodeActiveContext *nac, char *message); -#define MonitorFSM_SIZE 79 #define MonitorFSM_MultiStandbyCascadeResumeAfterPos 305 static const MonitorFSMSectionPath SectionApiTriggered = @@ -1301,13 +1321,20 @@ RuleMatches(const NodeActiveContext *nac, const MonitorFSMTransition *rule) * very start of whichever rows are under prefix", the ordinary case; a specific * pos is only ever passed for the one genuine mid-section resume point this * table has (see MonitorFSM_MultiStandbyCascadeResumeAfterPos). + * + * table is always MonitorFSM[] itself, terminated by a sentinel row whose + * .pos is left at its zero default (no real row ever has pos <= 0) -- the + * loop below stops there instead of at a separately hand-maintained count, + * which is exactly the mechanism that once let a row silently fall out of + * every bounded search in this file when it drifted out of sync (see the + * MonitorFSM[] array's own trailing comment, right after its last real row). */ static int -FindMatchingMonitorFSMRuleIndexUnderPath(const MonitorFSMTransition table[], int tableSize, +FindMatchingMonitorFSMRuleIndexUnderPath(const MonitorFSMTransition table[], const MonitorFSMSectionPath prefix, int afterPos, const NodeActiveContext *nac) { - for (int i = 0; i < tableSize; i++) + for (int i = 0; table[i].pos != 0; i++) { if (table[i].pos <= afterPos) { @@ -1436,8 +1463,7 @@ static bool FindAndDispatchMonitorFSMRuleUnderPath(GroupStateContext *ctx, NodeActiveContext *nac, const MonitorFSMSectionPath prefix, int afterPos) { - int index = FindMatchingMonitorFSMRuleIndexUnderPath(MonitorFSM, MonitorFSM_SIZE, - prefix, afterPos, nac); + int index = FindMatchingMonitorFSMRuleIndexUnderPath(MonitorFSM, prefix, afterPos, nac); if (index < 0) { @@ -2000,7 +2026,7 @@ ProceedGroupStateForApiTrigger(MonitorApiFunction apiFunction, BuildGroupStateContext(&ctx, activeNode); BuildApiTriggerNodeActiveContext(&ctx, apiFunction, activeNode, primaryNode, &nac); - int index = FindMatchingMonitorFSMRuleIndexUnderPath(MonitorFSM, MonitorFSM_SIZE, + int index = FindMatchingMonitorFSMRuleIndexUnderPath(MonitorFSM, SectionApiTriggered, 0, &nac); if (index < 0) @@ -2414,7 +2440,51 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNodeAssignedState = GOAL(REPLICATION_STATE_SINGLE), .comment = "alone in group, candidate-eligible -> single" }, - /* alone in group, not candidate-eligible */ + /* + * alone in group, not candidate-eligible, but already serving as primary + * -- candidatePriority=0 exists to steer a FUTURE election away from + * this node when an alternative exists; once every other node is gone, + * there is no alternative left to prefer, and demoting the cluster's + * only remaining writable node to report_lsn would strand it there + * indefinitely (no candidate will ever appear to promote instead) for + * no operational benefit -- the least-surprising behavior is to let it + * keep serving as SINGLE, exactly as a candidate-eligible lone primary + * already does via pos 209. Checked before pos 211 below (a strict + * subset of its own condition -- candidateEligible=FALSE, plus + * "already primary-role") so first-match-wins routes this specific case + * here instead. + * + * Gated on FSM_REPORTED_PRIMARY_ROLE_STATES (reportedState alone), not + * isInPrimaryState() (which also requires goalState to already agree): + * a live pgaftest run (keeper_fsm_gap_211_primary_priority_zero.pgaf) + * caught a real self-undermining oscillation with the isInPrimaryState + * version -- this row's own extraAction reassigns goalState to SINGLE, + * which makes isInPrimaryState() evaluate false on the very next + * dispatch (goalState=single no longer agrees with reportedState, which + * is still primary, and single isn't in isInPrimaryState()'s own + * {PRIMARY,APPLY_SETTINGS} second disjunct either) -- so this row + * stopped matching one dispatch after firing, and pos 211 (which has no + * such requirement) fired right behind it and overwrote the assignment + * back to REPORT_LSN. reportedState is untouched by this row's own + * action, so matching on it alone stays stable across dispatches until + * the keeper itself actually converges to single. + */ + { .pos = 210, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, + .activeNode = { .statePattern = FSM_REPORTED_PRIMARY_ROLE_STATES, + .candidateEligible = BOOL_FALSE }, + .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_SINGLE), + .comment = "alone in group, already primary despite candidatePriority " + "zero -> single" }, + + /* + * alone in group, not candidate-eligible, and was never already primary + * (a lone surviving standby) -- candidatePriority=0 is honored here: + * wait for an operator or a new peer, don't self-promote. + */ { .pos = 211, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS @@ -3226,10 +3296,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "primary maintenance" }, /* - * --- [MonitorFSM_PrimaryNodeSectionStart, MonitorFSM_SIZE): the - * declarative replacement for ProceedGroupStateForPrimaryNode()'s own - * sequential if-chain. Here .activeNode maps to the primaryNode parameter, - * not a reporting node -- reached either directly by the top-level driver + * --- the PRIMARY_NODE section (sectionPath[0] == + * MONITOR_FSM_SECTION_PRIMARY_NODE, pos 401 onward): the declarative + * replacement for ProceedGroupStateForPrimaryNode()'s own sequential + * if-chain. Here .activeNode maps to the primaryNode parameter, not a + * reporting node -- reached either directly by the top-level driver * (activeNode already primary-role) or via ActionRunPrimaryNodeTransition's * nested pass on primaryNode (the join_secondary cascade row above). */ @@ -3387,6 +3458,20 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), .comment = "backwards-compat: join_primary -> primary" }, + + /* + * Terminator, not a real rule -- .pos is deliberately left unset (its + * zero default), which no real row above ever has (every real .pos + * starts at 101). Every loop over MonitorFSM[] in this file stops here + * instead of at a separately hand-maintained size constant (see this + * array's own leading comment and AssertMonitorFSMWellFormed() for why + * that constant was removed): a row inserted anywhere above this one is + * automatically in scope for every one of those loops, and a row + * mistakenly inserted after this one instead is caught by + * AssertMonitorFSMWellFormed()'s own iteration-count sanity check. This + * row must always stay last. + */ + { .comment = "terminator -- do not add rows after this one" } }; /* @@ -3420,11 +3505,30 @@ static void AssertMonitorFSMWellFormed(void) { #ifdef USE_ASSERT_CHECKING + int previousPos = 0; bool foundResumeAnchor = false; - for (int i = 0; i < MonitorFSM_SIZE; i++) + /* + * MonitorFSM[] ends with a terminator row (.pos left at its zero + * default) rather than a separately maintained count -- a hand- + * maintained MonitorFSM_SIZE #define used to serve this purpose, and + * adding a row without also bumping it once silently dropped pos 421 + * (the actual last row at the time) out of every loop bounded by it, + * including dispatch itself -- caught only by chance, cross-checking + * dump_fsm_edges() output by hand against expected keeper edges, not by + * anything in this file. A terminator can't go stale the same way: it's + * part of the array's own literal initializer, so any row added before + * it is automatically in scope for every loop below. The iteration cap + * here is just a sanity net against the terminator itself ever being + * removed or a new row accidentally inserted after it, which would + * otherwise turn every one of these loops into an unbounded scan past + * the end of the array. + */ + for (int i = 0; MonitorFSM[i].pos != 0; i++) { + Assert(i < 1000); + int pos = MonitorFSM[i].pos; MonitorFSMSection top = MonitorFSM[i].sectionPath[0]; @@ -4146,7 +4250,7 @@ dump_fsm(PG_FUNCTION_ARGS) MemoryContextSwitchTo(oldContext); - for (int i = 0; i < MonitorFSM_SIZE; i++) + for (int i = 0; MonitorFSM[i].pos != 0; i++) { const MonitorFSMTransition *rule = &MonitorFSM[i]; Datum values[14]; @@ -4812,7 +4916,7 @@ dump_fsm_edges(PG_FUNCTION_ARGS) MemoryContextSwitchTo(oldContext); - for (int i = 0; i < MonitorFSM_SIZE; i++) + for (int i = 0; MonitorFSM[i].pos != 0; i++) { const MonitorFSMTransition *rule = &MonitorFSM[i]; @@ -5297,7 +5401,7 @@ TryFanOutReportLsnRow(GroupStateContext *ctx, AutoFailoverNode *node) static bool DispatchMonitorFSMRuleByPos(GroupStateContext *ctx, NodeActiveContext *nac, int pos) { - for (int i = 0; i < MonitorFSM_SIZE; i++) + for (int i = 0; MonitorFSM[i].pos != 0; i++) { if (MonitorFSM[i].pos == pos) { diff --git a/tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf b/tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf index ac0b40e4d..99e7e3132 100644 --- a/tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf +++ b/tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf @@ -1,42 +1,42 @@ -# Diagnostic spec for MonitorFSM[] pos 211 ("alone in group, -# candidatePriority zero -> report_lsn", src/monitor/group_state_machine.c) -# -- NOT in the CI schedule, same convention as debug_failover_pg19.pgaf, -# since it is expected to FAIL until the missing KeeperFSM[] transition -# documented below is implemented. +# Regression spec for MonitorFSM[] pos 210/211 ("alone in group, +# candidatePriority zero", src/monitor/group_state_machine.c). # -# src/monitor/sql/keeper_fsm_edges.sql's Step 2a found that pos 211's -# NodeStatePattern is willing to assign REPORT_LSN_STATE to a lone remaining -# node reporting PRIMARY_STATE as its current_state (among ten other -# fanned-out states), but KeeperFSM[] (src/bin/pg_autoctl/fsm.c) has no -# PRIMARY_STATE -> REPORT_LSN_STATE row at all. +# This spec originally documented a real gap: pos 211 ("alone in group, +# candidatePriority zero -> report_lsn") fired even when the lone remaining +# node was already serving as primary, and KeeperFSM[] (src/bin/pg_autoctl/ +# fsm.c) has no PRIMARY_STATE -> REPORT_LSN_STATE row -- so the node got +# stuck reporting "primary" forever, still accepting writes, while the +# monitor believed it had been demoted. # -# This spec manufactures that exact (current, assigned) pair deterministically -# (no timing race: candidate-priority and node role are both explicit, -# durable node properties, not transient states): +# The fix is on the monitor side, not the keeper side: demoting the +# cluster's only remaining writable node to report_lsn once every other +# node is gone serves no purpose (there is no alternative candidate left to +# promote instead), so pos 210 now assigns SINGLE instead, exactly as a +# candidate-eligible lone primary already gets via pos 209. +# +# This spec now asserts that fixed behavior: # # 1. node2 is promoted to primary, then its own candidate-priority is set # to 0. # 2. node1 (now the only secondary) is dropped, leaving node2 alone in the # group -- still reporting PRIMARY_STATE, since nothing prompts it to # change role on its own. -# 3. The monitor recomputes: groupHasExactlyOneNode is now true, node2's -# own reported current_state is "primary", and its candidate priority -# is 0 (not eligible), so pos 211 fires and the monitor assigns -# REPORT_LSN. -# 4. node2's keeper receives that assigned state, but -# keeper_fsm_reach_assigned_state() (fsm.c) finds no matching -# KeeperFSM[] row for (primary -> report_lsn) and logs -# "pg_autoctl does not know how to reach state \"report_lsn\" from -# \"primary\"" (fsm.c's own log_fatal call) -- node2 never converges, -# staying stuck reporting "primary" forever (still accepting writes, -# which the monitor no longer expects once "report_lsn" is assigned). +# 3. The monitor recomputes: groupHasExactlyOneNode is now true, node2 is +# still primary-role (reportedState in {primary, wait_primary, +# join_primary, apply_settings}), and its candidate priority is 0 (not +# eligible) -- pos 210 fires and the monitor assigns SINGLE. +# 4. node2's keeper has an ordinary PRIMARY_STATE -> SINGLE_STATE +# transition (KeeperFSM[]), so it actually converges: reportedstate +# becomes "single", matching the assigned goal. # -# Expected fix: add a PRIMARY_STATE -> REPORT_LSN_STATE row to KeeperFSM[], -# see this file's own header comment in fsm.c for the reusable-code question -# (a lone priority-zero primary demoting itself to report_lsn is a novel -# transition shape; check whether any existing transition function already -# does "stop being a writable primary without a promotion counterpart" that -# could be reused, e.g. the DEMOTED_STATE-family transitions). +# pos 210's own condition is gated on node2's *reported* state alone (not +# IsInPrimaryState(), which also requires goalState to already agree) -- +# an earlier version of this row used isInPrimaryState=true and, live in +# this exact scenario, self-undermined one dispatch after firing: assigning +# SINGLE changed goalState away from what IsInPrimaryState() requires, so +# the row stopped matching on the very next dispatch and pos 211 (no such +# requirement) fired right behind it, overwriting the assignment back to +# REPORT_LSN. This spec is what caught that oscillation live. cluster { monitor @@ -67,35 +67,24 @@ step test_002_drop_secondary_leaves_lone_priority_zero_primary { exec node1 pg_autoctl drop node --no-wait wait until node1 stopped timeout 60s # The monitor now sees exactly one node (node2) in the group, still - # reporting "primary", with candidate-priority 0 -- pos 211 must fire - # and assign "report_lsn". - wait until node2 assigned-state = report_lsn timeout 60s + # reporting "primary", with candidate-priority 0 -- pos 210 must fire + # and assign "single" (not pos 211's "report_lsn": that would strand a + # node with no alternative candidate left to hand off to). + wait until node2 assigned-state = single timeout 60s } -step test_003_node2_cannot_reach_report_lsn_from_primary { - # This is the failure this spec exists to demonstrate: node2 is assigned - # "report_lsn" (confirmed above) but has no KeeperFSM[] transition to get - # there from "primary", so it never converges -- it keeps reporting - # "primary" indefinitely instead, retrying the failed transition on every - # loop iteration. Queried directly against pgautofailover.node rather - # than via "assert ... state is ..."/"assert ... stays ... while { }": - # neither of those constructs fit here, since node2's *reported* state - # (reportedstate) and its *goal* state (goalstate) have genuinely and - # permanently diverged -- there's no single moment to "wait until", and - # no unchanged baseline to assert "stays" against. - # - # Once the missing transition is implemented, this query's second column - # is expected to change to "report_lsn" (node2 actually converges), at - # which point this spec should be rewritten to assert that convergence - # instead. - sleep 5s +step test_003_node2_converges_to_single { + # node2 has an ordinary PRIMARY_STATE -> SINGLE_STATE transition + # (KeeperFSM[]), so unlike the original report_lsn assignment, this one + # is actually reachable -- confirm it converges rather than getting + # stuck reporting "primary" forever. + wait until node2 state is single timeout 60s sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; } - expect { primary } + expect { single } sql monitor { SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node2'; } - expect { report_lsn } - logs node2 contains "does not know how to reach state" + expect { single } } From 08e7dbe48945a4ce78f958b9346d25d960102e85 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 22:35:52 +0200 Subject: [PATCH 27/52] tests: schedule keeper_fsm_gap_209/211 specs into CI (node.sch) Both specs were live-validated (individually and together via --schedule) against the pos 210/isInPrimaryState oscillation fix and the pos 209 wait_maintenance gap fix, and now assert the FIXED behavior rather than demonstrating a bug -- add them to tests/tap/schedules/node.sch (the schedule ci.yml's test_pgaftest job actually runs) alongside the other FSM-edge-case regression specs (demote_timeout_wait_primary_deadlock, timeline_fork_report_lsn_deadlock), and to the legacy tests/tap/schedule for documentation consistency (not read by CI, but kept in sync). Re-ran make installcheck against current HEAD to check for a new list of keeper-unreachable edges: none -- fsm.out, check_fsm_reachability.out, and keeper_fsm_edges.out are byte-identical to their committed expected versions (confirmed via explicit diff, not just pg_regress's own ok/not ok). pos 210's REPORTED-kind pattern rewrite produces the exact same set of assigned edges as before, just via a mechanism that doesn't self-undermine -- no new gap opened or closed. --- tests/tap/schedule | 2 ++ tests/tap/schedules/node.sch | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/tap/schedule b/tests/tap/schedule index 417560ee0..e6126b4d4 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -31,6 +31,8 @@ fast_forward demote_timeout_wait_primary_deadlock wait_primary_draining_deadlock timeline_fork_report_lsn_deadlock +keeper_fsm_gap_209_wait_maintenance +keeper_fsm_gap_211_primary_priority_zero extension_update tablespaces installcheck diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index 84ce20484..c835d793e 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -1,4 +1,4 @@ -# Node lifecycle, monitor operations, and Debian/tablespace layouts (~30 min). +# Node lifecycle, monitor operations, and Debian/tablespace layouts (~32 min). # Merged from former node, monitor, and node-extra schedules to reduce CI job # count and GitHub Actions runner queue pressure. create_standby_with_pgdata @@ -15,3 +15,5 @@ replication_stall_3dc demote_timeout_wait_primary_deadlock timeline_fork_report_lsn_deadlock timeline_fork_3node_auto_detect +keeper_fsm_gap_209_wait_maintenance +keeper_fsm_gap_211_primary_priority_zero From 506c2dc6d112bbac9535025223d08416c1445587 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 30 Jul 2026 23:23:48 +0200 Subject: [PATCH 28/52] monitor FSM: narrow pos 211 so dump_fsm_edges() reflects the pos 210 fix pos 210 (added in 35de216) makes an already-primary, priority-zero lone node go to SINGLE instead of pos 211's REPORT_LSN, and always intercepts those 4 states first (identical candidateEligible/groupHasExactlyOneNode conditions, checked earlier in the array) -- but dump_fsm_edges() kept listing pos 211 as still capable of producing primary/wait_primary/ join_primary/apply_settings -> report_lsn. Verified live via dump_fsm_edges() directly: the edge was still there, and Step 2a of keeper_fsm_edges.out still flagged it as a keeper-coverage gap, even though it's genuinely unreachable in practice. Root cause: dump_fsm_edges()'s shadow-detector (EdgeIsShadowedByEarlierRule / RuleUnconditionallyMatchesActiveNodeState) only recognizes an earlier row as shadowing a state when that earlier row is *fully* unconditional -- every field besides the state pattern must be the wildcard default. pos 210 isn't: it also requires candidateEligible=FALSE and groupHasExactlyOneNode=TRUE. Those happen to be identical to pos 211's own conditions, making the shadow real and provable for this specific pair, but the existing heuristic isn't built to prove that kind of matching- conditions subsumption, only true unconditionality. Rather than generalize the shadow-detector (invasive, and this project has been deliberately conservative there -- a false positive would silently hide a real edge), narrow pos 211's own pattern instead, making its code match what its own comment already claimed ("was never already primary"). Adds a new NodeStatusPattern field, reportedCanTakeWrites (CanTakeWritesInState applied to reportedState alone, mirroring the existing goalState-based canTakeWrites) -- deliberately reported-only, for the same reason pos 210's own fix needed a reported-only condition: gating on anything goalState-dependent risks the self-undermining oscillation documented on pos 210 (a row whose own extraAction changes goalState can't safely condition its own match on goalState). Wired into NodeMatchesPattern, the dump_fsm() conditions-text column, the shadow- detector's own "other fields are ANY" check, and both dump_fsm_edges() enumeration loops (activeNode/primaryNode). pos 211 now sets reportedCanTakeWrites=FALSE, dropping the 4 already- pos-210-owned states from its own resolved edge set. Verified: dump_fsm_edges()'s total edge count drops from 202 to 198 (exactly the 4 phantom edges), Step 2a's gap count drops from 98 to 94 rows, full regress (19/19) + isolation (6/6) suites pass with regenerated expected/fsm.out, check_fsm_reachability.out, and keeper_fsm_edges.out, and both keeper_fsm_gap_209/211 pgaftest specs still pass live. --- .../expected/check_fsm_reachability.out | 4 +- src/monitor/expected/fsm.out | 2 +- src/monitor/expected/keeper_fsm_edges.out | 8 +- src/monitor/group_state_machine.c | 78 ++++++++++++++++++- 4 files changed, 82 insertions(+), 10 deletions(-) diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index e7a569f7f..d2cc76df2 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 202 + 198 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 202 + 198 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index 926454ff0..3223350b0 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -342,7 +342,7 @@ section_path | early_checks active_node_current_state | other_node_current_state | candidate_node_current_state | -active_node_conditions | candidateEligible=false +active_node_conditions | candidateEligible=false, reportedCanTakeWrites=false other_node_conditions | candidate_node_conditions | group_conditions | groupHasExactlyOneNode=true diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index a95ebbbcd..cf8928a1a 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -185,15 +185,11 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 209 | 1 | wait_maintenance | single | alone in group, candidate-eligible -> single 209 | 1 | fast_forward | single | alone in group, candidate-eligible -> single 209 | 1 | join_secondary | single | alone in group, candidate-eligible -> single - 211 | 11 | | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | wait_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | primary | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 7 | | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | demote_timeout | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | prepare_promotion | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | wait_standby | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | join_primary | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | apply_settings | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | wait_maintenance | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | fast_forward | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | join_secondary | report_lsn | alone in group, candidatePriority zero -> report_lsn @@ -277,7 +273,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 351 | 1 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 391 | 1 | | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining 391 | 1 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining -(98 rows) +(94 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 839d3cb71..264593724 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -502,6 +502,7 @@ typedef struct NodeStatusPattern BoolPattern isInMaintenance; BoolPattern isDemotedPrimary; BoolPattern canTakeWrites; + BoolPattern reportedCanTakeWrites; BoolPattern isReadyToStreamWAL; BoolPattern drainTimeExpired; BoolPattern isCitusWorkerGroup; @@ -561,6 +562,16 @@ BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *stat * transition node (goalState allows writes, reportedState hasn't caught up * yet) still satisfies where isInPrimaryState (which additionally requires * stability) would not. + * + * reportedCanTakeWrites is CanTakeWritesInState applied to reportedState + * instead of goalState -- the reported-only counterpart pos 210's own + * comment explains the need for: a row whose extraAction reassigns + * goalState (any GOAL(...) assignment) can't gate its own match on anything + * that reads goalState (canTakeWrites, isInPrimaryState) without risking the + * exact self-undermining oscillation documented there, since the row's own + * action changes the very fact its condition is testing. reportedState is + * never written by this row's own action, so a condition built purely from + * it stays stable across dispatches until the keeper itself converges. */ static bool NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) @@ -585,6 +596,9 @@ NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) BoolMatchesPattern(status->node != NULL && CanTakeWritesInState(status->node->goalState), pattern->canTakeWrites) && + BoolMatchesPattern(status->node != NULL && + CanTakeWritesInState(status->node->reportedState), + pattern->reportedCanTakeWrites) && BoolMatchesPattern(CandidateNodeIsReadyToStreamWAL(status->node), pattern->isReadyToStreamWAL) && BoolMatchesPattern(NodeIsDrainTimeExpired(status->node, status->ctx), @@ -2484,13 +2498,35 @@ static const MonitorFSMTransition MonitorFSM[] = { * alone in group, not candidate-eligible, and was never already primary * (a lone surviving standby) -- candidatePriority=0 is honored here: * wait for an operator or a new peer, don't self-promote. + * + * reportedCanTakeWrites=FALSE explicitly excludes the 4 primary-role + * states pos 210 above already owns (a strict subset of this row's own + * candidateEligible=FALSE + groupHasExactlyOneNode=TRUE condition, so + * pos 210 always intercepts them first regardless of this exclusion -- + * first-match-wins alone already makes this row unreachable for those + * states in practice). The exclusion is added anyway because + * dump_fsm_edges()'s own shadow-detector (EdgeIsShadowedByEarlierRule) + * only recognizes an earlier row as shadowing when it's *fully* + * unconditional for the state -- pos 210 additionally requires + * candidateEligible=FALSE and groupHasExactlyOneNode=TRUE, so it doesn't + * qualify even though those conditions are identical to this row's own. + * Without this exclusion, dump_fsm_edges() (and hence + * keeper_fsm_edges.sql's Step 2a) kept listing "primary/wait_primary/ + * join_primary/apply_settings -> report_lsn" as a live keeper-coverage + * gap even after pos 210 made it unreachable at runtime -- a real + * runtime fix that the diagnostic didn't reflect. Making this row's own + * pattern match its comment ("was never already primary") directly is + * simpler and safer than generalizing the shadow-detector to prove + * conditional (not just unconditional) shadowing between arbitrary row + * pairs. */ { .pos = 211, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, - .candidateEligible = BOOL_FALSE }, + .candidateEligible = BOOL_FALSE, + .reportedCanTakeWrites = BOOL_FALSE }, .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), .comment = "alone in group, candidatePriority zero -> report_lsn" }, @@ -4088,6 +4124,7 @@ NodeStatusPatternConditionsText(const NodeStatusPattern *pattern, bool *isNull) APPEND_BOOL_CONDITION(&buf, "isInMaintenance", pattern->isInMaintenance); APPEND_BOOL_CONDITION(&buf, "isDemotedPrimary", pattern->isDemotedPrimary); APPEND_BOOL_CONDITION(&buf, "canTakeWrites", pattern->canTakeWrites); + APPEND_BOOL_CONDITION(&buf, "reportedCanTakeWrites", pattern->reportedCanTakeWrites); APPEND_BOOL_CONDITION(&buf, "isReadyToStreamWAL", pattern->isReadyToStreamWAL); APPEND_BOOL_CONDITION(&buf, "drainTimeExpired", pattern->drainTimeExpired); APPEND_BOOL_CONDITION(&buf, "isCitusWorkerGroup", pattern->isCitusWorkerGroup); @@ -4530,6 +4567,32 @@ NodeStatusPatternSurvivesIsInPrimaryState(const NodeStatusPattern *pattern, } +/* + * NodeStatusPatternSurvivesReportedCanTakeWrites filters a candidate + * edge-source state against pattern's own .reportedCanTakeWrites field + * (BOOL_ANY -- the vast majority of rows -- always survives). Unlike + * isInPrimaryState, this needs no separate "does some goal exist making this + * true" satisfiability proof: reportedCanTakeWrites is CanTakeWritesInState + * applied to reportedState alone (see NodeMatchesPattern's own comment), so + * for a hypothetical reportedState of state, whether the pattern's demand is + * satisfiable is just CanTakeWritesInState(state) itself -- no goalState + * involved at all. + */ +static bool +NodeStatusPatternSurvivesReportedCanTakeWrites(const NodeStatusPattern *pattern, + ReplicationState state) +{ + if (pattern->reportedCanTakeWrites == BOOL_ANY) + { + return true; + } + + bool required = (pattern->reportedCanTakeWrites == BOOL_TRUE); + + return CanTakeWritesInState(state) == required; +} + + /* * NodeStatePatternKindIsReportedStateOnly: true for the pattern kinds whose * match genuinely depends only on reportedState (ignoring, for STABLE, its @@ -4615,6 +4678,7 @@ NodeStatusPatternOtherFieldsAreAny(const NodeStatusPattern *pattern) pattern->isInMaintenance == BOOL_ANY && pattern->isDemotedPrimary == BOOL_ANY && pattern->canTakeWrites == BOOL_ANY && + pattern->reportedCanTakeWrites == BOOL_ANY && pattern->isReadyToStreamWAL == BOOL_ANY && pattern->drainTimeExpired == BOOL_ANY && pattern->isCitusWorkerGroup == BOOL_ANY && @@ -4946,6 +5010,12 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + if (!NodeStatusPatternSurvivesReportedCanTakeWrites(&rule->activeNode, + states[j])) + { + continue; + } + if (EdgeIsShadowedByEarlierRule(i, states[j], false, rule->sectionPath[0])) { continue; @@ -4994,6 +5064,12 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + if (!NodeStatusPatternSurvivesReportedCanTakeWrites(&rule->primaryNode, + states[j])) + { + continue; + } + if (EdgeIsShadowedByEarlierRule(i, states[j], true, rule->sectionPath[0])) { continue; From 2f008022bd252935e7a1b72ea81b63289b2bd78b Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Fri, 31 Jul 2026 02:02:00 +0200 Subject: [PATCH 29/52] keeper: fix wait_maintenance keeper gap; monitor: exclude wait_standby from pos 209/211 Follow-up to the pos 209/211 gap investigation: for each of the two already-proven-real gap states (wait_maintenance, wait_standby), fixed the one that had a real fix and documented the one that didn't. wait_maintenance (KeeperFSM[], fsm.c): a converged, actively-streaming standby whose only peer vanishes while it's waiting to enter maintenance had no WAIT_MAINTENANCE_STATE -> SINGLE_STATE (pos 209) or -> REPORT_LSN_ STATE (pos 211) row, so it got stuck. Added both, reusing fsm_promote_ standby and fsm_report_lsn respectively -- the same functions every other converged-standby source state (SECONDARY/CATCHINGUP/PREP_PROMOTION/ STOP_REPLICATION) already reuses for the same targets, since entering wait_maintenance itself runs no transition function and leaves Postgres running and replicating normally. Regenerated keeper_fsm_edges.json (the checked-in fixture keeper_fsm_edges.sql reads, generated via `pg_autoctl inspect fsm list --json` -- confirmed via Makefile inspection this has no build-time dependency wiring at all, so it silently goes stale unless manually regenerated whenever KeeperFSM[] changes, exactly as happened here). wait_standby: a node stuck there never actually started streaming (no pg_basebackup done -- fsm_init_standby, the WAIT_STANDBY_STATE -> CATCHINGUP_STATE transition, never ran). The only available local action for SINGLE_STATE (fsm_init_primary, reused by INIT_STATE/DROPPED_STATE -> SINGLE_STATE) does a fresh initdb, generating a new system_identifier -- which collides with the one already registered for this node, tripping the monitor's own same_system_identifier_within_group exclusion constraint (confirmed live: "Failed to transition from state \"wait_standby\" to state \"single\""). No safe keeper-side fix exists. Excluded wait_standby from pos 209/211 entirely instead (new reportedIsWaitStandby field, NodeStatusPattern) -- the monitor simply never assigns a goal the keeper can't safely reach, leaving the node visibly stuck (still unhealthy) for an operator to notice, rather than pretending a fix exists. The exclusion needed two iterations to get right. The first version folded WAIT_STANDBY into the existing FSM_NOT_STABLE_SINGLE pattern (a NOT_STABLE kind, which requires reported == goal to exclude a state) -- this passed regress but still failed live: pos 101 (remove_node()'s own fan-out, unconditionally assigning REPORT_LSN to every surviving non-maintenance standby) rewrites the lone node's goalState synchronously as part of dropping its peer, before pos 209/211 ever evaluate it on its own next heartbeat. That breaks reported == goal, reviving the NOT_STABLE exclusion's match -- the exact self-undermining class of bug pos 210 hit earlier in this same investigation. Fixed by making the field goal-independent (a plain reportedState equality check, mirroring reportedCanTakeWrites), which cannot be defeated by another row's own goalState write. Confirmed live this time: node2 stays safely parked (reportedstate never leaves wait_standby, logs "Still waiting for the monitor to drive us to state \"catchingup\"") regardless of what its own goalState ends up as via pos 101's independent fan-out. Added keeper_fsm_gap_211_wait_maintenance.pgaf, keeper_fsm_gap_209_ wait_standby.pgaf, keeper_fsm_gap_211_wait_standby.pgaf (new) and rewrote keeper_fsm_gap_209_wait_maintenance.pgaf (existing, now asserts the fixed convergence instead of the bug it used to demonstrate). Documented pos 209/211's other remaining gap states (prepare_maintenance, demote_timeout, prepare_promotion, stop_replication, fast_forward, join_secondary) and pos 325's remaining "single" state as investigated -- each either contrived (implies a multi-node context or a tension with candidateEligible=FALSE) or a genuine model contradiction (325) -- not pursued this pass. Verified: full regress (19/19) + isolation (6/6) pass with regenerated fsm.out/check_fsm_reachability.out/keeper_fsm_edges.out, all 5 gap specs (the 2 rewritten/new wait_maintenance ones, the 2 new wait_standby ones, and the existing primary_priority_zero one) pass live against a rebuilt Docker image. --- src/bin/pg_autoctl/fsm.c | 35 ++++++++ .../expected/check_fsm_reachability.out | 4 +- src/monitor/expected/fsm.out | 4 +- src/monitor/expected/keeper_fsm_edges.out | 42 +++++++-- src/monitor/group_state_machine.c | 85 +++++++++++++++++- src/monitor/keeper_fsm_edges.json | 8 ++ src/monitor/sql/keeper_fsm_edges.sql | 28 ++++++ .../keeper_fsm_gap_209_wait_maintenance.pgaf | 54 ++++++------ .../keeper_fsm_gap_209_wait_standby.pgaf | 84 ++++++++++++++++++ .../keeper_fsm_gap_211_wait_maintenance.pgaf | 86 +++++++++++++++++++ .../keeper_fsm_gap_211_wait_standby.pgaf | 78 +++++++++++++++++ 11 files changed, 466 insertions(+), 42 deletions(-) create mode 100644 tests/tap/specs/keeper_fsm_gap_209_wait_standby.pgaf create mode 100644 tests/tap/specs/keeper_fsm_gap_211_wait_maintenance.pgaf create mode 100644 tests/tap/specs/keeper_fsm_gap_211_wait_standby.pgaf diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index 8ffe189fd..d6fdfad71 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -161,6 +161,10 @@ #define COMMENT_REPORT_LSN_TO_SINGLE \ "There is no other node anymore, promote this node" +#define COMMENT_WAIT_MAINTENANCE_TO_SINGLE \ + "Was waiting to be sent to maintenance, but the primary vanished, " \ + "promote this node" + #define COMMENT_FOLLOW_NEW_PRIMARY \ "Switch replication to the new primary" @@ -517,6 +521,22 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_REMOVAL }, + /* + * was waiting for the primary to disable sync replication before going + * to maintenance (a converged, actively-streaming standby -- entering + * WAIT_MAINTENANCE_STATE itself runs no transition function, see its own + * comment below), but the primary was forcibly removed instead: reuse + * fsm_promote_standby exactly like every other converged-standby source + * state above (SECONDARY/CATCHINGUP/PREP_PROMOTION/STOP_REPLICATION/ + * REPORT_LSN), since Postgres here is already running and replicating, + * with no special handling wait_maintenance itself needs. + */ + { + WAIT_MAINTENANCE_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_WAIT_MAINTENANCE_TO_SINGLE, + &fsm_promote_standby, + FSM_PHASE_REMOVAL + }, /* * On the Primary, wait for a standby to be ready: WAIT_PRIMARY @@ -860,6 +880,21 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_MAINTENANCE }, + /* + * was waiting for the primary to disable sync replication before going + * to maintenance (a converged, actively-streaming standby, same as the + * other reused source states just above), but the primary was forcibly + * removed and this node's own candidate-priority is 0 -- reuse + * fsm_report_lsn exactly like SECONDARY/CATCHINGUP/MAINTENANCE/ + * PREPARE_MAINTENANCE above. + */ + { + WAIT_MAINTENANCE_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_MAINTENANCE + }, + { REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, COMMENT_REPORT_LSN_TO_PREP_PROMOTION, diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index d2cc76df2..86592d3a8 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 198 + 196 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 198 + 196 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index 3223350b0..d3f4a7055 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -312,7 +312,7 @@ section_path | early_checks active_node_current_state | other_node_current_state | candidate_node_current_state | -active_node_conditions | candidateEligible=true +active_node_conditions | candidateEligible=true, reportedIsWaitStandby=false other_node_conditions | candidate_node_conditions | group_conditions | groupHasExactlyOneNode=true @@ -342,7 +342,7 @@ section_path | early_checks active_node_current_state | other_node_current_state | candidate_node_current_state | -active_node_conditions | candidateEligible=false, reportedCanTakeWrites=false +active_node_conditions | candidateEligible=false, reportedCanTakeWrites=false, reportedIsWaitStandby=false other_node_conditions | candidate_node_conditions | group_conditions | groupHasExactlyOneNode=true diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index cf8928a1a..0620d3c1b 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -116,14 +116,16 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; single | wait_primary stop_replication | single stop_replication | wait_primary + wait_maintenance | single wait_maintenance | maintenance + wait_maintenance | report_lsn wait_primary | single wait_primary | primary wait_primary | demoted wait_primary | join_primary wait_primary | apply_settings wait_standby | catchingup -(77 rows) +(79 rows) -- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A @@ -163,6 +165,34 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- this is Step 2a's own structural artifact of enumerating a role/predicate -- gate across every syntactically possible current_state, not a sign of 10 -- separate functional bugs. +-- +-- Follow-up investigation of pos 209/211/325's own remaining gap states +-- (after wait_maintenance and wait_standby were resolved -- see +-- group_state_machine.c's reportedIsWaitStandby field (NodeStatusPattern) +-- and KeeperFSM[]'s new WAIT_MAINTENANCE_STATE rows, fsm.c): +-- +-- * pos 209/211's remaining states (prepare_maintenance, demote_timeout, +-- prepare_promotion, stop_replication, fast_forward, join_secondary) +-- were each checked for real reachability under "alone in group" and +-- found contrived: reaching them normally implies a multi-node context +-- (fast_forward/join_secondary need another standby to fetch WAL from +-- or a newly-elected primary to join, respectively) or an internal +-- tension with candidateEligible=FALSE (prepare_promotion/stop_ +-- replication imply having already been selected as a promotion +-- candidate; demote_timeout's genuinely-stuck case is already +-- intercepted earlier by pos 207). None ruled out as impossible, but +-- none reproducible via a single, realistic operator/network-failure +-- sequence the way wait_maintenance and wait_standby were -- left as +-- documented artifacts, not pursued further this pass. +-- * pos 325's remaining "single" state (the primaryNode side) is a +-- genuine model contradiction, not just a contrived scenario: it would +-- require the primary to report goalState == reportedState == single +-- while a *separate* node in the same group is simultaneously converged +-- and reporting secondary -- but the instant a second node registers, +-- the primary's own goal moves off single (to wait_primary) as part of +-- that registration, before the joining node could ever reach +-- secondary. No real sequence of monitor/keeper actions can produce +-- this combination. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos @@ -179,18 +209,14 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; rule | n | current_state | assigned_state | comment ------+----+---------------------+----------------+---------------------------------------------------------------------------------------------------------------------- - 209 | 5 | | single | alone in group, candidate-eligible -> single - 209 | 1 | wait_standby | single | alone in group, candidate-eligible -> single + 209 | 3 | | single | alone in group, candidate-eligible -> single 209 | 1 | prepare_maintenance | single | alone in group, candidate-eligible -> single - 209 | 1 | wait_maintenance | single | alone in group, candidate-eligible -> single 209 | 1 | fast_forward | single | alone in group, candidate-eligible -> single 209 | 1 | join_secondary | single | alone in group, candidate-eligible -> single - 211 | 7 | | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 5 | | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | demote_timeout | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | prepare_promotion | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | wait_standby | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | wait_maintenance | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | fast_forward | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | join_secondary | report_lsn | alone in group, candidatePriority zero -> report_lsn 325 | 1 | | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) @@ -273,7 +299,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 351 | 1 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 391 | 1 | | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining 391 | 1 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining -(94 rows) +(90 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 264593724..b6a892314 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -503,6 +503,7 @@ typedef struct NodeStatusPattern BoolPattern isDemotedPrimary; BoolPattern canTakeWrites; BoolPattern reportedCanTakeWrites; + BoolPattern reportedIsWaitStandby; BoolPattern isReadyToStreamWAL; BoolPattern drainTimeExpired; BoolPattern isCitusWorkerGroup; @@ -572,6 +573,22 @@ BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *stat * action changes the very fact its condition is testing. reportedState is * never written by this row's own action, so a condition built purely from * it stays stable across dispatches until the keeper itself converges. + * + * reportedIsWaitStandby is a plain reportedState equality check, for the + * same reported-only reason as reportedCanTakeWrites above, but forced by a + * DIFFERENT row's action this time, not this row's own: pos 101's own + * remove_node() fan-out (.otherNodeAssignedState = GOAL(REPORT_LSN), + * unconditional on every surviving non-maintenance standby) can rewrite a + * lone wait_standby node's own goalState to report_lsn synchronously, + * before pos 209/211's own early_checks evaluation ever runs on that node's + * next heartbeat -- a goalState-dependent exclusion (e.g. NOT_STABLE) would + * see reported != goal at that point and treat wait_standby as "no longer + * stable", reviving a match it was supposed to permanently exclude. + * Confirmed live: an earlier NOT_STABLE-based version of this exclusion + * looked correct in dump_fsm_edges()'s own static analysis (which never + * sees pos 101's cross-row goalState write) but still let pos 209/211 fire + * for a real wait_standby node in a live pgaftest run, exactly because of + * this. Matching on reportedState alone sidesteps it entirely. */ static bool NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) @@ -599,6 +616,9 @@ NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) BoolMatchesPattern(status->node != NULL && CanTakeWritesInState(status->node->reportedState), pattern->reportedCanTakeWrites) && + BoolMatchesPattern(status->node != NULL && + status->node->reportedState == REPLICATION_STATE_WAIT_STANDBY, + pattern->reportedIsWaitStandby) && BoolMatchesPattern(CandidateNodeIsReadyToStreamWAL(status->node), pattern->isReadyToStreamWAL) && BoolMatchesPattern(NodeIsDrainTimeExpired(status->node, status->ctx), @@ -2443,13 +2463,26 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), .comment = "reported demote_timeout, assigned goal can't reach it -> demoted" }, - /* alone in group, candidate-eligible */ + /* + * alone in group, candidate-eligible -- excludes WAIT_STANDBY_STATE too + * (reportedIsWaitStandby's own comment on NodeMatchesPattern): a node + * stuck there never actually started streaming, so there is no safe way + * for the keeper to reach SINGLE from it without risking a system_ + * identifier conflict with its own prior registration. Gated via the + * reported-only reportedIsWaitStandby field rather than folding + * WAIT_STANDBY into FSM_NOT_STABLE_SINGLE's own NOT_STABLE-kind pattern: + * pos 101's own remove_node() fan-out can rewrite this node's goalState + * to report_lsn before this row's own next evaluation, which would + * defeat a goalState-dependent (NOT_STABLE) exclusion the same way pos + * 210's own isInPrimaryState condition was defeated -- confirmed live. + */ { .pos = 209, .sectionPath = { MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, - .candidateEligible = BOOL_TRUE }, + .candidateEligible = BOOL_TRUE, + .reportedIsWaitStandby = BOOL_FALSE }, .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SINGLE), .comment = "alone in group, candidate-eligible -> single" }, @@ -2519,6 +2552,15 @@ static const MonitorFSMTransition MonitorFSM[] = { * simpler and safer than generalizing the shadow-detector to prove * conditional (not just unconditional) shadowing between arbitrary row * pairs. + * + * reportedIsWaitStandby=FALSE also excludes WAIT_STANDBY_STATE (see its + * own comment on NodeMatchesPattern): a node stuck there never actually + * started streaming, so there's no safe way for the keeper to reach + * REPORT_LSN from it either, for the same system_identifier-conflict + * reason pos 209 documents. Reported-only for the same reason as pos + * 209's own use of this field: pos 101's remove_node() fan-out can + * rewrite this node's goalState to report_lsn before this row's own + * next evaluation, which would defeat a goalState-dependent exclusion. */ { .pos = 211, .sectionPath = { @@ -2526,7 +2568,8 @@ static const MonitorFSMTransition MonitorFSM[] = { }, .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, .candidateEligible = BOOL_FALSE, - .reportedCanTakeWrites = BOOL_FALSE }, + .reportedCanTakeWrites = BOOL_FALSE, + .reportedIsWaitStandby = BOOL_FALSE }, .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), .comment = "alone in group, candidatePriority zero -> report_lsn" }, @@ -4125,6 +4168,7 @@ NodeStatusPatternConditionsText(const NodeStatusPattern *pattern, bool *isNull) APPEND_BOOL_CONDITION(&buf, "isDemotedPrimary", pattern->isDemotedPrimary); APPEND_BOOL_CONDITION(&buf, "canTakeWrites", pattern->canTakeWrites); APPEND_BOOL_CONDITION(&buf, "reportedCanTakeWrites", pattern->reportedCanTakeWrites); + APPEND_BOOL_CONDITION(&buf, "reportedIsWaitStandby", pattern->reportedIsWaitStandby); APPEND_BOOL_CONDITION(&buf, "isReadyToStreamWAL", pattern->isReadyToStreamWAL); APPEND_BOOL_CONDITION(&buf, "drainTimeExpired", pattern->drainTimeExpired); APPEND_BOOL_CONDITION(&buf, "isCitusWorkerGroup", pattern->isCitusWorkerGroup); @@ -4593,6 +4637,28 @@ NodeStatusPatternSurvivesReportedCanTakeWrites(const NodeStatusPattern *pattern, } +/* + * NodeStatusPatternSurvivesReportedIsWaitStandby filters a candidate + * edge-source state against pattern's own .reportedIsWaitStandby field, the + * same shape as NodeStatusPatternSurvivesReportedCanTakeWrites just above -- + * a plain equality on reportedState alone, no goalState-dependent + * satisfiability proof needed. + */ +static bool +NodeStatusPatternSurvivesReportedIsWaitStandby(const NodeStatusPattern *pattern, + ReplicationState state) +{ + if (pattern->reportedIsWaitStandby == BOOL_ANY) + { + return true; + } + + bool required = (pattern->reportedIsWaitStandby == BOOL_TRUE); + + return (state == REPLICATION_STATE_WAIT_STANDBY) == required; +} + + /* * NodeStatePatternKindIsReportedStateOnly: true for the pattern kinds whose * match genuinely depends only on reportedState (ignoring, for STABLE, its @@ -4679,6 +4745,7 @@ NodeStatusPatternOtherFieldsAreAny(const NodeStatusPattern *pattern) pattern->isDemotedPrimary == BOOL_ANY && pattern->canTakeWrites == BOOL_ANY && pattern->reportedCanTakeWrites == BOOL_ANY && + pattern->reportedIsWaitStandby == BOOL_ANY && pattern->isReadyToStreamWAL == BOOL_ANY && pattern->drainTimeExpired == BOOL_ANY && pattern->isCitusWorkerGroup == BOOL_ANY && @@ -5016,6 +5083,12 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + if (!NodeStatusPatternSurvivesReportedIsWaitStandby(&rule->activeNode, + states[j])) + { + continue; + } + if (EdgeIsShadowedByEarlierRule(i, states[j], false, rule->sectionPath[0])) { continue; @@ -5070,6 +5143,12 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + if (!NodeStatusPatternSurvivesReportedIsWaitStandby(&rule->primaryNode, + states[j])) + { + continue; + } + if (EdgeIsShadowedByEarlierRule(i, states[j], true, rule->sectionPath[0])) { continue; diff --git a/src/monitor/keeper_fsm_edges.json b/src/monitor/keeper_fsm_edges.json index 2eca675ce..8296e8012 100644 --- a/src/monitor/keeper_fsm_edges.json +++ b/src/monitor/keeper_fsm_edges.json @@ -398,5 +398,13 @@ { "current": "any", "assigned": "dropped" + }, + { + "current": "wait_maintenance", + "assigned": "single" + }, + { + "current": "wait_maintenance", + "assigned": "report_lsn" } ] diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index a3f1c8bb1..86a58f47a 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -85,6 +85,34 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- this is Step 2a's own structural artifact of enumerating a role/predicate -- gate across every syntactically possible current_state, not a sign of 10 -- separate functional bugs. +-- +-- Follow-up investigation of pos 209/211/325's own remaining gap states +-- (after wait_maintenance and wait_standby were resolved -- see +-- group_state_machine.c's reportedIsWaitStandby field (NodeStatusPattern) +-- and KeeperFSM[]'s new WAIT_MAINTENANCE_STATE rows, fsm.c): +-- +-- * pos 209/211's remaining states (prepare_maintenance, demote_timeout, +-- prepare_promotion, stop_replication, fast_forward, join_secondary) +-- were each checked for real reachability under "alone in group" and +-- found contrived: reaching them normally implies a multi-node context +-- (fast_forward/join_secondary need another standby to fetch WAL from +-- or a newly-elected primary to join, respectively) or an internal +-- tension with candidateEligible=FALSE (prepare_promotion/stop_ +-- replication imply having already been selected as a promotion +-- candidate; demote_timeout's genuinely-stuck case is already +-- intercepted earlier by pos 207). None ruled out as impossible, but +-- none reproducible via a single, realistic operator/network-failure +-- sequence the way wait_maintenance and wait_standby were -- left as +-- documented artifacts, not pursued further this pass. +-- * pos 325's remaining "single" state (the primaryNode side) is a +-- genuine model contradiction, not just a contrived scenario: it would +-- require the primary to report goalState == reportedState == single +-- while a *separate* node in the same group is simultaneously converged +-- and reporting secondary -- but the instant a second node registers, +-- the primary's own goal moves off single (to wait_primary) as part of +-- that registration, before the joining node could ever reach +-- secondary. No real sequence of monitor/keeper actions can produce +-- this combination. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos diff --git a/tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf b/tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf index 8d77fcffc..a2af9b69d 100644 --- a/tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf +++ b/tests/tap/specs/keeper_fsm_gap_209_wait_maintenance.pgaf @@ -1,16 +1,23 @@ -# Diagnostic spec for MonitorFSM[] pos 209 ("alone in group, -# candidate-eligible -> single", src/monitor/group_state_machine.c) -- NOT in -# the CI schedule, same convention as debug_failover_pg19.pgaf, since it is -# expected to FAIL until the missing KeeperFSM[] transition documented below -# is implemented. +# Regression spec for MonitorFSM[] pos 209 ("alone in group, +# candidate-eligible -> single", src/monitor/group_state_machine.c). # -# src/monitor/sql/keeper_fsm_edges.sql's Step 2a found that pos 209's -# NodeStatePattern is willing to assign SINGLE_STATE to a lone remaining -# node reporting WAIT_MAINTENANCE_STATE as its current_state, but KeeperFSM[] -# (src/bin/pg_autoctl/fsm.c) has no WAIT_MAINTENANCE_STATE -> SINGLE_STATE -# row at all. +# This spec originally documented a real gap: pos 209's NodeStatePattern is +# willing to assign SINGLE_STATE to a lone remaining node reporting +# WAIT_MAINTENANCE_STATE as its current_state, but KeeperFSM[] +# (src/bin/pg_autoctl/fsm.c) had no WAIT_MAINTENANCE_STATE -> SINGLE_STATE +# row at all -- node2 would get stuck reporting wait_maintenance forever, +# logging "pg_autoctl does not know how to reach state \"single\" from +# \"wait_maintenance\"". # -# This spec manufactures that exact (current, assigned) pair: +# Fixed by adding that row to KeeperFSM[], reusing fsm_promote_standby (the +# same function already reused by every other converged-standby source +# state: SECONDARY/CATCHINGUP/PREP_PROMOTION/STOP_REPLICATION/REPORT_LSN) -- +# entering WAIT_MAINTENANCE_STATE itself runs no transition function (a +# converged standby just marks intent to go to maintenance, Postgres keeps +# running and replicating normally), so promoting it exactly like any other +# actively-replicating standby is safe. +# +# This spec now asserts that fixed behavior: # # 1. node1 (primary) is network-disconnected, so it can never acknowledge # a replication-quorum change. @@ -28,18 +35,8 @@ # 4. The monitor recomputes: groupHasExactlyOneNode is now true, node2's # own reported current_state is "wait_maintenance", and its candidate # priority is > 0, so pos 209 fires and the monitor assigns SINGLE. -# 5. node2's keeper receives that assigned state, but -# keeper_fsm_reach_assigned_state() (fsm.c) finds no matching -# KeeperFSM[] row for (wait_maintenance -> single) and logs -# "pg_autoctl does not know how to reach state \"single\" from -# \"wait_maintenance\"" -- node2 never converges, staying stuck -# reporting wait_maintenance forever. -# -# Expected fix: add a WAIT_MAINTENANCE_STATE -> SINGLE_STATE row to -# KeeperFSM[]. Check for reuse first: WAIT_MAINTENANCE_STATE -> -# MAINTENANCE_STATE already exists (fsm.c) and its transition function may -# be adaptable, since both targets ultimately mean "stop waiting for a peer -# that isn't coming." +# 5. node2's keeper now has a matching KeeperFSM[] row and actually +# converges to single. cluster { monitor @@ -72,16 +69,19 @@ step test_002_secondary_stuck_at_wait_maintenance { wait until node2 state is wait_maintenance timeout 60s } -step test_003_drop_primary_bypass_and_check_stuck { +step test_003_drop_primary_converges_to_single { sql monitor { SELECT pgautofailover.remove_node(nodehost, nodeport) FROM pgautofailover.node WHERE nodename = 'node1'; } wait until node2 assigned-state = single timeout 60s - sleep 5s + wait until node2 state is single timeout 60s sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; } - expect { wait_maintenance } - logs node2 contains "does not know how to reach state" + expect { single } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { single } } diff --git a/tests/tap/specs/keeper_fsm_gap_209_wait_standby.pgaf b/tests/tap/specs/keeper_fsm_gap_209_wait_standby.pgaf new file mode 100644 index 000000000..5a9b2d331 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_209_wait_standby.pgaf @@ -0,0 +1,84 @@ +# Regression spec for MonitorFSM[] pos 209 ("alone in group, +# candidate-eligible -> single", src/monitor/group_state_machine.c), for its +# wait_standby current_state. +# +# This spec originally set out to reproduce the same kind of gap as +# keeper_fsm_gap_209_wait_maintenance.pgaf: pos 209's NodeStatePattern was +# willing to assign SINGLE_STATE to a lone remaining node reporting +# WAIT_STANDBY_STATE, but KeeperFSM[] had no matching row. Reproducing it +# live (network-disconnect node1 before node2 ever streams, then drop node1) +# confirmed the gap is real -- but also surfaced something worse than a +# missing keeper transition: a node stuck at WAIT_STANDBY_STATE has never +# actually started streaming (no pg_basebackup done, see fsm_init_standby, +# the WAIT_STANDBY_STATE -> CATCHINGUP_STATE transition, fsm.c), so there is +# no safe way to reach SINGLE_STATE from it at all. The only local action +# available (fsm_init_primary, the same one used for INIT_STATE/ +# DROPPED_STATE -> SINGLE_STATE) does a *fresh* initdb, generating a new +# system_identifier -- which conflicts with the one this node already +# registered under, tripping the monitor's own same_system_identifier_ +# within_group exclusion constraint (confirmed live: "Failed to transition +# from state \"wait_standby\" to state \"single\""). +# +# Rather than force this through on the keeper side, pos 209 was narrowed +# instead (the reportedIsWaitStandby field, group_state_machine.c) to stop +# assigning SINGLE to a wait_standby node at all. +# +# Note this does NOT mean node2's own goalstate stays "wait_standby" +# forever: pos 101 (remove_node()'s own fan-out, a separate, unconditional +# row -- ".otherNodeAssignedState = GOAL(REPORT_LSN)" for "every surviving +# non-maintenance standby") still reassigns it to report_lsn as a direct, +# synchronous side effect of dropping node1, regardless of pos 209/211. That +# reassignment is harmless here: KeeperFSM[] has no WAIT_STANDBY_STATE -> +# REPORT_LSN_STATE row either, so node2's keeper simply stays parked (its +# own reportedstate never leaves wait_standby) instead of attempting +# anything -- the actual safety property this spec cares about. An earlier +# version of this spec asserted goalstate also stays at wait_standby, which +# is not what actually happens and isn't the real invariant to check. +# +# 1. node2 is created with launch deferred, then started with node1 +# already network-disconnected -- it registers (reaching +# WAIT_STANDBY_STATE) but can never reach node1 to fetch its initial +# pg_basebackup, so it's durably stuck reporting wait_standby, +# unhealthy -- not a narrow timing window. +# 2. node1 is dropped directly via pgautofailover.remove_node(), leaving +# node2 alone in the group, still reporting wait_standby, with default +# (nonzero) candidate-priority. +# 3. The monitor recomputes: groupHasExactlyOneNode is now true, but pos +# 209 no longer matches wait_standby at all, so it never assigns +# SINGLE. node2's reportedstate stays wait_standby -- it never attempts +# (and never risks) an unsafe fresh-init transition. + +cluster { + monitor + formation { + node1 + node2 create and launch deferred + } +} + +setup { + wait until node1 state is single timeout 60s +} + +teardown { + compose down +} + +step test_001_start_node2_isolated_from_node1 { + network disconnect node1 + exec node2 pg_autoctl node start + wait until node2 state is wait_standby timeout 60s +} + +step test_002_drop_node1_leaves_node2_reportedstate_untouched { + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + sleep 10s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { wait_standby } + logs node2 contains "Still waiting for the monitor to drive us to state" +} diff --git a/tests/tap/specs/keeper_fsm_gap_211_wait_maintenance.pgaf b/tests/tap/specs/keeper_fsm_gap_211_wait_maintenance.pgaf new file mode 100644 index 000000000..558d1d404 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_211_wait_maintenance.pgaf @@ -0,0 +1,86 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, +# candidatePriority zero -> report_lsn", src/monitor/group_state_machine.c), +# for its wait_maintenance current_state -- a separate gap from +# keeper_fsm_gap_209_wait_maintenance.pgaf's own scenario (that one is pos +# 209's candidate-eligible sibling, assigning single instead of report_lsn). +# +# This spec originally documented a real gap: pos 211's NodeStatePattern is +# willing to assign REPORT_LSN_STATE to a lone remaining node reporting +# WAIT_MAINTENANCE_STATE as its current_state, but KeeperFSM[] +# (src/bin/pg_autoctl/fsm.c) had no WAIT_MAINTENANCE_STATE -> REPORT_LSN_STATE +# row at all -- node2 would get stuck reporting wait_maintenance forever. +# +# Fixed by adding that row to KeeperFSM[], reusing fsm_report_lsn (the same +# function already reused by SECONDARY/CATCHINGUP/MAINTENANCE/ +# PREPARE_MAINTENANCE_STATE -> REPORT_LSN_STATE) -- entering +# WAIT_MAINTENANCE_STATE itself runs no transition function (a converged +# standby just marks intent to go to maintenance, Postgres keeps running and +# replicating normally), so it's safe to treat exactly like those other +# actively-replicating standby states. +# +# Scenario: identical to keeper_fsm_gap_209_wait_maintenance.pgaf's own, +# except node2's candidate-priority is set to 0 first, so pos 211 (not pos +# 209) is the one that fires once node2 is alone: +# +# 1. node2's candidate-priority is set to 0. +# 2. node1 (primary) is network-disconnected, so it can never acknowledge +# a replication-quorum change. +# 3. node2 (the only secondary, hence the last quorum member) is told to +# enter maintenance, and gets durably stuck at WAIT_MAINTENANCE_STATE +# for the same reason as the pos 209 spec (node1 can never see or +# acknowledge the request). +# 4. node1 is dropped directly via pgautofailover.remove_node(), leaving +# node2 alone in the group, still reporting wait_maintenance, with +# candidate-priority 0 -- pos 211 fires and the monitor assigns +# REPORT_LSN. +# 5. node2's keeper now has a matching KeeperFSM[] row and actually +# converges to report_lsn. + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_001_zero_priority_and_disconnect_primary { + exec node2 pg_autoctl set node candidate-priority 0 + network disconnect node1 +} + +step test_002_secondary_stuck_at_wait_maintenance { + # pg_autoctl enable maintenance itself polls for convergence and exits + # non-zero when it can't reach "maintenance" quickly (the primary is + # unreachable, exactly the point of this scenario) -- the real state + # transition still happens regardless of this command's own exit code, + # confirmed by the wait below. + exec-fails node2 pg_autoctl enable maintenance + wait until node2 state is wait_maintenance timeout 60s +} + +step test_003_drop_primary_converges_to_report_lsn { + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + wait until node2 assigned-state = report_lsn timeout 60s + wait until node2 state is report_lsn timeout 60s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { report_lsn } +} diff --git a/tests/tap/specs/keeper_fsm_gap_211_wait_standby.pgaf b/tests/tap/specs/keeper_fsm_gap_211_wait_standby.pgaf new file mode 100644 index 000000000..005c0b751 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_211_wait_standby.pgaf @@ -0,0 +1,78 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, +# candidatePriority zero -> report_lsn", src/monitor/group_state_machine.c), +# for its wait_standby current_state -- the candidatePriority=0 sibling of +# keeper_fsm_gap_209_wait_standby.pgaf's own scenario. See that spec's header +# for the full story: a node stuck at WAIT_STANDBY_STATE never actually +# started streaming, so there is no safe way to reach REPORT_LSN_STATE (or +# SINGLE_STATE) from it without risking a system_identifier conflict with +# its own prior registration. +# +# pos 211 was narrowed the same way pos 209 was (the reportedIsWaitStandby +# field, group_state_machine.c) -- this spec asserts the same safe, +# conservative behavior for the candidatePriority=0 case. +# +# Note this does NOT mean node2's own goalstate stays "wait_standby" +# forever: pos 101 (remove_node()'s own fan-out, a separate, unconditional +# row -- ".otherNodeAssignedState = GOAL(REPORT_LSN)" for "every surviving +# non-maintenance standby") still reassigns it to report_lsn as a direct, +# synchronous side effect of dropping node1, regardless of pos 209/211 -- +# happens to be the very state pos 211 would itself assign anyway, so +# there's nothing to "correct" it. That reassignment is harmless here: +# KeeperFSM[] has no WAIT_STANDBY_STATE -> REPORT_LSN_STATE row either, so +# node2's keeper simply stays parked (its own reportedstate never leaves +# wait_standby) instead of attempting anything -- the actual safety +# property this spec cares about. +# +# 1. node2 is created with launch deferred, then started with node1 +# already network-disconnected -- it registers (reaching +# WAIT_STANDBY_STATE) but can never reach node1 to fetch its initial +# pg_basebackup, so it's durably stuck reporting wait_standby, +# unhealthy. +# 2. node2's candidate-priority is set to 0 (registration itself must +# complete first for this CLI call to have a monitor connection to +# use, even though the node is otherwise stuck), so pos 211 (not pos +# 209) is the one that would otherwise apply once node2 is alone. +# 3. node1 is dropped directly via pgautofailover.remove_node(), leaving +# node2 alone in the group, still reporting wait_standby, with +# candidate-priority 0. +# 4. The monitor recomputes: groupHasExactlyOneNode is now true, but pos +# 211 no longer matches wait_standby at all, so it never (re)assigns +# REPORT_LSN on its own account. node2's reportedstate stays +# wait_standby -- it never attempts (and never risks) an unsafe +# fresh-init transition. + +cluster { + monitor + formation { + node1 + node2 create and launch deferred + } +} + +setup { + wait until node1 state is single timeout 60s +} + +teardown { + compose down +} + +step test_001_start_node2_isolated_and_zero_priority { + network disconnect node1 + exec node2 pg_autoctl node start + wait until node2 state is wait_standby timeout 60s + exec node2 pg_autoctl set node candidate-priority 0 +} + +step test_002_drop_node1_leaves_node2_reportedstate_untouched { + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + sleep 10s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { wait_standby } + logs node2 contains "Still waiting for the monitor to drive us to state" +} From c0b9b07924370c0599a990f6b0cb7fc5200828e2 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Fri, 31 Jul 2026 02:02:13 +0200 Subject: [PATCH 30/52] tests: schedule the 3 new keeper_fsm_gap specs into CI (node.sch) keeper_fsm_gap_211_wait_maintenance, keeper_fsm_gap_209_wait_standby, and keeper_fsm_gap_211_wait_standby (added in 4f59a40) all pass live -- add them to node.sch (the schedule ci.yml's test_pgaftest job runs) alongside the existing gap specs, and to the legacy tests/tap/schedule for documentation consistency. --- tests/tap/schedule | 3 +++ tests/tap/schedules/node.sch | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/tap/schedule b/tests/tap/schedule index e6126b4d4..b0719f1d2 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -32,6 +32,9 @@ demote_timeout_wait_primary_deadlock wait_primary_draining_deadlock timeline_fork_report_lsn_deadlock keeper_fsm_gap_209_wait_maintenance +keeper_fsm_gap_211_wait_maintenance +keeper_fsm_gap_209_wait_standby +keeper_fsm_gap_211_wait_standby keeper_fsm_gap_211_primary_priority_zero extension_update tablespaces diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index c835d793e..b94ceaac6 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -1,4 +1,4 @@ -# Node lifecycle, monitor operations, and Debian/tablespace layouts (~32 min). +# Node lifecycle, monitor operations, and Debian/tablespace layouts (~34 min). # Merged from former node, monitor, and node-extra schedules to reduce CI job # count and GitHub Actions runner queue pressure. create_standby_with_pgdata @@ -16,4 +16,7 @@ demote_timeout_wait_primary_deadlock timeline_fork_report_lsn_deadlock timeline_fork_3node_auto_detect keeper_fsm_gap_209_wait_maintenance +keeper_fsm_gap_211_wait_maintenance +keeper_fsm_gap_209_wait_standby +keeper_fsm_gap_211_wait_standby keeper_fsm_gap_211_primary_priority_zero From 7ca2562fad30f6663d75157c286254b14617fe3f Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Fri, 31 Jul 2026 16:13:53 +0200 Subject: [PATCH 31/52] Fix keeper FSM gap: fast_forward left alone in group pos 209/211 ("alone in group" -> single/report_lsn) already correctly resolved fast_forward as a source state on the monitor side, but KeeperFSM[] had no matching row: a node left alone while genuinely reporting fast_forward (its WAL-source peer and the old primary both gone mid MS-failover) would get stuck logging "does not know how to reach state ... from \"fast_forward\"". Add both rows to KeeperFSM[] (fsm.c): - FAST_FORWARD_STATE -> SINGLE_STATE, via fsm_promote_standby - FAST_FORWARD_STATE -> REPORT_LSN_STATE, via fsm_report_lsn reusing the same transition functions already shared by every other converged-standby source state (SECONDARY/CATCHINGUP/PREP_PROMOTION/ STOP_REPLICATION/REPORT_LSN/WAIT_MAINTENANCE) -- fast_forward only means Postgres is already running as a caught-up-enough standby, and fsm_fast_forward's own "no upstream found" branch already accepts promoting with local data when nothing more advanced is available, so this is exactly as safe as the analogous existing rows. Add pgaftest coverage for both (keeper_fsm_gap_209_fast_forward.pgaf, keeper_fsm_gap_211_fast_forward.pgaf), reproducing a genuine MS-failover candidate falling behind, fetching real WAL from a peer, and then being left alone -- using the established network-disconnect + INSERT + CHECKPOINT divergence recipe (multi_ifdown.pgaf, debug_citus_worker_fast_forward.pgaf). Getting these to actually exercise the new rows (rather than the monitor's own ordinary cascade continuing past fast_forward to prepare_promotion within about a second of convergence) requires removing the other group members while the candidate is still mid-fetch, not after it reports fast_forward -- see the specs' own headers for the full explanation. Also add keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf, covering a related question raised during this investigation: once a lone node is parked at report_lsn with candidate-priority 0 (pos 211's own outcome), can the cluster ever recover when a new peer joins? Confirmed already fully working via RegisterNode()'s existing report_lsn-source handling, no code change needed -- this is a pure regression spec. Regenerate keeper_fsm_edges.json/expected/keeper_fsm_edges.out from the real KeeperFSM[] table, and update the "remaining gap states" follow-up comment in keeper_fsm_edges.sql to move fast_forward from "investigated, not pursued" to "fixed and covered". Add all three new specs to the CI schedules (tests/tap/schedule, tests/tap/schedules/node.sch). Verified: full regress (19/19) + isolation (6/6) installcheck, citus_indent clean, and all three new pgaftest specs passing live. --- src/bin/pg_autoctl/fsm.c | 41 +++++ src/monitor/expected/keeper_fsm_edges.out | 48 ++++-- src/monitor/keeper_fsm_edges.json | 8 + src/monitor/sql/keeper_fsm_edges.sql | 36 ++-- tests/tap/schedule | 3 + tests/tap/schedules/node.sch | 3 + .../keeper_fsm_gap_209_fast_forward.pgaf | 161 ++++++++++++++++++ .../keeper_fsm_gap_211_fast_forward.pgaf | 154 +++++++++++++++++ ...m_gap_new_node_joins_report_lsn_group.pgaf | 96 +++++++++++ 9 files changed, 520 insertions(+), 30 deletions(-) create mode 100644 tests/tap/specs/keeper_fsm_gap_209_fast_forward.pgaf create mode 100644 tests/tap/specs/keeper_fsm_gap_211_fast_forward.pgaf create mode 100644 tests/tap/specs/keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index d6fdfad71..7ee9cb723 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -165,6 +165,10 @@ "Was waiting to be sent to maintenance, but the primary vanished, " \ "promote this node" +#define COMMENT_FAST_FORWARD_TO_SINGLE \ + "Was fetching missing WAL from another standby, but every other node " \ + "vanished, promote this node with whatever it has" + #define COMMENT_FOLLOW_NEW_PRIMARY \ "Switch replication to the new primary" @@ -538,6 +542,27 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_REMOVAL }, + /* + * was fetching missing WAL from another standby to catch up before + * promotion (a converged-enough standby -- Postgres is already running + * and replicating, same shape as the other converged-standby source + * states above), but every other node vanished, including the peer we + * were fetching from: fsm_fast_forward's own "no upstream found" branch + * already accepts this (skips the fetch, treats local data as the best + * available -- there is nothing more advanced left anywhere to lose), + * so promoting with whatever this node already has is exactly as safe + * as it gets. Reuse fsm_promote_standby exactly like every other + * converged-standby source state -- no separate WAL fetch is attempted + * or needed here, matching PREP_PROMOTION_STATE/STOP_REPLICATION_STATE's + * own direct-to-SINGLE shortcut for "was mid-promotion, peer vanished". + */ + { + FAST_FORWARD_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_FAST_FORWARD_TO_SINGLE, + &fsm_promote_standby, + FSM_PHASE_REMOVAL + }, + /* * On the Primary, wait for a standby to be ready: WAIT_PRIMARY */ @@ -895,6 +920,22 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_MAINTENANCE }, + /* + * was fetching missing WAL from another standby to catch up before + * promotion (same converged-enough shape as the source states above), + * but every other node vanished and this node's own candidate-priority + * is 0 -- reuse fsm_report_lsn exactly like SECONDARY/CATCHINGUP/ + * MAINTENANCE/PREPARE_MAINTENANCE/WAIT_MAINTENANCE above. No WAL fetch + * is attempted here either, same reasoning as FAST_FORWARD_STATE -> + * SINGLE_STATE's own comment. + */ + { + FAST_FORWARD_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_FAILOVER + }, + { REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, COMMENT_REPORT_LSN_TO_PREP_PROMOTION, diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 0620d3c1b..0b5c94ac3 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -73,7 +73,9 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; dropped | single dropped | wait_standby dropped | report_lsn + fast_forward | single fast_forward | prepare_promotion + fast_forward | report_lsn init | single init | wait_standby init | report_lsn @@ -125,7 +127,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; wait_primary | join_primary wait_primary | apply_settings wait_standby | catchingup -(79 rows) +(81 rows) -- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A @@ -171,19 +173,31 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- group_state_machine.c's reportedIsWaitStandby field (NodeStatusPattern) -- and KeeperFSM[]'s new WAIT_MAINTENANCE_STATE rows, fsm.c): -- +-- * pos 209/211's fast_forward current_state is now genuinely fixed and +-- covered: a lone node reporting fast_forward (its WAL-source peer and +-- the old primary both gone mid MS-failover) now has matching +-- KeeperFSM[] rows (FAST_FORWARD_STATE -> SINGLE_STATE / +-- REPORT_LSN_STATE, fsm.c, reusing fsm_promote_standby/fsm_report_lsn +-- exactly like every other converged-standby source state) and real +-- pgaftest coverage (keeper_fsm_gap_209_fast_forward.pgaf, +-- keeper_fsm_gap_211_fast_forward.pgaf) reproducing a genuine MS-failover +-- candidate falling behind, fetching real WAL, and then being left alone +-- -- see those specs' own headers for why node1/node2 must be removed +-- from the group while node3 is still fetching (not after it reports +-- fast_forward) to actually exercise these rows instead of racing +-- against the monitor's own cascade continuation. -- * pos 209/211's remaining states (prepare_maintenance, demote_timeout, --- prepare_promotion, stop_replication, fast_forward, join_secondary) --- were each checked for real reachability under "alone in group" and --- found contrived: reaching them normally implies a multi-node context --- (fast_forward/join_secondary need another standby to fetch WAL from --- or a newly-elected primary to join, respectively) or an internal --- tension with candidateEligible=FALSE (prepare_promotion/stop_ --- replication imply having already been selected as a promotion --- candidate; demote_timeout's genuinely-stuck case is already --- intercepted earlier by pos 207). None ruled out as impossible, but --- none reproducible via a single, realistic operator/network-failure --- sequence the way wait_maintenance and wait_standby were -- left as --- documented artifacts, not pursued further this pass. +-- prepare_promotion, stop_replication, join_secondary) were each checked +-- for real reachability under "alone in group" and found contrived: +-- reaching them normally implies a multi-node context (join_secondary +-- needs a newly-elected primary to join) or an internal tension with +-- candidateEligible=FALSE (prepare_promotion/stop_replication imply +-- having already been selected as a promotion candidate; demote_timeout's +-- genuinely-stuck case is already intercepted earlier by pos 207). None +-- ruled out as impossible, but none reproducible via a single, realistic +-- operator/network-failure sequence the way wait_maintenance and +-- wait_standby were -- left as documented artifacts, not pursued further +-- this pass. -- * pos 325's remaining "single" state (the primaryNode side) is a -- genuine model contradiction, not just a contrived scenario: it would -- require the primary to report goalState == reportedState == single @@ -209,15 +223,13 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; rule | n | current_state | assigned_state | comment ------+----+---------------------+----------------+---------------------------------------------------------------------------------------------------------------------- - 209 | 3 | | single | alone in group, candidate-eligible -> single + 209 | 2 | | single | alone in group, candidate-eligible -> single 209 | 1 | prepare_maintenance | single | alone in group, candidate-eligible -> single - 209 | 1 | fast_forward | single | alone in group, candidate-eligible -> single 209 | 1 | join_secondary | single | alone in group, candidate-eligible -> single - 211 | 5 | | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 4 | | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | demote_timeout | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | prepare_promotion | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | fast_forward | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | join_secondary | report_lsn | alone in group, candidatePriority zero -> report_lsn 325 | 1 | | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) 325 | 1 | single | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) @@ -299,7 +311,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 351 | 1 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 391 | 1 | | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining 391 | 1 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining -(90 rows) +(88 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/keeper_fsm_edges.json b/src/monitor/keeper_fsm_edges.json index 8296e8012..88003efd1 100644 --- a/src/monitor/keeper_fsm_edges.json +++ b/src/monitor/keeper_fsm_edges.json @@ -406,5 +406,13 @@ { "current": "wait_maintenance", "assigned": "report_lsn" + }, + { + "current": "fast_forward", + "assigned": "single" + }, + { + "current": "fast_forward", + "assigned": "report_lsn" } ] diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 86a58f47a..8d3cc6fb0 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -91,19 +91,31 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- group_state_machine.c's reportedIsWaitStandby field (NodeStatusPattern) -- and KeeperFSM[]'s new WAIT_MAINTENANCE_STATE rows, fsm.c): -- +-- * pos 209/211's fast_forward current_state is now genuinely fixed and +-- covered: a lone node reporting fast_forward (its WAL-source peer and +-- the old primary both gone mid MS-failover) now has matching +-- KeeperFSM[] rows (FAST_FORWARD_STATE -> SINGLE_STATE / +-- REPORT_LSN_STATE, fsm.c, reusing fsm_promote_standby/fsm_report_lsn +-- exactly like every other converged-standby source state) and real +-- pgaftest coverage (keeper_fsm_gap_209_fast_forward.pgaf, +-- keeper_fsm_gap_211_fast_forward.pgaf) reproducing a genuine MS-failover +-- candidate falling behind, fetching real WAL, and then being left alone +-- -- see those specs' own headers for why node1/node2 must be removed +-- from the group while node3 is still fetching (not after it reports +-- fast_forward) to actually exercise these rows instead of racing +-- against the monitor's own cascade continuation. -- * pos 209/211's remaining states (prepare_maintenance, demote_timeout, --- prepare_promotion, stop_replication, fast_forward, join_secondary) --- were each checked for real reachability under "alone in group" and --- found contrived: reaching them normally implies a multi-node context --- (fast_forward/join_secondary need another standby to fetch WAL from --- or a newly-elected primary to join, respectively) or an internal --- tension with candidateEligible=FALSE (prepare_promotion/stop_ --- replication imply having already been selected as a promotion --- candidate; demote_timeout's genuinely-stuck case is already --- intercepted earlier by pos 207). None ruled out as impossible, but --- none reproducible via a single, realistic operator/network-failure --- sequence the way wait_maintenance and wait_standby were -- left as --- documented artifacts, not pursued further this pass. +-- prepare_promotion, stop_replication, join_secondary) were each checked +-- for real reachability under "alone in group" and found contrived: +-- reaching them normally implies a multi-node context (join_secondary +-- needs a newly-elected primary to join) or an internal tension with +-- candidateEligible=FALSE (prepare_promotion/stop_replication imply +-- having already been selected as a promotion candidate; demote_timeout's +-- genuinely-stuck case is already intercepted earlier by pos 207). None +-- ruled out as impossible, but none reproducible via a single, realistic +-- operator/network-failure sequence the way wait_maintenance and +-- wait_standby were -- left as documented artifacts, not pursued further +-- this pass. -- * pos 325's remaining "single" state (the primaryNode side) is a -- genuine model contradiction, not just a contrived scenario: it would -- require the primary to report goalState == reportedState == single diff --git a/tests/tap/schedule b/tests/tap/schedule index b0719f1d2..41e7523c5 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -36,6 +36,9 @@ keeper_fsm_gap_211_wait_maintenance keeper_fsm_gap_209_wait_standby keeper_fsm_gap_211_wait_standby keeper_fsm_gap_211_primary_priority_zero +keeper_fsm_gap_new_node_joins_report_lsn_group +keeper_fsm_gap_209_fast_forward +keeper_fsm_gap_211_fast_forward extension_update tablespaces installcheck diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index b94ceaac6..d8dab7e04 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -20,3 +20,6 @@ keeper_fsm_gap_211_wait_maintenance keeper_fsm_gap_209_wait_standby keeper_fsm_gap_211_wait_standby keeper_fsm_gap_211_primary_priority_zero +keeper_fsm_gap_new_node_joins_report_lsn_group +keeper_fsm_gap_209_fast_forward +keeper_fsm_gap_211_fast_forward diff --git a/tests/tap/specs/keeper_fsm_gap_209_fast_forward.pgaf b/tests/tap/specs/keeper_fsm_gap_209_fast_forward.pgaf new file mode 100644 index 000000000..78fe45177 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_209_fast_forward.pgaf @@ -0,0 +1,161 @@ +# Regression spec for MonitorFSM[] pos 209 ("alone in group, +# candidate-eligible -> single", src/monitor/group_state_machine.c), for its +# fast_forward current_state. +# +# Until this session, KeeperFSM[] (src/bin/pg_autoctl/fsm.c) had no +# FAST_FORWARD_STATE -> SINGLE_STATE row at all: a node that ends up alone +# in its group while genuinely reporting "fast_forward" (its WAL-source peer +# and the old primary both gone) would get assigned SINGLE by pos 209 (whose +# own NodeStatePattern, FSM_NOT_STABLE_SINGLE, is broad enough to match a +# node reporting fast_forward) but have no way to reach it -- stuck logging +# "pg_autoctl does not know how to reach state \"single\" from +# \"fast_forward\"". +# +# Fixed by adding that row to KeeperFSM[], reusing fsm_promote_standby (the +# same function already reused by every other converged-standby source +# state: SECONDARY/CATCHINGUP/PREP_PROMOTION/STOP_REPLICATION/REPORT_LSN/ +# WAIT_MAINTENANCE) -- fast_forward only means "Postgres is running as a +# caught-up-enough standby", so promoting it directly is exactly as safe as +# promoting any of those other states. +# +# A real fast_forward assignment only happens mid MS-failover, when the +# monitor's selected candidate is behind the most advanced available node +# and must fetch the missing WAL before it can safely promote (pos 377, +# ProceedGroupStateForMSFailover / PromoteSelectedNode). This spec +# reproduces that genuinely, using the same network-disconnect + +# INSERT + CHECKPOINT recipe already established by +# tests/tap/specs/multi_ifdown.pgaf and +# tests/tap/specs/debug_citus_worker_fast_forward.pgaf. +# +# Getting the KeeperFSM[] row under test (rather than the pre-existing +# STOP_REPLICATION_STATE -> SINGLE_STATE row) to actually be the one that +# fires took an extra subtlety, found by adding a diagnostic +# pgautofailover.event/rule_pos dump to an earlier draft of this spec: once +# node3 (the candidate) reports fast_forward for the first time, the +# monitor's own MS-failover cascade advances it to prepare_promotion within +# the same ~1s cycle (node2, still alive, keeps re-triggering evaluation of +# node3's candidate status) -- so waiting for "node3 state is fast_forward" +# and only THEN removing node1/node2 is already too late; by the time that +# external wait observes fast_forward, the monitor may have already moved +# node3 on to prepare_promotion/stop_replication, and it would reach single +# via that pre-existing row instead of the one this spec means to exercise. +# +# The fix is to make the group "alone" (pos 209's own NodeStatePattern +# check, evaluated in the early_checks section, ahead of the +# reporting_node.ms_failover section that drives the ordinary cascade) +# BEFORE node3 ever gets to report fast_forward for the first time, not +# after. node1's and node2's rows are therefore dropped immediately once +# the failover is triggered, while node3 is still mid-fetch (its own +# fsm_fast_forward already established a direct Postgres-level connection +# to node2 to stream the missing WAL, independent of the monitor's own +# bookkeeping -- removing node2's *row* doesn't interrupt that already +# in-flight transfer, since node2's actual Postgres process and network +# path are left untouched here). node2 is still alive and connected at +# that point, so pgautofailover.remove_node() needs force=true (without +# it, remove_node() only marks goalState=dropped and waits on node2's own +# cooperative shutdown, which both takes an indeterminate amount of time +# and would kill the WAL source's Postgres mid-transfer). +# +# 1. 3-node formation (node1 primary, node2 + node3 secondaries). +# node2's candidate-priority is set to 0 -- it stays a synchronous, +# always-caught-up standby that is never itself selected as a +# failover candidate, existing purely to be node3's WAL-fetch source +# later. node3 keeps its default (nonzero) candidate-priority -- +# pos 209's own candidate-eligible case. +# 2. node3 is network-disconnected, then node1 (still primary) receives +# real write traffic + a CHECKPOINT -- node2 (still connected, still +# synchronous) replicates and catches up to the new LSN; node3, cut +# off, is now genuinely behind. +# 3. node1 is network-disconnected (killing the primary) and node3 is +# reconnected in the same step -- this forces an MS-failover election: +# node2 (candidate-priority 0) is excluded from candidacy entirely +# (GroupListCandidates), so node3 is the only candidate, but it is +# behind node2's LSN -- the monitor assigns it FAST_FORWARD_STATE, +# pointing at node2 as the WAL source, and node3 starts fetching. +# 4. Immediately (node3 is still mid-fetch, not yet reporting +# fast_forward): node1's and node2's rows are dropped directly via +# pgautofailover.remove_node(..., true) -- "alone in group" is true +# from this point on, well before node3 finishes catching up. +# 5. Once node3's own local recovery genuinely completes and it reports +# "fast_forward" for the first time, the monitor evaluates pos 209 +# immediately on that exact report (alone=true, reported=fast_forward) +# and assigns SINGLE directly -- no cascade through prepare_promotion +# is possible, since node2's row is already gone. +# 6. node3's keeper now has a matching KeeperFSM[] row (the fix under +# test) and actually converges to single, instead of getting stuck +# logging "does not know how to reach state \"single\" from +# \"fast_forward\"". + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state is secondary timeout 60s + promote node1 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s +} + +teardown { + compose down +} + +step test_001_set_candidate_priorities { + exec node2 pg_autoctl set node candidate-priority 0 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 60s + sql node1 { CREATE TABLE t1 (a int); } +} + +step test_002_disconnect_node3_and_diverge { + network disconnect node3 + sql node1 { + INSERT INTO t1 SELECT x FROM generate_series(1, 5000) as gs(x); + } + sql node1 { CHECKPOINT; } +} + +step test_003_failover_and_remove_peers_while_still_fetching { + network disconnect node1 + network connect node3 + # Confirm the monitor has already selected node3 and pointed it at + # node2 as its WAL source (assigned, not yet reported/converged) + # before removing node1/node2 -- this is what guarantees a real fetch + # is already underway (or about to start) using node2's still-live + # Postgres instance, rather than short-circuiting straight to single + # off of node3's own stale pre-failover data. + wait until node3 assigned-state = fast_forward timeout 60s + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node2'; + } +} + +step test_004_node3_converges_straight_to_single { + wait until node3 assigned-state = single timeout 180s + wait until node3 state is single timeout 60s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { single } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { single } +} diff --git a/tests/tap/specs/keeper_fsm_gap_211_fast_forward.pgaf b/tests/tap/specs/keeper_fsm_gap_211_fast_forward.pgaf new file mode 100644 index 000000000..919240b90 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_211_fast_forward.pgaf @@ -0,0 +1,154 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, +# candidatePriority zero -> report_lsn", src/monitor/group_state_machine.c), +# for its fast_forward current_state -- the candidatePriority=0 sibling of +# keeper_fsm_gap_209_fast_forward.pgaf's own scenario. See that spec's header +# for the full story of how a real fast_forward state is forced (network +# disconnect + INSERT + CHECKPOINT to create genuine LSN divergence, the +# same recipe already established by tests/tap/specs/multi_ifdown.pgaf and +# tests/tap/specs/debug_citus_worker_fast_forward.pgaf) and, critically, why +# node1's and node2's rows must be dropped immediately once node3 is +# *assigned* fast_forward -- not after it *reports* fast_forward -- to +# reliably exercise the KeeperFSM[] row under test instead of racing against +# the monitor's own MS-failover cascade (which, left alone, advances +# fast_forward -> prepare_promotion -> stop_replication within about a +# second of node3 reporting convergence, converging to single via a +# different, pre-existing row instead). +# +# This spec asserts the candidatePriority=0 sibling row that was added +# alongside pos 209's: +# +# { +# FAST_FORWARD_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, +# COMMENT_SECONDARY_TO_REPORT_LSN, +# &fsm_report_lsn, +# FSM_PHASE_FAILOVER +# }, +# +# reusing fsm_report_lsn exactly like every other converged-standby source +# state that already had a REPORT_LSN_STATE-bound row (SECONDARY/CATCHINGUP/ +# MAINTENANCE/PREPARE_MAINTENANCE/WAIT_MAINTENANCE) -- no separate WAL fetch +# is attempted or needed, fast_forward already means Postgres is running as +# a caught-up-enough standby. +# +# node3 needs its ordinary (nonzero) candidate-priority to actually be +# selected as the failover candidate and assigned fast_forward in the first +# place -- GroupListCandidates excludes candidatePriority == 0 nodes from +# selection entirely, so a node can only reach fast_forward while still +# candidate-eligible. Its priority is only dropped to 0 afterwards, once it +# is already assigned fast_forward (confirmed via "assigned-state = +# fast_forward", the same checkpoint keeper_fsm_gap_209_fast_forward.pgaf +# uses before removing peers) -- and, same as the peer removal, done via +# the underlying pgautofailover.set_node_candidate_priority() SQL function +# directly rather than the `pg_autoctl set node candidate-priority` CLI: +# that CLI spawns a docker exec plus its own "wait for the settings to be +# applied to ... the primary node" confirmation loop, slow enough that the +# monitor's own MS-failover cascade (fast_forward -> prepare_promotion) +# already won the race by the time an earlier draft of this spec got around +# to removing node1/node2 below -- the plain SQL call is exactly as fast as +# those remove_node() calls, closing the window instead of losing the race +# to confirm against at that point: +# +# 1. Identical setup to keeper_fsm_gap_209_fast_forward.pgaf: node2 +# (candidate-priority 0, WAL source) stays connected and caught up; +# node3 (default nonzero candidate-priority) is disconnected while +# node1 receives writes, then reconnected as node1 is disconnected -- +# forcing an MS-failover election that selects node3 (the only +# candidate) and assigns it FAST_FORWARD_STATE to fetch the missing +# WAL from node2. +# 2. Once node3 is confirmed *assigned* fast_forward (still fetching, +# not yet converged): its own candidate-priority is set to 0, and +# node1's and node2's rows are dropped directly via +# pgautofailover.remove_node(..., true) (node2 is still alive and +# connected, so force=true is required; its actual Postgres instance +# and the already-established direct fetch from node3 are +# unaffected by removing its monitor row). This makes "alone in +# group" true well before node3 finishes catching up. +# 3. Once node3's own local recovery genuinely completes and it reports +# "fast_forward" for the first time, the monitor evaluates pos 211 +# immediately on that exact report (alone=true, candidate-priority=0, +# reported=fast_forward) and assigns REPORT_LSN directly -- no +# cascade through prepare_promotion is possible, since node2's row is +# already gone. +# 4. node3's keeper now has a matching KeeperFSM[] row (the fix under +# test) and actually converges to report_lsn, instead of getting +# stuck logging "does not know how to reach state \"report_lsn\" +# from \"fast_forward\"". + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state is secondary timeout 60s + promote node1 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s +} + +teardown { + compose down +} + +step test_001_set_candidate_priorities { + exec node2 pg_autoctl set node candidate-priority 0 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 60s + sql node1 { CREATE TABLE t1 (a int); } +} + +step test_002_disconnect_node3_and_diverge { + network disconnect node3 + sql node1 { + INSERT INTO t1 SELECT x FROM generate_series(1, 5000) as gs(x); + } + sql node1 { CHECKPOINT; } +} + +step test_003_failover_zero_priority_and_remove_peers_while_still_fetching { + network disconnect node1 + network connect node3 + wait until node3 assigned-state = fast_forward timeout 60s + # Zero node3's own candidate-priority via the underlying SQL function + # (pgautofailover.set_node_candidate_priority), not the + # `pg_autoctl set node candidate-priority` CLI: that CLI spawns a new + # docker exec + its own "wait for settings to be applied" confirmation + # loop, slow enough that the monitor's own cascade (fast_forward -> + # prepare_promotion) already wins the race by the time it would get to + # removing node1/node2 below. A direct SQL call is just as fast as the + # remove_node() calls that follow it. + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node2'; + } +} + +step test_004_node3_converges_straight_to_report_lsn { + wait until node3 assigned-state = report_lsn timeout 180s + wait until node3 state is report_lsn timeout 60s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} diff --git a/tests/tap/specs/keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf b/tests/tap/specs/keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf new file mode 100644 index 000000000..2af88068a --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf @@ -0,0 +1,96 @@ +# Regression spec confirming a real requirement flagged during the pos +# 209/211 "alone in group" investigation: once a lone node is parked at +# report_lsn with candidate-priority 0 (pos 211's own "wait for an operator +# or a new peer" outcome), is there actually a way for the cluster to +# recover once a new peer DOES show up? +# +# Answer: yes, already fully working, no monitor or keeper change needed -- +# confirmed live. register_node()'s own logic (RegisterNode, +# node_active_protocol.c ~line 662-694) already handles this exact case: if +# there's no primary and no promotion already in flight, it searches for a +# node with candidatePriority == 0 AND IsCurrentState(REPORT_LSN); if found, +# the new node is ALSO assigned initialState = REPORT_LSN (not +# WAIT_STANDBY), which uses fsm_init_from_standby / keeper_get_most_ +# advanced_standby (NOT keeper_get_primary, which would find nothing here -- +# GetPrimaryOrDemotedNodeInGroupFromList requires CanTakeWritesInState or +# StateBelongsToPrimary, neither of which report_lsn satisfies) to +# basebackup from the existing report_lsn node instead of a "real" primary. +# +# Once the new node registers, the group now has TWO report_lsn-reporting +# nodes -- exactly the ordinary MS-failover candidate-selection scenario +# (BuildCandidateList/SelectFailoverCandidateNode/PromoteSelectedNode). +# Since the new node has ordinary (nonzero) candidate-priority while the +# original node is still 0, the new node is selected as the failover +# candidate and promoted -- through fast_forward (if marginally behind the +# original node's exact LSN after basebackup), prepare_promotion, +# wait_primary, primary -- while the ORIGINAL report_lsn node transitions +# report_lsn -> join_secondary -> secondary, following the new node as its +# primary. The cluster ends up fully HA again with zero manual +# intervention beyond starting the new node. +# +# This was discovered by literally trying it live: an earlier draft of this +# spec asserted the new node would become a plain secondary of the +# original node, which is backwards -- it actually becomes the new +# primary, and the original (priority-zero) node becomes its secondary. +# +# 1. node1 (primary) + node2 (secondary) bootstrap normally. +# 2. node2's candidate-priority is set to 0. +# 3. node1 is network-disconnected, node2 told to enter maintenance (last +# quorum member -> wait_maintenance), then node1 is dropped directly -- +# leaving node2 alone, converged at report_lsn (candidate-priority 0). +# (Same mechanism as keeper_fsm_gap_211_wait_maintenance.pgaf.) +# 4. node3 (created with launch deferred) is started only now, once node2 +# is confirmed stuck at report_lsn. It registers, basebackups from +# node2, and the cluster elects it as the new primary. + +cluster { + monitor + formation { + node1 + node2 + node3 create and launch deferred + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_001_zero_priority_and_disconnect_primary { + exec node2 pg_autoctl set node candidate-priority 0 + network disconnect node1 +} + +step test_002_secondary_stuck_at_wait_maintenance { + exec-fails node2 pg_autoctl enable maintenance + wait until node2 state is wait_maintenance timeout 60s +} + +step test_003_drop_primary_converges_to_report_lsn { + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + wait until node2 assigned-state = report_lsn timeout 60s + wait until node2 state is report_lsn timeout 60s +} + +step test_004_node3_joins_and_becomes_primary { + exec node3 pg_autoctl node start + wait until node3 state is primary + and node2 state is secondary + timeout 90s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { primary } + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node2'; + } + expect { secondary } +} From c0f9277de069807971803f5b841778388e13b318 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Fri, 31 Jul 2026 17:05:56 +0200 Subject: [PATCH 32/52] Fix monitor FSM rule 209: exclude join_secondary and prepare_maintenance pos 209 ("alone in group, candidate-eligible -> single") was willing to match a node reporting join_secondary or prepare_maintenance and promote it straight to SINGLE. Investigating this surfaced a real data-loss risk for both, not just a missing keeper transition: - join_secondary: entering it stops Postgres (fsm_checkpoint_and_stop_ postgres) as part of switching allegiance to an already-elected new primary. The on-disk data is only a trustworthy copy of this node's own last moment as the OLD primary, frozen before that new primary ever took a write. If that new primary later also vanishes, promoting this node straight to SINGLE would silently discard everything it committed in the meantime. - prepare_maintenance: the same risk, reached one step earlier. pos 343 lets the candidate standby reach WAIT_PRIMARY/PRIMARY the moment this node's own reportedState merely converges to prepare_maintenance -- no requirement that this node's row ever be removed first. A different, already-promoted primary can be live and taking writes while this node still sits in prepare_maintenance indefinitely. Fixed by adding reportedIsJoinSecondary/reportedIsPrepareMaintenance exclusions to pos 209 (NodeStatusPattern, NodeMatchesPattern, and the dump_fsm_edges() shadow-detection helpers, mirroring the existing reportedIsWaitStandby field exactly) -- not by adding KeeperFSM[] rows, since there is nothing safe to promote either state to. prepare_maintenance additionally needed a new no-op row (pos 208): unlike join_secondary (already recognized by node_metadata.c's IsParticipatingInPromotion, so a lone node there safely no-ops via the ordinary heartbeat no-match fallthrough), a lone prepare_maintenance node is recognized by neither that function, IsBeingPromoted, nor IsInPrimaryState. Without an explicit match, ProceedGroupStateFromContext's own "couldn't find the primary node" guard would ereport(ERROR) on every single subsequent heartbeat from that node -- strictly worse than the original bug. pos 208 matches this exact "alone, reporting prepare_maintenance" combination with a plain no-op (same shape as pos 205's own "converged to maintenance" row). Add keeper_fsm_gap_209_prepare_maintenance.pgaf, verifying live that a lone prepare_maintenance primary stays safely parked (goalstate never becomes single) and keeps successfully checking in (reporttime advancing) rather than looping on that error. join_secondary's own safety is left to the existing static regress coverage (fsm.out/check_fsm_reachability.out/ keeper_fsm_edges.out, all regenerated) plus the code-level IsParticipatingInPromotion confirmation -- pos 211's own join_secondary/prepare_maintenance handling is intentionally untouched (report_lsn is a far less dangerous target than single). Regenerated fsm.out/check_fsm_reachability.out/keeper_fsm_edges.out from the real monitor extension, and updated keeper_fsm_edges.sql's own follow-up investigation comment. Verified: full regress (19/19) + isolation (6/6) installcheck, citus_indent clean, all existing pos 209/211 pgaftest specs re-verified passing, new spec passing live. --- .../expected/check_fsm_reachability.out | 4 +- src/monitor/expected/fsm.out | 137 +++++++------- src/monitor/expected/keeper_fsm_edges.out | 44 +++-- src/monitor/group_state_machine.c | 169 +++++++++++++++++- src/monitor/sql/keeper_fsm_edges.sql | 39 +++- tests/tap/schedule | 1 + tests/tap/schedules/node.sch | 1 + ...eeper_fsm_gap_209_prepare_maintenance.pgaf | 131 ++++++++++++++ 8 files changed, 446 insertions(+), 80 deletions(-) create mode 100644 tests/tap/specs/keeper_fsm_gap_209_prepare_maintenance.pgaf diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index 86592d3a8..e43bf2bf8 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 196 + 194 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 196 + 194 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index d3f4a7055..af2d2f2b6 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -306,13 +306,28 @@ other_node_assigned_state | has_extra_action | f comment | reported demote_timeout, assigned goal can't reach it -> demoted -[ RECORD 20 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 208 +section | early_checks +section_path | early_checks +active_node_current_state | prepare_maintenance +other_node_current_state | +candidate_node_current_state | +active_node_conditions | +other_node_conditions | +candidate_node_conditions | +group_conditions | groupHasExactlyOneNode=true +active_node_assigned_state | +other_node_assigned_state | +has_extra_action | f +comment | alone in group, still preparing for maintenance -> no-op +-[ RECORD 21 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 209 section | early_checks section_path | early_checks active_node_current_state | other_node_current_state | candidate_node_current_state | -active_node_conditions | candidateEligible=true, reportedIsWaitStandby=false +active_node_conditions | candidateEligible=true, reportedIsWaitStandby=false, reportedIsJoinSecondary=false, reportedIsPrepareMaintenance=false other_node_conditions | candidate_node_conditions | group_conditions | groupHasExactlyOneNode=true @@ -320,7 +335,7 @@ active_node_assigned_state | single other_node_assigned_state | has_extra_action | f comment | alone in group, candidate-eligible -> single --[ RECORD 21 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 22 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 210 section | early_checks section_path | early_checks @@ -335,7 +350,7 @@ active_node_assigned_state | single other_node_assigned_state | has_extra_action | f comment | alone in group, already primary despite candidatePriority zero -> single --[ RECORD 22 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 23 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 211 section | early_checks section_path | early_checks @@ -350,7 +365,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | alone in group, candidatePriority zero -> report_lsn --[ RECORD 23 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 24 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 301 section | reporting_node section_path | reporting_node.from_context @@ -365,7 +380,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | has_extra_action | f comment | converged secondary, reportedTLI not an ancestor of reference -> catchingup --[ RECORD 24 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 25 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 303 section | reporting_node section_path | reporting_node.from_context @@ -380,7 +395,7 @@ active_node_assigned_state | other_node_assigned_state | wait_primary has_extra_action | f comment | primary healthy, no standby past replication_stall_timeout -> wait_primary --[ RECORD 25 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 26 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 305 section | reporting_node section_path | reporting_node.from_context @@ -395,7 +410,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade --[ RECORD 26 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 27 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 307 section | reporting_node section_path | reporting_node.from_context @@ -410,7 +425,7 @@ active_node_assigned_state | secondary other_node_assigned_state | has_extra_action | f comment | report_lsn, primary converged wait/join_primary, healthy -> secondary --[ RECORD 27 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 28 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 309 section | reporting_node section_path | reporting_node.from_context @@ -425,7 +440,7 @@ active_node_assigned_state | secondary other_node_assigned_state | has_extra_action | f comment | report_lsn, primary converged primary, healthy -> secondary --[ RECORD 28 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 29 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 311 section | reporting_node section_path | reporting_node.from_context @@ -440,7 +455,7 @@ active_node_assigned_state | prepare_promotion other_node_assigned_state | has_extra_action | f comment | fast_forward done -> prepare_promotion --[ RECORD 29 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 30 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 313 section | reporting_node section_path | reporting_node.from_context @@ -455,7 +470,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | report_lsn or fast_forward, continuing an already-started failover -> MS-failover cascade --[ RECORD 30 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 31 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 315 section | reporting_node section_path | reporting_node.from_context @@ -470,7 +485,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | has_extra_action | f comment | wait_standby, primary converged wait/join_primary -> catchingup --[ RECORD 31 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 32 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 317 section | reporting_node section_path | reporting_node.from_context @@ -485,7 +500,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | apply_settings has_extra_action | f comment | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings --[ RECORD 32 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 33 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 319 section | reporting_node section_path | reporting_node.from_context @@ -500,7 +515,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | has_extra_action | f comment | wait_standby (not a quorum member), primary converged primary -> catchingup --[ RECORD 33 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 34 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 321 section | reporting_node section_path | reporting_node.from_context @@ -515,7 +530,7 @@ active_node_assigned_state | secondary other_node_assigned_state | has_extra_action | f comment | caught up, same TLI as primary, within sync threshold -> secondary --[ RECORD 34 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 35 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 323 section | reporting_node section_path | reporting_node.from_context @@ -530,7 +545,7 @@ active_node_assigned_state | prepare_promotion other_node_assigned_state | has_extra_action | f comment | primary fails, already converged wait_primary (issue #1168) -> secondary -> prepare_promotion only (1 of 2) --[ RECORD 35 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 36 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 325 section | reporting_node section_path | reporting_node.from_context @@ -545,7 +560,7 @@ active_node_assigned_state | prepare_promotion other_node_assigned_state | draining has_extra_action | f comment | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) --[ RECORD 36 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 37 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 327 section | reporting_node section_path | reporting_node.from_context @@ -560,7 +575,7 @@ active_node_assigned_state | maintenance other_node_assigned_state | has_extra_action | f comment | wait_maintenance, primary converged wait_primary -> maintenance --[ RECORD 37 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 38 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 329 section | reporting_node section_path | reporting_node.from_context @@ -575,7 +590,7 @@ active_node_assigned_state | maintenance other_node_assigned_state | has_extra_action | f comment | wait_maintenance, primary's goal no longer wait_primary -> maintenance --[ RECORD 38 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 39 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 331 section | reporting_node section_path | reporting_node.from_context @@ -590,7 +605,7 @@ active_node_assigned_state | stop_replication other_node_assigned_state | has_extra_action | f comment | prepare_promotion, primary converged prepare_maintenance -> stop_replication --[ RECORD 39 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 40 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 333 section | reporting_node section_path | reporting_node.from_context @@ -605,7 +620,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | demoted has_extra_action | f comment | Citus worker prepare_promotion, primary present -> wait_primary + demoted --[ RECORD 40 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 41 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 335 section | reporting_node section_path | reporting_node.from_context @@ -620,7 +635,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | Citus worker prepare_promotion, primary removed -> wait_primary --[ RECORD 41 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 42 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 337 section | reporting_node section_path | reporting_node.from_context @@ -635,7 +650,7 @@ active_node_assigned_state | stop_replication other_node_assigned_state | has_extra_action | f comment | prepare_promotion, primary already converged wait_primary (issue #1168) -> stop_replication only (1 of 2) --[ RECORD 42 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 43 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 339 section | reporting_node section_path | reporting_node.from_context @@ -650,7 +665,7 @@ active_node_assigned_state | stop_replication other_node_assigned_state | demote_timeout has_extra_action | f comment | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) --[ RECORD 43 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 44 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 341 section | reporting_node section_path | reporting_node.from_context @@ -665,7 +680,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | prepare_promotion, primary removed -> wait_primary --[ RECORD 44 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 45 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 343 section | reporting_node section_path | reporting_node.from_context @@ -680,7 +695,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | maintenance has_extra_action | f comment | stop_replication, primary converged prepare_maintenance -> wait_primary + maintenance --[ RECORD 45 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 46 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 345 section | reporting_node section_path | reporting_node.from_context @@ -695,7 +710,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | demoted has_extra_action | f comment | stop_replication, primary converged demote_timeout -> wait_primary + demoted (1 of 3) --[ RECORD 46 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 47 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 347 section | reporting_node section_path | reporting_node.from_context @@ -710,7 +725,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | demoted has_extra_action | f comment | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) --[ RECORD 47 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 48 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 349 section | reporting_node section_path | reporting_node.from_context @@ -725,7 +740,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | demoted has_extra_action | f comment | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) --[ RECORD 48 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 49 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 351 section | reporting_node section_path | reporting_node.from_context @@ -740,7 +755,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | demoted has_extra_action | f comment | Citus worker stop_replication, primary present -> wait_primary + demoted --[ RECORD 49 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 50 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 353 section | reporting_node section_path | reporting_node.from_context @@ -755,7 +770,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | Citus worker stop_replication, primary removed -> wait_primary --[ RECORD 50 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 51 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 355 section | reporting_node section_path | reporting_node.from_context @@ -770,7 +785,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | has_extra_action | f comment | demoted, primary reported wait/join_primary with goal primary -> catchingup --[ RECORD 51 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 52 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 357 section | reporting_node section_path | reporting_node.from_context @@ -785,7 +800,7 @@ active_node_assigned_state | catchingup other_node_assigned_state | has_extra_action | f comment | demoted, primary converged wait/join_primary/primary, healthy -> catchingup --[ RECORD 52 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 53 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 359 section | reporting_node section_path | reporting_node.from_context @@ -800,7 +815,7 @@ active_node_assigned_state | secondary other_node_assigned_state | has_extra_action | t comment | join_secondary, primary reported wait_primary with goal wait/primary -> secondary --[ RECORD 53 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 54 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 361 section | reporting_node section_path | reporting_node.from_context @@ -815,7 +830,7 @@ active_node_assigned_state | secondary other_node_assigned_state | has_extra_action | f comment | join_secondary, primary converged primary -> secondary --[ RECORD 54 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 55 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 363 section | reporting_node section_path | reporting_node.ms_failover.retry_reset @@ -830,7 +845,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | MS-failover: candidate stuck in fast_forward, all WAL sources unhealthy, guard_data_loss=true -> report_lsn (retry once a source recovers) --[ RECORD 55 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 56 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 365 section | reporting_node section_path | reporting_node.ms_failover.candidate_join @@ -845,7 +860,7 @@ active_node_assigned_state | join_secondary other_node_assigned_state | has_extra_action | f comment | MS-failover: activeNode in report_lsn, failover candidate ready to stream WAL -> join_secondary --[ RECORD 56 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 57 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 367 section | reporting_node section_path | reporting_node.ms_failover.candidate_fanout @@ -860,7 +875,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) --[ RECORD 57 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 58 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 369 section | reporting_node section_path | reporting_node.ms_failover.candidate_fanout @@ -875,7 +890,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | MS-failover fan-out: rejoining from maintenance -> report_lsn (2 of 4) --[ RECORD 58 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 59 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 371 section | reporting_node section_path | reporting_node.ms_failover.candidate_fanout @@ -890,7 +905,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | MS-failover fan-out: old primary converged draining or demoted -> report_lsn (3 of 4) --[ RECORD 59 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 60 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 373 section | reporting_node section_path | reporting_node.ms_failover.candidate_fanout @@ -905,7 +920,7 @@ active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f comment | MS-failover fan-out: old primary demoted, was rejoining a now-failed primary -> report_lsn (4 of 4) --[ RECORD 60 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 61 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 375 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome @@ -920,7 +935,7 @@ active_node_assigned_state | prepare_promotion other_node_assigned_state | has_extra_action | f comment | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate already has all WAL -> prepare_promotion (1 of 2 -- see this row's own comment on why both are listed) --[ RECORD 61 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 62 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 377 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome @@ -935,7 +950,7 @@ active_node_assigned_state | fast_forward other_node_assigned_state | has_extra_action | f comment | MS-failover: no promotion in flight, most-advanced candidate within threshold, selected candidate is lagging -> fast_forward (2 of 2) --[ RECORD 62 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 63 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 379 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.missing_nodes_gate @@ -950,7 +965,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=true -> decline, wait for more reports (1 of 2) --[ RECORD 63 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 64 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 381 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.missing_nodes_gate @@ -965,7 +980,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=false -> proceed despite possible data loss (2 of 2) --[ RECORD 64 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 65 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 383 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.candidate_count_gate @@ -980,7 +995,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | f comment | MS-failover: zero candidates have reported their LSN yet -> silent decline --[ RECORD 65 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 66 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 385 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.quorum_candidate_gate @@ -995,7 +1010,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | MS-failover: not enough quorum candidates reported yet, guard_data_loss=true -> decline (1 of 2) --[ RECORD 66 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 67 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 387 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.quorum_candidate_gate @@ -1010,7 +1025,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | t comment | MS-failover: not enough quorum candidates reported yet, guard_data_loss=false -> proceed with fewer than required (2 of 2) --[ RECORD 67 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 68 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 389 section | reporting_node section_path | reporting_node.ms_failover.promotion_outcome.no_candidate_yet @@ -1025,7 +1040,7 @@ active_node_assigned_state | other_node_assigned_state | has_extra_action | f comment | MS-failover: no promotion in flight, not enough (or not safe enough) candidates yet -> no-op besides the fan-out above --[ RECORD 68 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 69 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 391 section | reporting_node section_path | reporting_node.ms_failover.draining_or_maintenance @@ -1040,7 +1055,7 @@ active_node_assigned_state | other_node_assigned_state | draining has_extra_action | f comment | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining --[ RECORD 69 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 70 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 393 section | reporting_node section_path | reporting_node.ms_failover.draining_or_maintenance @@ -1055,7 +1070,7 @@ active_node_assigned_state | other_node_assigned_state | maintenance has_extra_action | f comment | nodesCount>2, primary unhealthy, converged prepare_maintenance -> primary maintenance --[ RECORD 70 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 401 section | primary_node section_path | primary_node @@ -1070,7 +1085,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | primary alone, another node reached wait_standby -> wait_primary --[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 403 section | primary_node section_path | primary_node @@ -1085,7 +1100,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | all nodes async, zero secondaries -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 405 section | primary_node section_path | primary_node @@ -1100,7 +1115,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | all nodes async, >=1 secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 407 section | primary_node section_path | primary_node @@ -1115,7 +1130,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 75 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 409 section | primary_node section_path | primary_node @@ -1130,7 +1145,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 75 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 411 section | primary_node section_path | primary_node @@ -1145,7 +1160,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 413 section | primary_node section_path | primary_node @@ -1160,7 +1175,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, both zero -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 415 section | primary_node section_path | primary_node @@ -1175,7 +1190,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 417 section | primary_node section_path | primary_node @@ -1190,7 +1205,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 80 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 419 section | primary_node section_path | primary_node @@ -1205,7 +1220,7 @@ active_node_assigned_state | other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/wait_primary/apply_settings, no other condition applies -> no-op besides the unhealthy-secondary fan-out to catchingup --[ RECORD 80 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 81 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 421 section | primary_node section_path | primary_node diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 0b5c94ac3..106d2a0f7 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -186,12 +186,39 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- from the group while node3 is still fetching (not after it reports -- fast_forward) to actually exercise these rows instead of racing -- against the monitor's own cascade continuation. --- * pos 209/211's remaining states (prepare_maintenance, demote_timeout, --- prepare_promotion, stop_replication, join_secondary) were each checked --- for real reachability under "alone in group" and found contrived: --- reaching them normally implies a multi-node context (join_secondary --- needs a newly-elected primary to join) or an internal tension with --- candidateEligible=FALSE (prepare_promotion/stop_replication imply +-- * pos 209's join_secondary and prepare_maintenance current_states are no +-- longer even in this gap list at all: both were found to be a genuine +-- data-loss risk, not a missing convenience -- promoting either straight +-- to SINGLE if left alone can silently discard writes a *different, +-- already-promoted* primary made in the meantime (join_secondary: that +-- new primary already exists by the time this node reaches +-- join_secondary; prepare_maintenance: pos 343 lets the candidate +-- standby reach primary the moment this node's own reportedState merely +-- converges to prepare_maintenance, no removal required). Fixed by +-- excluding both from pos 209 itself (reportedIsJoinSecondary, +-- reportedIsPrepareMaintenance on NodeMatchesPattern) rather than adding +-- KeeperFSM[] rows -- there is nothing safe to promote either one to. +-- prepare_maintenance additionally needed a new no-op row (pos 208): +-- unlike join_secondary (already recognized by node_metadata.c's +-- IsParticipatingInPromotion, so a lone node there safely no-ops on its +-- own), a lone prepare_maintenance node isn't recognized by that +-- function, IsBeingPromoted, or IsInPrimaryState, so excluding it from +-- pos 209 alone would have left ProceedGroupStateFromContext's own +-- "couldn't find the primary node" guard to ereport(ERROR) on every +-- single subsequent heartbeat -- worse than the original bug. Both +-- fixes verified live: keeper_fsm_gap_209_prepare_maintenance.pgaf +-- reproduces a lone prepare_maintenance primary staying safely parked +-- (goalstate never becomes single) and confirms it keeps successfully +-- checking in (reporttime advancing) rather than looping on that error. +-- * pos 211's own join_secondary/prepare_maintenance current_states are +-- untouched by the above -- report_lsn is a far less dangerous target +-- than single (it doesn't let the node accept writes), so the same +-- split-brain argument doesn't automatically carry over; not +-- investigated further this pass. +-- * pos 209/211's remaining states (demote_timeout, prepare_promotion, +-- stop_replication) were each checked for real reachability under +-- "alone in group" and found contrived: each implies an internal tension +-- with candidateEligible=FALSE (prepare_promotion/stop_replication imply -- having already been selected as a promotion candidate; demote_timeout's -- genuinely-stuck case is already intercepted earlier by pos 207). None -- ruled out as impossible, but none reproducible via a single, realistic @@ -223,9 +250,6 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; rule | n | current_state | assigned_state | comment ------+----+---------------------+----------------+---------------------------------------------------------------------------------------------------------------------- - 209 | 2 | | single | alone in group, candidate-eligible -> single - 209 | 1 | prepare_maintenance | single | alone in group, candidate-eligible -> single - 209 | 1 | join_secondary | single | alone in group, candidate-eligible -> single 211 | 4 | | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | demote_timeout | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | prepare_promotion | report_lsn | alone in group, candidatePriority zero -> report_lsn @@ -311,7 +335,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 351 | 1 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 391 | 1 | | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining 391 | 1 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining -(88 rows) +(85 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index b6a892314..babacfa3e 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -504,6 +504,8 @@ typedef struct NodeStatusPattern BoolPattern canTakeWrites; BoolPattern reportedCanTakeWrites; BoolPattern reportedIsWaitStandby; + BoolPattern reportedIsJoinSecondary; + BoolPattern reportedIsPrepareMaintenance; BoolPattern isReadyToStreamWAL; BoolPattern drainTimeExpired; BoolPattern isCitusWorkerGroup; @@ -589,6 +591,36 @@ BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *stat * sees pos 101's cross-row goalState write) but still let pos 209/211 fire * for a real wait_standby node in a live pgaftest run, exactly because of * this. Matching on reportedState alone sidesteps it entirely. + * + * reportedIsJoinSecondary, same reported-only shape as reportedIsWaitStandby + * just above, excludes JOIN_SECONDARY_STATE from pos 209's own "alone in + * group, candidate-eligible -> single" match. A node reporting + * join_secondary has already had Postgres cleanly checkpointed and stopped + * (fsm_checkpoint_and_stop_postgres, fsm.c) as part of switching its + * replication target to a newly-elected primary -- its on-disk data is a + * trustworthy copy, but only of *this node's own* last moment as the old + * primary, frozen before that new primary ever took a single write. + * Promoting it straight to SINGLE if it ends up alone risks silently + * discarding whatever the new primary committed in the meantime -- a real + * split-brain/data-loss risk, unlike every other source state pos 209 + * matches, where the reporting node's own data is either already the most + * advanced available or was safely fetched from whichever node was. + * + * reportedIsPrepareMaintenance, same reported-only shape and same pos 209 + * exclusion, for the primary-role counterpart of join_secondary's own risk. + * start_maintenance() assigns PREPARE_MAINTENANCE_STATE to the primary and + * PREPARE_PROMOTION_STATE to its chosen standby in the very same call (see + * pos 109/111's own comment) -- and pos 343 lets that standby advance all + * the way to WAIT_PRIMARY/PRIMARY the moment the old primary's own + * reportedState merely *converges* to prepare_maintenance (Postgres + * cleanly stopped, fsm_stop_postgres_for_primary_maintenance), with no + * requirement that the old primary's row ever be removed first. So a node + * can sit in prepare_maintenance indefinitely while a different, already + * fully-promoted primary is live and taking writes elsewhere -- if that new + * primary later also vanishes, promoting the OLD node straight to SINGLE + * would discard everything the new primary committed in between, the exact + * same split-brain risk reportedIsJoinSecondary guards against, just + * reached one step earlier in the handoff. */ static bool NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) @@ -619,6 +651,13 @@ NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) BoolMatchesPattern(status->node != NULL && status->node->reportedState == REPLICATION_STATE_WAIT_STANDBY, pattern->reportedIsWaitStandby) && + BoolMatchesPattern(status->node != NULL && + status->node->reportedState == REPLICATION_STATE_JOIN_SECONDARY, + pattern->reportedIsJoinSecondary) && + BoolMatchesPattern(status->node != NULL && + status->node->reportedState == + REPLICATION_STATE_PREPARE_MAINTENANCE, + pattern->reportedIsPrepareMaintenance) && BoolMatchesPattern(CandidateNodeIsReadyToStreamWAL(status->node), pattern->isReadyToStreamWAL) && BoolMatchesPattern(NodeIsDrainTimeExpired(status->node, status->ctx), @@ -2463,6 +2502,35 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNodeAssignedState = GOAL(REPLICATION_STATE_DEMOTED), .comment = "reported demote_timeout, assigned goal can't reach it -> demoted" }, + /* + * alone in group, still preparing for primary maintenance -- no-op, + * same shape as pos 205's own "converged to maintenance" row. This row + * exists purely to keep pos 209's own reportedIsPrepareMaintenance + * exclusion (see its comment on NodeMatchesPattern) from turning into a + * *worse* bug than the one it fixes: join_secondary is safely tolerated + * when left alone because IsParticipatingInPromotion (node_metadata.c) + * already recognizes it, but prepare_maintenance isn't recognized by + * that function, by IsBeingPromoted, or by IsInPrimaryState + * (CanTakeWritesInState(prepare_maintenance) is false) -- so without an + * explicit match here, ProceedGroupStateFromContext's own "primaryNode + * == NULL && !IsFailoverInProgress(...)" guard would ereport(ERROR) on + * every single subsequent heartbeat from this node, instead of safely + * leaving it stuck. Frozen here (no assignment) until an operator + * intervenes -- the same "wait for an operator" outcome pos 211's own + * candidatePriority=0 case already accepts for other source states. + * Only fires when alone: in an ordinary (non-alone) group, the + * candidate standby being promoted in parallel already satisfies + * IsFailoverInProgress for the whole group, so this row must not + * intercept that already-working path. + */ + { .pos = 208, + .sectionPath = { + MONITOR_FSM_SECTION_EARLY_CHECKS + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, + .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, + .comment = "alone in group, still preparing for maintenance -> no-op" }, + /* * alone in group, candidate-eligible -- excludes WAIT_STANDBY_STATE too * (reportedIsWaitStandby's own comment on NodeMatchesPattern): a node @@ -2475,6 +2543,30 @@ static const MonitorFSMTransition MonitorFSM[] = { * to report_lsn before this row's own next evaluation, which would * defeat a goalState-dependent (NOT_STABLE) exclusion the same way pos * 210's own isInPrimaryState condition was defeated -- confirmed live. + * + * Also excludes JOIN_SECONDARY_STATE (reportedIsJoinSecondary's own + * comment on NodeMatchesPattern): unlike every other source state this + * row matches, a node reporting join_secondary has already had Postgres + * cleanly checkpointed and stopped as part of switching its replication + * target to a newly-elected primary. Its on-disk data is only a + * trustworthy copy of its own last moment as the OLD primary, frozen + * before that new primary ever took a single write -- resuming it + * straight to SINGLE if it ends up alone risks silently discarding + * whatever the new primary committed in the meantime, a real + * split-brain/data-loss risk this row must not create. Same reported- + * only reasoning as reportedIsWaitStandby above for why this is gated on + * reportedState alone rather than folded into FSM_NOT_STABLE_SINGLE. + * + * Also excludes PREPARE_MAINTENANCE_STATE (reportedIsPrepareMaintenance's + * own comment on NodeMatchesPattern): the same split-brain risk as + * join_secondary, reached one step earlier -- start_maintenance() + * assigns this node PREPARE_MAINTENANCE_STATE and its chosen standby + * PREPARE_PROMOTION_STATE in the very same call, and pos 343 lets that + * standby advance all the way to primary the moment this node's own + * reportedState merely converges to prepare_maintenance, with no + * requirement that this node's row ever be removed first. A different, + * already fully-promoted primary can be live and taking writes while + * this node still sits in prepare_maintenance indefinitely. */ { .pos = 209, .sectionPath = { @@ -2482,7 +2574,9 @@ static const MonitorFSMTransition MonitorFSM[] = { }, .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, .candidateEligible = BOOL_TRUE, - .reportedIsWaitStandby = BOOL_FALSE }, + .reportedIsWaitStandby = BOOL_FALSE, + .reportedIsJoinSecondary = BOOL_FALSE, + .reportedIsPrepareMaintenance = BOOL_FALSE }, .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SINGLE), .comment = "alone in group, candidate-eligible -> single" }, @@ -4169,6 +4263,9 @@ NodeStatusPatternConditionsText(const NodeStatusPattern *pattern, bool *isNull) APPEND_BOOL_CONDITION(&buf, "canTakeWrites", pattern->canTakeWrites); APPEND_BOOL_CONDITION(&buf, "reportedCanTakeWrites", pattern->reportedCanTakeWrites); APPEND_BOOL_CONDITION(&buf, "reportedIsWaitStandby", pattern->reportedIsWaitStandby); + APPEND_BOOL_CONDITION(&buf, "reportedIsJoinSecondary", pattern->reportedIsJoinSecondary); + APPEND_BOOL_CONDITION(&buf, "reportedIsPrepareMaintenance", + pattern->reportedIsPrepareMaintenance); APPEND_BOOL_CONDITION(&buf, "isReadyToStreamWAL", pattern->isReadyToStreamWAL); APPEND_BOOL_CONDITION(&buf, "drainTimeExpired", pattern->drainTimeExpired); APPEND_BOOL_CONDITION(&buf, "isCitusWorkerGroup", pattern->isCitusWorkerGroup); @@ -4659,6 +4756,50 @@ NodeStatusPatternSurvivesReportedIsWaitStandby(const NodeStatusPattern *pattern, } +/* + * NodeStatusPatternSurvivesReportedIsJoinSecondary filters a candidate + * edge-source state against pattern's own .reportedIsJoinSecondary field, + * the same shape as NodeStatusPatternSurvivesReportedIsWaitStandby just + * above -- a plain equality on reportedState alone, no goalState-dependent + * satisfiability proof needed. + */ +static bool +NodeStatusPatternSurvivesReportedIsJoinSecondary(const NodeStatusPattern *pattern, + ReplicationState state) +{ + if (pattern->reportedIsJoinSecondary == BOOL_ANY) + { + return true; + } + + bool required = (pattern->reportedIsJoinSecondary == BOOL_TRUE); + + return (state == REPLICATION_STATE_JOIN_SECONDARY) == required; +} + + +/* + * NodeStatusPatternSurvivesReportedIsPrepareMaintenance filters a candidate + * edge-source state against pattern's own .reportedIsPrepareMaintenance + * field, the same shape as NodeStatusPatternSurvivesReportedIsJoinSecondary + * just above -- a plain equality on reportedState alone, no goalState- + * dependent satisfiability proof needed. + */ +static bool +NodeStatusPatternSurvivesReportedIsPrepareMaintenance(const NodeStatusPattern *pattern, + ReplicationState state) +{ + if (pattern->reportedIsPrepareMaintenance == BOOL_ANY) + { + return true; + } + + bool required = (pattern->reportedIsPrepareMaintenance == BOOL_TRUE); + + return (state == REPLICATION_STATE_PREPARE_MAINTENANCE) == required; +} + + /* * NodeStatePatternKindIsReportedStateOnly: true for the pattern kinds whose * match genuinely depends only on reportedState (ignoring, for STABLE, its @@ -4746,6 +4887,8 @@ NodeStatusPatternOtherFieldsAreAny(const NodeStatusPattern *pattern) pattern->canTakeWrites == BOOL_ANY && pattern->reportedCanTakeWrites == BOOL_ANY && pattern->reportedIsWaitStandby == BOOL_ANY && + pattern->reportedIsJoinSecondary == BOOL_ANY && + pattern->reportedIsPrepareMaintenance == BOOL_ANY && pattern->isReadyToStreamWAL == BOOL_ANY && pattern->drainTimeExpired == BOOL_ANY && pattern->isCitusWorkerGroup == BOOL_ANY && @@ -5089,6 +5232,18 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + if (!NodeStatusPatternSurvivesReportedIsJoinSecondary(&rule->activeNode, + states[j])) + { + continue; + } + + if (!NodeStatusPatternSurvivesReportedIsPrepareMaintenance( + &rule->activeNode, states[j])) + { + continue; + } + if (EdgeIsShadowedByEarlierRule(i, states[j], false, rule->sectionPath[0])) { continue; @@ -5149,6 +5304,18 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + if (!NodeStatusPatternSurvivesReportedIsJoinSecondary(&rule->primaryNode, + states[j])) + { + continue; + } + + if (!NodeStatusPatternSurvivesReportedIsPrepareMaintenance( + &rule->primaryNode, states[j])) + { + continue; + } + if (EdgeIsShadowedByEarlierRule(i, states[j], true, rule->sectionPath[0])) { continue; diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 8d3cc6fb0..c0bbabad4 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -104,12 +104,39 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- from the group while node3 is still fetching (not after it reports -- fast_forward) to actually exercise these rows instead of racing -- against the monitor's own cascade continuation. --- * pos 209/211's remaining states (prepare_maintenance, demote_timeout, --- prepare_promotion, stop_replication, join_secondary) were each checked --- for real reachability under "alone in group" and found contrived: --- reaching them normally implies a multi-node context (join_secondary --- needs a newly-elected primary to join) or an internal tension with --- candidateEligible=FALSE (prepare_promotion/stop_replication imply +-- * pos 209's join_secondary and prepare_maintenance current_states are no +-- longer even in this gap list at all: both were found to be a genuine +-- data-loss risk, not a missing convenience -- promoting either straight +-- to SINGLE if left alone can silently discard writes a *different, +-- already-promoted* primary made in the meantime (join_secondary: that +-- new primary already exists by the time this node reaches +-- join_secondary; prepare_maintenance: pos 343 lets the candidate +-- standby reach primary the moment this node's own reportedState merely +-- converges to prepare_maintenance, no removal required). Fixed by +-- excluding both from pos 209 itself (reportedIsJoinSecondary, +-- reportedIsPrepareMaintenance on NodeMatchesPattern) rather than adding +-- KeeperFSM[] rows -- there is nothing safe to promote either one to. +-- prepare_maintenance additionally needed a new no-op row (pos 208): +-- unlike join_secondary (already recognized by node_metadata.c's +-- IsParticipatingInPromotion, so a lone node there safely no-ops on its +-- own), a lone prepare_maintenance node isn't recognized by that +-- function, IsBeingPromoted, or IsInPrimaryState, so excluding it from +-- pos 209 alone would have left ProceedGroupStateFromContext's own +-- "couldn't find the primary node" guard to ereport(ERROR) on every +-- single subsequent heartbeat -- worse than the original bug. Both +-- fixes verified live: keeper_fsm_gap_209_prepare_maintenance.pgaf +-- reproduces a lone prepare_maintenance primary staying safely parked +-- (goalstate never becomes single) and confirms it keeps successfully +-- checking in (reporttime advancing) rather than looping on that error. +-- * pos 211's own join_secondary/prepare_maintenance current_states are +-- untouched by the above -- report_lsn is a far less dangerous target +-- than single (it doesn't let the node accept writes), so the same +-- split-brain argument doesn't automatically carry over; not +-- investigated further this pass. +-- * pos 209/211's remaining states (demote_timeout, prepare_promotion, +-- stop_replication) were each checked for real reachability under +-- "alone in group" and found contrived: each implies an internal tension +-- with candidateEligible=FALSE (prepare_promotion/stop_replication imply -- having already been selected as a promotion candidate; demote_timeout's -- genuinely-stuck case is already intercepted earlier by pos 207). None -- ruled out as impossible, but none reproducible via a single, realistic diff --git a/tests/tap/schedule b/tests/tap/schedule index 41e7523c5..f15bd43f2 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -39,6 +39,7 @@ keeper_fsm_gap_211_primary_priority_zero keeper_fsm_gap_new_node_joins_report_lsn_group keeper_fsm_gap_209_fast_forward keeper_fsm_gap_211_fast_forward +keeper_fsm_gap_209_prepare_maintenance extension_update tablespaces installcheck diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index d8dab7e04..761806a83 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -23,3 +23,4 @@ keeper_fsm_gap_211_primary_priority_zero keeper_fsm_gap_new_node_joins_report_lsn_group keeper_fsm_gap_209_fast_forward keeper_fsm_gap_211_fast_forward +keeper_fsm_gap_209_prepare_maintenance diff --git a/tests/tap/specs/keeper_fsm_gap_209_prepare_maintenance.pgaf b/tests/tap/specs/keeper_fsm_gap_209_prepare_maintenance.pgaf new file mode 100644 index 000000000..ced7e0ccc --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_209_prepare_maintenance.pgaf @@ -0,0 +1,131 @@ +# Regression spec for MonitorFSM[] pos 209 ("alone in group, +# candidate-eligible -> single", src/monitor/group_state_machine.c), for its +# prepare_maintenance current_state -- and for the new pos 208 no-op row +# added alongside it. +# +# Unlike keeper_fsm_gap_209_wait_maintenance.pgaf's own secondary-side +# scenario (already fixed by adding a KeeperFSM[] row), prepare_maintenance +# is deliberately NOT wired up the same way: it's the PRIMARY's own +# pre-maintenance state (start_maintenance() only ever assigns it to a node +# that .isInPrimaryState, see pos 109/111), entered via +# fsm_stop_postgres_for_primary_maintenance (fsm.c) -- which cleanly +# checkpoints and STOPS Postgres, in preparation for handing off to a +# standby that was assigned PREPARE_PROMOTION_STATE in the very same +# start_maintenance() call. +# +# Investigating this while looking at pos 209's own remaining Step 2a gaps +# found that "prepare_maintenance -> single" would be a real data-loss bug, +# not a missing convenience: pos 343 (this same file) lets that standby +# advance all the way to WAIT_PRIMARY/PRIMARY the moment the old primary's +# own reportedState merely *converges* to prepare_maintenance, with no +# requirement that the old primary's row ever be removed first. So a node +# can sit in prepare_maintenance indefinitely while a different, +# already-fully-promoted primary is live and taking writes elsewhere -- if +# that new primary later also vanishes, promoting the OLD node straight to +# SINGLE would discard everything the new primary committed in between. +# This is the exact same split-brain risk pos 209's own +# reportedIsJoinSecondary exclusion already guards against for +# join_secondary, just reached one step earlier in the handoff. +# +# Fixed by adding reportedIsPrepareMaintenance = BOOL_FALSE to pos 209 +# (excluding this source state, mirroring reportedIsJoinSecondary) -- NOT by +# adding a KeeperFSM[] row. No keeper change was made at all: there's +# nothing safe to promote it to. +# +# That exclusion alone would have been an improvement over a data-loss bug, +# but a strictly worse *availability* bug on its own: unlike join_secondary +# (already recognized by node_metadata.c's IsParticipatingInPromotion, so a +# lone join_secondary node safely no-ops), a lone node reporting +# prepare_maintenance is recognized by neither that function, IsBeingPromoted, +# nor IsInPrimaryState (CanTakeWritesInState(prepare_maintenance) is false). +# Without an explicit match, ProceedGroupStateFromContext's own "primaryNode +# == NULL && !IsFailoverInProgress(...)" guard would ereport(ERROR) on +# *every single subsequent heartbeat* from this node -- worse than the +# original bug, which merely got the node stuck silently. Fixed by adding +# pos 208, a plain no-op row (same shape as pos 205's own "converged to +# maintenance" row) matching exactly this "alone, reporting +# prepare_maintenance" combination. +# +# This spec proves both fixes live: +# +# 1. node1 (primary) is put into maintenance directly via +# pgautofailover.start_maintenance() (the underlying SQL function, not +# the `pg_autoctl enable maintenance` CLI -- that CLI blocks polling +# for full convergence to "maintenance", which would race against the +# very next step below; calling the SQL function directly gives full +# control over timing, the same lesson learned constructing +# keeper_fsm_gap_211_fast_forward.pgaf). This assigns node1 +# PREPARE_MAINTENANCE_STATE and node2 (the only standby) +# PREPARE_PROMOTION_STATE in the same call. +# 2. Once node1 is confirmed *assigned* prepare_maintenance (still +# converging, Postgres not yet stopped): node2's row is force-removed +# (it's still alive and connected, so force=true is required) -- +# before node2 could ever advance to wait_primary/primary. This makes +# "alone in group" true well before any handoff race could resolve +# either way. +# 3. node1 finishes converging (Postgres stops, current_state becomes +# prepare_maintenance) with nothing left in its group. Before this +# session's fix, pos 209 would have assigned it SINGLE (a data-loss +# risk); after the fix, pos 209 declines (reportedIsPrepareMaintenance +# exclusion) and pos 208 matches instead (plain no-op). +# 4. Assert node1 stays parked at prepare_maintenance (goalstate never +# becomes single) -- and, to prove pos 208 actually prevents the +# ereport(ERROR) rather than just happening not to fire yet, assert +# node1's own reporttime keeps advancing well after being left alone: +# if pos 208 were missing, every subsequent node_active() call from +# node1 would hit that ereport(ERROR) instead of successfully updating +# its own report time. + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_001_start_maintenance_and_remove_standby_while_still_converging { + sql monitor { + SELECT pgautofailover.start_maintenance(nodeid) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + wait until node1 assigned-state = prepare_maintenance timeout 60s + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node2'; + } +} + +step test_002_node1_converges_and_stays_parked { + wait until node1 state is prepare_maintenance timeout 60s + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node1'; + } + expect { prepare_maintenance } + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node1'; + } + expect { prepare_maintenance } +} + +step test_003_node1_keeps_checking_in_not_stuck_erroring { + sleep 10s + sql monitor { + SELECT (now() - reporttime) < interval '10 seconds' + FROM pgautofailover.node WHERE nodename = 'node1'; + } + expect { t } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node1'; + } + expect { prepare_maintenance } +} From c63f2beb3c90523779654387ddaded5ba2f72205 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Fri, 31 Jul 2026 18:53:03 +0200 Subject: [PATCH 33/52] Fix keeper FSM gap: pos 211's prepare_promotion/demote_timeout/join_secondary pos 211 ("alone in group, candidatePriority zero -> report_lsn") was willing to assign report_lsn to a node reporting prepare_promotion, demote_timeout, or join_secondary, but KeeperFSM[] had no matching row for any of the three -- the exact same shape of gap fast_forward had, just with a much lower-stakes target: report_lsn never grants write access, so none of pos 209's split-brain argument applies here. Add all three to KeeperFSM[] (fsm.c), all reusing fsm_report_lsn directly: - PREP_PROMOTION_STATE -> REPORT_LSN_STATE: entering prepare_promotion (fsm_prepare_standby_for_promotion) is a no-op -- Postgres is untouched, still an ordinary streaming standby. - DEMOTE_TIMEOUT_STATE -> REPORT_LSN_STATE: fsm_stop_replication already sets default_transaction_read_only=on before this state is ever reported, so no writes can have landed here that a real primary elsewhere wouldn't also already have. - JOIN_SECONDARY_STATE -> REPORT_LSN_STATE: Postgres was cleanly checkpointed and stopped (fsm_checkpoint_and_stop_postgres) before reaching this state -- a trustworthy copy of data that, unlike pos 209's own join_secondary concern, has nothing to have fallen behind (candidatePriority=0 was never in the running to become primary). fsm_report_lsn's own restart (standby_restart_with_current_replication_ source, primary_standby.c) handles all three uniformly: it stops Postgres if running, rewrites the recovery config with no primary_conninfo, and restarts -- it never needs to reach any peer. pos 211's stop_replication current_state remains genuinely unfixed: fsm_stop_replication doesn't just stop a process, it promotes the replica (Postgres has already left recovery onto a new timeline by the time this state is reported), and the only existing path back to an ordinary standby (fsm_restart_standby -> fsm_rewind_or_init) hard-requires a live, reachable primary via keeper_get_primary() -- which cannot exist by pos 211's own "alone in group" precondition. Left as a documented, unfixed gap. Verification for these three is static + code-level, not a live pgaftest reproduction: fsm_prepare_standby_for_promotion's own no-op nature means the monitor's cascade advances from assigned=prepare_promotion straight through to stop_replication within the same heartbeat, before any external test script's own peer-removal SQL can land in between -- a tighter race than fast_forward's own genuine WAL-fetch delay gave room for. That live attempt did incidentally confirm the stop_replication dead-end for real: the candidate sat reporting stop_replication, endlessly reassigned report_lsn by this same row, with no KeeperFSM[] row able to reach it. Also, per this session's own investigation: pos 209 ("alone in group, candidate-eligible -> single") no longer assigns single to a node reporting join_secondary or prepare_maintenance -- both found to be a genuine split-brain/data-loss risk (a different, already-promoted primary can be live elsewhere by the time either state is reached), fixed by excluding them from pos 209 directly rather than adding keeper rows. prepare_maintenance additionally needed a new pos 208 no-op row: unlike join_secondary (already recognized by node_metadata.c's IsParticipatingInPromotion), a lone prepare_maintenance node wasn't recognized by that function, IsBeingPromoted, or IsInPrimaryState, so excluding it from pos 209 alone would have left ProceedGroupStateFromContext's own "couldn't find the primary node" guard to ereport(ERROR) on every subsequent heartbeat -- worse than the original bug. Verified live via keeper_fsm_gap_primary_left_alone_mid_maintenance_ handoff.pgaf: a lone prepare_maintenance primary stays safely parked (goalstate never becomes single) and keeps successfully checking in (reporttime advancing) rather than looping on that error. Renamed this session's own new pgaftest spec files away from per-rule naming (keeper_fsm_gap_209_fast_forward.pgaf -> keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf, keeper_fsm_gap_211_fast_forward.pgaf -> keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf, keeper_fsm_gap_209_prepare_maintenance.pgaf -> keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf), re-verified passing under their new names, and updated every reference (schedules, keeper_fsm_edges.sql's own comment). Regenerated keeper_fsm_edges.json/expected/keeper_fsm_edges.out from the real monitor extension. Verified: full regress (19/19) + isolation (6/6) installcheck, citus_indent clean, all renamed and pre-existing pos 209/211 pgaftest specs re-verified passing live. --- src/bin/pg_autoctl/fsm.c | 64 +++++++++++ src/monitor/expected/keeper_fsm_edges.out | 102 +++++++++++++----- src/monitor/keeper_fsm_edges.json | 12 +++ src/monitor/sql/keeper_fsm_edges.sql | 90 ++++++++++++---- tests/tap/schedule | 6 +- tests/tap/schedules/node.sch | 6 +- ...ap_candidate_fast_forward_left_alone.pgaf} | 0 ...y_left_alone_mid_maintenance_handoff.pgaf} | 2 +- ...riority_zero_fast_forward_left_alone.pgaf} | 6 +- 9 files changed, 226 insertions(+), 62 deletions(-) rename tests/tap/specs/{keeper_fsm_gap_209_fast_forward.pgaf => keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf} (100%) rename tests/tap/specs/{keeper_fsm_gap_209_prepare_maintenance.pgaf => keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf} (98%) rename tests/tap/specs/{keeper_fsm_gap_211_fast_forward.pgaf => keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf} (96%) diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index 7ee9cb723..c7566bad7 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -936,6 +936,70 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_FAILOVER }, + /* + * was mid-promotion (selected as the MS-failover candidate, assigned + * prepare_promotion) but every other node vanished and this node's own + * candidate-priority is 0 -- reuse fsm_report_lsn exactly like every + * other converged-standby source state above. Safe to reuse directly, + * with no intermediate hop: entering prepare_promotion itself runs + * fsm_prepare_standby_for_promotion, which is a no-op (see its own + * comment, fsm_transition.c) -- Postgres is still running, still an + * ordinary streaming standby, completely untouched. fsm_report_lsn's + * own restart (standby_restart_with_current_replication_source, + * primary_standby.c) writes a fresh disconnected-standby recovery + * config and restarts Postgres itself, so it doesn't matter that there + * is no live upstream to reach -- it never tries to reach one. + */ + { + PREP_PROMOTION_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_FAILOVER + }, + + /* + * was demoting after losing the primary role (draining timed out, or a + * manual failover put another node in charge) but every other node + * vanished and this node's own candidate-priority is 0 -- reuse + * fsm_report_lsn exactly like every other converged-standby source + * state above. Safe for the same reason as prepare_promotion just + * above: fsm_stop_replication's own default_transaction_read_only=on + * already blocks this node from taking writes while in demote_timeout, + * so no writes have landed here that a real primary elsewhere + * wouldn't also have; fsm_report_lsn's own restart reconfigures and + * restarts Postgres unconditionally, with no live peer required. + */ + { + DEMOTE_TIMEOUT_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_FAILOVER + }, + + /* + * was following a newly-elected primary (report_lsn -> join_secondary, + * Postgres cleanly checkpointed and stopped by + * fsm_checkpoint_and_stop_postgres while switching allegiance) but + * every other node vanished before it could finish, including the new + * primary it was about to follow, and this node's own candidate- + * priority is 0 -- reuse fsm_report_lsn exactly like every other + * converged-standby source state above. Safe for the same reason as + * prepare_promotion above despite Postgres currently being stopped: + * fsm_report_lsn's own restart (standby_restart_with_current_ + * replication_source) stops Postgres if running, reconfigures it as a + * disconnected standby, and starts it back up -- it handles "already + * stopped" and "still running" identically, and never needs to reach + * any peer to do it. The data itself is trustworthy: the checkpoint + * that stopped Postgres happened before any new primary could have + * taken a single write this node doesn't already have. + */ + { + JOIN_SECONDARY_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_FAILOVER + }, + { REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, COMMENT_REPORT_LSN_TO_PREP_PROMOTION, diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 106d2a0f7..b223578f3 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -63,6 +63,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; demote_timeout | single demote_timeout | primary demote_timeout | demoted + demote_timeout | report_lsn demoted | single demoted | catchingup demoted | report_lsn @@ -86,6 +87,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; join_primary | demote_timeout join_primary | demoted join_secondary | secondary + join_secondary | report_lsn maintenance | catchingup maintenance | report_lsn prepare_maintenance | catchingup @@ -94,6 +96,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; prepare_promotion | single prepare_promotion | wait_primary prepare_promotion | stop_replication + prepare_promotion | report_lsn primary | single primary | wait_primary primary | draining @@ -127,7 +130,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; wait_primary | join_primary wait_primary | apply_settings wait_standby | catchingup -(81 rows) +(84 rows) -- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A @@ -179,13 +182,14 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- KeeperFSM[] rows (FAST_FORWARD_STATE -> SINGLE_STATE / -- REPORT_LSN_STATE, fsm.c, reusing fsm_promote_standby/fsm_report_lsn -- exactly like every other converged-standby source state) and real --- pgaftest coverage (keeper_fsm_gap_209_fast_forward.pgaf, --- keeper_fsm_gap_211_fast_forward.pgaf) reproducing a genuine MS-failover --- candidate falling behind, fetching real WAL, and then being left alone --- -- see those specs' own headers for why node1/node2 must be removed --- from the group while node3 is still fetching (not after it reports --- fast_forward) to actually exercise these rows instead of racing --- against the monitor's own cascade continuation. +-- pgaftest coverage (keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf, +-- keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf) reproducing +-- a genuine MS-failover candidate falling behind, fetching real WAL, and +-- then being left alone -- see those specs' own headers for why +-- node1/node2 must be removed from the group while node3 is still +-- fetching (not after it reports fast_forward) to actually exercise +-- these rows instead of racing against the monitor's own cascade +-- continuation. -- * pos 209's join_secondary and prepare_maintenance current_states are no -- longer even in this gap list at all: both were found to be a genuine -- data-loss risk, not a missing convenience -- promoting either straight @@ -206,25 +210,68 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- pos 209 alone would have left ProceedGroupStateFromContext's own -- "couldn't find the primary node" guard to ereport(ERROR) on every -- single subsequent heartbeat -- worse than the original bug. Both --- fixes verified live: keeper_fsm_gap_209_prepare_maintenance.pgaf +-- fixes verified live: keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf -- reproduces a lone prepare_maintenance primary staying safely parked -- (goalstate never becomes single) and confirms it keeps successfully -- checking in (reporttime advancing) rather than looping on that error. --- * pos 211's own join_secondary/prepare_maintenance current_states are --- untouched by the above -- report_lsn is a far less dangerous target --- than single (it doesn't let the node accept writes), so the same --- split-brain argument doesn't automatically carry over; not --- investigated further this pass. --- * pos 209/211's remaining states (demote_timeout, prepare_promotion, --- stop_replication) were each checked for real reachability under --- "alone in group" and found contrived: each implies an internal tension --- with candidateEligible=FALSE (prepare_promotion/stop_replication imply --- having already been selected as a promotion candidate; demote_timeout's --- genuinely-stuck case is already intercepted earlier by pos 207). None --- ruled out as impossible, but none reproducible via a single, realistic --- operator/network-failure sequence the way wait_maintenance and --- wait_standby were -- left as documented artifacts, not pursued further --- this pass. +-- * pos 211's own join_secondary/prepare_promotion/demote_timeout +-- current_states are now genuinely fixed and covered too, via a much +-- simpler path than pos 209's: report_lsn never grants write access, so +-- none of pos 209's split-brain argument carries over here -- there was +-- nothing *unsafe* about these, they were simply missing KeeperFSM[] +-- rows, the exact same shape of gap fast_forward had. All three reuse +-- fsm_report_lsn directly: +-- - prepare_promotion: entering it (fsm_prepare_standby_for_promotion) +-- is a no-op -- Postgres is untouched, still an ordinary streaming +-- standby -- so this is exactly as safe as SECONDARY/CATCHINGUP's +-- own existing rows. +-- - demote_timeout: fsm_stop_replication already sets +-- default_transaction_read_only=on before this state is ever +-- reported, so no writes can have landed here that a real primary +-- elsewhere wouldn't also already have. +-- - join_secondary: Postgres was cleanly checkpointed and stopped +-- (fsm_checkpoint_and_stop_postgres) before reaching this state -- +-- a trustworthy, consistent copy of data that hasn't been +-- superseded by anything (unlike pos 209's own join_secondary +-- concern, there is no new primary for this data to have fallen +-- behind, since candidatePriority=0 was never in the running to +-- become one). +-- fsm_report_lsn's own restart (standby_restart_with_current_ +-- replication_source, primary_standby.c) handles all three uniformly: +-- it stops Postgres if running, rewrites the recovery config with no +-- primary_conninfo, and restarts -- it never needs to reach any peer, +-- so it doesn't matter that none exist. +-- +-- Verified via this test (Step 1/2a, the keeper_fsm_edges.json fixture +-- and dump_fsm_edges() both resolving these three edges consistently) +-- and the code reasoning above, not a live pgaftest reproduction: a +-- live attempt for prepare_promotion specifically found the race +-- fast_forward's own fix doesn't have -- fsm_prepare_standby_for_ +-- promotion is a no-op, so the monitor's cascade advances from +-- assigned=prepare_promotion straight through to stop_replication +-- within the same heartbeat, before any external test script's own +-- "remove the other peers" SQL call can land in between. (That same +-- attempt did incidentally confirm live that stop_replication really +-- is a dead end below: node3 sat reporting stop_replication, +-- endlessly reassigned report_lsn by this same pos 211 row, with no +-- KeeperFSM[] row able to reach it.) demote_timeout and join_secondary +-- were not attempted live given the identical instantaneous-transition +-- shape. +-- * pos 211's stop_replication current_state is the one exception left +-- unfixed, and unlike the three above it is not simply "missing a row": +-- fsm_stop_replication doesn't just stop a process, its own comment +-- says it shuts down the replication stream "by promoting the +-- replica" -- Postgres has already left recovery onto a new timeline +-- by the time this state is reported. Getting back to an ordinary, +-- disconnected-standby report_lsn state from there needs a real +-- pg_rewind/basebackup style re-sync (fsm_restart_standby -> +-- fsm_rewind_or_init, already used for MAINTENANCE_STATE/ +-- PREPARE_MAINTENANCE_STATE -> CATCHINGUP_STATE), and that function's +-- own first step, keeper_get_primary(), hard-requires a live, +-- reachable primary to rewind against or basebackup from -- which +-- cannot exist by pos 211's own "alone in group" precondition. No +-- existing function can do this safely when truly alone; left as a +-- documented, unfixed gap. -- * pos 325's remaining "single" state (the primaryNode side) is a -- genuine model contradiction, not just a contrived scenario: it would -- require the primary to report goalState == reportedState == single @@ -250,11 +297,8 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; rule | n | current_state | assigned_state | comment ------+----+---------------------+----------------+---------------------------------------------------------------------------------------------------------------------- - 211 | 4 | | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | demote_timeout | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | prepare_promotion | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | join_secondary | report_lsn | alone in group, candidatePriority zero -> report_lsn 325 | 1 | | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) 325 | 1 | single | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) 333 | 14 | | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted @@ -335,7 +379,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 351 | 1 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 391 | 1 | | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining 391 | 1 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining -(85 rows) +(82 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/keeper_fsm_edges.json b/src/monitor/keeper_fsm_edges.json index 88003efd1..f752c0058 100644 --- a/src/monitor/keeper_fsm_edges.json +++ b/src/monitor/keeper_fsm_edges.json @@ -414,5 +414,17 @@ { "current": "fast_forward", "assigned": "report_lsn" + }, + { + "current": "prepare_promotion", + "assigned": "report_lsn" + }, + { + "current": "demote_timeout", + "assigned": "report_lsn" + }, + { + "current": "join_secondary", + "assigned": "report_lsn" } ] diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index c0bbabad4..46afef076 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -97,13 +97,14 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- KeeperFSM[] rows (FAST_FORWARD_STATE -> SINGLE_STATE / -- REPORT_LSN_STATE, fsm.c, reusing fsm_promote_standby/fsm_report_lsn -- exactly like every other converged-standby source state) and real --- pgaftest coverage (keeper_fsm_gap_209_fast_forward.pgaf, --- keeper_fsm_gap_211_fast_forward.pgaf) reproducing a genuine MS-failover --- candidate falling behind, fetching real WAL, and then being left alone --- -- see those specs' own headers for why node1/node2 must be removed --- from the group while node3 is still fetching (not after it reports --- fast_forward) to actually exercise these rows instead of racing --- against the monitor's own cascade continuation. +-- pgaftest coverage (keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf, +-- keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf) reproducing +-- a genuine MS-failover candidate falling behind, fetching real WAL, and +-- then being left alone -- see those specs' own headers for why +-- node1/node2 must be removed from the group while node3 is still +-- fetching (not after it reports fast_forward) to actually exercise +-- these rows instead of racing against the monitor's own cascade +-- continuation. -- * pos 209's join_secondary and prepare_maintenance current_states are no -- longer even in this gap list at all: both were found to be a genuine -- data-loss risk, not a missing convenience -- promoting either straight @@ -124,25 +125,68 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- pos 209 alone would have left ProceedGroupStateFromContext's own -- "couldn't find the primary node" guard to ereport(ERROR) on every -- single subsequent heartbeat -- worse than the original bug. Both --- fixes verified live: keeper_fsm_gap_209_prepare_maintenance.pgaf +-- fixes verified live: keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf -- reproduces a lone prepare_maintenance primary staying safely parked -- (goalstate never becomes single) and confirms it keeps successfully -- checking in (reporttime advancing) rather than looping on that error. --- * pos 211's own join_secondary/prepare_maintenance current_states are --- untouched by the above -- report_lsn is a far less dangerous target --- than single (it doesn't let the node accept writes), so the same --- split-brain argument doesn't automatically carry over; not --- investigated further this pass. --- * pos 209/211's remaining states (demote_timeout, prepare_promotion, --- stop_replication) were each checked for real reachability under --- "alone in group" and found contrived: each implies an internal tension --- with candidateEligible=FALSE (prepare_promotion/stop_replication imply --- having already been selected as a promotion candidate; demote_timeout's --- genuinely-stuck case is already intercepted earlier by pos 207). None --- ruled out as impossible, but none reproducible via a single, realistic --- operator/network-failure sequence the way wait_maintenance and --- wait_standby were -- left as documented artifacts, not pursued further --- this pass. +-- * pos 211's own join_secondary/prepare_promotion/demote_timeout +-- current_states are now genuinely fixed and covered too, via a much +-- simpler path than pos 209's: report_lsn never grants write access, so +-- none of pos 209's split-brain argument carries over here -- there was +-- nothing *unsafe* about these, they were simply missing KeeperFSM[] +-- rows, the exact same shape of gap fast_forward had. All three reuse +-- fsm_report_lsn directly: +-- - prepare_promotion: entering it (fsm_prepare_standby_for_promotion) +-- is a no-op -- Postgres is untouched, still an ordinary streaming +-- standby -- so this is exactly as safe as SECONDARY/CATCHINGUP's +-- own existing rows. +-- - demote_timeout: fsm_stop_replication already sets +-- default_transaction_read_only=on before this state is ever +-- reported, so no writes can have landed here that a real primary +-- elsewhere wouldn't also already have. +-- - join_secondary: Postgres was cleanly checkpointed and stopped +-- (fsm_checkpoint_and_stop_postgres) before reaching this state -- +-- a trustworthy, consistent copy of data that hasn't been +-- superseded by anything (unlike pos 209's own join_secondary +-- concern, there is no new primary for this data to have fallen +-- behind, since candidatePriority=0 was never in the running to +-- become one). +-- fsm_report_lsn's own restart (standby_restart_with_current_ +-- replication_source, primary_standby.c) handles all three uniformly: +-- it stops Postgres if running, rewrites the recovery config with no +-- primary_conninfo, and restarts -- it never needs to reach any peer, +-- so it doesn't matter that none exist. +-- +-- Verified via this test (Step 1/2a, the keeper_fsm_edges.json fixture +-- and dump_fsm_edges() both resolving these three edges consistently) +-- and the code reasoning above, not a live pgaftest reproduction: a +-- live attempt for prepare_promotion specifically found the race +-- fast_forward's own fix doesn't have -- fsm_prepare_standby_for_ +-- promotion is a no-op, so the monitor's cascade advances from +-- assigned=prepare_promotion straight through to stop_replication +-- within the same heartbeat, before any external test script's own +-- "remove the other peers" SQL call can land in between. (That same +-- attempt did incidentally confirm live that stop_replication really +-- is a dead end below: node3 sat reporting stop_replication, +-- endlessly reassigned report_lsn by this same pos 211 row, with no +-- KeeperFSM[] row able to reach it.) demote_timeout and join_secondary +-- were not attempted live given the identical instantaneous-transition +-- shape. +-- * pos 211's stop_replication current_state is the one exception left +-- unfixed, and unlike the three above it is not simply "missing a row": +-- fsm_stop_replication doesn't just stop a process, its own comment +-- says it shuts down the replication stream "by promoting the +-- replica" -- Postgres has already left recovery onto a new timeline +-- by the time this state is reported. Getting back to an ordinary, +-- disconnected-standby report_lsn state from there needs a real +-- pg_rewind/basebackup style re-sync (fsm_restart_standby -> +-- fsm_rewind_or_init, already used for MAINTENANCE_STATE/ +-- PREPARE_MAINTENANCE_STATE -> CATCHINGUP_STATE), and that function's +-- own first step, keeper_get_primary(), hard-requires a live, +-- reachable primary to rewind against or basebackup from -- which +-- cannot exist by pos 211's own "alone in group" precondition. No +-- existing function can do this safely when truly alone; left as a +-- documented, unfixed gap. -- * pos 325's remaining "single" state (the primaryNode side) is a -- genuine model contradiction, not just a contrived scenario: it would -- require the primary to report goalState == reportedState == single diff --git a/tests/tap/schedule b/tests/tap/schedule index f15bd43f2..739be4e2f 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -37,9 +37,9 @@ keeper_fsm_gap_209_wait_standby keeper_fsm_gap_211_wait_standby keeper_fsm_gap_211_primary_priority_zero keeper_fsm_gap_new_node_joins_report_lsn_group -keeper_fsm_gap_209_fast_forward -keeper_fsm_gap_211_fast_forward -keeper_fsm_gap_209_prepare_maintenance +keeper_fsm_gap_candidate_fast_forward_left_alone +keeper_fsm_gap_priority_zero_fast_forward_left_alone +keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff extension_update tablespaces installcheck diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index 761806a83..edbe577db 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -21,6 +21,6 @@ keeper_fsm_gap_209_wait_standby keeper_fsm_gap_211_wait_standby keeper_fsm_gap_211_primary_priority_zero keeper_fsm_gap_new_node_joins_report_lsn_group -keeper_fsm_gap_209_fast_forward -keeper_fsm_gap_211_fast_forward -keeper_fsm_gap_209_prepare_maintenance +keeper_fsm_gap_candidate_fast_forward_left_alone +keeper_fsm_gap_priority_zero_fast_forward_left_alone +keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff diff --git a/tests/tap/specs/keeper_fsm_gap_209_fast_forward.pgaf b/tests/tap/specs/keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf similarity index 100% rename from tests/tap/specs/keeper_fsm_gap_209_fast_forward.pgaf rename to tests/tap/specs/keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf diff --git a/tests/tap/specs/keeper_fsm_gap_209_prepare_maintenance.pgaf b/tests/tap/specs/keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf similarity index 98% rename from tests/tap/specs/keeper_fsm_gap_209_prepare_maintenance.pgaf rename to tests/tap/specs/keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf index ced7e0ccc..ab62d6693 100644 --- a/tests/tap/specs/keeper_fsm_gap_209_prepare_maintenance.pgaf +++ b/tests/tap/specs/keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf @@ -54,7 +54,7 @@ # for full convergence to "maintenance", which would race against the # very next step below; calling the SQL function directly gives full # control over timing, the same lesson learned constructing -# keeper_fsm_gap_211_fast_forward.pgaf). This assigns node1 +# keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf). This assigns node1 # PREPARE_MAINTENANCE_STATE and node2 (the only standby) # PREPARE_PROMOTION_STATE in the same call. # 2. Once node1 is confirmed *assigned* prepare_maintenance (still diff --git a/tests/tap/specs/keeper_fsm_gap_211_fast_forward.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf similarity index 96% rename from tests/tap/specs/keeper_fsm_gap_211_fast_forward.pgaf rename to tests/tap/specs/keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf index 919240b90..2bd7ecf5b 100644 --- a/tests/tap/specs/keeper_fsm_gap_211_fast_forward.pgaf +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf @@ -1,7 +1,7 @@ # Regression spec for MonitorFSM[] pos 211 ("alone in group, # candidatePriority zero -> report_lsn", src/monitor/group_state_machine.c), # for its fast_forward current_state -- the candidatePriority=0 sibling of -# keeper_fsm_gap_209_fast_forward.pgaf's own scenario. See that spec's header +# keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf's own scenario. See that spec's header # for the full story of how a real fast_forward state is forced (network # disconnect + INSERT + CHECKPOINT to create genuine LSN divergence, the # same recipe already established by tests/tap/specs/multi_ifdown.pgaf and @@ -36,7 +36,7 @@ # selection entirely, so a node can only reach fast_forward while still # candidate-eligible. Its priority is only dropped to 0 afterwards, once it # is already assigned fast_forward (confirmed via "assigned-state = -# fast_forward", the same checkpoint keeper_fsm_gap_209_fast_forward.pgaf +# fast_forward", the same checkpoint keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf # uses before removing peers) -- and, same as the peer removal, done via # the underlying pgautofailover.set_node_candidate_priority() SQL function # directly rather than the `pg_autoctl set node candidate-priority` CLI: @@ -48,7 +48,7 @@ # those remove_node() calls, closing the window instead of losing the race # to confirm against at that point: # -# 1. Identical setup to keeper_fsm_gap_209_fast_forward.pgaf: node2 +# 1. Identical setup to keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf: node2 # (candidate-priority 0, WAL source) stays connected and caught up; # node3 (default nonzero candidate-priority) is disconnected while # node1 receives writes, then reconnected as node1 is disconnected -- From 075b80e72254b889db8718e355f44afde6791296 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sat, 1 Aug 2026 06:54:24 +0200 Subject: [PATCH 34/52] Add live pgaftest coverage for pos 211's three previously-static-only rows Replaces the "static + code proof only" verification this session previously settled for on pos 211's prepare_promotion/demote_timeout/ join_secondary -> report_lsn fixes (fe71d50) with real, deterministic live pgaftest reproduction of all three, using the new no-autopilot + "fsm step " DSL primitives: - keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf -- a lone standby left mid-promotion (secondary -> prepare_promotion) after its only primary is killed. - keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf -- the primary itself left mid-demotion (primary -> draining -> demote_timeout) after perform_failover() hands off to its standby. - keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf -- the losing side of a genuine two-candidate MS-failover election (report_lsn -> join_secondary), needing three nodes and both standbys under step-mode control to pace the election by hand. Each spec drives its node to converge locally to the state under test via one explicit "fsm step", stops there -- frozen, unreported to the monitor, for as long as the spec likes, since nothing ticks without an explicit command -- zeroes that node's own candidate priority and force-removes its former peers, then one further "fsm step" reports the frozen state, receives the newly-applicable report_lsn assignment from pos 211, and performs the transition under test in the very same call. This is exactly the race that made live reproduction impossible last session (fsm_prepare_standby_for_promotion's own no-op nature let the monitor's cascade outrun any external test script's timing); step mode removes the race instead of trying to win it. Notable non-obvious findings from getting these live: - A hard kill/removal of the primary never gives it a chance to self-report (unlike a graceful shutdown, which reports prepare_maintenance on its own way out and, as a side effect of processing that report, cascades a fresh goal to standbys too) -- so nothing recalculates a standby's own goal state until the standby itself makes contact. Each spec's kill step is followed by a 30s sleep (letting the monitor's own health-check worker notice) and then an explicit "fsm step", not a bare wait. - That same "fsm step" call discovers a fresh assignment and performs the transition into it in one shot -- there is no way to observe an assignment without also applying it. This means priority-zeroing and peer-removal must happen *before* the node's next contact, not after: report_lsn's own precondition (pos 211) must already be true the moment the node reports being in prepare_promotion/ demote_timeout/join_secondary, or the monitor's ordinary cascade (e.g. prepare_promotion -> stop_replication, a dead end -- see this file's own updated comment) wins instead. - The "wait until X and Y timeout Ns" multi-condition form only supports "state is", not "assigned-state =" -- a real grammar gap (wait_multi_condition, test_spec_parse.y), worked around here with separate single-condition waits rather than extending the grammar, since only these new specs currently need assigned-state waits at all. Regenerated expected/keeper_fsm_edges.out for the updated comment (pg_regress echoes .sql comments verbatim) via a targeted three-test pg_regress run (create_extension, fsm, check_fsm_reachability, keeper_fsm_edges) rather than the full schedule, which hit an unrelated pre-existing ordering issue (dummy_update's fake extension version bleeding into an earlier test) reproducible even with --no-cache -- not investigated further as out of scope for this change. citus_indent clean; all three new specs verified passing live multiple times each. --- src/monitor/expected/keeper_fsm_edges.out | 42 +++-- src/monitor/sql/keeper_fsm_edges.sql | 42 +++-- tests/tap/schedule | 3 + tests/tap/schedules/node.sch | 3 + ...ro_candidate_left_alone_mid_promotion.pgaf | 120 +++++++++++++++ ...sing_candidate_left_alone_mid_handoff.pgaf | 145 ++++++++++++++++++ ..._zero_primary_left_alone_mid_demotion.pgaf | 123 +++++++++++++++ 7 files changed, 456 insertions(+), 22 deletions(-) create mode 100644 tests/tap/specs/keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf create mode 100644 tests/tap/specs/keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf create mode 100644 tests/tap/specs/keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index b223578f3..6878ceab8 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -243,20 +243,40 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- so it doesn't matter that none exist. -- -- Verified via this test (Step 1/2a, the keeper_fsm_edges.json fixture --- and dump_fsm_edges() both resolving these three edges consistently) --- and the code reasoning above, not a live pgaftest reproduction: a --- live attempt for prepare_promotion specifically found the race --- fast_forward's own fix doesn't have -- fsm_prepare_standby_for_ --- promotion is a no-op, so the monitor's cascade advances from --- assigned=prepare_promotion straight through to stop_replication +-- and dump_fsm_edges() both resolving these three edges consistently), +-- the code reasoning above, AND real live pgaftest reproduction for all +-- three (keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf, +-- keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf, +-- keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf). +-- A first attempt at live reproduction (an earlier session) found +-- exactly the race fast_forward's own fix doesn't have: fsm_prepare_ +-- standby_for_promotion is a no-op, so the monitor's cascade advances +-- from assigned=prepare_promotion straight through to stop_replication -- within the same heartbeat, before any external test script's own -- "remove the other peers" SQL call can land in between. (That same -- attempt did incidentally confirm live that stop_replication really --- is a dead end below: node3 sat reporting stop_replication, --- endlessly reassigned report_lsn by this same pos 211 row, with no --- KeeperFSM[] row able to reach it.) demote_timeout and join_secondary --- were not attempted live given the identical instantaneous-transition --- shape. +-- is a dead end below: node3 sat reporting stop_replication, endlessly +-- reassigned report_lsn by this same pos 211 row, with no KeeperFSM[] +-- row able to reach it.) +-- +-- Step mode (PG_AUTOCTL_STEP_MODE, src/bin/pg_autoctl/step_socket.c) +-- removes that race entirely: a "no-autopilot" node's node-active +-- service never ticks on its own, so a pgaftest spec can drive it to +-- converge locally to prepare_promotion/demote_timeout/join_secondary +-- via one explicit "fsm step " (see fsm_step_cmd, +-- src/bin/pgaftest/test_spec_parse.y) and then simply not issue the +-- next one for as long as it likes -- there is no cascade to race +-- against, because nothing happens without an explicit command. Once +-- frozen there, unreported to the monitor, each spec zeroes the node's +-- own candidate priority and force-removes its former peers, then a +-- single further "fsm step" reports the frozen state, receives the +-- newly-applicable report_lsn assignment from pos 211, and performs +-- the transition under test in the very same call. See those three +-- specs' own headers for the full mechanics, including why each one's +-- particular current_state needs a different cluster shape to reach in +-- the first place (a lone standby for prepare_promotion, the original +-- primary itself for demote_timeout, and a losing MS-failover +-- candidate for join_secondary). -- * pos 211's stop_replication current_state is the one exception left -- unfixed, and unlike the three above it is not simply "missing a row": -- fsm_stop_replication doesn't just stop a process, its own comment diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 46afef076..27cd7a845 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -158,20 +158,40 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- so it doesn't matter that none exist. -- -- Verified via this test (Step 1/2a, the keeper_fsm_edges.json fixture --- and dump_fsm_edges() both resolving these three edges consistently) --- and the code reasoning above, not a live pgaftest reproduction: a --- live attempt for prepare_promotion specifically found the race --- fast_forward's own fix doesn't have -- fsm_prepare_standby_for_ --- promotion is a no-op, so the monitor's cascade advances from --- assigned=prepare_promotion straight through to stop_replication +-- and dump_fsm_edges() both resolving these three edges consistently), +-- the code reasoning above, AND real live pgaftest reproduction for all +-- three (keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf, +-- keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf, +-- keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf). +-- A first attempt at live reproduction (an earlier session) found +-- exactly the race fast_forward's own fix doesn't have: fsm_prepare_ +-- standby_for_promotion is a no-op, so the monitor's cascade advances +-- from assigned=prepare_promotion straight through to stop_replication -- within the same heartbeat, before any external test script's own -- "remove the other peers" SQL call can land in between. (That same -- attempt did incidentally confirm live that stop_replication really --- is a dead end below: node3 sat reporting stop_replication, --- endlessly reassigned report_lsn by this same pos 211 row, with no --- KeeperFSM[] row able to reach it.) demote_timeout and join_secondary --- were not attempted live given the identical instantaneous-transition --- shape. +-- is a dead end below: node3 sat reporting stop_replication, endlessly +-- reassigned report_lsn by this same pos 211 row, with no KeeperFSM[] +-- row able to reach it.) +-- +-- Step mode (PG_AUTOCTL_STEP_MODE, src/bin/pg_autoctl/step_socket.c) +-- removes that race entirely: a "no-autopilot" node's node-active +-- service never ticks on its own, so a pgaftest spec can drive it to +-- converge locally to prepare_promotion/demote_timeout/join_secondary +-- via one explicit "fsm step " (see fsm_step_cmd, +-- src/bin/pgaftest/test_spec_parse.y) and then simply not issue the +-- next one for as long as it likes -- there is no cascade to race +-- against, because nothing happens without an explicit command. Once +-- frozen there, unreported to the monitor, each spec zeroes the node's +-- own candidate priority and force-removes its former peers, then a +-- single further "fsm step" reports the frozen state, receives the +-- newly-applicable report_lsn assignment from pos 211, and performs +-- the transition under test in the very same call. See those three +-- specs' own headers for the full mechanics, including why each one's +-- particular current_state needs a different cluster shape to reach in +-- the first place (a lone standby for prepare_promotion, the original +-- primary itself for demote_timeout, and a losing MS-failover +-- candidate for join_secondary). -- * pos 211's stop_replication current_state is the one exception left -- unfixed, and unlike the three above it is not simply "missing a row": -- fsm_stop_replication doesn't just stop a process, its own comment diff --git a/tests/tap/schedule b/tests/tap/schedule index 739be4e2f..f31660dfe 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -40,6 +40,9 @@ keeper_fsm_gap_new_node_joins_report_lsn_group keeper_fsm_gap_candidate_fast_forward_left_alone keeper_fsm_gap_priority_zero_fast_forward_left_alone keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff +keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion +keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion +keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff extension_update tablespaces installcheck diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index edbe577db..a37f52636 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -24,3 +24,6 @@ keeper_fsm_gap_new_node_joins_report_lsn_group keeper_fsm_gap_candidate_fast_forward_left_alone keeper_fsm_gap_priority_zero_fast_forward_left_alone keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff +keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion +keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion +keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff diff --git a/tests/tap/specs/keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf new file mode 100644 index 000000000..55953a69f --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf @@ -0,0 +1,120 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, candidatePriority +# zero -> report_lsn", src/monitor/group_state_machine.c), for its +# prepare_promotion current_state, and for the matching KeeperFSM[] row +# added in fsm.c: +# +# { +# PREP_PROMOTION_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, +# COMMENT_SECONDARY_TO_REPORT_LSN, +# &fsm_report_lsn, +# FSM_PHASE_FAILOVER +# }, +# +# When this fix was first written, live pgaftest reproduction was attempted +# and abandoned: fsm_prepare_standby_for_promotion (the transition INTO +# prepare_promotion) is a no-op, so in ordinary autonomous operation the +# monitor's own cascade advances assigned=prepare_promotion straight through +# to stop_replication within the same heartbeat -- faster than any external +# test script's own "remove the other peer" SQL call could land in between. +# See src/monitor/sql/keeper_fsm_edges.sql's own comment for the full story +# and the static/code-level verification that was used instead at the time. +# +# Step mode (PG_AUTOCTL_STEP_MODE, src/bin/pg_autoctl/step_socket.c) removes +# that race entirely: node3's node-active service never ticks on its own, so +# the spec can drive it to converge locally to prepare_promotion via one +# explicit "fsm step node3" and then simply *not issue the next one* for as +# long as it likes -- there is no cascade to race against, because nothing +# happens without an explicit command. Once node3 is confirmed frozen there +# (still unreported to the monitor), the spec zeroes its own candidate +# priority and force-removes the old primary's row, then a single further +# "fsm step node3" reports prepare_promotion, receives the newly-applicable +# report_lsn assignment from pos 211, and performs the transition under +# test in the very same call. +# +# Cluster shape: plain 2-node (no MS-failover candidate-selection ambiguity +# needed here, unlike the join_secondary sibling spec) -- node3 is the +# formation's only standby, so failing node1 assigns it straight through +# secondary -> prepare_promotion with no report_lsn hop at all (confirmed +# live: querying node3's own assigned-state right after node1 goes away +# already shows "prepare_promotion", never "report_lsn" first). + +cluster { + monitor + formation { + node1 + node3 no-autopilot candidate-priority 50 + } +} + +setup { + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node3 state is secondary + timeout 60s +} + +teardown { + compose down +} + +step test_001_kill_primary_and_converge_locally_to_prepare_promotion { + # A hard kill never gives node1 a chance to self-report (unlike a + # graceful shutdown, which would report prepare_maintenance on its own + # way out and, as a side effect of processing *that* report, cascade a + # fresh goal to node3 too) -- so nothing recalculates node3's own goal + # state until node3 itself makes contact. node3 never ticks on its own + # (no-autopilot), so this explicit "fsm step node3" is what prompts + # that contact -- and, in the very same call, discovers the fresh + # prepare_promotion assignment and performs the transition into it. + # There's no way to observe the assignment without also applying it. + # The sleep gives the monitor's own health-check worker (a periodic + # background process, unrelated to and slower than any of our + # explicit steps) time to actually notice node1 is gone before we + # step node3 -- stepping too early would just be a same-state no-op, + # since the monitor wouldn't yet have anything new to assign. + compose kill node1 + sleep 30s + fsm step node3 + wait until node3 assigned-state = prepare_promotion timeout 30s +} + +step test_002_zero_priority_and_remove_old_primary_row { + # node3's own current_role is already prepare_promotion (from the step + # above), but that has not been reported back to the monitor yet -- the + # step above reported the *old* "secondary" state, before transitioning. + # This has to happen *before* node3's next contact, not after: the very + # next time node3 reports "prepare_promotion" as its current state, the + # monitor's ordinary cascade would advance it straight to + # stop_replication (a dead end -- see the header comment) unless pos + # 211's "alone in group, candidatePriority zero" condition is already + # true by then. So the priority drop and the peer removal both need to + # land while node3 is still sitting here, frozen, unreported. + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } +} + +step test_003_node3_converges_straight_to_report_lsn { + fsm step node3 + # The call above already performed the prepare_promotion -> report_lsn + # transition under test, but keeper_fsm_step() only reports the state + # it had *before* that transition -- one more call is needed for node3 + # to actually report "report_lsn" back to the monitor. + fsm step node3 + wait until node3 state = report_lsn timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} diff --git a/tests/tap/specs/keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf new file mode 100644 index 000000000..37f15ad41 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf @@ -0,0 +1,145 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, candidatePriority +# zero -> report_lsn", src/monitor/group_state_machine.c), for its +# join_secondary current_state, and for the matching KeeperFSM[] row added in +# fsm.c: +# +# { +# JOIN_SECONDARY_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, +# COMMENT_SECONDARY_TO_REPORT_LSN, +# &fsm_report_lsn, +# FSM_PHASE_FAILOVER +# }, +# +# Unlike its prepare_promotion and demote_timeout siblings, join_secondary +# only exists on the *losing* side of a genuine MS-failover candidate +# election, so this scenario needs three nodes and two candidates: node1 is +# the original primary; node2 and node3 both start as ordinary (nonzero +# candidate-priority) standbys, so both are eligible and get sent through +# report_lsn to have their LSNs compared once node1 is gone. Both are +# declared "no-autopilot" so the spec can pace the whole election by hand -- +# node2 is driven to actually win (report_lsn -> prepare_promotion), and +# node3, the loser, is driven only as far as report_lsn and then frozen +# there while node2 pulls ahead, exactly reproducing the real window in +# which the monitor assigns join_secondary to the node that didn't win. +# +# Every "fsm step" call the setup/steps below make is deterministic and +# non-racing precisely because both node2 and node3 are step-mode: nothing +# advances except in response to an explicit command, so there is no +# autonomous cascade to lose a race against, unlike attempting this same +# scenario against ordinary (autonomous) standbys. +# +# test_003's first call steps node2 from prepare_promotion onward; whether +# that particular attempt (reaching stop_replication) itself succeeds or +# fails is irrelevant to what this spec is testing, and by this point node1 +# (the old primary) has already been force-killed, so it can go either way. +# What matters is the side effect that already landed regardless: node2's +# report of "I am now in prepare_promotion" reached the monitor, which is +# what makes it assign join_secondary to node3 in the first place. + +cluster { + monitor + formation { + node1 + node2 no-autopilot candidate-priority 50 + node3 no-autopilot candidate-priority 50 + } +} + +setup { + wait until node2 state = catchingup timeout 60s + wait until node3 state = catchingup timeout 60s + fsm step node2 + fsm step node3 + fsm step node2 + fsm step node3 + fsm step node2 + fsm step node3 + fsm step node2 + fsm step node3 + wait until node1 state is primary timeout 60s + wait until node2 state is secondary timeout 60s + wait until node3 state is secondary timeout 60s +} + +teardown { + compose down +} + +step test_001_kill_primary_and_converge_both_locally_to_report_lsn { + # A hard kill never gives node1 a chance to self-report (unlike a + # graceful shutdown), so nothing recalculates either standby's own goal + # state until each makes its own contact -- neither node2 nor node3 + # ticks on its own (no-autopilot), so it takes one explicit "fsm step" + # per node here, not a bare wait. Each call both discovers the new + # report_lsn assignment *and* performs the transition into it, in the + # same call -- there's no way to observe the assignment without also + # applying it. The sleep gives the monitor's own health-check worker + # (a periodic background process, unrelated to and slower than any of + # our explicit steps) time to actually notice node1 is gone before we + # step either standby -- stepping too early would just be a + # same-state no-op, since the monitor wouldn't yet have anything new + # to assign. + compose kill node1 + sleep 30s + fsm step node2 + fsm step node3 + wait until node2 assigned-state = report_lsn timeout 30s + wait until node3 assigned-state = report_lsn timeout 30s +} + +step test_002_winner_reports_report_lsn_and_converges_to_prepare_promotion { + # This call reports node2's own report_lsn state to the monitor (which + # is what lets it be selected) and, in the same call, converges node2 + # locally to prepare_promotion -- still unreported at this point. + fsm step node2 +} + +step test_003_winner_reports_prepare_promotion_and_loser_gets_join_secondary { + # This call reports node2's own prepare_promotion state to the monitor + # and then attempts the prepare_promotion -> stop_replication + # transition. Whether that attempt itself succeeds or fails (it can go + # either way depending on exactly how reachable node1 still looks at + # this point) is irrelevant to what this spec is testing -- plain exec + # (not the "fsm step" sugar, which retries on failure) accepts either + # outcome without burning its retry budget. What matters is the report + # of "prepare_promotion" that already landed before that attempt: that + # is what makes the monitor assign join_secondary to node3 next. + exec node2 pg_autoctl manual fsm step --pgdata /var/lib/postgres/pgaf + # node3 still reports "report_lsn" (unreported since test_001); this + # call refreshes that report, and only now -- with node2's own + # prepare_promotion report already landed above -- does the monitor + # assign join_secondary in response. + fsm step node3 +} + +step test_004_zero_priority_and_remove_other_two_nodes { + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node2'; + } +} + +step test_005_node3_converges_straight_to_report_lsn { + fsm step node3 + # The call above already performed the join_secondary -> report_lsn + # transition under test, but keeper_fsm_step() only reports the state + # it had *before* that transition -- one more call is needed for node3 + # to actually report "report_lsn" back to the monitor. + fsm step node3 + wait until node3 state = report_lsn timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} diff --git a/tests/tap/specs/keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf new file mode 100644 index 000000000..62db67867 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf @@ -0,0 +1,123 @@ +# Regression spec for MonitorFSM[] pos 211 ("alone in group, candidatePriority +# zero -> report_lsn", src/monitor/group_state_machine.c), for its +# demote_timeout current_state, and for the matching KeeperFSM[] row added in +# fsm.c: +# +# { +# DEMOTE_TIMEOUT_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, +# COMMENT_SECONDARY_TO_REPORT_LSN, +# &fsm_report_lsn, +# FSM_PHASE_FAILOVER +# }, +# +# Unlike the prepare_promotion sibling spec, here the node under test is the +# primary being demoted, not a standby being promoted. node3 (no-autopilot) +# is declared *second*, not first: compose_gen only attaches a healthcheck +# to the first data node (so that later nodes can depend_on it being +# healthy, making the initial election deterministic) -- a no-autopilot +# node can never satisfy that healthcheck on its own (`pg_autoctl status` +# requires Postgres to be running, and nothing restarts Postgres after the +# one-shot node-init phase stops it until the very first "fsm step" call), +# so it must never be the first node in the formation. +# +# This means node3 starts life as an ordinary standby and has to be +# *promoted* into place before it can be demoted -- two separate +# single-candidate MS-failover cycles chained together, both driven the +# same deterministic way the sibling specs use: node3's node-active +# service never ticks on its own, so every step below is an explicit, +# non-racing "fsm step node3" call, and the spec is free to pause for as +# long as it likes between any two of them. + +cluster { + monitor + formation { + node1 + node3 no-autopilot candidate-priority 50 + } +} + +setup { + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node3 state is secondary + timeout 60s +} + +teardown { + compose down +} + +step test_001_promote_node3 { + sql monitor { + SELECT pgautofailover.perform_promotion('default', 'node3'); + } + wait until node3 assigned-state = prepare_promotion timeout 60s +} + +step test_002_step_node3_through_to_primary { + # node3 needs to walk secondary -> prepare_promotion -> stop_replication + # -> wait_primary -> primary, each hop its own explicit "fsm step". + # There's no harm in calling it a few more times than strictly + # necessary -- once a hop is already applied, the next call is simply a + # no-op "ensure current state" -- so blast enough of them to reach + # "primary" regardless of exactly how many hops this happened to take + # this time around, rather than hand-counting an exact number that's + # sensitive to timing. + fsm step node3 + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is secondary timeout 90s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node3 state is primary timeout 60s + # One more step to have node3 actually *report* primary to the monitor + # -- perform_failover() below requires a node already confirmed primary. + fsm step node3 +} + +step test_003_perform_failover_and_wait_demote_timeout_assignment { + sql monitor { + SELECT pgautofailover.perform_failover('default', 0); + } + wait until node3 assigned-state = demote_timeout timeout 60s +} + +step test_004_converge_locally_without_reporting { + # node3's own current_role is now demote_timeout, but this has not been + # reported back to the monitor yet, and won't be until the spec issues + # another explicit "fsm step node3" below. + fsm step node3 +} + +step test_005_zero_priority_and_remove_new_primary_row { + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } +} + +step test_006_node3_converges_straight_to_report_lsn { + fsm step node3 + # The call above already performed the demote_timeout -> report_lsn + # transition under test, but keeper_fsm_step() only reports the state + # it had *before* that transition -- one more call is needed for node3 + # to actually report "report_lsn" back to the monitor. + fsm step node3 + wait until node3 state = report_lsn timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} From 881d0f4afda2b6bde885219bfcdb921088620f77 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sat, 1 Aug 2026 16:46:51 +0200 Subject: [PATCH 35/52] Fix monitor FSM rule 325: exclude SINGLE from primaryNode's reachable states Rule 325 (primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining) matched its primaryNode role via isInPrimaryState=TRUE, which admits SINGLE as one of the 5 states CanTakeWritesInState allows. But this row's own precondition already requires a second, distinctly-matched node (activeNode, in SECONDARY state) to exist in the same group -- so primaryNode genuinely reporting SINGLE ("alone in my own group") is a real model contradiction, not just an unlikely scenario: the instant a second node registers, the primary's own goal moves off single before the joining node could ever reach secondary. dump_fsm_edges() had no way to know this: NodeStatePatternResolveFromStates() only ever reads a row's own .statePattern, so isInPrimaryState's own 5-state set (the field responsible for admitting SINGLE at all) was invisible to it by construction, and Step 2a's keeper/monitor gap comparison spuriously flagged primary->single as an unexplored edge. Fix in two parts: - pos 325 now spells out the invariant explicitly via .conditions.groupHasExactlyOneNode = BOOL_FALSE. - StateCanSatisfyIsInPrimaryState() gains a new singleExcluded parameter, fed by a new MonitorFSMTransitionExcludesSingleNode() helper that reads groupHasExactlyOneNode/groupHasMoreThanTwoNodes off a rule's own .conditions, and excludes SINGLE from the reachable state set whenever either is set. pos 391 (MS-failover cascade) needed no rule change: it already carried groupHasMoreThanTwoNodes = BOOL_TRUE for an unrelated reason (its own "more than two nodes" gate), so it benefits from the same narrowing for free. Regenerated fsm.out, check_fsm_reachability.out (194 -> 192 total edges), and keeper_fsm_edges.out (Step 2a gap rows 82 -> 78, pos 325's and 391's single-state gaps both gone); updated keeper_fsm_edges.sql's own commentary to match. Verified: full regress (19/19) + isolation (6/6) suites pass with zero unexpected diffs, multi_standbys.pgaf (27/27) confirms the MS-failover cascade is unaffected, citus_indent clean. --- .../expected/check_fsm_reachability.out | 4 +- src/monitor/expected/fsm.out | 2 +- src/monitor/expected/keeper_fsm_edges.out | 95 ++++++++++++++----- src/monitor/group_state_machine.c | 74 +++++++++++++-- src/monitor/sql/keeper_fsm_edges.sql | 85 +++++++++++++---- 5 files changed, 203 insertions(+), 57 deletions(-) diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index e43bf2bf8..ecb325d02 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 194 + 192 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 194 + 192 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index af2d2f2b6..f0040eb11 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -555,7 +555,7 @@ candidate_node_current_state | active_node_conditions | isHealthy=true, candidateEligible=true other_node_conditions | isUnhealthy=true, isInPrimaryState=true candidate_node_conditions | -group_conditions | walWithinPromoteThreshold=true +group_conditions | groupHasExactlyOneNode=false, walWithinPromoteThreshold=true active_node_assigned_state | prepare_promotion other_node_assigned_state | draining has_extra_action | f diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 6878ceab8..523f5e4ed 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -154,22 +154,32 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. -- --- As of this writing the gap list is exactly 10 rules this way (134 detail --- rows total), each an "alone in group"/failover/Citus-worker rule whose +-- As of this writing the reporting_node role/predicate cohort below is down +-- to 5 rules this way (71 detail rows total: 333/351's Citus-worker rows +-- and 339/347/349's generic siblings -- see each one's own discussion +-- further down), each an "alone in group"/failover/Citus-worker rule whose -- NodeStatePattern is a role or predicate check (e.g. !IsCurrentState(...), -- an opaque NodeIsXxx() helper, or no state restriction at all) rather than -- an enumerated state list -- investigated rule by rule against the --- pre-refactor hand-written code (commit 9c9c9b9^): in every one of the 10, +-- pre-refactor hand-written code (commit 9c9c9b9^): in every one of the 5, -- that breadth already existed before this refactor (this is a faithful, -- behavior-preserving translation, not a widening introduced here). Real -- regression/tap-spec precedent exists for the "obvious" current_state each --- rule is clearly meant for (e.g. issue #997 for pos 303, issue #1168 for --- pos 325/347/349's sibling branches), but none of the 10 has a test --- exercising the transition from one of the other, more exotic fanned-out --- current_states (dropped, fast_forward, join_secondary, and similar) -- --- this is Step 2a's own structural artifact of enumerating a role/predicate --- gate across every syntactically possible current_state, not a sign of 10 --- separate functional bugs. +-- rule is clearly meant for (e.g. issue #1168 for 339/347/349's sibling +-- branches), but none of the 5 has a test exercising the transition from +-- one of the other, more exotic fanned-out current_states (dropped, +-- fast_forward, join_secondary, and similar) -- this is Step 2a's own +-- structural artifact of enumerating a role/predicate gate across every +-- syntactically possible current_state, not a sign of 5 separate +-- functional bugs. This cohort was originally 10 rules/134 detail rows +-- (including pos 303 and pos 325); both have since been narrowed to zero +-- remaining gap rows -- pos 303 by teaching StateCanSatisfyIsInPrimaryState() +-- the 5-state set IsInPrimaryState() can ever admit at all, pos 325 (and, +-- as a side effect, pos 391 in the MS-failover section below, which was +-- never part of this specific cohort but shares the same isInPrimaryState +-- field) by additionally teaching it to exclude SINGLE when a rule's own +-- .conditions already prove the group has more than one node -- see each's +-- own discussion further down. -- -- Follow-up investigation of pos 209/211/325's own remaining gap states -- (after wait_maintenance and wait_standby were resolved -- see @@ -292,15 +302,52 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- cannot exist by pos 211's own "alone in group" precondition. No -- existing function can do this safely when truly alone; left as a -- documented, unfixed gap. --- * pos 325's remaining "single" state (the primaryNode side) is a --- genuine model contradiction, not just a contrived scenario: it would --- require the primary to report goalState == reportedState == single --- while a *separate* node in the same group is simultaneously converged --- and reporting secondary -- but the instant a second node registers, --- the primary's own goal moves off single (to wait_primary) as part of --- that registration, before the joining node could ever reach --- secondary. No real sequence of monitor/keeper actions can produce --- this combination. +-- * pos 325's and pos 391's own "single" states (the primaryNode side, in +-- both cases) are now fixed at the source rather than merely explained +-- away: both rows' own preconditions already require a *second, +-- distinctly-matched* node (activeNode for 325, the healthy candidate +-- counted by atLeastOneHealthyCandidate for 391) to exist in the same +-- group as primaryNode, which makes primaryNode genuinely reporting +-- SINGLE ("alone in my own group") a real model contradiction -- the +-- instant a second node registers, the primary's own goal moves off +-- single (to wait_primary) as part of that registration, before the +-- joining node could ever reach secondary. No real sequence of +-- monitor/keeper actions can produce this combination. +-- +-- Previously this was left as documented-but-unfixed, the same way the +-- stop_replication gap right above stays unfixed today: correct, but +-- dump_fsm_edges() itself had no way to know it, since +-- NodeStatePatternResolveFromStates() only ever reads a row's own +-- .statePattern -- every other NodeStatusPattern field, including +-- isInPrimaryState (the field responsible for admitting "single" as a +-- candidate primaryNode state at all), is invisible to it by +-- construction. pos 325 now spells out explicitly, via its own +-- .conditions, exactly the invariant its shape already implied +-- (groupHasExactlyOneNode = BOOL_FALSE -- two distinctly-matched roles +-- can't coexist in a one-node group); StateCanSatisfyIsInPrimaryState() +-- was taught to read that (and the stronger, already-present +-- groupHasMoreThanTwoNodes = BOOL_TRUE, which implies it) via a new +-- singleExcluded parameter, and exclude SINGLE from the reachable state +-- set whenever it's set. pos 391 needed no rule change at all: it +-- already carried groupHasMoreThanTwoNodes = BOOL_TRUE for an unrelated +-- reason (the MS-failover cascade's own "more than two nodes" gate), +-- so it started benefiting from the same narrowing immediately. +-- +-- This narrowing is deliberately still scoped to isInPrimaryState only, +-- same restraint StateCanSatisfyIsInPrimaryState's own comment already +-- documents for pos 303: several sibling NodeStatusPattern fields +-- (isInMaintenance, canTakeWrites, drainTimeExpired, +-- unreachableFromDemoteTimeout -- see pos 333/339/347/349/351's own +-- still-wide gap lists below) are just as state-dependent in principle, +-- but each is goal-state- or wall-clock-dependent rather than a plain +-- reportedState equality check, and needs its own from-scratch +-- satisfiability proof before narrowing it the same way is safe -- +-- isInMaintenance in particular already has a documented counterexample +-- (pos 369, via EdgeIsShadowedByEarlierRule's own investigation) of a +-- reportedState assumed incompatible with a goal-dependent condition +-- turning out to be reachable anyway. Narrowing those without doing +-- that same diligence for each risks reintroducing exactly that class +-- of bug, so they remain unnarrowed for now. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos @@ -315,12 +362,10 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen (e.pos, e.assigned_state, f.comment) ) ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; - rule | n | current_state | assigned_state | comment -------+----+---------------------+----------------+---------------------------------------------------------------------------------------------------------------------- + rule | n | current_state | assigned_state | comment +------+----+---------------------+----------------+------------------------------------------------------------------------------------------------------ 211 | 1 | | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn - 325 | 1 | | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) - 325 | 1 | single | draining | primary fails, not already wait_primary -> secondary -> prepare_promotion, primary -> draining (2 of 2) 333 | 14 | | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | init | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | single | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted @@ -397,9 +442,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 351 | 1 | fast_forward | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | join_secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 391 | 1 | | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining - 391 | 1 | single | draining | nodesCount>2, primary unhealthy, in primary role but not yet wait_primary, >=1 healthy candidate -> primary draining -(82 rows) +(78 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index babacfa3e..4f2d45623 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -2845,7 +2845,18 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "primary fails, already converged wait_primary (issue #1168) -> " "secondary -> prepare_promotion only (1 of 2)" }, - /* primary fails, not already wait_primary */ + /* + * primary fails, not already wait_primary. activeNode (secondary) and + * primaryNode are necessarily two distinct nodes in the same group, so + * groupHasExactlyOneNode = BOOL_FALSE is always true whenever this row + * can match at all -- spelled out explicitly (rather than left implicit) + * because dump_fsm_edges() reads it to prove primaryNode can never be + * genuinely reporting SINGLE here (see + * MonitorFSMTransitionExcludesSingleNode's own comment): SINGLE means + * "alone in my own group," which can't coexist with this row's own + * requirement that a second, distinctly-matched node (activeNode) also + * exists in that same group. + */ { .pos = 325, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2857,7 +2868,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .primaryNode = { .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, .isInPrimaryState = BOOL_TRUE, .isUnhealthy = BOOL_TRUE }, - .conditions = { .walWithinPromoteThreshold = BOOL_TRUE }, + .conditions = { .walWithinPromoteThreshold = BOOL_TRUE, + .groupHasExactlyOneNode = BOOL_FALSE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), .otherNodeAssignedState = GOAL(REPLICATION_STATE_DRAINING), .comment = @@ -4661,7 +4673,22 @@ NodeStatePatternIncludesState(const NodeStatePattern *pattern, ReplicationState * single, primary, wait_primary, join_primary, apply_settings (from * CanTakeWritesInState's own set). * - * Deliberately narrow in scope: only .isInPrimaryState is modeled this way. + * singleExcluded narrows that 5-state set by one more, conditionally: a rule + * whose own .conditions already prove the group has more than one node + * (groupHasExactlyOneNode = BOOL_FALSE, or the stronger + * groupHasMoreThanTwoNodes = BOOL_TRUE which implies it -- see + * MonitorFSMTransitionExcludesSingleNode) can never have this same node + * simultaneously reporting SINGLE, since SINGLE means "alone in my own + * group" and the row's own precondition already requires a second, + * distinctly-matched node (activeNode/primaryNode's counterpart role) to + * exist in that same group. This is a genuine, provable exclusion (unlike + * the sibling fields discussed below) precisely because it's derived from + * .conditions rather than guessed: pos 325 is the confirmed real-world + * case (its own .conditions carries groupHasExactlyOneNode = BOOL_FALSE for + * exactly this reason). + * + * Deliberately narrow in scope beyond that: only .isInPrimaryState (plus the + * one group-cardinality-derived refinement above) is modeled this way. * Several sibling fields (isInMaintenance, canTakeWrites, drainTimeExpired, * unreachableFromDemoteTimeout) are ALSO state-dependent in the same sense * and could in principle be filtered the same way, but each needs its own @@ -4676,35 +4703,60 @@ NodeStatePatternIncludesState(const NodeStatePattern *pattern, ReplicationState * each risks reintroducing exactly that class of bug. */ static bool -StateCanSatisfyIsInPrimaryState(ReplicationState state, bool required) +StateCanSatisfyIsInPrimaryState(ReplicationState state, bool required, + bool singleExcluded) { if (!required) { return true; } + if (singleExcluded && state == REPLICATION_STATE_SINGLE) + { + return false; + } + return CanTakeWritesInState(state) || state == REPLICATION_STATE_PRIMARY || state == REPLICATION_STATE_APPLY_SETTINGS; } +/* + * MonitorFSMTransitionExcludesSingleNode returns true when a rule's own + * .conditions already guarantee the group has more than one node -- either + * directly (groupHasExactlyOneNode = BOOL_FALSE) or via the stronger + * groupHasMoreThanTwoNodes = BOOL_TRUE, which implies it (more than two + * nodes can't be exactly one). Feeds StateCanSatisfyIsInPrimaryState's own + * singleExcluded parameter; see its comment for why this is a sound, + * provable narrowing rather than a guess. + */ +static bool +MonitorFSMTransitionExcludesSingleNode(const NodeActiveContextPattern *cond) +{ + return cond->groupHasExactlyOneNode == BOOL_FALSE || + cond->groupHasMoreThanTwoNodes == BOOL_TRUE; +} + + /* * NodeStatusPatternSurvivesIsInPrimaryState filters a candidate edge-source * state against pattern's own .isInPrimaryState field (BOOL_ANY -- the vast * majority of rows -- always survives; see StateCanSatisfyIsInPrimaryState's - * own comment for BOOL_TRUE/BOOL_FALSE). + * own comment for BOOL_TRUE/BOOL_FALSE, and for singleExcluded). */ static bool NodeStatusPatternSurvivesIsInPrimaryState(const NodeStatusPattern *pattern, - ReplicationState state) + ReplicationState state, + bool singleExcluded) { if (pattern->isInPrimaryState == BOOL_ANY) { return true; } - return StateCanSatisfyIsInPrimaryState(state, pattern->isInPrimaryState == BOOL_TRUE); + return StateCanSatisfyIsInPrimaryState(state, pattern->isInPrimaryState == BOOL_TRUE, + singleExcluded); } @@ -5199,6 +5251,8 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + bool singleExcluded = MonitorFSMTransitionExcludesSingleNode(&rule->conditions); + if (rule->activeNodeAssignedState.kind == GOAL_STATE_SET) { int count; @@ -5215,7 +5269,8 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } - if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->activeNode, states[j])) + if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->activeNode, states[j], + singleExcluded)) { continue; } @@ -5287,7 +5342,8 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } - if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->primaryNode, states[j])) + if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->primaryNode, states[j], + singleExcluded)) { continue; } diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 27cd7a845..265710eb1 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -69,22 +69,32 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. -- --- As of this writing the gap list is exactly 10 rules this way (134 detail --- rows total), each an "alone in group"/failover/Citus-worker rule whose +-- As of this writing the reporting_node role/predicate cohort below is down +-- to 5 rules this way (71 detail rows total: 333/351's Citus-worker rows +-- and 339/347/349's generic siblings -- see each one's own discussion +-- further down), each an "alone in group"/failover/Citus-worker rule whose -- NodeStatePattern is a role or predicate check (e.g. !IsCurrentState(...), -- an opaque NodeIsXxx() helper, or no state restriction at all) rather than -- an enumerated state list -- investigated rule by rule against the --- pre-refactor hand-written code (commit 9c9c9b9^): in every one of the 10, +-- pre-refactor hand-written code (commit 9c9c9b9^): in every one of the 5, -- that breadth already existed before this refactor (this is a faithful, -- behavior-preserving translation, not a widening introduced here). Real -- regression/tap-spec precedent exists for the "obvious" current_state each --- rule is clearly meant for (e.g. issue #997 for pos 303, issue #1168 for --- pos 325/347/349's sibling branches), but none of the 10 has a test --- exercising the transition from one of the other, more exotic fanned-out --- current_states (dropped, fast_forward, join_secondary, and similar) -- --- this is Step 2a's own structural artifact of enumerating a role/predicate --- gate across every syntactically possible current_state, not a sign of 10 --- separate functional bugs. +-- rule is clearly meant for (e.g. issue #1168 for 339/347/349's sibling +-- branches), but none of the 5 has a test exercising the transition from +-- one of the other, more exotic fanned-out current_states (dropped, +-- fast_forward, join_secondary, and similar) -- this is Step 2a's own +-- structural artifact of enumerating a role/predicate gate across every +-- syntactically possible current_state, not a sign of 5 separate +-- functional bugs. This cohort was originally 10 rules/134 detail rows +-- (including pos 303 and pos 325); both have since been narrowed to zero +-- remaining gap rows -- pos 303 by teaching StateCanSatisfyIsInPrimaryState() +-- the 5-state set IsInPrimaryState() can ever admit at all, pos 325 (and, +-- as a side effect, pos 391 in the MS-failover section below, which was +-- never part of this specific cohort but shares the same isInPrimaryState +-- field) by additionally teaching it to exclude SINGLE when a rule's own +-- .conditions already prove the group has more than one node -- see each's +-- own discussion further down. -- -- Follow-up investigation of pos 209/211/325's own remaining gap states -- (after wait_maintenance and wait_standby were resolved -- see @@ -207,15 +217,52 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- cannot exist by pos 211's own "alone in group" precondition. No -- existing function can do this safely when truly alone; left as a -- documented, unfixed gap. --- * pos 325's remaining "single" state (the primaryNode side) is a --- genuine model contradiction, not just a contrived scenario: it would --- require the primary to report goalState == reportedState == single --- while a *separate* node in the same group is simultaneously converged --- and reporting secondary -- but the instant a second node registers, --- the primary's own goal moves off single (to wait_primary) as part of --- that registration, before the joining node could ever reach --- secondary. No real sequence of monitor/keeper actions can produce --- this combination. +-- * pos 325's and pos 391's own "single" states (the primaryNode side, in +-- both cases) are now fixed at the source rather than merely explained +-- away: both rows' own preconditions already require a *second, +-- distinctly-matched* node (activeNode for 325, the healthy candidate +-- counted by atLeastOneHealthyCandidate for 391) to exist in the same +-- group as primaryNode, which makes primaryNode genuinely reporting +-- SINGLE ("alone in my own group") a real model contradiction -- the +-- instant a second node registers, the primary's own goal moves off +-- single (to wait_primary) as part of that registration, before the +-- joining node could ever reach secondary. No real sequence of +-- monitor/keeper actions can produce this combination. +-- +-- Previously this was left as documented-but-unfixed, the same way the +-- stop_replication gap right above stays unfixed today: correct, but +-- dump_fsm_edges() itself had no way to know it, since +-- NodeStatePatternResolveFromStates() only ever reads a row's own +-- .statePattern -- every other NodeStatusPattern field, including +-- isInPrimaryState (the field responsible for admitting "single" as a +-- candidate primaryNode state at all), is invisible to it by +-- construction. pos 325 now spells out explicitly, via its own +-- .conditions, exactly the invariant its shape already implied +-- (groupHasExactlyOneNode = BOOL_FALSE -- two distinctly-matched roles +-- can't coexist in a one-node group); StateCanSatisfyIsInPrimaryState() +-- was taught to read that (and the stronger, already-present +-- groupHasMoreThanTwoNodes = BOOL_TRUE, which implies it) via a new +-- singleExcluded parameter, and exclude SINGLE from the reachable state +-- set whenever it's set. pos 391 needed no rule change at all: it +-- already carried groupHasMoreThanTwoNodes = BOOL_TRUE for an unrelated +-- reason (the MS-failover cascade's own "more than two nodes" gate), +-- so it started benefiting from the same narrowing immediately. +-- +-- This narrowing is deliberately still scoped to isInPrimaryState only, +-- same restraint StateCanSatisfyIsInPrimaryState's own comment already +-- documents for pos 303: several sibling NodeStatusPattern fields +-- (isInMaintenance, canTakeWrites, drainTimeExpired, +-- unreachableFromDemoteTimeout -- see pos 333/339/347/349/351's own +-- still-wide gap lists below) are just as state-dependent in principle, +-- but each is goal-state- or wall-clock-dependent rather than a plain +-- reportedState equality check, and needs its own from-scratch +-- satisfiability proof before narrowing it the same way is safe -- +-- isInMaintenance in particular already has a documented counterexample +-- (pos 369, via EdgeIsShadowedByEarlierRule's own investigation) of a +-- reportedState assumed incompatible with a goal-dependent condition +-- turning out to be reachable anyway. Narrowing those without doing +-- that same diligence for each risks reintroducing exactly that class +-- of bug, so they remain unnarrowed for now. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos From d90c1bab1975186bcac96c080316df29a6bb4478 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sat, 1 Aug 2026 17:38:22 +0200 Subject: [PATCH 36/52] Fix dump_fsm_edges() blindness to primaryNode's resolver-derived DROPPED exclusion Rules 333/339/347/349/351 all match primaryNode/otherNode against whatever node GetPrimaryOrDemotedNodeInGroupFromList() resolves -- called once at the top of ProceedGroupStateFromContext() and threaded unchanged through every nested dispatch, including the MS-failover cascade. That resolver can never return a node reporting DROPPED: its own two-phase logic excludes it outright (phase 1 requires a writable goalState; phase 2's fallback target set doesn't include it either), and it's structurally unreachable besides -- a node's reportedState only becomes DROPPED once its own goalState already is, and pos 201 (early_checks) deletes that row from the catalog atomically, in the very same node_active() call that converges it, so a DROPPED-reporting node never persists long enough for a later node's own call to see it. dump_fsm_edges() didn't know this: it enumerated the full, unconstrained reportedState universe for any row whose primaryNode/otherNode pattern doesn't otherwise narrow it (true of all 5 of these rows), spuriously flagging primaryNode=dropped as an unexplored Step 2a gap. Fixed via a new, unconditional filter, PrimaryNodeReportedStateCanBeResolved(), applied only to the primaryNode/otherNode candidate-state loop (never activeNode's own, where DROPPED is an entirely ordinary current_state -- see pos 201). Investigated whether these same 5 rows' SINGLE gap could be closed the same way pos 325/391 were (via .isInPrimaryState + groupHasExactlyOneNode): it can't. Unlike pos 325/391, none of these rows requires primaryNode's own convergence (isInPrimaryState), so a primary that converged to SINGLE, then had a second node register (bumping its own *goal* to WAIT_PRIMARY as part of that registration), then died before ever reporting the new goal, leaves a row GetPrimaryOrDemotedNodeInGroupFromList() would still resolve as primaryNode -- a genuinely reachable case, not a false positive. Deliberately left unnarrowed and documented as such, rather than force a real, behavior-changing isInPrimaryState requirement onto rules that don't actually need one. Regenerated check_fsm_reachability.out (192 -> 187 total edges) and keeper_fsm_edges.out (Step 2a's 5 remaining wide-gap rows drop one DROPPED row each, 78 -> 73); updated keeper_fsm_edges.sql's own commentary to match (71 -> 66 detail rows) and to explain both the DROPPED fix and why SINGLE stays. Verified: full regress (19/19) + isolation (6/6) pass with zero unexpected diffs, multi_standbys.pgaf (27/27) confirms the MS-failover cascade is unaffected, citus_indent clean. --- .../expected/check_fsm_reachability.out | 4 +- src/monitor/expected/keeper_fsm_edges.out | 82 +++++++++++----- src/monitor/group_state_machine.c | 94 +++++++++++++++++-- src/monitor/sql/keeper_fsm_edges.sql | 65 ++++++++++--- 4 files changed, 198 insertions(+), 47 deletions(-) diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index ecb325d02..17f750e21 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 192 + 187 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 192 + 187 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 523f5e4ed..663df136e 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -155,7 +155,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- summary row right before its own detail rows, as a header. -- -- As of this writing the reporting_node role/predicate cohort below is down --- to 5 rules this way (71 detail rows total: 333/351's Citus-worker rows +-- to 5 rules this way (66 detail rows total: 333/351's Citus-worker rows -- and 339/347/349's generic siblings -- see each one's own discussion -- further down), each an "alone in group"/failover/Citus-worker rule whose -- NodeStatePattern is a role or predicate check (e.g. !IsCurrentState(...), @@ -167,19 +167,56 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- regression/tap-spec precedent exists for the "obvious" current_state each -- rule is clearly meant for (e.g. issue #1168 for 339/347/349's sibling -- branches), but none of the 5 has a test exercising the transition from --- one of the other, more exotic fanned-out current_states (dropped, --- fast_forward, join_secondary, and similar) -- this is Step 2a's own --- structural artifact of enumerating a role/predicate gate across every --- syntactically possible current_state, not a sign of 5 separate --- functional bugs. This cohort was originally 10 rules/134 detail rows --- (including pos 303 and pos 325); both have since been narrowed to zero --- remaining gap rows -- pos 303 by teaching StateCanSatisfyIsInPrimaryState() --- the 5-state set IsInPrimaryState() can ever admit at all, pos 325 (and, --- as a side effect, pos 391 in the MS-failover section below, which was --- never part of this specific cohort but shares the same isInPrimaryState --- field) by additionally teaching it to exclude SINGLE when a rule's own --- .conditions already prove the group has more than one node -- see each's --- own discussion further down. +-- one of the other, more exotic fanned-out current_states (fast_forward, +-- join_secondary, and similar) -- this is Step 2a's own structural artifact +-- of enumerating a role/predicate gate across every syntactically possible +-- current_state, not a sign of 5 separate functional bugs. This cohort was +-- originally 10 rules/134 detail rows (including pos 303 and pos 325); both +-- have since been narrowed to zero remaining gap rows -- pos 303 by teaching +-- StateCanSatisfyIsInPrimaryState() the 5-state set IsInPrimaryState() can +-- ever admit at all, pos 325 (and, as a side effect, pos 391 in the +-- MS-failover section below, which was never part of this specific cohort +-- but shares the same isInPrimaryState field) by additionally teaching it to +-- exclude SINGLE when a rule's own .conditions already prove the group has +-- more than one node -- see each's own discussion further down. +-- +-- The remaining 5 rows' own DROPPED current_state was also a false +-- positive, fixed at the source rather than merely explained: every one of +-- them matches .primaryNode/.otherNode against +-- GetPrimaryOrDemotedNodeInGroupFromList()'s own resolved node (see +-- ProceedGroupStateFromContext's own call to it), and that resolver can +-- never return a node reporting DROPPED -- its own two-phase logic +-- excludes it outright (phase 1 requires a writable goalState, phase 2's +-- fallback target set doesn't include it either), and it's structurally +-- unreachable besides: a node's reportedState only becomes DROPPED once its +-- own goalState is already DROPPED, and pos 201 (early_checks) removes that +-- row from the catalog atomically, in the very same node_active() call that +-- converges it -- so a DROPPED-reporting node never persists long enough +-- for a later node's own node_active() call to see it. dump_fsm_edges() +-- didn't know this before, since it enumerated the full, unconstrained +-- reportedState universe for any row whose primaryNode/otherNode pattern +-- doesn't otherwise narrow it (as none of these 5 do) -- see +-- PrimaryNodeReportedStateCanBeResolved's own comment +-- (group_state_machine.c) for the full argument. This is unconditional, not +-- gated on any row's own .conditions, so it applies to every row reaching +-- that part of dump_fsm_edges() -- confirmed by grep that every such row +-- (i.e. every row surviving both the api_triggered skip at the top of the +-- function and its own otherNodesFn skip) lives in this same reporting_node +-- section and traces back to that identical resolver. +-- +-- These 5 rows' own SINGLE current_state, by contrast, genuinely is +-- reachable and was deliberately left alone: unlike pos 325/391, none of +-- these 5 rows requires primaryNode's own .isInPrimaryState (a convergence +-- requirement -- reportedState == goalState -- see +-- StateCanSatisfyIsInPrimaryState's own comment), so nothing rules out a +-- primary that converged to SINGLE, then had a second node register +-- (bumping the primary's own *goal* to WAIT_PRIMARY as part of that +-- registration), then died or partitioned before ever reporting that new +-- goal -- its row would sit with reportedState=SINGLE/goalState=WAIT_PRIMARY +-- indefinitely, and GetPrimaryOrDemotedNodeInGroupFromList()'s own phase 1 +-- checks only goalState, so it would still resolve this stale node as +-- primaryNode. A real, if narrow, case -- not a false positive to narrow +-- away the same way DROPPED was. -- -- Follow-up investigation of pos 209/211/325's own remaining gap states -- (after wait_maintenance and wait_standby were resolved -- see @@ -366,7 +403,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen ------+----+---------------------+----------------+------------------------------------------------------------------------------------------------------ 211 | 1 | | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn - 333 | 14 | | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 13 | | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | init | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | single | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | catchingup | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted @@ -380,8 +417,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 333 | 1 | report_lsn | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | fast_forward | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | join_secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | dropped | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 339 | 15 | | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 14 | | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | init | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | single | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | demoted | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) @@ -396,8 +432,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 339 | 1 | report_lsn | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | fast_forward | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | join_secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | dropped | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 347 | 14 | | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 13 | | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | init | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | single | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | catchingup | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) @@ -411,8 +446,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 347 | 1 | report_lsn | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | fast_forward | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | join_secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | dropped | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 349 | 14 | | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 13 | | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | init | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | single | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | catchingup | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) @@ -426,8 +460,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 349 | 1 | report_lsn | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | fast_forward | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | join_secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | dropped | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 351 | 14 | | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 13 | | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | init | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | single | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | catchingup | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted @@ -441,8 +474,7 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 351 | 1 | report_lsn | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | fast_forward | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | join_secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | dropped | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted -(78 rows) +(73 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 4f2d45623..0b338105d 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -2911,7 +2911,20 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "prepare_promotion, primary converged prepare_maintenance -> stop_replication" }, - /* Citus worker, primary present */ + /* + * Citus worker, primary present. Unlike pos 325, this row's primaryNode + * carries no .isInPrimaryState requirement, so its SINGLE gap can't be + * ruled out the same way: a primary can converge to SINGLE, then a + * second node register (bumping the primary's own *goal* to + * WAIT_PRIMARY as part of that registration) -- and if the primary dies + * or partitions in that exact window, before ever reporting the new + * goal, its row sits with reportedState=SINGLE/goalState=WAIT_PRIMARY + * indefinitely. GetPrimaryOrDemotedNodeInGroupFromList()'s own phase 1 + * checks only goalState, so it would still resolve this stale node as + * primaryNode -- a genuinely reachable case, not a false positive (see + * PrimaryNodeReportedStateCanBeResolved's own comment for the one + * exclusion -- DROPPED -- that IS sound here, on different grounds). + */ { .pos = 333, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -2957,7 +2970,9 @@ static const MonitorFSMTransition MonitorFSM[] = { /* * prepare_promotion, primary present, not in maintenance, not already - * wait_primary + * wait_primary. Same "no isInPrimaryState convergence requirement, so + * SINGLE stays a genuinely reachable gap" reasoning as pos 333 -- see + * its own comment. */ { .pos = 339, .sectionPath = { @@ -3010,7 +3025,14 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "stop_replication, primary converged demote_timeout -> " "wait_primary + demoted (1 of 3)" }, - /* stop_replication, primary's drain time expired (3-way OR, 2 of 3) */ + /* + * stop_replication, primary's drain time expired (3-way OR, 2 of 3). + * Same "no isInPrimaryState convergence requirement, so SINGLE stays a + * genuinely reachable gap" reasoning as pos 333 -- see its own comment. + * dump_fsm_edges() also has no filter for .drainTimeExpired itself + * (unlike isInPrimaryState/reportedCanTakeWrites/etc.), so this row's + * primaryNode candidate universe is the same unnarrowed ANY set. + */ { .pos = 347, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -3024,8 +3046,15 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_primary + demoted (2 of 3)" }, /* - * stop_replication, primary's goal is wait_primary but presumed dead (3-way - * OR, 3 of 3) + * stop_replication, primary's goal is wait_primary but presumed dead + * (3-way OR, 3 of 3). primaryIsWaitPrimaryPresumedDead is a .conditions + * field, not a NodeStatusPattern on .primaryNode -- dump_fsm_edges()'s + * state enumeration doesn't read .conditions at all when resolving + * primaryNode's candidate states (only .groupHasExactlyOneNode/ + * .groupHasMoreThanTwoNodes ever feed into it, via singleExcluded), so + * this row's primaryNode candidate universe is the full unnarrowed ANY + * set, same as pos 333/347/351 -- see pos 333's own comment for why + * SINGLE in particular stays a genuinely reachable gap here too. */ { .pos = 349, .sectionPath = { @@ -3039,7 +3068,11 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "stop_replication, primary's goal wait_primary but presumed dead -> " "wait_primary + demoted (3 of 3)" }, - /* Citus worker, primary present */ + /* + * Citus worker, primary present. Same "no isInPrimaryState convergence + * requirement, so SINGLE stays a genuinely reachable gap" reasoning as + * pos 333 -- see its own comment. + */ { .pos = 351, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -4760,6 +4793,50 @@ NodeStatusPatternSurvivesIsInPrimaryState(const NodeStatusPattern *pattern, } +/* + * PrimaryNodeReportedStateCanBeResolved excludes REPLICATION_STATE_DROPPED + * from the primaryNode/otherNode candidate-state loop only (never + * activeNode's own loop, where DROPPED is a perfectly ordinary, real + * current_state -- see pos 201's own row). Every row dump_fsm_edges() + * reaches through that second loop (i.e. survives both the api_triggered + * skip at the top of the function and the otherNodesFn skip guarding the + * loop itself) is a reporting_node-section row whose primaryNode ultimately + * comes from the single call to GetPrimaryOrDemotedNodeInGroupFromList() at + * the top of ProceedGroupStateFromContext() -- threaded through unchanged + * into every nested dispatch (BuildFromContextNodeActiveContext's own + * primaryNode/otherNode, and, via ActionRunMultiStandbyFailoverCascade/ + * ActionRunPlainMSFailoverCascade passing nac->primaryNode.node straight + * through, the MS-failover cluster's own use of the same role). That + * resolver can never return a node reporting DROPPED, on two independent + * grounds: + * + * - Its own two-phase logic excludes it outright: phase 1 requires + * CanTakeWritesInState(goalState) (DROPPED is not one of the 5 writable + * states), and phase 2's fallback target set (StateBelongsToPrimary() + * plus an explicit REPLICATION_STATE_DEMOTED check) doesn't include + * DROPPED either. + * + * - It's structurally unreachable in the first place: a node's own + * reportedState only ever becomes DROPPED once its own goalState has + * already been set to DROPPED (the API-triggered remove_node row is the + * only place that ever assigns that goal), and pos 201 (early_checks) + * removes the row from the catalog entirely, atomically, in that exact + * same node_active() call the moment its own reportedState converges to + * DROPPED -- so a DROPPED-reporting node never persists long enough for + * a later, different node's own node_active() call to see it sitting in + * ctx->groupNodeList at all. + * + * Unlike singleExcluded, this doesn't depend on any row's own .conditions -- + * it's an invariant of the resolver itself, so it applies unconditionally to + * every row reaching this loop, not just ones that happen to declare it. + */ +static bool +PrimaryNodeReportedStateCanBeResolved(ReplicationState state) +{ + return state != REPLICATION_STATE_DROPPED; +} + + /* * NodeStatusPatternSurvivesReportedCanTakeWrites filters a candidate * edge-source state against pattern's own .reportedCanTakeWrites field @@ -5348,6 +5425,11 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } + if (!PrimaryNodeReportedStateCanBeResolved(states[j])) + { + continue; + } + if (!NodeStatusPatternSurvivesReportedCanTakeWrites(&rule->primaryNode, states[j])) { diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 265710eb1..55cf0f998 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -70,7 +70,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- summary row right before its own detail rows, as a header. -- -- As of this writing the reporting_node role/predicate cohort below is down --- to 5 rules this way (71 detail rows total: 333/351's Citus-worker rows +-- to 5 rules this way (66 detail rows total: 333/351's Citus-worker rows -- and 339/347/349's generic siblings -- see each one's own discussion -- further down), each an "alone in group"/failover/Citus-worker rule whose -- NodeStatePattern is a role or predicate check (e.g. !IsCurrentState(...), @@ -82,19 +82,56 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- regression/tap-spec precedent exists for the "obvious" current_state each -- rule is clearly meant for (e.g. issue #1168 for 339/347/349's sibling -- branches), but none of the 5 has a test exercising the transition from --- one of the other, more exotic fanned-out current_states (dropped, --- fast_forward, join_secondary, and similar) -- this is Step 2a's own --- structural artifact of enumerating a role/predicate gate across every --- syntactically possible current_state, not a sign of 5 separate --- functional bugs. This cohort was originally 10 rules/134 detail rows --- (including pos 303 and pos 325); both have since been narrowed to zero --- remaining gap rows -- pos 303 by teaching StateCanSatisfyIsInPrimaryState() --- the 5-state set IsInPrimaryState() can ever admit at all, pos 325 (and, --- as a side effect, pos 391 in the MS-failover section below, which was --- never part of this specific cohort but shares the same isInPrimaryState --- field) by additionally teaching it to exclude SINGLE when a rule's own --- .conditions already prove the group has more than one node -- see each's --- own discussion further down. +-- one of the other, more exotic fanned-out current_states (fast_forward, +-- join_secondary, and similar) -- this is Step 2a's own structural artifact +-- of enumerating a role/predicate gate across every syntactically possible +-- current_state, not a sign of 5 separate functional bugs. This cohort was +-- originally 10 rules/134 detail rows (including pos 303 and pos 325); both +-- have since been narrowed to zero remaining gap rows -- pos 303 by teaching +-- StateCanSatisfyIsInPrimaryState() the 5-state set IsInPrimaryState() can +-- ever admit at all, pos 325 (and, as a side effect, pos 391 in the +-- MS-failover section below, which was never part of this specific cohort +-- but shares the same isInPrimaryState field) by additionally teaching it to +-- exclude SINGLE when a rule's own .conditions already prove the group has +-- more than one node -- see each's own discussion further down. +-- +-- The remaining 5 rows' own DROPPED current_state was also a false +-- positive, fixed at the source rather than merely explained: every one of +-- them matches .primaryNode/.otherNode against +-- GetPrimaryOrDemotedNodeInGroupFromList()'s own resolved node (see +-- ProceedGroupStateFromContext's own call to it), and that resolver can +-- never return a node reporting DROPPED -- its own two-phase logic +-- excludes it outright (phase 1 requires a writable goalState, phase 2's +-- fallback target set doesn't include it either), and it's structurally +-- unreachable besides: a node's reportedState only becomes DROPPED once its +-- own goalState is already DROPPED, and pos 201 (early_checks) removes that +-- row from the catalog atomically, in the very same node_active() call that +-- converges it -- so a DROPPED-reporting node never persists long enough +-- for a later node's own node_active() call to see it. dump_fsm_edges() +-- didn't know this before, since it enumerated the full, unconstrained +-- reportedState universe for any row whose primaryNode/otherNode pattern +-- doesn't otherwise narrow it (as none of these 5 do) -- see +-- PrimaryNodeReportedStateCanBeResolved's own comment +-- (group_state_machine.c) for the full argument. This is unconditional, not +-- gated on any row's own .conditions, so it applies to every row reaching +-- that part of dump_fsm_edges() -- confirmed by grep that every such row +-- (i.e. every row surviving both the api_triggered skip at the top of the +-- function and its own otherNodesFn skip) lives in this same reporting_node +-- section and traces back to that identical resolver. +-- +-- These 5 rows' own SINGLE current_state, by contrast, genuinely is +-- reachable and was deliberately left alone: unlike pos 325/391, none of +-- these 5 rows requires primaryNode's own .isInPrimaryState (a convergence +-- requirement -- reportedState == goalState -- see +-- StateCanSatisfyIsInPrimaryState's own comment), so nothing rules out a +-- primary that converged to SINGLE, then had a second node register +-- (bumping the primary's own *goal* to WAIT_PRIMARY as part of that +-- registration), then died or partitioned before ever reporting that new +-- goal -- its row would sit with reportedState=SINGLE/goalState=WAIT_PRIMARY +-- indefinitely, and GetPrimaryOrDemotedNodeInGroupFromList()'s own phase 1 +-- checks only goalState, so it would still resolve this stale node as +-- primaryNode. A real, if narrow, case -- not a false positive to narrow +-- away the same way DROPPED was. -- -- Follow-up investigation of pos 209/211/325's own remaining gap states -- (after wait_maintenance and wait_standby were resolved -- see From 21d0eea8cedf8b441d3b6c8a21c356fa78d07880 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sat, 1 Aug 2026 23:31:45 +0200 Subject: [PATCH 37/52] Fix dump_fsm_edges() blindness to primaryNode's WAIT_STANDBY/JOIN_SECONDARY exclusion Reviewing rule 333's remaining Step 2a gap states surfaced two more false positives, on top of the DROPPED exclusion fixed previously. GetPrimaryOrDemotedNodeInGroupFromList()'s phase 1 admits any reportedState paired with a writable goalState -- pos 209's own broad "alone in group -> SINGLE" row (which fires for almost any reportedState) is what makes most of 333/339/347/349/351's remaining gap states genuinely reachable via a stale/unconverged primary. But pos 209 itself explicitly excludes 3 states for split-brain/data-loss reasons: WAIT_STANDBY, JOIN_SECONDARY, and PREPARE_MAINTENANCE. PREPARE_MAINTENANCE stays reachable via the resolver's own phase 2 fallback regardless. WAIT_STANDBY and JOIN_SECONDARY are not -- and an exhaustive grep confirms no other row in the whole table ever assigns a writable goal to a node reporting either (pos 315/317/319 assign only CATCHINGUP from WAIT_STANDBY; pos 359/361 assign only SECONDARY from JOIN_SECONDARY). So the resolver can never actually select such a node, and dump_fsm_edges() had no way to know it. Extended PrimaryNodeReportedStateCanBeResolved() to exclude both, alongside DROPPED. Unlike DROPPED's structural argument, this one rests on the current table's own contents (documented in the function's own comment, with a note to revisit if a future row is ever added assigning a writable goal from either state). Regenerated check_fsm_reachability.out (187 -> 177 total edges) and keeper_fsm_edges.out (Step 2a's overall total 73 -> 63 rows; the 333/339/347/349/351 cohort specifically drops 2 detail rows each, 66 -> 56); updated keeper_fsm_edges.sql's own commentary to match. Verified: full regress (19/19) + isolation (6/6) pass with zero unexpected diffs, multi_standbys.pgaf (27/27) confirms the Citus-worker/MS-failover paths are unaffected, citus_indent clean. --- .../expected/check_fsm_reachability.out | 4 +- src/monitor/expected/keeper_fsm_edges.out | 53 +++++++++++-------- src/monitor/group_state_machine.c | 49 ++++++++++++----- src/monitor/sql/keeper_fsm_edges.sql | 31 +++++++++-- 4 files changed, 95 insertions(+), 42 deletions(-) diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index 17f750e21..467daebb8 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 187 + 177 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 187 + 177 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 663df136e..b2573510b 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -155,7 +155,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- summary row right before its own detail rows, as a header. -- -- As of this writing the reporting_node role/predicate cohort below is down --- to 5 rules this way (66 detail rows total: 333/351's Citus-worker rows +-- to 5 rules this way (56 detail rows total: 333/351's Citus-worker rows -- and 339/347/349's generic siblings -- see each one's own discussion -- further down), each an "alone in group"/failover/Citus-worker rule whose -- NodeStatePattern is a role or predicate check (e.g. !IsCurrentState(...), @@ -167,10 +167,10 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- regression/tap-spec precedent exists for the "obvious" current_state each -- rule is clearly meant for (e.g. issue #1168 for 339/347/349's sibling -- branches), but none of the 5 has a test exercising the transition from --- one of the other, more exotic fanned-out current_states (fast_forward, --- join_secondary, and similar) -- this is Step 2a's own structural artifact --- of enumerating a role/predicate gate across every syntactically possible --- current_state, not a sign of 5 separate functional bugs. This cohort was +-- one of the other, more exotic fanned-out current_states (fast_forward and +-- similar) -- this is Step 2a's own structural artifact of enumerating a +-- role/predicate gate across every syntactically possible current_state, +-- not a sign of 5 separate functional bugs. This cohort was -- originally 10 rules/134 detail rows (including pos 303 and pos 325); both -- have since been narrowed to zero remaining gap rows -- pos 303 by teaching -- StateCanSatisfyIsInPrimaryState() the 5-state set IsInPrimaryState() can @@ -218,6 +218,27 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- primaryNode. A real, if narrow, case -- not a false positive to narrow -- away the same way DROPPED was. -- +-- WAIT_STANDBY and JOIN_SECONDARY were two more false positives in this +-- same cohort, found by asking a sharper version of the SINGLE question +-- above: is there any row *anywhere* in MonitorFSM[] that ever assigns one +-- of the 5 writable goals to a node currently reporting this state? For +-- most of these 5 rows' remaining current_states the answer is yes (pos 209 +-- alone, "alone in group -> SINGLE", covers most of them -- see its own +-- comment), which is exactly what keeps them real, reachable gaps rather +-- than bugs. But WAIT_STANDBY and JOIN_SECONDARY are two of the three +-- states pos 209 itself explicitly excludes (split-brain/data-loss risk, +-- same reasoning as its own header comment), and an exhaustive grep of +-- every other row matching either state (pos 315/317/319 for WAIT_STANDBY, +-- pos 359/361 for JOIN_SECONDARY) shows all five assign only +-- CATCHINGUP/SECONDARY, never a writable goal. So no row anywhere ever +-- gives GetPrimaryOrDemotedNodeInGroupFromList()'s phase 1 a way to select +-- a node reporting either, and neither is in phase 2's own target set +-- either -- a genuine false positive, fixed the same way DROPPED was (see +-- PrimaryNodeReportedStateCanBeResolved's own comment). Unlike DROPPED's +-- exclusion, this one rests on the current table's own contents rather +-- than a structural invariant, so it needs revisiting if a future row ever +-- assigns a writable goal from either state. +-- -- Follow-up investigation of pos 209/211/325's own remaining gap states -- (after wait_maintenance and wait_standby were resolved -- see -- group_state_machine.c's reportedIsWaitStandby field (NodeStatusPattern) @@ -403,21 +424,19 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen ------+----+---------------------+----------------+------------------------------------------------------------------------------------------------------ 211 | 1 | | report_lsn | alone in group, candidatePriority zero -> report_lsn 211 | 1 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn - 333 | 13 | | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted + 333 | 11 | | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | init | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | single | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | catchingup | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | prepare_promotion | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | stop_replication | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | wait_standby | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | prepare_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | wait_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | report_lsn | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted 333 | 1 | fast_forward | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | join_secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 339 | 14 | | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) + 339 | 12 | | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | init | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | single | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | demoted | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) @@ -425,56 +444,48 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen 339 | 1 | secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | prepare_promotion | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | stop_replication | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | wait_standby | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | prepare_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | wait_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | report_lsn | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) 339 | 1 | fast_forward | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | join_secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 347 | 13 | | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) + 347 | 11 | | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | init | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | single | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | catchingup | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | prepare_promotion | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | stop_replication | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | wait_standby | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | prepare_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | wait_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | report_lsn | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) 347 | 1 | fast_forward | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | join_secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 349 | 13 | | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) + 349 | 11 | | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | init | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | single | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | catchingup | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | prepare_promotion | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | stop_replication | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | wait_standby | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | prepare_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | wait_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | report_lsn | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) 349 | 1 | fast_forward | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | join_secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 351 | 13 | | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted + 351 | 11 | | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | init | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | single | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | catchingup | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | prepare_promotion | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | stop_replication | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | wait_standby | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | prepare_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | wait_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | report_lsn | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted 351 | 1 | fast_forward | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | join_secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted -(73 rows) +(63 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 0b338105d..0799a3c41 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -4794,21 +4794,22 @@ NodeStatusPatternSurvivesIsInPrimaryState(const NodeStatusPattern *pattern, /* - * PrimaryNodeReportedStateCanBeResolved excludes REPLICATION_STATE_DROPPED - * from the primaryNode/otherNode candidate-state loop only (never - * activeNode's own loop, where DROPPED is a perfectly ordinary, real - * current_state -- see pos 201's own row). Every row dump_fsm_edges() - * reaches through that second loop (i.e. survives both the api_triggered - * skip at the top of the function and the otherNodesFn skip guarding the - * loop itself) is a reporting_node-section row whose primaryNode ultimately + * PrimaryNodeReportedStateCanBeResolved excludes 3 states from the + * primaryNode/otherNode candidate-state loop only (never activeNode's own + * loop, where all 3 are perfectly ordinary, real current_states -- see pos + * 201/315-319/359-361's own rows). Every row dump_fsm_edges() reaches + * through that second loop (i.e. survives both the api_triggered skip at + * the top of the function and the otherNodesFn skip guarding the loop + * itself) is a reporting_node-section row whose primaryNode ultimately * comes from the single call to GetPrimaryOrDemotedNodeInGroupFromList() at * the top of ProceedGroupStateFromContext() -- threaded through unchanged * into every nested dispatch (BuildFromContextNodeActiveContext's own * primaryNode/otherNode, and, via ActionRunMultiStandbyFailoverCascade/ * ActionRunPlainMSFailoverCascade passing nac->primaryNode.node straight - * through, the MS-failover cluster's own use of the same role). That - * resolver can never return a node reporting DROPPED, on two independent - * grounds: + * through, the MS-failover cluster's own use of the same role). + * + * REPLICATION_STATE_DROPPED: that resolver can never return a node + * reporting it, on two independent, purely structural grounds -- * * - Its own two-phase logic excludes it outright: phase 1 requires * CanTakeWritesInState(goalState) (DROPPED is not one of the 5 writable @@ -4826,14 +4827,34 @@ NodeStatusPatternSurvivesIsInPrimaryState(const NodeStatusPattern *pattern, * a later, different node's own node_active() call to see it sitting in * ctx->groupNodeList at all. * - * Unlike singleExcluded, this doesn't depend on any row's own .conditions -- - * it's an invariant of the resolver itself, so it applies unconditionally to - * every row reaching this loop, not just ones that happen to declare it. + * REPLICATION_STATE_WAIT_STANDBY and REPLICATION_STATE_JOIN_SECONDARY: a + * different, table-content argument, but resting on the same design intent + * pos 209's own exclusion of both already documents (a node reporting + * either has either not yet started streaming, or already stopped Postgres + * as the OLD primary mid-handoff -- promoting either straight to a writable + * role is exactly the split-brain/data-loss risk pos 209's own comment + * describes). Confirmed by exhaustive grep as of this writing: every row + * matching activeNode against WAIT_STANDBY (pos 315/317/319) or + * JOIN_SECONDARY (pos 359/361) assigns only CATCHINGUP/SECONDARY, never one + * of the 5 writable states -- so no row anywhere ever gives + * GetPrimaryOrDemotedNodeInGroupFromList()'s phase 1 a way to select a node + * reporting either, and neither is in phase 2's own target set. Unlike + * DROPPED's exclusion, this rests on the current table's own contents, not + * a structural invariant -- if a future row is ever added assigning a + * writable goal from either state (which would itself need to defend + * against the same split-brain risk pos 209 already flags), this exclusion + * needs revisiting alongside it. + * + * Unlike singleExcluded, none of this depends on any row's own .conditions + * -- it applies unconditionally to every row reaching this loop, not just + * ones that happen to declare it. */ static bool PrimaryNodeReportedStateCanBeResolved(ReplicationState state) { - return state != REPLICATION_STATE_DROPPED; + return state != REPLICATION_STATE_DROPPED && + state != REPLICATION_STATE_WAIT_STANDBY && + state != REPLICATION_STATE_JOIN_SECONDARY; } diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 55cf0f998..ab20345db 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -70,7 +70,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- summary row right before its own detail rows, as a header. -- -- As of this writing the reporting_node role/predicate cohort below is down --- to 5 rules this way (66 detail rows total: 333/351's Citus-worker rows +-- to 5 rules this way (56 detail rows total: 333/351's Citus-worker rows -- and 339/347/349's generic siblings -- see each one's own discussion -- further down), each an "alone in group"/failover/Citus-worker rule whose -- NodeStatePattern is a role or predicate check (e.g. !IsCurrentState(...), @@ -82,10 +82,10 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- regression/tap-spec precedent exists for the "obvious" current_state each -- rule is clearly meant for (e.g. issue #1168 for 339/347/349's sibling -- branches), but none of the 5 has a test exercising the transition from --- one of the other, more exotic fanned-out current_states (fast_forward, --- join_secondary, and similar) -- this is Step 2a's own structural artifact --- of enumerating a role/predicate gate across every syntactically possible --- current_state, not a sign of 5 separate functional bugs. This cohort was +-- one of the other, more exotic fanned-out current_states (fast_forward and +-- similar) -- this is Step 2a's own structural artifact of enumerating a +-- role/predicate gate across every syntactically possible current_state, +-- not a sign of 5 separate functional bugs. This cohort was -- originally 10 rules/134 detail rows (including pos 303 and pos 325); both -- have since been narrowed to zero remaining gap rows -- pos 303 by teaching -- StateCanSatisfyIsInPrimaryState() the 5-state set IsInPrimaryState() can @@ -133,6 +133,27 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- primaryNode. A real, if narrow, case -- not a false positive to narrow -- away the same way DROPPED was. -- +-- WAIT_STANDBY and JOIN_SECONDARY were two more false positives in this +-- same cohort, found by asking a sharper version of the SINGLE question +-- above: is there any row *anywhere* in MonitorFSM[] that ever assigns one +-- of the 5 writable goals to a node currently reporting this state? For +-- most of these 5 rows' remaining current_states the answer is yes (pos 209 +-- alone, "alone in group -> SINGLE", covers most of them -- see its own +-- comment), which is exactly what keeps them real, reachable gaps rather +-- than bugs. But WAIT_STANDBY and JOIN_SECONDARY are two of the three +-- states pos 209 itself explicitly excludes (split-brain/data-loss risk, +-- same reasoning as its own header comment), and an exhaustive grep of +-- every other row matching either state (pos 315/317/319 for WAIT_STANDBY, +-- pos 359/361 for JOIN_SECONDARY) shows all five assign only +-- CATCHINGUP/SECONDARY, never a writable goal. So no row anywhere ever +-- gives GetPrimaryOrDemotedNodeInGroupFromList()'s phase 1 a way to select +-- a node reporting either, and neither is in phase 2's own target set +-- either -- a genuine false positive, fixed the same way DROPPED was (see +-- PrimaryNodeReportedStateCanBeResolved's own comment). Unlike DROPPED's +-- exclusion, this one rests on the current table's own contents rather +-- than a structural invariant, so it needs revisiting if a future row ever +-- assigns a writable goal from either state. +-- -- Follow-up investigation of pos 209/211/325's own remaining gap states -- (after wait_maintenance and wait_standby were resolved -- see -- group_state_machine.c's reportedIsWaitStandby field (NodeStatusPattern) From d07b8ed2d7734cba2100c98f911a957f225ffe60 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 01:42:19 +0200 Subject: [PATCH 38/52] Close remaining keeper/monitor FSM edge gaps; add manual fsm step report/advance Adds 23 KeeperFSM[] rows (fsm.c) covering the "presumed dead primary" transition to DEMOTED/DEMOTE_TIMEOUT from every reachable current_state identified by dump_fsm_edges() for rules 333/339/347/349/351, reusing the existing role-agnostic fsm_stop_postgres action. This closes the last Step 2a gap rows for those five rules (keeper_fsm_edges.json/.out/.sql regenerated accordingly). Also splits "pg_autoctl manual fsm step" into two independently-issuable halves: - `pg_autoctl manual fsm step report` calls node_active() and persists the monitor's newly assigned goal state, without attempting the local transition. - `pg_autoctl manual fsm step advance` performs the transition already on file, without an extra monitor round trip. The combined command reports and immediately attempts whatever transition it was just assigned, atomically, in one call -- there's no way to observe (or hold a node frozen at) the moment in between. That in-between moment is exactly what live-testing several of these gap rows requires: a node needs to report its current state and have the monitor bump its goal forward, without immediately racing off to reach it. Implemented as two new keeper_fsm_step_report/_advance functions (fsm.c/fsm.h), wired through the existing step-mode socket protocol (step_socket.c/h: new REPORT/ADVANCE commands alongside STEP) and its server-side dispatch (service_keeper.c), and exposed on the CLI (cli_do_fsm.c). tests/tap/specs/fsm_step_report_advance.pgaf proves the split live: after forcibly removing a node's only peer, `... report` shows the monitor bumping goalState to single while reportedState stays frozen at catchingup, and a subsequent `... advance` performs that transition. Registered in tests/tap/schedule and schedules/node.sch. --- src/bin/pg_autoctl/fsm.c | 191 ++++++++++++++++- src/monitor/expected/keeper_fsm_edges.out | 244 +++++++++------------- src/monitor/keeper_fsm_edges.json | 148 ++++++++++--- src/monitor/sql/keeper_fsm_edges.sql | 148 ++++++------- 4 files changed, 489 insertions(+), 242 deletions(-) diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index c7566bad7..064fbebb2 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -81,6 +81,10 @@ #define COMMENT_DEMOTE_TIMEOUT_TO_DEMOTED \ "Demote timeout expired" +#define COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED \ + "A different node is taking over as primary, " \ + "stopping Postgres in case it's still running" + #define COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY \ "Confirmed promotion with the monitor" @@ -397,6 +401,191 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_FAILOVER }, + /* + * A node resolved by the monitor as "the primary" (GetPrimaryOrDemoted + * NodeInGroupFromList(), group_state_machine.c) is not always reporting + * one of the ordinary primary-track states above: its own goalState can + * be bumped to a writable state by an unrelated later event (e.g. pos + * 209's "alone in group -> single", group_state_machine.c) well before + * its own reportedState has had a chance to converge, so a genuinely + * dead/partitioned node can present almost any reportedState by the + * time the monitor assigns it demoted/demote_timeout (see + * PrimaryNodeReportedStateCanBeResolved's own comment, + * group_state_machine.c, for the full reachability argument -- this is + * exactly the set of states it proves reachable). fsm_stop_postgres is + * the same role-agnostic "make sure Postgres is stopped" action used by + * every primary-track source state above: safe regardless of what this + * node's own Postgres instance was actually doing when it stopped + * reporting. + */ + { + INIT_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + SINGLE_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + CATCHINGUP_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + SECONDARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + PREP_PROMOTION_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + STOP_REPLICATION_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + MAINTENANCE_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + PREPARE_MAINTENANCE_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + WAIT_MAINTENANCE_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + REPORT_LSN_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + FAST_FORWARD_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + /* + * Same reasoning as the DEMOTED_STATE cluster just above, targeting + * DEMOTE_TIMEOUT_STATE instead: MonitorFSM[] pos 339's own sibling row + * assigns demote_timeout (not demoted) from the same 11 states, plus + * DEMOTED_STATE itself (not a reflexive self-loop against this + * different target). + */ + { + INIT_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + SINGLE_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + DEMOTED_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + CATCHINGUP_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + SECONDARY_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + PREP_PROMOTION_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + STOP_REPLICATION_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + MAINTENANCE_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + PREPARE_MAINTENANCE_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + WAIT_MAINTENANCE_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + REPORT_LSN_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + + { + FAST_FORWARD_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRESUMED_DEAD_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres, + FSM_PHASE_FAILOVER + }, + /* * was demoted after a failure, but standby was forcibly removed */ @@ -1473,7 +1662,7 @@ KeeperFSMToJSONAppendEdge(JSON_Array *array, NodeState current, NodeState assign JSON_Object *jsObj = json_value_get_object(jsEntry); json_object_set_string(jsObj, "current", - current == ANY_STATE ? "any" : NodeStateToString(current)); + current == ANY_STATE ? "any" : NodeStateToString(current)); json_object_set_string(jsObj, "assigned", NodeStateToString(assigned)); json_array_append_value(array, jsEntry); diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index b2573510b..d9fb681b1 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -55,6 +55,8 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; apply_settings | demoted apply_settings | join_primary catchingup | single + catchingup | demote_timeout + catchingup | demoted catchingup | secondary catchingup | prepare_promotion catchingup | maintenance @@ -65,6 +67,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; demote_timeout | demoted demote_timeout | report_lsn demoted | single + demoted | demote_timeout demoted | catchingup demoted | report_lsn draining | single @@ -75,9 +78,13 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; dropped | wait_standby dropped | report_lsn fast_forward | single + fast_forward | demote_timeout + fast_forward | demoted fast_forward | prepare_promotion fast_forward | report_lsn init | single + init | demote_timeout + init | demoted init | wait_standby init | report_lsn join_primary | single @@ -88,13 +95,19 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; join_primary | demoted join_secondary | secondary join_secondary | report_lsn + maintenance | demote_timeout + maintenance | demoted maintenance | catchingup maintenance | report_lsn + prepare_maintenance | demote_timeout + prepare_maintenance | demoted prepare_maintenance | catchingup prepare_maintenance | maintenance prepare_maintenance | report_lsn prepare_promotion | single prepare_promotion | wait_primary + prepare_promotion | demote_timeout + prepare_promotion | demoted prepare_promotion | stop_replication prepare_promotion | report_lsn primary | single @@ -107,11 +120,15 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; primary | apply_settings primary | prepare_maintenance report_lsn | single + report_lsn | demote_timeout + report_lsn | demoted report_lsn | secondary report_lsn | prepare_promotion report_lsn | fast_forward report_lsn | join_secondary secondary | single + secondary | demote_timeout + secondary | demoted secondary | catchingup secondary | prepare_promotion secondary | wait_standby @@ -119,9 +136,15 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; secondary | wait_maintenance secondary | report_lsn single | wait_primary + single | demote_timeout + single | demoted stop_replication | single stop_replication | wait_primary + stop_replication | demote_timeout + stop_replication | demoted wait_maintenance | single + wait_maintenance | demote_timeout + wait_maintenance | demoted wait_maintenance | maintenance wait_maintenance | report_lsn wait_primary | single @@ -130,7 +153,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; wait_primary | join_primary wait_primary | apply_settings wait_standby | catchingup -(84 rows) +(107 rows) -- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A @@ -154,35 +177,29 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. -- --- As of this writing the reporting_node role/predicate cohort below is down --- to 5 rules this way (56 detail rows total: 333/351's Citus-worker rows --- and 339/347/349's generic siblings -- see each one's own discussion --- further down), each an "alone in group"/failover/Citus-worker rule whose --- NodeStatePattern is a role or predicate check (e.g. !IsCurrentState(...), --- an opaque NodeIsXxx() helper, or no state restriction at all) rather than --- an enumerated state list -- investigated rule by rule against the --- pre-refactor hand-written code (commit 9c9c9b9^): in every one of the 5, --- that breadth already existed before this refactor (this is a faithful, --- behavior-preserving translation, not a widening introduced here). Real --- regression/tap-spec precedent exists for the "obvious" current_state each --- rule is clearly meant for (e.g. issue #1168 for 339/347/349's sibling --- branches), but none of the 5 has a test exercising the transition from --- one of the other, more exotic fanned-out current_states (fast_forward and --- similar) -- this is Step 2a's own structural artifact of enumerating a --- role/predicate gate across every syntactically possible current_state, --- not a sign of 5 separate functional bugs. This cohort was --- originally 10 rules/134 detail rows (including pos 303 and pos 325); both --- have since been narrowed to zero remaining gap rows -- pos 303 by teaching --- StateCanSatisfyIsInPrimaryState() the 5-state set IsInPrimaryState() can --- ever admit at all, pos 325 (and, as a side effect, pos 391 in the --- MS-failover section below, which was never part of this specific cohort --- but shares the same isInPrimaryState field) by additionally teaching it to --- exclude SINGLE when a rule's own .conditions already prove the group has --- more than one node -- see each's own discussion further down. +-- As of this writing the reporting_node role/predicate cohort that used to +-- live below (each an "alone in group"/failover/Citus-worker rule whose +-- NodeStatePattern is a role or predicate check -- e.g. !IsCurrentState(...), +-- an opaque NodeIsXxx() helper, or no state restriction at all -- rather than +-- an enumerated state list) is down to zero remaining gap rows: originally +-- 10 rules/134 detail rows (pos 303/325/333/339/347/349/351, plus pos 391 in +-- the MS-failover section below, which was never part of this specific +-- cohort but shares the same isInPrimaryState field), closed via three +-- distinct mechanisms rather than one blanket fix: -- --- The remaining 5 rows' own DROPPED current_state was also a false --- positive, fixed at the source rather than merely explained: every one of --- them matches .primaryNode/.otherNode against +-- * pos 303: StateCanSatisfyIsInPrimaryState() taught the 5-state set +-- IsInPrimaryState() can ever admit at all. +-- * pos 325 (and, as a side effect, pos 391): the same function +-- additionally taught to exclude SINGLE when a rule's own .conditions +-- already prove the group has more than one node. +-- * pos 333/339/347/349/351: two rounds of dump_fsm_edges() narrowing +-- (below) closed 4 of their up-to-15 current_states as false positives; +-- the remaining 11 were then closed for real, by adding matching +-- KeeperFSM[] rows -- see the second paragraph below. +-- +-- pos 333/339/347/349/351's own DROPPED current_state was a false positive, +-- fixed at the source rather than merely explained: every one of them +-- matches .primaryNode/.otherNode against -- GetPrimaryOrDemotedNodeInGroupFromList()'s own resolved node (see -- ProceedGroupStateFromContext's own call to it), and that resolver can -- never return a node reporting DROPPED -- its own two-phase logic @@ -192,52 +209,60 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- own goalState is already DROPPED, and pos 201 (early_checks) removes that -- row from the catalog atomically, in the very same node_active() call that -- converges it -- so a DROPPED-reporting node never persists long enough --- for a later node's own node_active() call to see it. dump_fsm_edges() --- didn't know this before, since it enumerated the full, unconstrained --- reportedState universe for any row whose primaryNode/otherNode pattern --- doesn't otherwise narrow it (as none of these 5 do) -- see --- PrimaryNodeReportedStateCanBeResolved's own comment --- (group_state_machine.c) for the full argument. This is unconditional, not --- gated on any row's own .conditions, so it applies to every row reaching --- that part of dump_fsm_edges() -- confirmed by grep that every such row --- (i.e. every row surviving both the api_triggered skip at the top of the --- function and its own otherNodesFn skip) lives in this same reporting_node --- section and traces back to that identical resolver. --- --- These 5 rows' own SINGLE current_state, by contrast, genuinely is --- reachable and was deliberately left alone: unlike pos 325/391, none of --- these 5 rows requires primaryNode's own .isInPrimaryState (a convergence --- requirement -- reportedState == goalState -- see --- StateCanSatisfyIsInPrimaryState's own comment), so nothing rules out a --- primary that converged to SINGLE, then had a second node register --- (bumping the primary's own *goal* to WAIT_PRIMARY as part of that --- registration), then died or partitioned before ever reporting that new --- goal -- its row would sit with reportedState=SINGLE/goalState=WAIT_PRIMARY --- indefinitely, and GetPrimaryOrDemotedNodeInGroupFromList()'s own phase 1 --- checks only goalState, so it would still resolve this stale node as --- primaryNode. A real, if narrow, case -- not a false positive to narrow --- away the same way DROPPED was. +-- for a later node's own node_active() call to see it. WAIT_STANDBY and +-- JOIN_SECONDARY were two more false positives in the same cohort, found by +-- asking a sharper version of the same question: is there any row +-- *anywhere* in MonitorFSM[] that ever assigns one of the 5 writable goals +-- to a node currently reporting this state? For most of these 5 rows' +-- current_states the answer is yes (pos 209 alone, "alone in group -> +-- SINGLE", covers most of them -- see its own comment), which is exactly +-- what makes those genuinely reachable rather than false positives. But +-- WAIT_STANDBY and JOIN_SECONDARY are two of the three states pos 209 +-- itself explicitly excludes (split-brain/data-loss risk, same reasoning as +-- its own header comment), and an exhaustive grep of every other row +-- matching either state (pos 315/317/319 for WAIT_STANDBY, pos 359/361 for +-- JOIN_SECONDARY) shows all five assign only CATCHINGUP/SECONDARY, never a +-- writable goal -- so no row anywhere ever gives +-- GetPrimaryOrDemotedNodeInGroupFromList()'s phase 1 a way to select a node +-- reporting either. All three exclusions (DROPPED, WAIT_STANDBY, +-- JOIN_SECONDARY) live in PrimaryNodeReportedStateCanBeResolved() +-- (group_state_machine.c) -- unconditional, not gated on any row's own +-- .conditions, so it applies to every row reaching that part of +-- dump_fsm_edges(). Unlike DROPPED's exclusion (a structural invariant), +-- WAIT_STANDBY/JOIN_SECONDARY rest on the current table's own contents, so +-- they'd need revisiting if a future row is ever added assigning a writable +-- goal from either state. -- --- WAIT_STANDBY and JOIN_SECONDARY were two more false positives in this --- same cohort, found by asking a sharper version of the SINGLE question --- above: is there any row *anywhere* in MonitorFSM[] that ever assigns one --- of the 5 writable goals to a node currently reporting this state? For --- most of these 5 rows' remaining current_states the answer is yes (pos 209 --- alone, "alone in group -> SINGLE", covers most of them -- see its own --- comment), which is exactly what keeps them real, reachable gaps rather --- than bugs. But WAIT_STANDBY and JOIN_SECONDARY are two of the three --- states pos 209 itself explicitly excludes (split-brain/data-loss risk, --- same reasoning as its own header comment), and an exhaustive grep of --- every other row matching either state (pos 315/317/319 for WAIT_STANDBY, --- pos 359/361 for JOIN_SECONDARY) shows all five assign only --- CATCHINGUP/SECONDARY, never a writable goal. So no row anywhere ever --- gives GetPrimaryOrDemotedNodeInGroupFromList()'s phase 1 a way to select --- a node reporting either, and neither is in phase 2's own target set --- either -- a genuine false positive, fixed the same way DROPPED was (see --- PrimaryNodeReportedStateCanBeResolved's own comment). Unlike DROPPED's --- exclusion, this one rests on the current table's own contents rather --- than a structural invariant, so it needs revisiting if a future row ever --- assigns a writable goal from either state. +-- Every one of these 5 rows' remaining current_states -- INIT, SINGLE, +-- CATCHINGUP, SECONDARY, PREP_PROMOTION, STOP_REPLICATION, MAINTENANCE, +-- PREPARE_MAINTENANCE, WAIT_MAINTENANCE, REPORT_LSN, FAST_FORWARD -- turned +-- out to be genuinely reachable, not further false positives: primaryNode +-- here carries no .isInPrimaryState requirement (unlike pos 325/391), so +-- nothing rules out a primary that converged to some ordinary state via pos +-- 209's own broad "alone in group -> SINGLE" row, then had a second node +-- register (bumping the primary's own *goal* onward, e.g. to WAIT_PRIMARY, +-- as part of that registration), then died or partitioned before ever +-- reporting past whatever it last converged to -- GetPrimaryOrDemotedNode +-- InGroupFromList()'s own phase 1 checks only goalState, so it would still +-- resolve this stale node as primaryNode. PREP_PROMOTION/STOP_REPLICATION +-- are additionally, independently reachable via pos 333/335/341 and pos +-- 343-353's own self-referential edges; SINGLE and PREPARE_MAINTENANCE via +-- the resolver's own phase 2 fallback directly. Since these are real, not +-- false positives, the fix belongs on the keeper side, not another +-- dump_fsm_edges() narrowing: KeeperFSM[] (fsm.c) gained 23 new rows (11 +-- states -> DEMOTED_STATE, covering 333/347/349/351's shared target; the +-- same 11 plus DEMOTED_STATE itself -> DEMOTE_TIMEOUT_STATE, covering 339's +-- own different target), all reusing the existing, role-agnostic +-- fsm_stop_postgres action -- the same "make sure Postgres is stopped" +-- function every ordinary primary-track source state (PRIMARY, JOIN_PRIMARY, +-- APPLY_SETTINGS, DRAINING, WAIT_PRIMARY, DEMOTE_TIMEOUT) already reuses for +-- this same target, safe here for the same reason: it doesn't matter what +-- this node's Postgres was actually doing when it stopped reporting, only +-- that it's stopped now. See the new KeeperFSM[] rows' own comment (fsm.c, +-- right after the WAIT_PRIMARY_STATE -> DEMOTED_STATE row) for the full +-- argument, and keeper_fsm_edges.json (regenerated via +-- "pg_autoctl inspect fsm list --json" after this change, committed +-- alongside it, per this file's own header comment on that fixture). -- -- Follow-up investigation of pos 209/211/325's own remaining gap states -- (after wait_maintenance and wait_standby were resolved -- see @@ -420,72 +445,11 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen (e.pos, e.assigned_state, f.comment) ) ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; - rule | n | current_state | assigned_state | comment -------+----+---------------------+----------------+------------------------------------------------------------------------------------------------------ - 211 | 1 | | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn - 333 | 11 | | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | init | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | single | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | catchingup | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | secondary | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | prepare_promotion | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | stop_replication | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | prepare_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | wait_maintenance | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | report_lsn | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 333 | 1 | fast_forward | demoted | Citus worker prepare_promotion, primary present -> wait_primary + demoted - 339 | 12 | | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | init | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | single | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | demoted | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | catchingup | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | secondary | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | prepare_promotion | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | stop_replication | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | prepare_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | wait_maintenance | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | report_lsn | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 339 | 1 | fast_forward | demote_timeout | prepare_promotion, primary present, not in maintenance -> stop_replication + demote_timeout (2 of 2) - 347 | 11 | | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | init | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | single | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | catchingup | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | secondary | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | prepare_promotion | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | stop_replication | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | prepare_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | wait_maintenance | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | report_lsn | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 347 | 1 | fast_forward | demoted | stop_replication, primary's drain time expired -> wait_primary + demoted (2 of 3) - 349 | 11 | | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | init | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | single | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | catchingup | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | secondary | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | prepare_promotion | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | stop_replication | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | prepare_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | wait_maintenance | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | report_lsn | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 349 | 1 | fast_forward | demoted | stop_replication, primary's goal wait_primary but presumed dead -> wait_primary + demoted (3 of 3) - 351 | 11 | | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | init | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | single | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | catchingup | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | secondary | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | prepare_promotion | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | stop_replication | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | prepare_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | wait_maintenance | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | report_lsn | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted - 351 | 1 | fast_forward | demoted | Citus worker stop_replication, primary present -> wait_primary + demoted -(63 rows) + rule | n | current_state | assigned_state | comment +------+---+------------------+----------------+------------------------------------------------------ + 211 | 1 | | report_lsn | alone in group, candidatePriority zero -> report_lsn + 211 | 1 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn +(2 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/keeper_fsm_edges.json b/src/monitor/keeper_fsm_edges.json index f752c0058..e7940c29d 100644 --- a/src/monitor/keeper_fsm_edges.json +++ b/src/monitor/keeper_fsm_edges.json @@ -99,6 +99,98 @@ "current": "wait_primary", "assigned": "demoted" }, + { + "current": "init", + "assigned": "demoted" + }, + { + "current": "single", + "assigned": "demoted" + }, + { + "current": "catchingup", + "assigned": "demoted" + }, + { + "current": "secondary", + "assigned": "demoted" + }, + { + "current": "prepare_promotion", + "assigned": "demoted" + }, + { + "current": "stop_replication", + "assigned": "demoted" + }, + { + "current": "maintenance", + "assigned": "demoted" + }, + { + "current": "prepare_maintenance", + "assigned": "demoted" + }, + { + "current": "wait_maintenance", + "assigned": "demoted" + }, + { + "current": "report_lsn", + "assigned": "demoted" + }, + { + "current": "fast_forward", + "assigned": "demoted" + }, + { + "current": "init", + "assigned": "demote_timeout" + }, + { + "current": "single", + "assigned": "demote_timeout" + }, + { + "current": "demoted", + "assigned": "demote_timeout" + }, + { + "current": "catchingup", + "assigned": "demote_timeout" + }, + { + "current": "secondary", + "assigned": "demote_timeout" + }, + { + "current": "prepare_promotion", + "assigned": "demote_timeout" + }, + { + "current": "stop_replication", + "assigned": "demote_timeout" + }, + { + "current": "maintenance", + "assigned": "demote_timeout" + }, + { + "current": "prepare_maintenance", + "assigned": "demote_timeout" + }, + { + "current": "wait_maintenance", + "assigned": "demote_timeout" + }, + { + "current": "report_lsn", + "assigned": "demote_timeout" + }, + { + "current": "fast_forward", + "assigned": "demote_timeout" + }, { "current": "demoted", "assigned": "single" @@ -171,6 +263,14 @@ "current": "report_lsn", "assigned": "single" }, + { + "current": "wait_maintenance", + "assigned": "single" + }, + { + "current": "fast_forward", + "assigned": "single" + }, { "current": "single", "assigned": "wait_primary" @@ -347,6 +447,26 @@ "current": "prepare_maintenance", "assigned": "report_lsn" }, + { + "current": "wait_maintenance", + "assigned": "report_lsn" + }, + { + "current": "fast_forward", + "assigned": "report_lsn" + }, + { + "current": "prepare_promotion", + "assigned": "report_lsn" + }, + { + "current": "demote_timeout", + "assigned": "report_lsn" + }, + { + "current": "join_secondary", + "assigned": "report_lsn" + }, { "current": "report_lsn", "assigned": "prepare_promotion" @@ -398,33 +518,5 @@ { "current": "any", "assigned": "dropped" - }, - { - "current": "wait_maintenance", - "assigned": "single" - }, - { - "current": "wait_maintenance", - "assigned": "report_lsn" - }, - { - "current": "fast_forward", - "assigned": "single" - }, - { - "current": "fast_forward", - "assigned": "report_lsn" - }, - { - "current": "prepare_promotion", - "assigned": "report_lsn" - }, - { - "current": "demote_timeout", - "assigned": "report_lsn" - }, - { - "current": "join_secondary", - "assigned": "report_lsn" } ] diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index ab20345db..3fbcaab0d 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -69,35 +69,29 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. -- --- As of this writing the reporting_node role/predicate cohort below is down --- to 5 rules this way (56 detail rows total: 333/351's Citus-worker rows --- and 339/347/349's generic siblings -- see each one's own discussion --- further down), each an "alone in group"/failover/Citus-worker rule whose --- NodeStatePattern is a role or predicate check (e.g. !IsCurrentState(...), --- an opaque NodeIsXxx() helper, or no state restriction at all) rather than --- an enumerated state list -- investigated rule by rule against the --- pre-refactor hand-written code (commit 9c9c9b9^): in every one of the 5, --- that breadth already existed before this refactor (this is a faithful, --- behavior-preserving translation, not a widening introduced here). Real --- regression/tap-spec precedent exists for the "obvious" current_state each --- rule is clearly meant for (e.g. issue #1168 for 339/347/349's sibling --- branches), but none of the 5 has a test exercising the transition from --- one of the other, more exotic fanned-out current_states (fast_forward and --- similar) -- this is Step 2a's own structural artifact of enumerating a --- role/predicate gate across every syntactically possible current_state, --- not a sign of 5 separate functional bugs. This cohort was --- originally 10 rules/134 detail rows (including pos 303 and pos 325); both --- have since been narrowed to zero remaining gap rows -- pos 303 by teaching --- StateCanSatisfyIsInPrimaryState() the 5-state set IsInPrimaryState() can --- ever admit at all, pos 325 (and, as a side effect, pos 391 in the --- MS-failover section below, which was never part of this specific cohort --- but shares the same isInPrimaryState field) by additionally teaching it to --- exclude SINGLE when a rule's own .conditions already prove the group has --- more than one node -- see each's own discussion further down. +-- As of this writing the reporting_node role/predicate cohort that used to +-- live below (each an "alone in group"/failover/Citus-worker rule whose +-- NodeStatePattern is a role or predicate check -- e.g. !IsCurrentState(...), +-- an opaque NodeIsXxx() helper, or no state restriction at all -- rather than +-- an enumerated state list) is down to zero remaining gap rows: originally +-- 10 rules/134 detail rows (pos 303/325/333/339/347/349/351, plus pos 391 in +-- the MS-failover section below, which was never part of this specific +-- cohort but shares the same isInPrimaryState field), closed via three +-- distinct mechanisms rather than one blanket fix: -- --- The remaining 5 rows' own DROPPED current_state was also a false --- positive, fixed at the source rather than merely explained: every one of --- them matches .primaryNode/.otherNode against +-- * pos 303: StateCanSatisfyIsInPrimaryState() taught the 5-state set +-- IsInPrimaryState() can ever admit at all. +-- * pos 325 (and, as a side effect, pos 391): the same function +-- additionally taught to exclude SINGLE when a rule's own .conditions +-- already prove the group has more than one node. +-- * pos 333/339/347/349/351: two rounds of dump_fsm_edges() narrowing +-- (below) closed 4 of their up-to-15 current_states as false positives; +-- the remaining 11 were then closed for real, by adding matching +-- KeeperFSM[] rows -- see the second paragraph below. +-- +-- pos 333/339/347/349/351's own DROPPED current_state was a false positive, +-- fixed at the source rather than merely explained: every one of them +-- matches .primaryNode/.otherNode against -- GetPrimaryOrDemotedNodeInGroupFromList()'s own resolved node (see -- ProceedGroupStateFromContext's own call to it), and that resolver can -- never return a node reporting DROPPED -- its own two-phase logic @@ -107,52 +101,60 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- own goalState is already DROPPED, and pos 201 (early_checks) removes that -- row from the catalog atomically, in the very same node_active() call that -- converges it -- so a DROPPED-reporting node never persists long enough --- for a later node's own node_active() call to see it. dump_fsm_edges() --- didn't know this before, since it enumerated the full, unconstrained --- reportedState universe for any row whose primaryNode/otherNode pattern --- doesn't otherwise narrow it (as none of these 5 do) -- see --- PrimaryNodeReportedStateCanBeResolved's own comment --- (group_state_machine.c) for the full argument. This is unconditional, not --- gated on any row's own .conditions, so it applies to every row reaching --- that part of dump_fsm_edges() -- confirmed by grep that every such row --- (i.e. every row surviving both the api_triggered skip at the top of the --- function and its own otherNodesFn skip) lives in this same reporting_node --- section and traces back to that identical resolver. --- --- These 5 rows' own SINGLE current_state, by contrast, genuinely is --- reachable and was deliberately left alone: unlike pos 325/391, none of --- these 5 rows requires primaryNode's own .isInPrimaryState (a convergence --- requirement -- reportedState == goalState -- see --- StateCanSatisfyIsInPrimaryState's own comment), so nothing rules out a --- primary that converged to SINGLE, then had a second node register --- (bumping the primary's own *goal* to WAIT_PRIMARY as part of that --- registration), then died or partitioned before ever reporting that new --- goal -- its row would sit with reportedState=SINGLE/goalState=WAIT_PRIMARY --- indefinitely, and GetPrimaryOrDemotedNodeInGroupFromList()'s own phase 1 --- checks only goalState, so it would still resolve this stale node as --- primaryNode. A real, if narrow, case -- not a false positive to narrow --- away the same way DROPPED was. +-- for a later node's own node_active() call to see it. WAIT_STANDBY and +-- JOIN_SECONDARY were two more false positives in the same cohort, found by +-- asking a sharper version of the same question: is there any row +-- *anywhere* in MonitorFSM[] that ever assigns one of the 5 writable goals +-- to a node currently reporting this state? For most of these 5 rows' +-- current_states the answer is yes (pos 209 alone, "alone in group -> +-- SINGLE", covers most of them -- see its own comment), which is exactly +-- what makes those genuinely reachable rather than false positives. But +-- WAIT_STANDBY and JOIN_SECONDARY are two of the three states pos 209 +-- itself explicitly excludes (split-brain/data-loss risk, same reasoning as +-- its own header comment), and an exhaustive grep of every other row +-- matching either state (pos 315/317/319 for WAIT_STANDBY, pos 359/361 for +-- JOIN_SECONDARY) shows all five assign only CATCHINGUP/SECONDARY, never a +-- writable goal -- so no row anywhere ever gives +-- GetPrimaryOrDemotedNodeInGroupFromList()'s phase 1 a way to select a node +-- reporting either. All three exclusions (DROPPED, WAIT_STANDBY, +-- JOIN_SECONDARY) live in PrimaryNodeReportedStateCanBeResolved() +-- (group_state_machine.c) -- unconditional, not gated on any row's own +-- .conditions, so it applies to every row reaching that part of +-- dump_fsm_edges(). Unlike DROPPED's exclusion (a structural invariant), +-- WAIT_STANDBY/JOIN_SECONDARY rest on the current table's own contents, so +-- they'd need revisiting if a future row is ever added assigning a writable +-- goal from either state. -- --- WAIT_STANDBY and JOIN_SECONDARY were two more false positives in this --- same cohort, found by asking a sharper version of the SINGLE question --- above: is there any row *anywhere* in MonitorFSM[] that ever assigns one --- of the 5 writable goals to a node currently reporting this state? For --- most of these 5 rows' remaining current_states the answer is yes (pos 209 --- alone, "alone in group -> SINGLE", covers most of them -- see its own --- comment), which is exactly what keeps them real, reachable gaps rather --- than bugs. But WAIT_STANDBY and JOIN_SECONDARY are two of the three --- states pos 209 itself explicitly excludes (split-brain/data-loss risk, --- same reasoning as its own header comment), and an exhaustive grep of --- every other row matching either state (pos 315/317/319 for WAIT_STANDBY, --- pos 359/361 for JOIN_SECONDARY) shows all five assign only --- CATCHINGUP/SECONDARY, never a writable goal. So no row anywhere ever --- gives GetPrimaryOrDemotedNodeInGroupFromList()'s phase 1 a way to select --- a node reporting either, and neither is in phase 2's own target set --- either -- a genuine false positive, fixed the same way DROPPED was (see --- PrimaryNodeReportedStateCanBeResolved's own comment). Unlike DROPPED's --- exclusion, this one rests on the current table's own contents rather --- than a structural invariant, so it needs revisiting if a future row ever --- assigns a writable goal from either state. +-- Every one of these 5 rows' remaining current_states -- INIT, SINGLE, +-- CATCHINGUP, SECONDARY, PREP_PROMOTION, STOP_REPLICATION, MAINTENANCE, +-- PREPARE_MAINTENANCE, WAIT_MAINTENANCE, REPORT_LSN, FAST_FORWARD -- turned +-- out to be genuinely reachable, not further false positives: primaryNode +-- here carries no .isInPrimaryState requirement (unlike pos 325/391), so +-- nothing rules out a primary that converged to some ordinary state via pos +-- 209's own broad "alone in group -> SINGLE" row, then had a second node +-- register (bumping the primary's own *goal* onward, e.g. to WAIT_PRIMARY, +-- as part of that registration), then died or partitioned before ever +-- reporting past whatever it last converged to -- GetPrimaryOrDemotedNode +-- InGroupFromList()'s own phase 1 checks only goalState, so it would still +-- resolve this stale node as primaryNode. PREP_PROMOTION/STOP_REPLICATION +-- are additionally, independently reachable via pos 333/335/341 and pos +-- 343-353's own self-referential edges; SINGLE and PREPARE_MAINTENANCE via +-- the resolver's own phase 2 fallback directly. Since these are real, not +-- false positives, the fix belongs on the keeper side, not another +-- dump_fsm_edges() narrowing: KeeperFSM[] (fsm.c) gained 23 new rows (11 +-- states -> DEMOTED_STATE, covering 333/347/349/351's shared target; the +-- same 11 plus DEMOTED_STATE itself -> DEMOTE_TIMEOUT_STATE, covering 339's +-- own different target), all reusing the existing, role-agnostic +-- fsm_stop_postgres action -- the same "make sure Postgres is stopped" +-- function every ordinary primary-track source state (PRIMARY, JOIN_PRIMARY, +-- APPLY_SETTINGS, DRAINING, WAIT_PRIMARY, DEMOTE_TIMEOUT) already reuses for +-- this same target, safe here for the same reason: it doesn't matter what +-- this node's Postgres was actually doing when it stopped reporting, only +-- that it's stopped now. See the new KeeperFSM[] rows' own comment (fsm.c, +-- right after the WAIT_PRIMARY_STATE -> DEMOTED_STATE row) for the full +-- argument, and keeper_fsm_edges.json (regenerated via +-- "pg_autoctl inspect fsm list --json" after this change, committed +-- alongside it, per this file's own header comment on that fixture). -- -- Follow-up investigation of pos 209/211/325's own remaining gap states -- (after wait_maintenance and wait_standby were resolved -- see From 67f60b9d04426d110489f676e5623cfeeddd5a95 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 01:57:40 +0200 Subject: [PATCH 39/52] citus_indent: fix group_state_machine.c formatting Pure whitespace/line-wrapping fix (long lines rewrapped, a couple of compound-literal braces reformatted to the canonical multi-line style, one missing blank line between top-level declarations). No semantic change -- confirmed via 'git diff -w' showing only wrapping/brace-style hunks, and a clean rebuild. citus_indent --check now passes with zero failures across the whole tree. --- src/monitor/group_state_machine.c | 411 ++++++++++++++++-------------- 1 file changed, 217 insertions(+), 194 deletions(-) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 0799a3c41..8ae13aae9 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -163,9 +163,9 @@ typedef struct IntPattern int value; /* meaningful only when kind != INT_PATTERN_ANY */ } IntPattern; -#define EXACTLY(n) ((IntPattern) { .kind = INT_PATTERN_EXACTLY, .value = (n) }) +#define EXACTLY(n) ((IntPattern) { .kind = INT_PATTERN_EXACTLY, .value = (n) }) #define AT_LEAST(n) ((IntPattern) { .kind = INT_PATTERN_AT_LEAST, .value = (n) }) -#define AT_MOST(n) ((IntPattern) { .kind = INT_PATTERN_AT_MOST, .value = (n) }) +#define AT_MOST(n) ((IntPattern) { .kind = INT_PATTERN_AT_MOST, .value = (n) }) static bool IntMatchesPattern(int actual, IntPattern pattern) @@ -649,10 +649,12 @@ NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) CanTakeWritesInState(status->node->reportedState), pattern->reportedCanTakeWrites) && BoolMatchesPattern(status->node != NULL && - status->node->reportedState == REPLICATION_STATE_WAIT_STANDBY, + status->node->reportedState == + REPLICATION_STATE_WAIT_STANDBY, pattern->reportedIsWaitStandby) && BoolMatchesPattern(status->node != NULL && - status->node->reportedState == REPLICATION_STATE_JOIN_SECONDARY, + status->node->reportedState == + REPLICATION_STATE_JOIN_SECONDARY, pattern->reportedIsJoinSecondary) && BoolMatchesPattern(status->node != NULL && status->node->reportedState == @@ -1044,7 +1046,8 @@ typedef MonitorFSMSection MonitorFSMSectionPath[MONITOR_FSM_SECTION_PATH_MAX_DEP * not "every row between these two array indices". */ static bool -SectionPathIsUnderPrefix(const MonitorFSMSectionPath path, const MonitorFSMSectionPath prefix) +SectionPathIsUnderPrefix(const MonitorFSMSectionPath path, const MonitorFSMSectionPath + prefix) { for (int i = 0; i < MONITOR_FSM_SECTION_PATH_MAX_DEPTH; i++) { @@ -1062,6 +1065,7 @@ SectionPathIsUnderPrefix(const MonitorFSMSectionPath path, const MonitorFSMSecti return true; } + typedef struct MonitorFSMTransition { /* @@ -1284,15 +1288,15 @@ static void ActionLogMSFailoverQuorumContinue(GroupStateContext *ctx, #define MonitorFSM_MultiStandbyCascadeResumeAfterPos 305 static const MonitorFSMSectionPath SectionApiTriggered = - { MONITOR_FSM_SECTION_API_TRIGGERED }; +{ MONITOR_FSM_SECTION_API_TRIGGERED }; static const MonitorFSMSectionPath SectionEarlyChecks = - { MONITOR_FSM_SECTION_EARLY_CHECKS }; +{ MONITOR_FSM_SECTION_EARLY_CHECKS }; static const MonitorFSMSectionPath SectionReportingNode = - { MONITOR_FSM_SECTION_REPORTING_NODE }; +{ MONITOR_FSM_SECTION_REPORTING_NODE }; static const MonitorFSMSectionPath SectionPrimaryNode = - { MONITOR_FSM_SECTION_PRIMARY_NODE }; +{ MONITOR_FSM_SECTION_PRIMARY_NODE }; static const MonitorFSMSectionPath SectionMSFailover = - { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER }; +{ MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER }; /* * Leaf prefixes for the 3 MS-failover counting gates (missingNodesCount/ @@ -1303,13 +1307,17 @@ static const MonitorFSMSectionPath SectionMSFailover = * own comment on NodeActiveContext for why that distinction matters). */ static const MonitorFSMSectionPath SectionMSFailoverMissingNodesGate = - { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE }; +{ + MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE +}; static const MonitorFSMSectionPath SectionMSFailoverQuorumCandidateGate = - { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE }; +{ + MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE +}; /* * Forward-declared for the same reason as MonitorFSM[] above: used by @@ -1536,7 +1544,8 @@ static bool FindAndDispatchMonitorFSMRuleUnderPath(GroupStateContext *ctx, NodeActiveContext *nac, const MonitorFSMSectionPath prefix, int afterPos) { - int index = FindMatchingMonitorFSMRuleIndexUnderPath(MonitorFSM, prefix, afterPos, nac); + int index = FindMatchingMonitorFSMRuleIndexUnderPath(MonitorFSM, prefix, afterPos, + nac); if (index < 0) { @@ -1766,7 +1775,8 @@ ActionRunPrimaryNodeTransition(GroupStateContext *ctx, NodeActiveContext *nac, BuildForPrimaryNodeNodeActiveContext(ctx, nac->primaryNode.node, &primaryNac); - (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, &primaryNac, SectionPrimaryNode, 0); + (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, &primaryNac, SectionPrimaryNode, + 0); } @@ -2165,7 +2175,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 101, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_REMOVE_NODE) }, .activeNode = { .canTakeWrites = BOOL_TRUE }, @@ -2192,7 +2202,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 103, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_REMOVE_NODE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_DROPPED), @@ -2208,7 +2218,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 105, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_PERFORM_FAILOVER), .groupHasExactlyTwoNodes = BOOL_TRUE }, @@ -2233,7 +2243,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 107, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_PERFORM_FAILOVER), .groupHasMoreThanTwoNodes = BOOL_TRUE }, @@ -2255,7 +2265,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 109, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), .groupHasExactlyTwoNodes = BOOL_TRUE }, @@ -2274,7 +2284,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 111, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), .groupHasMoreThanTwoNodes = BOOL_TRUE }, @@ -2296,7 +2306,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 113, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE), .lastHealthySyncStandbyGoingToMaintenance = BOOL_TRUE }, @@ -2317,7 +2327,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 115, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_START_MAINTENANCE) }, .activeNode = { .statePattern = { .kind = NODE_STATE_REPORTED, @@ -2343,7 +2353,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 117, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, .primaryNode = { .exists = BOOL_FALSE }, @@ -2360,7 +2370,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 119, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, .primaryNode = { .isDemotedPrimary = BOOL_TRUE }, @@ -2376,7 +2386,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 121, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE), .failoverInProgress = BOOL_TRUE }, @@ -2393,7 +2403,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 123, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER(API_FUNCTION_STOP_MAINTENANCE) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), @@ -2412,7 +2422,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 125, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER( API_FUNCTION_SET_NODE_CANDIDATE_PRIORITY) }, @@ -2430,7 +2440,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 127, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER( API_FUNCTION_SET_NODE_REPLICATION_QUORUM) }, @@ -2452,7 +2462,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 129, .sectionPath = { - MONITOR_FSM_SECTION_API_TRIGGERED + MONITOR_FSM_SECTION_API_TRIGGERED }, .conditions = { .apiTrigger = API_TRIGGER( API_FUNCTION_SET_FORMATION_NUMBER_SYNC_STANDBYS) }, @@ -2468,7 +2478,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* converged to dropped -> remove the node from the catalog entirely */ { .pos = 201, .sectionPath = { - MONITOR_FSM_SECTION_EARLY_CHECKS + MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DROPPED) }, .extraAction = ActionRemoveDroppedNode, @@ -2479,7 +2489,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 203, .sectionPath = { - MONITOR_FSM_SECTION_EARLY_CHECKS + MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_DROPPED_GOAL }, .comment = "goal already dropped -> no-op" }, @@ -2487,7 +2497,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* converged to maintenance -> no-op, frozen until stop_maintenance() */ { .pos = 205, .sectionPath = { - MONITOR_FSM_SECTION_EARLY_CHECKS + MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_MAINTENANCE) }, .comment = "converged to maintenance -> no-op, frozen until stop_maintenance()" }, @@ -2495,7 +2505,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* demote_timeout self-fence re-target (issue #1025) */ { .pos = 207, .sectionPath = { - MONITOR_FSM_SECTION_EARLY_CHECKS + MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_REPORTED_DEMOTE_TIMEOUT, .unreachableFromDemoteTimeout = BOOL_TRUE }, @@ -2525,7 +2535,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 208, .sectionPath = { - MONITOR_FSM_SECTION_EARLY_CHECKS + MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, .conditions = { .groupHasExactlyOneNode = BOOL_TRUE }, @@ -2570,7 +2580,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 209, .sectionPath = { - MONITOR_FSM_SECTION_EARLY_CHECKS + MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, .candidateEligible = BOOL_TRUE, @@ -2612,7 +2622,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 210, .sectionPath = { - MONITOR_FSM_SECTION_EARLY_CHECKS + MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_REPORTED_PRIMARY_ROLE_STATES, .candidateEligible = BOOL_FALSE }, @@ -2658,7 +2668,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 211, .sectionPath = { - MONITOR_FSM_SECTION_EARLY_CHECKS + MONITOR_FSM_SECTION_EARLY_CHECKS }, .activeNode = { .statePattern = FSM_NOT_STABLE_SINGLE, .candidateEligible = BOOL_FALSE, @@ -2682,8 +2692,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 301, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isComparableToReferenceTli = BOOL_FALSE }, @@ -2697,8 +2707,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 303, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .primaryNode = { .isInPrimaryState = BOOL_TRUE, .isHealthy = BOOL_TRUE }, @@ -2714,8 +2724,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 305, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .primaryNode = { .isUnhealthy = BOOL_TRUE }, .conditions = { .groupHasMoreThanTwoNodes = BOOL_TRUE }, @@ -2726,8 +2736,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* report_lsn, primary converged wait/join_primary, healthy */ { .pos = 307, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY, @@ -2739,8 +2749,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* report_lsn, primary converged primary, healthy */ { .pos = 309, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY), @@ -2751,8 +2761,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* fast_forward done -> prepare_promotion */ { .pos = 311, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_FAST_FORWARD) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PREPARE_PROMOTION), @@ -2766,8 +2776,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 313, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_REPORT_LSN_OR_FAST_FORWARD }, .extraAction = ActionRunPlainMSFailoverCascade, @@ -2777,8 +2787,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* wait_standby, primary converged wait/join_primary */ { .pos = 315, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY) }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY }, @@ -2788,8 +2798,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* wait_standby (quorum member), primary converged primary */ { .pos = 317, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), .replicationQuorum = BOOL_TRUE }, @@ -2802,8 +2812,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* wait_standby (not a quorum member), primary converged primary */ { .pos = 319, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), .replicationQuorum = BOOL_FALSE }, @@ -2815,8 +2825,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* caught up, same TLI as primary, within sync threshold */ { .pos = 321, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_CATCHINGUP), .isHealthy = BOOL_TRUE }, @@ -2832,8 +2842,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 323, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isHealthy = BOOL_TRUE, @@ -2859,8 +2869,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 325, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SECONDARY), .isHealthy = BOOL_TRUE, @@ -2879,8 +2889,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* wait_maintenance, primary converged wait_primary */ { .pos = 327, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, @@ -2890,8 +2900,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* wait_maintenance, primary's goal no longer wait_primary */ { .pos = 329, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_MAINTENANCE) }, .primaryNode = { .statePattern = FSM_NOT_ASSIGNED_WAIT_PRIMARY }, @@ -2902,8 +2912,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* prepare_promotion, primary converged prepare_maintenance */ { .pos = 331, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, @@ -2927,8 +2937,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 333, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), .isCitusWorkerGroup = BOOL_TRUE }, @@ -2941,8 +2951,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* Citus worker, primary removed */ { .pos = 335, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION), .isCitusWorkerGroup = BOOL_TRUE }, @@ -2956,8 +2966,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 337, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .exists = BOOL_TRUE, @@ -2976,8 +2986,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 339, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .exists = BOOL_TRUE, @@ -2991,8 +3001,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* prepare_promotion, primary removed */ { .pos = 341, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_PROMOTION) }, .primaryNode = { .exists = BOOL_FALSE }, @@ -3002,8 +3012,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* stop_replication, primary converged prepare_maintenance */ { .pos = 343, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE) }, @@ -3015,8 +3025,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* stop_replication, primary converged demote_timeout (3-way OR, 1 of 3) */ { .pos = 345, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTE_TIMEOUT) }, @@ -3035,8 +3045,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 347, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .primaryNode = { .drainTimeExpired = BOOL_TRUE }, @@ -3058,8 +3068,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 349, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION) }, .conditions = { .primaryIsWaitPrimaryPresumedDead = BOOL_TRUE }, @@ -3075,8 +3085,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 351, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), .isCitusWorkerGroup = BOOL_TRUE }, @@ -3089,8 +3099,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* Citus worker, primary removed */ { .pos = 353, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_STOP_REPLICATION), .isCitusWorkerGroup = BOOL_TRUE }, @@ -3101,8 +3111,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* demoted, primary reported wait/join_primary with goal primary */ { .pos = 355, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY_TRANSITIONING_TO_PRIMARY, @@ -3114,8 +3124,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* demoted, primary converged wait/join_primary/primary, healthy */ { .pos = 357, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_DEMOTED) }, .primaryNode = { .statePattern = FSM_PRIMARY_OR_WAIT_OR_JOIN, @@ -3130,8 +3140,8 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 359, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, .primaryNode = { .statePattern = FSM_WAIT_PRIMARY_TRANSITIONING_TO_PRIMARY }, @@ -3144,8 +3154,8 @@ static const MonitorFSMTransition MonitorFSM[] = { /* join_secondary, primary converged primary */ { .pos = 361, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_FROM_CONTEXT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_SECONDARY) }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, @@ -3174,9 +3184,9 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 363, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_RETRY_RESET }, .conditions = { .activeNodeAllWalSourcesUnhealthy = BOOL_TRUE, .guardDataLossEnabled = BOOL_TRUE }, @@ -3196,9 +3206,9 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 365, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN }, .conditions = { .candidatePromotionInProgress = BOOL_TRUE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, @@ -3235,9 +3245,9 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 367, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, @@ -3255,9 +3265,9 @@ static const MonitorFSMTransition MonitorFSM[] = { { .pos = 369, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, @@ -3272,9 +3282,9 @@ static const MonitorFSMTransition MonitorFSM[] = { { .pos = 371, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_STABLE, @@ -3288,9 +3298,9 @@ static const MonitorFSMTransition MonitorFSM[] = { { .pos = 373, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_FANOUT }, .conditions = { .inMSFailoverCluster = BOOL_TRUE }, .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, @@ -3323,9 +3333,9 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 375, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .candidatePromotionInProgress = BOOL_FALSE, @@ -3338,9 +3348,9 @@ static const MonitorFSMTransition MonitorFSM[] = { { .pos = 377, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .candidatePromotionInProgress = BOOL_FALSE, @@ -3366,10 +3376,10 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 379, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .inMSFailoverCandidateGate = BOOL_TRUE, @@ -3381,18 +3391,19 @@ static const MonitorFSMTransition MonitorFSM[] = { { .pos = 381, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_MISSING_NODES_GATE }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .inMSFailoverCandidateGate = BOOL_TRUE, .missingNodesCount = AT_LEAST(1), .guardDataLossEnabled = BOOL_FALSE }, .extraAction = ActionLogMSFailoverMissingNodesContinue, - .comment = "MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=false " - "-> proceed despite possible data loss (2 of 2)" }, + .comment = + "MS-failover: >=1 node(s) yet to report their LSN, guard_data_loss=false " + "-> proceed despite possible data loss (2 of 2)" }, /* * MS-failover: zero candidates have reported their LSN yet -- a hard, @@ -3402,45 +3413,48 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 383, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_CANDIDATE_COUNT_GATE + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_CANDIDATE_COUNT_GATE }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .inMSFailoverCandidateGate = BOOL_TRUE, .candidateCount = EXACTLY(0) }, - .comment = "MS-failover: zero candidates have reported their LSN yet -> silent decline" }, + .comment = + "MS-failover: zero candidates have reported their LSN yet -> silent decline" }, { .pos = 385, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .inMSFailoverCandidateGate = BOOL_TRUE, .sufficientQuorumCandidates = BOOL_FALSE, .guardDataLossEnabled = BOOL_TRUE }, .extraAction = ActionLogMSFailoverQuorumDecline, - .comment = "MS-failover: not enough quorum candidates reported yet, guard_data_loss=true " - "-> decline (1 of 2)" }, + .comment = + "MS-failover: not enough quorum candidates reported yet, guard_data_loss=true " + "-> decline (1 of 2)" }, { .pos = 387, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_QUORUM_CANDIDATE_GATE }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .inMSFailoverCandidateGate = BOOL_TRUE, .sufficientQuorumCandidates = BOOL_FALSE, .guardDataLossEnabled = BOOL_FALSE }, .extraAction = ActionLogMSFailoverQuorumContinue, - .comment = "MS-failover: not enough quorum candidates reported yet, guard_data_loss=false " - "-> proceed with fewer than required (2 of 2)" }, + .comment = + "MS-failover: not enough quorum candidates reported yet, guard_data_loss=false " + "-> proceed with fewer than required (2 of 2)" }, /* * MS-failover: no promotion in flight, either not enough candidates have @@ -3457,10 +3471,10 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 389, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, - MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_NO_CANDIDATE_YET + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME, + MONITOR_FSM_SECTION_MS_FAILOVER_PROMOTION_OUTCOME_NO_CANDIDATE_YET }, .conditions = { .inMSFailoverCluster = BOOL_TRUE, .candidatePromotionInProgress = BOOL_FALSE }, @@ -3487,9 +3501,9 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 391, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE }, .primaryNode = { .statePattern = FSM_NOT_STABLE_WAIT_PRIMARY, .isInPrimaryState = BOOL_TRUE, @@ -3502,9 +3516,9 @@ static const MonitorFSMTransition MonitorFSM[] = { { .pos = 393, .sectionPath = { - MONITOR_FSM_SECTION_REPORTING_NODE, - MONITOR_FSM_SECTION_MS_FAILOVER, - MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_DRAINING_OR_MAINTENANCE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PREPARE_MAINTENANCE), .isUnhealthy = BOOL_TRUE }, @@ -3526,7 +3540,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* primary alone, another node reached wait_standby */ { .pos = 401, .sectionPath = { - MONITOR_FSM_SECTION_PRIMARY_NODE + MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_SINGLE) }, .conditions = { .anyOtherNodeWaitingStandby = BOOL_TRUE }, @@ -3536,7 +3550,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* all nodes async, zero secondaries */ { .pos = 403, .sectionPath = { - MONITOR_FSM_SECTION_PRIMARY_NODE + MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, @@ -3550,7 +3564,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* all nodes async, >=1 secondary */ { .pos = 405, .sectionPath = { - MONITOR_FSM_SECTION_PRIMARY_NODE + MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, .conditions = { .replicationQuorumCountIsZero = BOOL_TRUE, @@ -3567,7 +3581,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 407, .sectionPath = { - MONITOR_FSM_SECTION_PRIMARY_NODE + MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, @@ -3584,7 +3598,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* same, but number_sync_standbys>0 -> block writes on primary */ { .pos = 409, .sectionPath = { - MONITOR_FSM_SECTION_PRIMARY_NODE + MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_PRIMARY_OR_APPLY_SETTINGS_ONLY }, .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_TRUE, @@ -3601,7 +3615,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* wait_primary, >=1 quorum secondary */ { .pos = 411, .sectionPath = { - MONITOR_FSM_SECTION_PRIMARY_NODE + MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_PRIMARY) }, .conditions = { .secondaryQuorumNodesCountIsZero = BOOL_FALSE }, @@ -3614,7 +3628,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* apply_settings, both zero */ { .pos = 413, .sectionPath = { - MONITOR_FSM_SECTION_PRIMARY_NODE + MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, @@ -3628,7 +3642,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* apply_settings, number_sync_standbys != 0 (1 of 2 disjuncts) */ { .pos = 415, .sectionPath = { - MONITOR_FSM_SECTION_PRIMARY_NODE + MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, .conditions = { .numberSyncStandbysIsZero = BOOL_FALSE }, @@ -3642,7 +3656,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* apply_settings, sync_standbys=0 but >=1 quorum secondary (2 of 2) */ { .pos = 417, .sectionPath = { - MONITOR_FSM_SECTION_PRIMARY_NODE + MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_APPLY_SETTINGS) }, .conditions = { .numberSyncStandbysIsZero = BOOL_TRUE, @@ -3659,7 +3673,7 @@ static const MonitorFSMTransition MonitorFSM[] = { */ { .pos = 419, .sectionPath = { - MONITOR_FSM_SECTION_PRIMARY_NODE + MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_PRIMARY_ROLE_STATES }, .otherNodesFn = OtherNodesDueForCatchingUp, @@ -3671,7 +3685,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* backwards-compat: join_primary -> primary */ { .pos = 421, .sectionPath = { - MONITOR_FSM_SECTION_PRIMARY_NODE + MONITOR_FSM_SECTION_PRIMARY_NODE }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_JOIN_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_PRIMARY), @@ -3777,7 +3791,8 @@ AssertMonitorFSMWellFormed(void) if (pos == MonitorFSM_MultiStandbyCascadeResumeAfterPos) { foundResumeAnchor = true; - Assert(SectionPathIsUnderPrefix(MonitorFSM[i].sectionPath, SectionReportingNode)); + Assert(SectionPathIsUnderPrefix(MonitorFSM[i].sectionPath, + SectionReportingNode)); } } @@ -4308,7 +4323,8 @@ NodeStatusPatternConditionsText(const NodeStatusPattern *pattern, bool *isNull) APPEND_BOOL_CONDITION(&buf, "canTakeWrites", pattern->canTakeWrites); APPEND_BOOL_CONDITION(&buf, "reportedCanTakeWrites", pattern->reportedCanTakeWrites); APPEND_BOOL_CONDITION(&buf, "reportedIsWaitStandby", pattern->reportedIsWaitStandby); - APPEND_BOOL_CONDITION(&buf, "reportedIsJoinSecondary", pattern->reportedIsJoinSecondary); + APPEND_BOOL_CONDITION(&buf, "reportedIsJoinSecondary", + pattern->reportedIsJoinSecondary); APPEND_BOOL_CONDITION(&buf, "reportedIsPrepareMaintenance", pattern->reportedIsPrepareMaintenance); APPEND_BOOL_CONDITION(&buf, "isReadyToStreamWAL", pattern->isReadyToStreamWAL); @@ -4871,7 +4887,7 @@ PrimaryNodeReportedStateCanBeResolved(ReplicationState state) */ static bool NodeStatusPatternSurvivesReportedCanTakeWrites(const NodeStatusPattern *pattern, - ReplicationState state) + ReplicationState state) { if (pattern->reportedCanTakeWrites == BOOL_ANY) { @@ -4893,7 +4909,7 @@ NodeStatusPatternSurvivesReportedCanTakeWrites(const NodeStatusPattern *pattern, */ static bool NodeStatusPatternSurvivesReportedIsWaitStandby(const NodeStatusPattern *pattern, - ReplicationState state) + ReplicationState state) { if (pattern->reportedIsWaitStandby == BOOL_ANY) { @@ -4915,7 +4931,7 @@ NodeStatusPatternSurvivesReportedIsWaitStandby(const NodeStatusPattern *pattern, */ static bool NodeStatusPatternSurvivesReportedIsJoinSecondary(const NodeStatusPattern *pattern, - ReplicationState state) + ReplicationState state) { if (pattern->reportedIsJoinSecondary == BOOL_ANY) { @@ -4937,7 +4953,7 @@ NodeStatusPatternSurvivesReportedIsJoinSecondary(const NodeStatusPattern *patter */ static bool NodeStatusPatternSurvivesReportedIsPrepareMaintenance(const NodeStatusPattern *pattern, - ReplicationState state) + ReplicationState state) { if (pattern->reportedIsPrepareMaintenance == BOOL_ANY) { @@ -5184,7 +5200,7 @@ RuleUnconditionallyMatchesPrimaryNodeState(const MonitorFSMTransition *rule, */ static bool EdgeIsShadowedByEarlierRule(int beforeIndex, ReplicationState state, bool primaryNodeSide, - MonitorFSMSection topLevelSection) + MonitorFSMSection topLevelSection) { for (int j = 0; j < beforeIndex; j++) { @@ -5367,7 +5383,8 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } - if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->activeNode, states[j], + if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->activeNode, + states[j], singleExcluded)) { continue; @@ -5397,7 +5414,8 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } - if (EdgeIsShadowedByEarlierRule(i, states[j], false, rule->sectionPath[0])) + if (EdgeIsShadowedByEarlierRule(i, states[j], false, + rule->sectionPath[0])) { continue; } @@ -5440,7 +5458,8 @@ dump_fsm_edges(PG_FUNCTION_ARGS) continue; } - if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->primaryNode, states[j], + if (!NodeStatusPatternSurvivesIsInPrimaryState(&rule->primaryNode, + states[j], singleExcluded)) { continue; @@ -5559,7 +5578,8 @@ ProceedGroupStateFromContext(GroupStateContext *ctx) BuildForPrimaryNodeNodeActiveContext(ctx, activeNode, &primaryNac); - return FindAndDispatchMonitorFSMRuleUnderPath(ctx, &primaryNac, SectionPrimaryNode, 0); + return FindAndDispatchMonitorFSMRuleUnderPath(ctx, &primaryNac, + SectionPrimaryNode, 0); } /* @@ -5737,7 +5757,7 @@ BuildMSFailoverCandidateGateNodeActiveContext(GroupStateContext *ctx, */ static void ActionLogMSFailoverMissingNodesDecline(GroupStateContext *ctx, NodeActiveContext *nac, - char *message) + char *message) { AutoFailoverNode *activeNode = nac->activeNode.node; @@ -5756,7 +5776,7 @@ ActionLogMSFailoverMissingNodesDecline(GroupStateContext *ctx, NodeActiveContext static void ActionLogMSFailoverMissingNodesContinue(GroupStateContext *ctx, NodeActiveContext *nac, - char *message) + char *message) { AutoFailoverNode *activeNode = nac->activeNode.node; @@ -5774,7 +5794,7 @@ ActionLogMSFailoverMissingNodesContinue(GroupStateContext *ctx, NodeActiveContex static void ActionLogMSFailoverQuorumDecline(GroupStateContext *ctx, NodeActiveContext *nac, - char *message) + char *message) { AutoFailoverNode *activeNode = nac->activeNode.node; int minCandidates = ctx->formation->number_sync_standbys + 1; @@ -5797,7 +5817,7 @@ ActionLogMSFailoverQuorumDecline(GroupStateContext *ctx, NodeActiveContext *nac, static void ActionLogMSFailoverQuorumContinue(GroupStateContext *ctx, NodeActiveContext *nac, - char *message) + char *message) { AutoFailoverNode *activeNode = nac->activeNode.node; int minCandidates = ctx->formation->number_sync_standbys + 1; @@ -6085,7 +6105,8 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, */ NodeActiveContext gateNac; - BuildMSFailoverCandidateGateNodeActiveContext(ctx, primaryNode, &candidateList, &gateNac); + BuildMSFailoverCandidateGateNodeActiveContext(ctx, primaryNode, &candidateList, + &gateNac); /* * Time to select a candidate? @@ -6101,7 +6122,8 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, if (candidateList.missingNodesCount > 0) { (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, &gateNac, - SectionMSFailoverMissingNodesGate, 0); + SectionMSFailoverMissingNodesGate, + 0); if (GuardDataLoss) { @@ -6137,7 +6159,8 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, if (candidateList.quorumCandidateCount < minCandidates) { (void) FindAndDispatchMonitorFSMRuleUnderPath(ctx, &gateNac, - SectionMSFailoverQuorumCandidateGate, 0); + SectionMSFailoverQuorumCandidateGate, + 0); if (GuardDataLoss) { @@ -6372,7 +6395,7 @@ BuildCandidateList(GroupStateContext *ctx, List *nodesGroupList, ereport(ERROR, (errmsg("BUG: no MS-failover fan-out row matched " NODE_FORMAT " although its own conditions " - "should always hold here", + "should always hold here", NODE_FORMAT_ARGS(node)))); } From 2a1e61c6c225f9fef27652f5aa018c67df131a59 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 03:23:45 +0200 Subject: [PATCH 40/52] Close the last Step 2a gap: rule 211's stop_replication -> report_lsn Adds the missing KeeperFSM[] row (fsm.c) reusing fsm_report_lsn directly, closing MonitorFSM[] rule 211's only remaining gap ("alone in group, candidatePriority zero -> report_lsn", stop_replication current_state). This was previously left as a documented, believed-unfixable gap on the mistaken assumption that reaching report_lsn from stop_replication would need the same live-primary-dependent rewind/basebackup machinery used elsewhere (fsm_restart_standby/fsm_rewind_or_init, for reaching catchingup -- a materially different target that genuinely does need to stream from someone). It doesn't: fsm_report_lsn's own restart (standby_restart_with_current_replication_source) is called with an all-zeroed upstream, so it never contacts a peer at all -- it just stops Postgres, writes a disconnected standby.signal, and restarts, exactly like every other source state already reusing this same action. Postgres itself decides "am I in recovery" purely from standby.signal's presence at this startup, not from the fact that fsm_stop_replication had already promoted it onto a new timeline moments earlier. Verified live with two pgaftest specs covering both documented ways out of the resulting parked report_lsn state (group_state_machine.c's own pos 211 comment): - keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf: raising candidate-priority back above 0 promotes straight to single via the already-existing pos 209 + report_lsn -> single edge, with nothing auto-promoting the node while priority stays 0. - keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf: a new node registers, basebackups from the parked node via RegisterNode's existing report_lsn-candidate-priority-0 special case, and takes over as primary while the parked node follows it back in as a secondary (report_lsn -> secondary directly, not via join_secondary as first assumed). keeper_fsm_edges.json/.out and the sql comment narrative regenerated and updated accordingly; both new specs registered in tests/tap/schedule and schedules/node.sch. --- src/bin/pg_autoctl/fsm.c | 51 ++++++ src/monitor/expected/keeper_fsm_edges.out | 11 +- src/monitor/keeper_fsm_edges.json | 4 + src/monitor/sql/keeper_fsm_edges.sql | 54 +++++-- tests/tap/schedule | 2 + tests/tap/schedules/node.sch | 2 + ..._stop_replication_report_lsn_new_node.pgaf | 153 ++++++++++++++++++ ..._stop_replication_report_lsn_priority.pgaf | 137 ++++++++++++++++ 8 files changed, 391 insertions(+), 23 deletions(-) create mode 100644 tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf create mode 100644 tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index 064fbebb2..587755882 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -1146,6 +1146,57 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_FAILOVER }, + /* + * was one step further into that same promotion (prepare_promotion's own + * no-op already gave way to fsm_stop_replication, which really did + * promote Postgres onto a new timeline this time) but every other node + * vanished before this node could finish becoming the new primary, and + * its own candidate-priority is 0 -- reuse fsm_report_lsn exactly like + * every other converged-standby source state above. + * + * Unlike prepare_promotion, Postgres here is no longer an ordinary + * streaming standby -- fsm_stop_replication already promoted it, so it's + * a genuinely writable, disconnected primary on its own new timeline. + * That's still safe to hand to fsm_report_lsn unmodified: its own + * restart (standby_restart_with_current_replication_source, + * primary_standby.c) never contacts any peer at all -- it's a purely + * local stop/reconfigure/restart sequence, entirely oblivious to + * whether this instance was ever promoted. Concretely: it calls + * standby_init_replication_source with an all-zeroed upstream (no + * host), so the subsequent restart's own primaryNode.host check + * (IS_EMPTY_STRING_BUFFER) skips the identify-system connection + * attempt entirely, and pg_setup_standby_mode's own identical check + * skips it too -- neither ever needs a reachable primary. It just + * stops Postgres, writes a fresh standby.signal with no + * primary_conninfo, and restarts -- exactly what turns an ordinary + * disconnected standby into a report_lsn candidate elsewhere in this + * table, and Postgres itself doesn't care that this data directory's + * own history includes a promotion: recovery-on-restart is decided + * purely by standby.signal's presence, not by promotion history. + * + * This was previously left as a documented, unfixed gap based on a + * mistaken assumption that reaching report_lsn from here would need + * the same live-primary-dependent rewind/basebackup machinery + * fsm_restart_standby/fsm_rewind_or_init uses elsewhere (for reaching + * catchingup, a materially different target that actually does need to + * stream from someone) -- it doesn't; fsm_report_lsn was already the + * right tool for this row the whole time. See + * keeper_fsm_gap_stop_replication_report_lsn.pgaf for the live + * reproduction, including both ways out documented on pos 211's own + * comment (group_state_machine.c): raising candidate-priority back + * above 0 (report_lsn -> single, already an existing edge, via pos + * 209), or a new node registering and RegisterNode's own existing + * report_lsn-candidate-priority-0 special case (node_active_protocol.c) + * basebackupping from this node and taking over as primary while this + * node follows it back in as a secondary. + */ + { + STOP_REPLICATION_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn, + FSM_PHASE_FAILOVER + }, + /* * was demoting after losing the primary role (draining timed out, or a * manual failover put another node in charge) but every other node diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index d9fb681b1..643dee86f 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -142,6 +142,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; stop_replication | wait_primary stop_replication | demote_timeout stop_replication | demoted + stop_replication | report_lsn wait_maintenance | single wait_maintenance | demote_timeout wait_maintenance | demoted @@ -153,7 +154,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; wait_primary | join_primary wait_primary | apply_settings wait_standby | catchingup -(107 rows) +(108 rows) -- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A @@ -445,11 +446,9 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen (e.pos, e.assigned_state, f.comment) ) ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; - rule | n | current_state | assigned_state | comment -------+---+------------------+----------------+------------------------------------------------------ - 211 | 1 | | report_lsn | alone in group, candidatePriority zero -> report_lsn - 211 | 1 | stop_replication | report_lsn | alone in group, candidatePriority zero -> report_lsn -(2 rows) + rule | n | current_state | assigned_state | comment +------+---+---------------+----------------+--------- +(0 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/keeper_fsm_edges.json b/src/monitor/keeper_fsm_edges.json index e7940c29d..20ab42ac9 100644 --- a/src/monitor/keeper_fsm_edges.json +++ b/src/monitor/keeper_fsm_edges.json @@ -459,6 +459,10 @@ "current": "prepare_promotion", "assigned": "report_lsn" }, + { + "current": "stop_replication", + "assigned": "report_lsn" + }, { "current": "demote_timeout", "assigned": "report_lsn" diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 3fbcaab0d..24c7aa796 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -262,21 +262,40 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- the first place (a lone standby for prepare_promotion, the original -- primary itself for demote_timeout, and a losing MS-failover -- candidate for join_secondary). --- * pos 211's stop_replication current_state is the one exception left --- unfixed, and unlike the three above it is not simply "missing a row": --- fsm_stop_replication doesn't just stop a process, its own comment --- says it shuts down the replication stream "by promoting the --- replica" -- Postgres has already left recovery onto a new timeline --- by the time this state is reported. Getting back to an ordinary, --- disconnected-standby report_lsn state from there needs a real --- pg_rewind/basebackup style re-sync (fsm_restart_standby -> --- fsm_rewind_or_init, already used for MAINTENANCE_STATE/ --- PREPARE_MAINTENANCE_STATE -> CATCHINGUP_STATE), and that function's --- own first step, keeper_get_primary(), hard-requires a live, --- reachable primary to rewind against or basebackup from -- which --- cannot exist by pos 211's own "alone in group" precondition. No --- existing function can do this safely when truly alone; left as a --- documented, unfixed gap. +-- * pos 211's stop_replication current_state is now fixed too, the same +-- "missing a row" shape as the three above: reuse fsm_report_lsn +-- directly, no new keeper code. This one was briefly misdiagnosed as +-- unfixable, on the assumption that reaching report_lsn from here +-- would need the same live-primary-dependent rewind/basebackup +-- machinery fsm_restart_standby/fsm_rewind_or_init use elsewhere (for +-- MAINTENANCE_STATE/PREPARE_MAINTENANCE_STATE -> CATCHINGUP_STATE, a +-- materially different target that genuinely does need to stream from +-- someone). fsm_report_lsn is not that function: its own restart +-- (standby_restart_with_current_replication_source, primary_standby.c) +-- is called with an all-zeroed upstream, so its own primaryNode.host +-- check (IS_EMPTY_STRING_BUFFER) -- and pg_setup_standby_mode's own +-- identical check -- skip the identify-system connection attempt +-- entirely; it never contacts a peer. Concretely it just stops +-- Postgres, writes a fresh standby.signal with no primary_conninfo, +-- and restarts -- exactly what already turns every other source state +-- in this list into a report_lsn candidate, and it's oblivious to +-- whether this data directory's own history includes a promotion +-- (fsm_stop_replication really did call fsm_promote_standby to get +-- here, so Postgres is a genuinely writable, disconnected primary on +-- its own new timeline by this point -- but Postgres itself decides +-- "am I in recovery" purely from standby.signal's presence at this +-- startup, not from promotion history). See fsm.c's own comment on +-- this KeeperFSM[] row for the full argument, and +-- keeper_fsm_gap_stop_replication_report_lsn.pgaf for the live +-- reproduction of both ways out documented on pos 211's own comment +-- (group_state_machine.c): raising candidate-priority back above 0 +-- (report_lsn -> single, already an existing edge, via pos 209), or a +-- new node registering and RegisterNode's own existing +-- report_lsn-candidate-priority-0 special case (node_active_ +-- protocol.c, already exercised by +-- keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf for a different +-- source state) basebackupping from this node and taking over as +-- primary while this node follows it back in as a secondary. -- * pos 325's and pos 391's own "single" states (the primaryNode side, in -- both cases) are now fixed at the source rather than merely explained -- away: both rows' own preconditions already require a *second, @@ -290,8 +309,9 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- monitor/keeper actions can produce this combination. -- -- Previously this was left as documented-but-unfixed, the same way the --- stop_replication gap right above stays unfixed today: correct, but --- dump_fsm_edges() itself had no way to know it, since +-- stop_replication gap right above was (both are since fixed, by +-- different mechanisms): correct, but dump_fsm_edges() itself had no +-- way to know it, since -- NodeStatePatternResolveFromStates() only ever reads a row's own -- .statePattern -- every other NodeStatusPattern field, including -- isInPrimaryState (the field responsible for admitting "single" as a diff --git a/tests/tap/schedule b/tests/tap/schedule index f31660dfe..3b29df16a 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -43,6 +43,8 @@ keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff +keeper_fsm_gap_stop_replication_report_lsn_priority +keeper_fsm_gap_stop_replication_report_lsn_new_node extension_update tablespaces installcheck diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index a37f52636..0c879b3f0 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -27,3 +27,5 @@ keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff +keeper_fsm_gap_stop_replication_report_lsn_priority +keeper_fsm_gap_stop_replication_report_lsn_new_node diff --git a/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf new file mode 100644 index 000000000..aa7306936 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf @@ -0,0 +1,153 @@ +# Regression spec for MonitorFSM[] pos 211's stop_replication current_state +# ("alone in group, candidatePriority zero -> report_lsn", +# src/monitor/group_state_machine.c), and for the matching KeeperFSM[] row +# added in fsm.c (see keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf's +# own header for the full row and the shared construction technique -- this +# spec's setup/test_001/test_002/test_003 steps are identical to that one, +# reaching the exact same "node3 parked at report_lsn, candidate-priority +# 0, alone in group" state). +# +# This exercises the *second* documented way out of that parked state: a +# new node registers, and RegisterNode's own existing report_lsn- +# candidate-priority-0 special case (node_active_protocol.c, already +# proven live for a different source state by +# keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf) basebackups it from +# node3 instead of from a "real" primary, then the ordinary MS-failover +# candidate-selection machinery (BuildCandidateList/ +# SelectFailoverCandidateNode/PromoteSelectedNode) picks the new node +# (ordinary nonzero priority) over node3 (priority 0) and promotes it, +# while node3 follows it back in as a secondary -- respecting node3's own +# candidate-priority=0 the whole time, exactly like the priority-raise +# sibling spec respects it by simply never promoting node3 on its own. +# +# node4 is basebacked up directly from node3's own frozen, disconnected +# report_lsn snapshot; by the time node4 finishes and reports in, node3 +# has moved a little further along in real time, so node4 *does* pass +# through a brief fast_forward hop (report_lsn -> fast_forward -> +# prepare_promotion -> wait_primary -> primary) to pick up the small +# remaining gap before promoting -- unlike +# keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf's own sibling +# scenario, where the comment notes fast_forward is conditional on +# exactly this kind of timing. node3 itself, once node4 is selected as +# the winning candidate, goes straight report_lsn -> secondary -- not +# through join_secondary as originally guessed here (join_secondary is +# for a node still mid-handoff from an old primary it hasn't finished +# checkpointing away from; node3 has no such old-primary handoff pending, +# it's simply been sitting parked, so PromoteSelectedNode's own fan-out +# takes the more direct edge). Confirmed live, the same way the sibling +# spec's own header already flags having had to fix its first guess about +# which node ends up primary. + +cluster { + monitor + formation { + node1 + node3 no-autopilot candidate-priority 50 + node4 create and launch deferred + } +} + +setup { + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node3 state is secondary + timeout 60s +} + +teardown { + compose down +} + +step test_001_kill_primary_and_converge_locally_to_stop_replication { + compose kill node1 + sleep 30s + fsm step node3 + wait until node3 assigned-state = prepare_promotion timeout 30s + fsm step node3 + wait until node3 assigned-state = stop_replication timeout 30s +} + +step test_002_zero_priority_and_remove_old_primary_row { + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } +} + +step test_003_node3_converges_to_report_lsn { + fsm step node3 + fsm step node3 + wait until node3 state = report_lsn timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} + +step test_004_node4_joins_and_becomes_primary { + # node3 stays no-autopilot for this whole spec (step mode is a + # startup-time setting, not something a running spec can toggle off + # again once the earlier steps no longer need it), and that turns out + # to matter here in a way it didn't in the earlier steps: + # node_active()'s own HBA refresh (keeper_refresh_other_nodes, + # service_keeper.c's step-mode dispatch calls it before every step, + # same as autopilot's own per-tick loop does) is what grants node4's + # IP the pg_hba.conf rule it needs to even *reach* node3 for its + # initial basebackup -- and node3 never runs that refresh on its own + # while parked, unreported. Confirmed live: without an explicit "fsm + # step node3" right after starting node4, node4 spins forever on "no + # pg_hba.conf entry for replication connection", never even reaching + # report_lsn itself, let alone getting basebacked-up-from and + # promoted. So node3 needs a step immediately, before node4 can make + # any progress at all -- not just later, once node4's own report + # fans out a fresh join_secondary goal to node3 (which still needs + # its own later explicit steps too, same as every earlier step in + # this spec). One immediate step turned out not to be enough either: + # "pg_autoctl node start" returns as soon as the container process is + # up, before node4 has necessarily finished registering with the + # monitor -- stepping node3 too early just finds nothing new to + # refresh yet. node4's own supervisor keeps retrying every few + # seconds regardless, so a couple of staggered steps a few seconds + # apart reliably lands at least one after node4's row exists. + # + # node4 can't reach primary until node3 has actually converged to + # secondary and is counted as a synced standby -- so node3's own + # steps have to happen, and land, *before* waiting on node4's own + # final state, not after (confirmed live: waiting for node4 first + # just times out with node4 stuck at wait_primary forever, since + # nothing ever drives node3's own report_lsn -> secondary hop). + exec node4 pg_autoctl node start + sleep 10s + fsm step node3 + sleep 10s + fsm step node3 + wait until node3 assigned-state = secondary timeout 60s + fsm step node3 + fsm step node3 + wait until node3 state is secondary timeout 30s + wait until node4 state is primary timeout 90s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node4'; + } + expect { primary } + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { secondary } +} + +sequence + test_001_kill_primary_and_converge_locally_to_stop_replication + test_002_zero_priority_and_remove_old_primary_row + test_003_node3_converges_to_report_lsn + test_004_node4_joins_and_becomes_primary diff --git a/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf new file mode 100644 index 000000000..136a2d228 --- /dev/null +++ b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf @@ -0,0 +1,137 @@ +# Regression spec for MonitorFSM[] pos 211's stop_replication current_state +# ("alone in group, candidatePriority zero -> report_lsn", +# src/monitor/group_state_machine.c), and for the matching KeeperFSM[] row +# added in fsm.c: +# +# { +# STOP_REPLICATION_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, +# COMMENT_SECONDARY_TO_REPORT_LSN, +# &fsm_report_lsn, +# FSM_PHASE_FAILOVER +# }, +# +# This exercises the first of the two documented ways out of that parked +# report_lsn state: raising candidate-priority back above 0. The second +# way out (a new node registers and takes over as primary) is +# keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf's own job. +# +# Same construction technique as +# keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf, one +# step further: that spec freezes node3 at prepare_promotion and lets +# pos 211 assign it report_lsn directly (a no-op hop in, so nothing here +# needs step-mode to hold still for it). Reaching stop_replication instead +# needs one more explicit "fsm step node3" first, since entering +# stop_replication is *not* a no-op -- fsm_stop_replication really does +# call fsm_promote_standby, physically promoting node3's own Postgres onto +# a new timeline -- and step mode is exactly what lets the spec hold node3 +# frozen there, unreported, for as long as it takes to zero its priority +# and remove the old primary's row before node3's next contact. +# +# Cluster shape: plain 2-node, same reasoning as the prepare_promotion +# sibling spec -- node3 is the formation's only standby, so failing node1 +# assigns it straight through secondary -> prepare_promotion -> (this +# spec's own extra step) stop_replication, with no report_lsn hop in +# between. + +cluster { + monitor + formation { + node1 + node3 no-autopilot candidate-priority 50 + } +} + +setup { + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node3 state is secondary + timeout 60s +} + +teardown { + compose down +} + +step test_001_kill_primary_and_converge_locally_to_stop_replication { + # First "fsm step node3": reports "secondary" (unchanged), receives + # the fresh prepare_promotion assignment, and performs that hop in the + # same call (fsm_prepare_standby_for_promotion is a no-op -- Postgres + # is untouched, still an ordinary streaming standby). + # + # Second "fsm step node3": reports "prepare_promotion" (the state the + # first call just reached, not yet reported before now), receives the + # ordinary promotion cascade's next assignment (stop_replication -- + # node1's row still exists at this point, just unhealthy, so this is + # the everyday 2-node failover path, not pos 211 yet), and performs + # *that* hop in the same call -- this is where node3's own Postgres + # actually gets promoted (fsm_stop_replication -> fsm_promote_standby). + compose kill node1 + sleep 30s + fsm step node3 + wait until node3 assigned-state = prepare_promotion timeout 30s + fsm step node3 + wait until node3 assigned-state = stop_replication timeout 30s +} + +step test_002_zero_priority_and_remove_old_primary_row { + # Same timing requirement as the prepare_promotion sibling spec: both + # of these have to land before node3's next contact, while it's still + # frozen at stop_replication, unreported -- otherwise the ordinary + # cascade would carry on past stop_replication instead of pos 211's + # "alone in group, candidatePriority zero" condition ever getting a + # chance to apply. + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); + } + sql monitor { + SELECT pgautofailover.remove_node(nodehost, nodeport, true) + FROM pgautofailover.node WHERE nodename = 'node1'; + } +} + +step test_003_node3_converges_to_report_lsn { + # First call performs the stop_replication -> report_lsn transition + # under test (reports "stop_replication", receives report_lsn from + # pos 211, applies it via the new KeeperFSM[] row -- fsm_report_lsn, + # no live peer contacted). Second call reports the result back. + fsm step node3 + fsm step node3 + wait until node3 state = report_lsn timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } + sql monitor { + SELECT goalstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { report_lsn } +} + +step test_004_raise_priority_and_converge_to_single { + # Respects the operator's own candidate-priority=0 declaration the + # whole time up to here: nothing auto-promotes node3 while it stays + # 0 (BuildCandidateList/GroupListCandidates would never select it). + # Only once priority is explicitly raised back above 0 does pos 209 + # ("alone in group, candidate-eligible -> single") start matching, and + # report_lsn -> single is already an ordinary existing KeeperFSM[] + # edge -- no new keeper code needed for this half. + sql monitor { + SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 50); + } + fsm step node3 + fsm step node3 + wait until node3 state = single timeout 30s + sql monitor { + SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; + } + expect { single } +} + +sequence + test_001_kill_primary_and_converge_locally_to_stop_replication + test_002_zero_priority_and_remove_old_primary_row + test_003_node3_converges_to_report_lsn + test_004_raise_priority_and_converge_to_single From c8fb371154d9f0464f5d155c8b5839e0d732d54f Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 04:44:46 +0200 Subject: [PATCH 41/52] keeper_fsm_edges.sql: trim comments to mechanics, drop fix history The Step 2a/2b comments had accumulated a full narrative of every investigation and fix across many sessions (pos 303, 325, 333/339/347/ 349/351, 209/211's fast_forward/join_secondary/prepare_maintenance/ stop_replication, ...). None of that is needed to understand what this test does or how to read its output, and since expected/keeper_fsm_edges.out echoes the .sql file's own comments verbatim, every one of those historical asides was also a latent maintenance trap: editing a comment without regenerating the expected file breaks the test on the next run (as just happened here). Trimmed both files down to what a reader actually needs: what Step 1's fixture load and "any" sentinel do, what a non-empty Step 2a/2b result means, and the one standing expected exception (Step 2b's "any -> dropped" row). 398 lines of sql/keeper_fsm_edges.sql down to 116; no change to query logic or expected results otherwise. --- src/monitor/expected/keeper_fsm_edges.out | 288 +------------------- src/monitor/sql/keeper_fsm_edges.sql | 308 +--------------------- 2 files changed, 28 insertions(+), 568 deletions(-) diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 643dee86f..90060dbeb 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -158,17 +158,14 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A --- non-empty result here is a real, actionable gap (unlike --- check_fsm_reachability.sql's own synthetic-input test, which only --- exercises the comparison mechanism itself, not real data): see this --- project's own investigation of these mismatches (dump_fsm_edges()'s own --- comment, group_state_machine.c, and the design doc) for which of them --- are genuine keeper gaps versus artifacts already excluded upstream. +-- non-empty result here is a real, actionable gap: the monitor can assign a +-- transition the keeper has no KeeperFSM[] row to perform. See +-- dump_fsm_edges()'s own comment (group_state_machine.c) for exactly what +-- it resolves and what it deliberately excludes (e.g. the api_triggered +-- section, resolved via hand-written C rather than a NodeStatePattern). -- -- k.current_state = 'any' matches every e.current_state -- a keeper row --- covering every current state also covers this specific one, so it counts --- as a match here exactly like a literal (e.current_state, e.assigned_state) --- row would. +-- covering every current state also covers this specific one. -- -- GROUPING SETS adds one summary row per rule (pos, assigned_state, comment) -- -- current_state NULL, n = how many current_states that single @@ -178,260 +175,8 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. -- --- As of this writing the reporting_node role/predicate cohort that used to --- live below (each an "alone in group"/failover/Citus-worker rule whose --- NodeStatePattern is a role or predicate check -- e.g. !IsCurrentState(...), --- an opaque NodeIsXxx() helper, or no state restriction at all -- rather than --- an enumerated state list) is down to zero remaining gap rows: originally --- 10 rules/134 detail rows (pos 303/325/333/339/347/349/351, plus pos 391 in --- the MS-failover section below, which was never part of this specific --- cohort but shares the same isInPrimaryState field), closed via three --- distinct mechanisms rather than one blanket fix: --- --- * pos 303: StateCanSatisfyIsInPrimaryState() taught the 5-state set --- IsInPrimaryState() can ever admit at all. --- * pos 325 (and, as a side effect, pos 391): the same function --- additionally taught to exclude SINGLE when a rule's own .conditions --- already prove the group has more than one node. --- * pos 333/339/347/349/351: two rounds of dump_fsm_edges() narrowing --- (below) closed 4 of their up-to-15 current_states as false positives; --- the remaining 11 were then closed for real, by adding matching --- KeeperFSM[] rows -- see the second paragraph below. --- --- pos 333/339/347/349/351's own DROPPED current_state was a false positive, --- fixed at the source rather than merely explained: every one of them --- matches .primaryNode/.otherNode against --- GetPrimaryOrDemotedNodeInGroupFromList()'s own resolved node (see --- ProceedGroupStateFromContext's own call to it), and that resolver can --- never return a node reporting DROPPED -- its own two-phase logic --- excludes it outright (phase 1 requires a writable goalState, phase 2's --- fallback target set doesn't include it either), and it's structurally --- unreachable besides: a node's reportedState only becomes DROPPED once its --- own goalState is already DROPPED, and pos 201 (early_checks) removes that --- row from the catalog atomically, in the very same node_active() call that --- converges it -- so a DROPPED-reporting node never persists long enough --- for a later node's own node_active() call to see it. WAIT_STANDBY and --- JOIN_SECONDARY were two more false positives in the same cohort, found by --- asking a sharper version of the same question: is there any row --- *anywhere* in MonitorFSM[] that ever assigns one of the 5 writable goals --- to a node currently reporting this state? For most of these 5 rows' --- current_states the answer is yes (pos 209 alone, "alone in group -> --- SINGLE", covers most of them -- see its own comment), which is exactly --- what makes those genuinely reachable rather than false positives. But --- WAIT_STANDBY and JOIN_SECONDARY are two of the three states pos 209 --- itself explicitly excludes (split-brain/data-loss risk, same reasoning as --- its own header comment), and an exhaustive grep of every other row --- matching either state (pos 315/317/319 for WAIT_STANDBY, pos 359/361 for --- JOIN_SECONDARY) shows all five assign only CATCHINGUP/SECONDARY, never a --- writable goal -- so no row anywhere ever gives --- GetPrimaryOrDemotedNodeInGroupFromList()'s phase 1 a way to select a node --- reporting either. All three exclusions (DROPPED, WAIT_STANDBY, --- JOIN_SECONDARY) live in PrimaryNodeReportedStateCanBeResolved() --- (group_state_machine.c) -- unconditional, not gated on any row's own --- .conditions, so it applies to every row reaching that part of --- dump_fsm_edges(). Unlike DROPPED's exclusion (a structural invariant), --- WAIT_STANDBY/JOIN_SECONDARY rest on the current table's own contents, so --- they'd need revisiting if a future row is ever added assigning a writable --- goal from either state. --- --- Every one of these 5 rows' remaining current_states -- INIT, SINGLE, --- CATCHINGUP, SECONDARY, PREP_PROMOTION, STOP_REPLICATION, MAINTENANCE, --- PREPARE_MAINTENANCE, WAIT_MAINTENANCE, REPORT_LSN, FAST_FORWARD -- turned --- out to be genuinely reachable, not further false positives: primaryNode --- here carries no .isInPrimaryState requirement (unlike pos 325/391), so --- nothing rules out a primary that converged to some ordinary state via pos --- 209's own broad "alone in group -> SINGLE" row, then had a second node --- register (bumping the primary's own *goal* onward, e.g. to WAIT_PRIMARY, --- as part of that registration), then died or partitioned before ever --- reporting past whatever it last converged to -- GetPrimaryOrDemotedNode --- InGroupFromList()'s own phase 1 checks only goalState, so it would still --- resolve this stale node as primaryNode. PREP_PROMOTION/STOP_REPLICATION --- are additionally, independently reachable via pos 333/335/341 and pos --- 343-353's own self-referential edges; SINGLE and PREPARE_MAINTENANCE via --- the resolver's own phase 2 fallback directly. Since these are real, not --- false positives, the fix belongs on the keeper side, not another --- dump_fsm_edges() narrowing: KeeperFSM[] (fsm.c) gained 23 new rows (11 --- states -> DEMOTED_STATE, covering 333/347/349/351's shared target; the --- same 11 plus DEMOTED_STATE itself -> DEMOTE_TIMEOUT_STATE, covering 339's --- own different target), all reusing the existing, role-agnostic --- fsm_stop_postgres action -- the same "make sure Postgres is stopped" --- function every ordinary primary-track source state (PRIMARY, JOIN_PRIMARY, --- APPLY_SETTINGS, DRAINING, WAIT_PRIMARY, DEMOTE_TIMEOUT) already reuses for --- this same target, safe here for the same reason: it doesn't matter what --- this node's Postgres was actually doing when it stopped reporting, only --- that it's stopped now. See the new KeeperFSM[] rows' own comment (fsm.c, --- right after the WAIT_PRIMARY_STATE -> DEMOTED_STATE row) for the full --- argument, and keeper_fsm_edges.json (regenerated via --- "pg_autoctl inspect fsm list --json" after this change, committed --- alongside it, per this file's own header comment on that fixture). --- --- Follow-up investigation of pos 209/211/325's own remaining gap states --- (after wait_maintenance and wait_standby were resolved -- see --- group_state_machine.c's reportedIsWaitStandby field (NodeStatusPattern) --- and KeeperFSM[]'s new WAIT_MAINTENANCE_STATE rows, fsm.c): --- --- * pos 209/211's fast_forward current_state is now genuinely fixed and --- covered: a lone node reporting fast_forward (its WAL-source peer and --- the old primary both gone mid MS-failover) now has matching --- KeeperFSM[] rows (FAST_FORWARD_STATE -> SINGLE_STATE / --- REPORT_LSN_STATE, fsm.c, reusing fsm_promote_standby/fsm_report_lsn --- exactly like every other converged-standby source state) and real --- pgaftest coverage (keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf, --- keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf) reproducing --- a genuine MS-failover candidate falling behind, fetching real WAL, and --- then being left alone -- see those specs' own headers for why --- node1/node2 must be removed from the group while node3 is still --- fetching (not after it reports fast_forward) to actually exercise --- these rows instead of racing against the monitor's own cascade --- continuation. --- * pos 209's join_secondary and prepare_maintenance current_states are no --- longer even in this gap list at all: both were found to be a genuine --- data-loss risk, not a missing convenience -- promoting either straight --- to SINGLE if left alone can silently discard writes a *different, --- already-promoted* primary made in the meantime (join_secondary: that --- new primary already exists by the time this node reaches --- join_secondary; prepare_maintenance: pos 343 lets the candidate --- standby reach primary the moment this node's own reportedState merely --- converges to prepare_maintenance, no removal required). Fixed by --- excluding both from pos 209 itself (reportedIsJoinSecondary, --- reportedIsPrepareMaintenance on NodeMatchesPattern) rather than adding --- KeeperFSM[] rows -- there is nothing safe to promote either one to. --- prepare_maintenance additionally needed a new no-op row (pos 208): --- unlike join_secondary (already recognized by node_metadata.c's --- IsParticipatingInPromotion, so a lone node there safely no-ops on its --- own), a lone prepare_maintenance node isn't recognized by that --- function, IsBeingPromoted, or IsInPrimaryState, so excluding it from --- pos 209 alone would have left ProceedGroupStateFromContext's own --- "couldn't find the primary node" guard to ereport(ERROR) on every --- single subsequent heartbeat -- worse than the original bug. Both --- fixes verified live: keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf --- reproduces a lone prepare_maintenance primary staying safely parked --- (goalstate never becomes single) and confirms it keeps successfully --- checking in (reporttime advancing) rather than looping on that error. --- * pos 211's own join_secondary/prepare_promotion/demote_timeout --- current_states are now genuinely fixed and covered too, via a much --- simpler path than pos 209's: report_lsn never grants write access, so --- none of pos 209's split-brain argument carries over here -- there was --- nothing *unsafe* about these, they were simply missing KeeperFSM[] --- rows, the exact same shape of gap fast_forward had. All three reuse --- fsm_report_lsn directly: --- - prepare_promotion: entering it (fsm_prepare_standby_for_promotion) --- is a no-op -- Postgres is untouched, still an ordinary streaming --- standby -- so this is exactly as safe as SECONDARY/CATCHINGUP's --- own existing rows. --- - demote_timeout: fsm_stop_replication already sets --- default_transaction_read_only=on before this state is ever --- reported, so no writes can have landed here that a real primary --- elsewhere wouldn't also already have. --- - join_secondary: Postgres was cleanly checkpointed and stopped --- (fsm_checkpoint_and_stop_postgres) before reaching this state -- --- a trustworthy, consistent copy of data that hasn't been --- superseded by anything (unlike pos 209's own join_secondary --- concern, there is no new primary for this data to have fallen --- behind, since candidatePriority=0 was never in the running to --- become one). --- fsm_report_lsn's own restart (standby_restart_with_current_ --- replication_source, primary_standby.c) handles all three uniformly: --- it stops Postgres if running, rewrites the recovery config with no --- primary_conninfo, and restarts -- it never needs to reach any peer, --- so it doesn't matter that none exist. --- --- Verified via this test (Step 1/2a, the keeper_fsm_edges.json fixture --- and dump_fsm_edges() both resolving these three edges consistently), --- the code reasoning above, AND real live pgaftest reproduction for all --- three (keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf, --- keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf, --- keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf). --- A first attempt at live reproduction (an earlier session) found --- exactly the race fast_forward's own fix doesn't have: fsm_prepare_ --- standby_for_promotion is a no-op, so the monitor's cascade advances --- from assigned=prepare_promotion straight through to stop_replication --- within the same heartbeat, before any external test script's own --- "remove the other peers" SQL call can land in between. (That same --- attempt did incidentally confirm live that stop_replication really --- is a dead end below: node3 sat reporting stop_replication, endlessly --- reassigned report_lsn by this same pos 211 row, with no KeeperFSM[] --- row able to reach it.) --- --- Step mode (PG_AUTOCTL_STEP_MODE, src/bin/pg_autoctl/step_socket.c) --- removes that race entirely: a "no-autopilot" node's node-active --- service never ticks on its own, so a pgaftest spec can drive it to --- converge locally to prepare_promotion/demote_timeout/join_secondary --- via one explicit "fsm step " (see fsm_step_cmd, --- src/bin/pgaftest/test_spec_parse.y) and then simply not issue the --- next one for as long as it likes -- there is no cascade to race --- against, because nothing happens without an explicit command. Once --- frozen there, unreported to the monitor, each spec zeroes the node's --- own candidate priority and force-removes its former peers, then a --- single further "fsm step" reports the frozen state, receives the --- newly-applicable report_lsn assignment from pos 211, and performs --- the transition under test in the very same call. See those three --- specs' own headers for the full mechanics, including why each one's --- particular current_state needs a different cluster shape to reach in --- the first place (a lone standby for prepare_promotion, the original --- primary itself for demote_timeout, and a losing MS-failover --- candidate for join_secondary). --- * pos 211's stop_replication current_state is the one exception left --- unfixed, and unlike the three above it is not simply "missing a row": --- fsm_stop_replication doesn't just stop a process, its own comment --- says it shuts down the replication stream "by promoting the --- replica" -- Postgres has already left recovery onto a new timeline --- by the time this state is reported. Getting back to an ordinary, --- disconnected-standby report_lsn state from there needs a real --- pg_rewind/basebackup style re-sync (fsm_restart_standby -> --- fsm_rewind_or_init, already used for MAINTENANCE_STATE/ --- PREPARE_MAINTENANCE_STATE -> CATCHINGUP_STATE), and that function's --- own first step, keeper_get_primary(), hard-requires a live, --- reachable primary to rewind against or basebackup from -- which --- cannot exist by pos 211's own "alone in group" precondition. No --- existing function can do this safely when truly alone; left as a --- documented, unfixed gap. --- * pos 325's and pos 391's own "single" states (the primaryNode side, in --- both cases) are now fixed at the source rather than merely explained --- away: both rows' own preconditions already require a *second, --- distinctly-matched* node (activeNode for 325, the healthy candidate --- counted by atLeastOneHealthyCandidate for 391) to exist in the same --- group as primaryNode, which makes primaryNode genuinely reporting --- SINGLE ("alone in my own group") a real model contradiction -- the --- instant a second node registers, the primary's own goal moves off --- single (to wait_primary) as part of that registration, before the --- joining node could ever reach secondary. No real sequence of --- monitor/keeper actions can produce this combination. --- --- Previously this was left as documented-but-unfixed, the same way the --- stop_replication gap right above stays unfixed today: correct, but --- dump_fsm_edges() itself had no way to know it, since --- NodeStatePatternResolveFromStates() only ever reads a row's own --- .statePattern -- every other NodeStatusPattern field, including --- isInPrimaryState (the field responsible for admitting "single" as a --- candidate primaryNode state at all), is invisible to it by --- construction. pos 325 now spells out explicitly, via its own --- .conditions, exactly the invariant its shape already implied --- (groupHasExactlyOneNode = BOOL_FALSE -- two distinctly-matched roles --- can't coexist in a one-node group); StateCanSatisfyIsInPrimaryState() --- was taught to read that (and the stronger, already-present --- groupHasMoreThanTwoNodes = BOOL_TRUE, which implies it) via a new --- singleExcluded parameter, and exclude SINGLE from the reachable state --- set whenever it's set. pos 391 needed no rule change at all: it --- already carried groupHasMoreThanTwoNodes = BOOL_TRUE for an unrelated --- reason (the MS-failover cascade's own "more than two nodes" gate), --- so it started benefiting from the same narrowing immediately. --- --- This narrowing is deliberately still scoped to isInPrimaryState only, --- same restraint StateCanSatisfyIsInPrimaryState's own comment already --- documents for pos 303: several sibling NodeStatusPattern fields --- (isInMaintenance, canTakeWrites, drainTimeExpired, --- unreachableFromDemoteTimeout -- see pos 333/339/347/349/351's own --- still-wide gap lists below) are just as state-dependent in principle, --- but each is goal-state- or wall-clock-dependent rather than a plain --- reportedState equality check, and needs its own from-scratch --- satisfiability proof before narrowing it the same way is safe -- --- isInMaintenance in particular already has a documented counterexample --- (pos 369, via EdgeIsShadowedByEarlierRule's own investigation) of a --- reportedState assumed incompatible with a goal-dependent condition --- turning out to be reachable anyway. Narrowing those without doing --- that same diligence for each risks reintroducing exactly that class --- of bug, so they remain unnarrowed for now. +-- Expected result: empty. Every MonitorFSM[] rule currently has a matching +-- KeeperFSM[] row for every current_state it can assign a transition from. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos @@ -463,18 +208,13 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen -- current_state for which the monitor assigns this same target is enough -- to say the target is implementable at all, so a gap here means the -- monitor can NEVER produce this assigned_state from ANY current state -- --- e.g. "any -> dropped": KeeperFSM[]'s two ANY_STATE -> DROPPED rows --- collapse to this single row (see Step 1's own comment on why "any" isn't --- expanded per-state anymore -- the previous per-state expansion turned --- this one keeper rule into 21 separate flagged rows here, a lot of --- redundant noise for a single underlying fact), and it stays flagged --- because dump_fsm_edges() really can never produce a 'dropped' edge at --- all: the monitor's own equivalent (remove_node(), pos 101/103) lives +-- e.g. "any -> dropped" (KeeperFSM[]'s two ANY_STATE -> DROPPED rows +-- collapse to this single row) is a standing, expected exception: the +-- monitor's own DROPPED assignment (remove_node(), pos 101/103) lives -- entirely in the api_triggered section, which dump_fsm_edges() --- deliberately excludes (see its own comment) since those rows resolve --- their target via hand-written C, not a NodeStatePattern. Same --- "investigate before assuming which" caveat as the rest of this file --- applies to every row below, "any" or not. +-- deliberately excludes (see its own comment), so it can never appear +-- here. Same "investigate before assuming which" caveat as the rest of +-- this file applies to every other row below. SELECT k.current_state, k.assigned_state FROM keeper_fsm_edges k WHERE NOT EXISTS ( diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 24c7aa796..276738592 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -49,17 +49,14 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A --- non-empty result here is a real, actionable gap (unlike --- check_fsm_reachability.sql's own synthetic-input test, which only --- exercises the comparison mechanism itself, not real data): see this --- project's own investigation of these mismatches (dump_fsm_edges()'s own --- comment, group_state_machine.c, and the design doc) for which of them --- are genuine keeper gaps versus artifacts already excluded upstream. +-- non-empty result here is a real, actionable gap: the monitor can assign a +-- transition the keeper has no KeeperFSM[] row to perform. See +-- dump_fsm_edges()'s own comment (group_state_machine.c) for exactly what +-- it resolves and what it deliberately excludes (e.g. the api_triggered +-- section, resolved via hand-written C rather than a NodeStatePattern). -- -- k.current_state = 'any' matches every e.current_state -- a keeper row --- covering every current state also covers this specific one, so it counts --- as a match here exactly like a literal (e.current_state, e.assigned_state) --- row would. +-- covering every current state also covers this specific one. -- -- GROUPING SETS adds one summary row per rule (pos, assigned_state, comment) -- -- current_state NULL, n = how many current_states that single @@ -69,280 +66,8 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. -- --- As of this writing the reporting_node role/predicate cohort that used to --- live below (each an "alone in group"/failover/Citus-worker rule whose --- NodeStatePattern is a role or predicate check -- e.g. !IsCurrentState(...), --- an opaque NodeIsXxx() helper, or no state restriction at all -- rather than --- an enumerated state list) is down to zero remaining gap rows: originally --- 10 rules/134 detail rows (pos 303/325/333/339/347/349/351, plus pos 391 in --- the MS-failover section below, which was never part of this specific --- cohort but shares the same isInPrimaryState field), closed via three --- distinct mechanisms rather than one blanket fix: --- --- * pos 303: StateCanSatisfyIsInPrimaryState() taught the 5-state set --- IsInPrimaryState() can ever admit at all. --- * pos 325 (and, as a side effect, pos 391): the same function --- additionally taught to exclude SINGLE when a rule's own .conditions --- already prove the group has more than one node. --- * pos 333/339/347/349/351: two rounds of dump_fsm_edges() narrowing --- (below) closed 4 of their up-to-15 current_states as false positives; --- the remaining 11 were then closed for real, by adding matching --- KeeperFSM[] rows -- see the second paragraph below. --- --- pos 333/339/347/349/351's own DROPPED current_state was a false positive, --- fixed at the source rather than merely explained: every one of them --- matches .primaryNode/.otherNode against --- GetPrimaryOrDemotedNodeInGroupFromList()'s own resolved node (see --- ProceedGroupStateFromContext's own call to it), and that resolver can --- never return a node reporting DROPPED -- its own two-phase logic --- excludes it outright (phase 1 requires a writable goalState, phase 2's --- fallback target set doesn't include it either), and it's structurally --- unreachable besides: a node's reportedState only becomes DROPPED once its --- own goalState is already DROPPED, and pos 201 (early_checks) removes that --- row from the catalog atomically, in the very same node_active() call that --- converges it -- so a DROPPED-reporting node never persists long enough --- for a later node's own node_active() call to see it. WAIT_STANDBY and --- JOIN_SECONDARY were two more false positives in the same cohort, found by --- asking a sharper version of the same question: is there any row --- *anywhere* in MonitorFSM[] that ever assigns one of the 5 writable goals --- to a node currently reporting this state? For most of these 5 rows' --- current_states the answer is yes (pos 209 alone, "alone in group -> --- SINGLE", covers most of them -- see its own comment), which is exactly --- what makes those genuinely reachable rather than false positives. But --- WAIT_STANDBY and JOIN_SECONDARY are two of the three states pos 209 --- itself explicitly excludes (split-brain/data-loss risk, same reasoning as --- its own header comment), and an exhaustive grep of every other row --- matching either state (pos 315/317/319 for WAIT_STANDBY, pos 359/361 for --- JOIN_SECONDARY) shows all five assign only CATCHINGUP/SECONDARY, never a --- writable goal -- so no row anywhere ever gives --- GetPrimaryOrDemotedNodeInGroupFromList()'s phase 1 a way to select a node --- reporting either. All three exclusions (DROPPED, WAIT_STANDBY, --- JOIN_SECONDARY) live in PrimaryNodeReportedStateCanBeResolved() --- (group_state_machine.c) -- unconditional, not gated on any row's own --- .conditions, so it applies to every row reaching that part of --- dump_fsm_edges(). Unlike DROPPED's exclusion (a structural invariant), --- WAIT_STANDBY/JOIN_SECONDARY rest on the current table's own contents, so --- they'd need revisiting if a future row is ever added assigning a writable --- goal from either state. --- --- Every one of these 5 rows' remaining current_states -- INIT, SINGLE, --- CATCHINGUP, SECONDARY, PREP_PROMOTION, STOP_REPLICATION, MAINTENANCE, --- PREPARE_MAINTENANCE, WAIT_MAINTENANCE, REPORT_LSN, FAST_FORWARD -- turned --- out to be genuinely reachable, not further false positives: primaryNode --- here carries no .isInPrimaryState requirement (unlike pos 325/391), so --- nothing rules out a primary that converged to some ordinary state via pos --- 209's own broad "alone in group -> SINGLE" row, then had a second node --- register (bumping the primary's own *goal* onward, e.g. to WAIT_PRIMARY, --- as part of that registration), then died or partitioned before ever --- reporting past whatever it last converged to -- GetPrimaryOrDemotedNode --- InGroupFromList()'s own phase 1 checks only goalState, so it would still --- resolve this stale node as primaryNode. PREP_PROMOTION/STOP_REPLICATION --- are additionally, independently reachable via pos 333/335/341 and pos --- 343-353's own self-referential edges; SINGLE and PREPARE_MAINTENANCE via --- the resolver's own phase 2 fallback directly. Since these are real, not --- false positives, the fix belongs on the keeper side, not another --- dump_fsm_edges() narrowing: KeeperFSM[] (fsm.c) gained 23 new rows (11 --- states -> DEMOTED_STATE, covering 333/347/349/351's shared target; the --- same 11 plus DEMOTED_STATE itself -> DEMOTE_TIMEOUT_STATE, covering 339's --- own different target), all reusing the existing, role-agnostic --- fsm_stop_postgres action -- the same "make sure Postgres is stopped" --- function every ordinary primary-track source state (PRIMARY, JOIN_PRIMARY, --- APPLY_SETTINGS, DRAINING, WAIT_PRIMARY, DEMOTE_TIMEOUT) already reuses for --- this same target, safe here for the same reason: it doesn't matter what --- this node's Postgres was actually doing when it stopped reporting, only --- that it's stopped now. See the new KeeperFSM[] rows' own comment (fsm.c, --- right after the WAIT_PRIMARY_STATE -> DEMOTED_STATE row) for the full --- argument, and keeper_fsm_edges.json (regenerated via --- "pg_autoctl inspect fsm list --json" after this change, committed --- alongside it, per this file's own header comment on that fixture). --- --- Follow-up investigation of pos 209/211/325's own remaining gap states --- (after wait_maintenance and wait_standby were resolved -- see --- group_state_machine.c's reportedIsWaitStandby field (NodeStatusPattern) --- and KeeperFSM[]'s new WAIT_MAINTENANCE_STATE rows, fsm.c): --- --- * pos 209/211's fast_forward current_state is now genuinely fixed and --- covered: a lone node reporting fast_forward (its WAL-source peer and --- the old primary both gone mid MS-failover) now has matching --- KeeperFSM[] rows (FAST_FORWARD_STATE -> SINGLE_STATE / --- REPORT_LSN_STATE, fsm.c, reusing fsm_promote_standby/fsm_report_lsn --- exactly like every other converged-standby source state) and real --- pgaftest coverage (keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf, --- keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf) reproducing --- a genuine MS-failover candidate falling behind, fetching real WAL, and --- then being left alone -- see those specs' own headers for why --- node1/node2 must be removed from the group while node3 is still --- fetching (not after it reports fast_forward) to actually exercise --- these rows instead of racing against the monitor's own cascade --- continuation. --- * pos 209's join_secondary and prepare_maintenance current_states are no --- longer even in this gap list at all: both were found to be a genuine --- data-loss risk, not a missing convenience -- promoting either straight --- to SINGLE if left alone can silently discard writes a *different, --- already-promoted* primary made in the meantime (join_secondary: that --- new primary already exists by the time this node reaches --- join_secondary; prepare_maintenance: pos 343 lets the candidate --- standby reach primary the moment this node's own reportedState merely --- converges to prepare_maintenance, no removal required). Fixed by --- excluding both from pos 209 itself (reportedIsJoinSecondary, --- reportedIsPrepareMaintenance on NodeMatchesPattern) rather than adding --- KeeperFSM[] rows -- there is nothing safe to promote either one to. --- prepare_maintenance additionally needed a new no-op row (pos 208): --- unlike join_secondary (already recognized by node_metadata.c's --- IsParticipatingInPromotion, so a lone node there safely no-ops on its --- own), a lone prepare_maintenance node isn't recognized by that --- function, IsBeingPromoted, or IsInPrimaryState, so excluding it from --- pos 209 alone would have left ProceedGroupStateFromContext's own --- "couldn't find the primary node" guard to ereport(ERROR) on every --- single subsequent heartbeat -- worse than the original bug. Both --- fixes verified live: keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff.pgaf --- reproduces a lone prepare_maintenance primary staying safely parked --- (goalstate never becomes single) and confirms it keeps successfully --- checking in (reporttime advancing) rather than looping on that error. --- * pos 211's own join_secondary/prepare_promotion/demote_timeout --- current_states are now genuinely fixed and covered too, via a much --- simpler path than pos 209's: report_lsn never grants write access, so --- none of pos 209's split-brain argument carries over here -- there was --- nothing *unsafe* about these, they were simply missing KeeperFSM[] --- rows, the exact same shape of gap fast_forward had. All three reuse --- fsm_report_lsn directly: --- - prepare_promotion: entering it (fsm_prepare_standby_for_promotion) --- is a no-op -- Postgres is untouched, still an ordinary streaming --- standby -- so this is exactly as safe as SECONDARY/CATCHINGUP's --- own existing rows. --- - demote_timeout: fsm_stop_replication already sets --- default_transaction_read_only=on before this state is ever --- reported, so no writes can have landed here that a real primary --- elsewhere wouldn't also already have. --- - join_secondary: Postgres was cleanly checkpointed and stopped --- (fsm_checkpoint_and_stop_postgres) before reaching this state -- --- a trustworthy, consistent copy of data that hasn't been --- superseded by anything (unlike pos 209's own join_secondary --- concern, there is no new primary for this data to have fallen --- behind, since candidatePriority=0 was never in the running to --- become one). --- fsm_report_lsn's own restart (standby_restart_with_current_ --- replication_source, primary_standby.c) handles all three uniformly: --- it stops Postgres if running, rewrites the recovery config with no --- primary_conninfo, and restarts -- it never needs to reach any peer, --- so it doesn't matter that none exist. --- --- Verified via this test (Step 1/2a, the keeper_fsm_edges.json fixture --- and dump_fsm_edges() both resolving these three edges consistently), --- the code reasoning above, AND real live pgaftest reproduction for all --- three (keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf, --- keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf, --- keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf). --- A first attempt at live reproduction (an earlier session) found --- exactly the race fast_forward's own fix doesn't have: fsm_prepare_ --- standby_for_promotion is a no-op, so the monitor's cascade advances --- from assigned=prepare_promotion straight through to stop_replication --- within the same heartbeat, before any external test script's own --- "remove the other peers" SQL call can land in between. (That same --- attempt did incidentally confirm live that stop_replication really --- is a dead end below: node3 sat reporting stop_replication, endlessly --- reassigned report_lsn by this same pos 211 row, with no KeeperFSM[] --- row able to reach it.) --- --- Step mode (PG_AUTOCTL_STEP_MODE, src/bin/pg_autoctl/step_socket.c) --- removes that race entirely: a "no-autopilot" node's node-active --- service never ticks on its own, so a pgaftest spec can drive it to --- converge locally to prepare_promotion/demote_timeout/join_secondary --- via one explicit "fsm step " (see fsm_step_cmd, --- src/bin/pgaftest/test_spec_parse.y) and then simply not issue the --- next one for as long as it likes -- there is no cascade to race --- against, because nothing happens without an explicit command. Once --- frozen there, unreported to the monitor, each spec zeroes the node's --- own candidate priority and force-removes its former peers, then a --- single further "fsm step" reports the frozen state, receives the --- newly-applicable report_lsn assignment from pos 211, and performs --- the transition under test in the very same call. See those three --- specs' own headers for the full mechanics, including why each one's --- particular current_state needs a different cluster shape to reach in --- the first place (a lone standby for prepare_promotion, the original --- primary itself for demote_timeout, and a losing MS-failover --- candidate for join_secondary). --- * pos 211's stop_replication current_state is now fixed too, the same --- "missing a row" shape as the three above: reuse fsm_report_lsn --- directly, no new keeper code. This one was briefly misdiagnosed as --- unfixable, on the assumption that reaching report_lsn from here --- would need the same live-primary-dependent rewind/basebackup --- machinery fsm_restart_standby/fsm_rewind_or_init use elsewhere (for --- MAINTENANCE_STATE/PREPARE_MAINTENANCE_STATE -> CATCHINGUP_STATE, a --- materially different target that genuinely does need to stream from --- someone). fsm_report_lsn is not that function: its own restart --- (standby_restart_with_current_replication_source, primary_standby.c) --- is called with an all-zeroed upstream, so its own primaryNode.host --- check (IS_EMPTY_STRING_BUFFER) -- and pg_setup_standby_mode's own --- identical check -- skip the identify-system connection attempt --- entirely; it never contacts a peer. Concretely it just stops --- Postgres, writes a fresh standby.signal with no primary_conninfo, --- and restarts -- exactly what already turns every other source state --- in this list into a report_lsn candidate, and it's oblivious to --- whether this data directory's own history includes a promotion --- (fsm_stop_replication really did call fsm_promote_standby to get --- here, so Postgres is a genuinely writable, disconnected primary on --- its own new timeline by this point -- but Postgres itself decides --- "am I in recovery" purely from standby.signal's presence at this --- startup, not from promotion history). See fsm.c's own comment on --- this KeeperFSM[] row for the full argument, and --- keeper_fsm_gap_stop_replication_report_lsn.pgaf for the live --- reproduction of both ways out documented on pos 211's own comment --- (group_state_machine.c): raising candidate-priority back above 0 --- (report_lsn -> single, already an existing edge, via pos 209), or a --- new node registering and RegisterNode's own existing --- report_lsn-candidate-priority-0 special case (node_active_ --- protocol.c, already exercised by --- keeper_fsm_gap_new_node_joins_report_lsn_group.pgaf for a different --- source state) basebackupping from this node and taking over as --- primary while this node follows it back in as a secondary. --- * pos 325's and pos 391's own "single" states (the primaryNode side, in --- both cases) are now fixed at the source rather than merely explained --- away: both rows' own preconditions already require a *second, --- distinctly-matched* node (activeNode for 325, the healthy candidate --- counted by atLeastOneHealthyCandidate for 391) to exist in the same --- group as primaryNode, which makes primaryNode genuinely reporting --- SINGLE ("alone in my own group") a real model contradiction -- the --- instant a second node registers, the primary's own goal moves off --- single (to wait_primary) as part of that registration, before the --- joining node could ever reach secondary. No real sequence of --- monitor/keeper actions can produce this combination. --- --- Previously this was left as documented-but-unfixed, the same way the --- stop_replication gap right above was (both are since fixed, by --- different mechanisms): correct, but dump_fsm_edges() itself had no --- way to know it, since --- NodeStatePatternResolveFromStates() only ever reads a row's own --- .statePattern -- every other NodeStatusPattern field, including --- isInPrimaryState (the field responsible for admitting "single" as a --- candidate primaryNode state at all), is invisible to it by --- construction. pos 325 now spells out explicitly, via its own --- .conditions, exactly the invariant its shape already implied --- (groupHasExactlyOneNode = BOOL_FALSE -- two distinctly-matched roles --- can't coexist in a one-node group); StateCanSatisfyIsInPrimaryState() --- was taught to read that (and the stronger, already-present --- groupHasMoreThanTwoNodes = BOOL_TRUE, which implies it) via a new --- singleExcluded parameter, and exclude SINGLE from the reachable state --- set whenever it's set. pos 391 needed no rule change at all: it --- already carried groupHasMoreThanTwoNodes = BOOL_TRUE for an unrelated --- reason (the MS-failover cascade's own "more than two nodes" gate), --- so it started benefiting from the same narrowing immediately. --- --- This narrowing is deliberately still scoped to isInPrimaryState only, --- same restraint StateCanSatisfyIsInPrimaryState's own comment already --- documents for pos 303: several sibling NodeStatusPattern fields --- (isInMaintenance, canTakeWrites, drainTimeExpired, --- unreachableFromDemoteTimeout -- see pos 333/339/347/349/351's own --- still-wide gap lists below) are just as state-dependent in principle, --- but each is goal-state- or wall-clock-dependent rather than a plain --- reportedState equality check, and needs its own from-scratch --- satisfiability proof before narrowing it the same way is safe -- --- isInMaintenance in particular already has a documented counterexample --- (pos 369, via EdgeIsShadowedByEarlierRule's own investigation) of a --- reportedState assumed incompatible with a goal-dependent condition --- turning out to be reachable anyway. Narrowing those without doing --- that same diligence for each risks reintroducing exactly that class --- of bug, so they remain unnarrowed for now. +-- Expected result: empty. Every MonitorFSM[] rule currently has a matching +-- KeeperFSM[] row for every current_state it can assign a transition from. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos @@ -371,18 +96,13 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen -- current_state for which the monitor assigns this same target is enough -- to say the target is implementable at all, so a gap here means the -- monitor can NEVER produce this assigned_state from ANY current state -- --- e.g. "any -> dropped": KeeperFSM[]'s two ANY_STATE -> DROPPED rows --- collapse to this single row (see Step 1's own comment on why "any" isn't --- expanded per-state anymore -- the previous per-state expansion turned --- this one keeper rule into 21 separate flagged rows here, a lot of --- redundant noise for a single underlying fact), and it stays flagged --- because dump_fsm_edges() really can never produce a 'dropped' edge at --- all: the monitor's own equivalent (remove_node(), pos 101/103) lives +-- e.g. "any -> dropped" (KeeperFSM[]'s two ANY_STATE -> DROPPED rows +-- collapse to this single row) is a standing, expected exception: the +-- monitor's own DROPPED assignment (remove_node(), pos 101/103) lives -- entirely in the api_triggered section, which dump_fsm_edges() --- deliberately excludes (see its own comment) since those rows resolve --- their target via hand-written C, not a NodeStatePattern. Same --- "investigate before assuming which" caveat as the rest of this file --- applies to every row below, "any" or not. +-- deliberately excludes (see its own comment), so it can never appear +-- here. Same "investigate before assuming which" caveat as the rest of +-- this file applies to every other row below. SELECT k.current_state, k.assigned_state FROM keeper_fsm_edges k WHERE NOT EXISTS ( From 6ada84eb96aa82d03c9d522013319b2da5f3c2e2 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 17:59:53 +0200 Subject: [PATCH 42/52] ci: refresh stale node-schedule comment in the pgaftest matrix The comment above the node schedule's matrix entries still listed its original contents from before #1183 (fsm_step_report_advance) and this branch's own FSM edge-gap specs were appended to tests/tap/schedules/node.sch. --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa70651d0..a2f24e7b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -314,7 +314,10 @@ jobs: - { PGVERSION: 19, schedule: quick } # node: create_standby_with_pgdata, maintenance_and_drop, auth, # monitor_disabled, replace_monitor, extension_update, - # debian_clusters, tablespaces + # debian_clusters, tablespaces, fsm_step_report_advance, + # replication_stall/demote_timeout/timeline_fork deadlocks, + # and the keeper/monitor FSM edge-gap specs (keeper_fsm_gap_*) + # — see tests/tap/schedules/node.sch for the full list - { PGVERSION: 14, schedule: node } - { PGVERSION: 15, schedule: node } - { PGVERSION: 16, schedule: node } From c1fd50a9bb94ffb31deb1d5dc1218bdda4051e80 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 17:59:59 +0200 Subject: [PATCH 43/52] group_state_machine.c: replace banned memcpy/snprintf calls NodeStatePatternResolveFromStates's NODE_STATE_ANY/ASSIGNED/NOT_ASSIGNED branch used memcpy() to copy AllReplicationStates; its own sibling branches in the same function already do this via an explicit element-by-element loop, so match that instead of reaching for IGNORE-BANNED. DispatchMonitorFSMRule's rule->comment copy used snprintf(...,"%s", ...) where strlcpy() (already used elsewhere in this same file for fixed-buffer string copies) is the simpler, non-banned fit. --- src/monitor/group_state_machine.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 8ae13aae9..8c9d6f5e0 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -1485,7 +1485,7 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, if (rule->comment != NULL) { - snprintf(message, BUFSIZE, "%s", rule->comment); + strlcpy(message, rule->comment, BUFSIZE); } if (rule->extraAction != NULL) @@ -4650,7 +4650,12 @@ NodeStatePatternResolveFromStates(const NodeStatePattern *pattern, int *outCount { out = (ReplicationState *) palloc(ALL_REPLICATION_STATES_COUNT * sizeof(ReplicationState)); - memcpy(out, AllReplicationStates, sizeof(AllReplicationStates)); + + for (int i = 0; i < ALL_REPLICATION_STATES_COUNT; i++) + { + out[i] = AllReplicationStates[i]; + } + *outCount = ALL_REPLICATION_STATES_COUNT; return out; } From 445ca6734f09cac0711fa62baa4ac12d8679af98 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 18:11:58 +0200 Subject: [PATCH 44/52] docs: document pgautofailover.fsm, the reachability cross-check, and its CI coverage Adds two sections to failover-state-machine.rst, right after the existing keeper FSM mermaid diagrams: - "The monitor's FSM: pgautofailover.fsm" -- sample query output (both a narrow tabular slice and a full \x record), and how section_path (ltree) supports hierarchical queries. - "Cross-checking the monitor and keeper FSMs" -- explains dump_fsm_edges()/check_fsm_reachability()/pg_autoctl inspect fsm check, and the "How this is tested in CI" subsection describing the three regress tests (fsm.sql, check_fsm_reachability.sql, keeper_fsm_edges.sql) that run on every installcheck, for every PG version, as part of the build_run_images job. Also refreshes pg_autoctl_inspect.rst's "pg_autoctl inspect fsm" sub-command listing, which still only showed state/list/gv -- missing check and mermaid entirely. --- docs/failover-state-machine.rst | 154 ++++++++++++++++++++++++++++++++ docs/ref/pg_autoctl_inspect.rst | 15 +++- 2 files changed, 166 insertions(+), 3 deletions(-) diff --git a/docs/failover-state-machine.rst b/docs/failover-state-machine.rst index 37806b851..da3130bc2 100644 --- a/docs/failover-state-machine.rst +++ b/docs/failover-state-machine.rst @@ -630,3 +630,157 @@ node reacting to the other side of that removal. the full graph, ``join_primary`` included, as one Graphviz file (e.g. to pipe into their own tooling), but it is no longer the documented way to visualize the FSM. + +The monitor's FSM: ``pgautofailover.fsm`` +------------------------------------------ + +The diagrams above are rendered from the *keeper's* side of the FSM +(``KeeperFSM[]``, ``src/bin/pg_autoctl/fsm.c``): the transitions a node +knows how to perform. The *monitor's* side is a separate, matching +declarative table, ``MonitorFSM[]`` (``src/monitor/group_state_machine.c``): +the rules deciding, for every combination of reported states and cluster +conditions, which goal state to assign next. It's exposed read-only via the +``pgautofailover.fsm`` view, one row per rule, ordered by ``pos``: + +:: + + =# SELECT pos, section, comment FROM pgautofailover.fsm + WHERE pos BETWEEN 301 AND 305; + pos | section | comment + -----+--------------------+--------------------------------------------------------------------------------- + 301 | reporting_node | converged secondary, reportedTLI not an ancestor of reference -> catchingup + 303 | reporting_node | converged secondary/catchingup, primary reachable, primary not in the primary + | | states -> catchingup + 305 | reporting_node | multi-standby cascade resume point (see MonitorFSM_MultiStandbyCascadeResumeAfterPos) + (3 rows) + +Every column of a single rule, expanded (``\x on``): + +:: + + =# \x on + =# SELECT * FROM pgautofailover.fsm WHERE pos = 301; + -[ RECORD 1 ]-----------------+---------------------------------------------------------------- + pos | 301 + section | reporting_node + comment | converged secondary, reportedTLI not an ancestor of reference -> catchingup + active_node_current_state | secondary + other_node_current_state | + candidate_node_current_state | + active_node_conditions | isComparableToReferenceTli=false + other_node_conditions | + candidate_node_conditions | + group_conditions | + active_node_assigned_state | catchingup + other_node_assigned_state | + has_extra_action | f + section_path | reporting_node.from_context + +``section_path`` is an ``ltree`` column, so the table's rows can be queried +hierarchically instead of by exact section name -- for example, every rule +belonging to the multi-standby candidate-election machinery, regardless of +how deep its own sub-leaf goes:: + + =# SELECT pos, comment FROM pgautofailover.fsm + WHERE section_path <@ 'reporting_node.ms_failover'::ltree + ORDER BY pos; + +Like the keeper diagrams above, this view is generated straight from the +compiled-in ``MonitorFSM[]`` table -- it's the same on every fresh monitor of +a given pg_auto_failover version, unaffected by any node or formation state, +and changes only when a rule is added, removed, or edited in a new release. + +Cross-checking the monitor and keeper FSMs +------------------------------------------- + +The monitor and the keeper are two different programs (the monitor extension +runs inside Postgres, the keeper is the ``pg_autoctl run`` process on each +node) with two independently-maintained tables: ``MonitorFSM[]`` decides +*what* goal state to assign, ``KeeperFSM[]`` decides whether a node *can +execute* the transition it's just been assigned. If a monitor rule is +changed or added without a matching keeper edge, the keeper has no way to +perform what it's told and fails at runtime with an error like +``pg_autoctl does not know how to reach state "X" from "Y"`` -- historically +only ever discovered when an operator's cluster actually reached that +specific combination of states in production. + +Three building blocks turn that from a runtime surprise into something +checked ahead of time: + +``pgautofailover.dump_fsm_edges()`` + Resolves every ``MonitorFSM[]`` row's state pattern into its concrete + ``(pos, current_state, assigned_state)`` edges -- the same information + ``pgautofailover.fsm`` shows as a pattern, fully expanded one row per + reachable current state. Reflexive edges (current state == assigned + state) and the ``api_triggered`` section are deliberately excluded: the + former are no-ops, and the latter resolves which node plays which role + via hand-written C ahead of dispatch, so its own state pattern was never + meant to double as a full reachability precondition. + +``pgautofailover.check_fsm_reachability(keeper_edges jsonb)`` + Takes a JSON array of ``{"current": ..., "assigned": ...}`` edges -- the + transitions *some* keeper knows how to perform -- and returns every + ``dump_fsm_edges()`` edge missing from it: every transition the monitor + could assign that this particular keeper has no edge for. + +``pg_autoctl inspect fsm check`` + The live, end-to-end version of the same check, run from a node against + its own monitor: it serializes the *real*, compiled-in ``KeeperFSM[]`` + (``KeeperFSMToJSON()``) and passes it straight to + ``check_fsm_reachability()`` above -- no synthetic input, no assumptions + about what the keeper can do. A clean cluster reports:: + + $ pg_autoctl inspect fsm check + 12:00:00 1 INFO OK: every monitor FSM transition has a matching keeper edge + + A gap reports one line per missing edge and exits non-zero, so it can be + used as a build gate. For illustration, here is what running an + *older* keeper binary against a *newer* monitor -- one that has since + learned a transition the old keeper predates -- would report (this is a + hypothetical mismatch for illustration, not a gap that exists in the + current tables):: + + $ pg_autoctl inspect fsm check + 12:00:00 1 ERROR pos 381: draining -> single has no matching keeper edge + (other node was forcibly removed, now single) + $ echo $? + 1 + + ``--json`` is also available, returning ``{"ok": false, "mismatches": [...]}`` + for scripting. + +How this is tested in CI +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Three regress tests exercise this mechanism on every build, for every +supported Postgres version, as part of the ``make -C src/monitor +installcheck`` step run while building each ``pgaf:run-pgN`` Docker image +(see the ``build_run_images`` job) -- so a gap fails CI directly, without +needing a live two-process cluster: + +- ``fsm.sql`` -- a plain dump of the whole ``pgautofailover.fsm`` view. + Since the table is compile-time-fixed, its expected output changes only + when a rule is added, removed, or edited, giving that change an explicit, + reviewable diff. +- ``check_fsm_reachability.sql`` -- exercises the SQL-side mechanism itself + against small, synthetic keeper-edge inputs (an edge present drops out of + the mismatch list, an edge absent stays in, an unrecognized state name + fails loudly). It doesn't touch the real ``KeeperFSM[]``, which lives in + the ``pg_autoctl`` binary, not the database -- it only proves the + comparison logic itself is correct. +- ``keeper_fsm_edges.sql`` -- the real end-to-end static check, without + needing a live cluster. It loads ``keeper_fsm_edges.json``, a fixture + generated from the actual ``KeeperFSM[]`` via ``pg_autoctl inspect fsm + list --json`` and committed alongside the test (regenerated by hand + whenever ``KeeperFSM[]`` changes), then cross-references it against + ``dump_fsm_edges()`` in both directions: every monitor edge with no + matching keeper row (a real, actionable gap), and every keeper row the + monitor never actually dispatches to (dead weight worth a second look, + not a build failure). + +``pg_autoctl inspect fsm check`` itself -- talking to a real monitor over +the network -- is exercised live rather than in the regress suite: it's +part of the ``fsm_step_report_advance`` pgaftest spec (see +:ref:`pg_autoctl_manual_fsm_step`) and is also the tool to reach for by hand +after any manual edit to either FSM table, or when investigating a report +that looks like a reachability gap. diff --git a/docs/ref/pg_autoctl_inspect.rst b/docs/ref/pg_autoctl_inspect.rst index a9c7c009d..239e2e37f 100644 --- a/docs/ref/pg_autoctl_inspect.rst +++ b/docs/ref/pg_autoctl_inspect.rst @@ -34,9 +34,18 @@ variable is required. All commands in this group are safe to run while tune Compute and log some Postgres tuning options pg_autoctl inspect fsm - state Read the keeper's state from disk and display it - list List reachable FSM states from current state - gv Output the FSM as a .gv program suitable for graphviz/dot + state Read the keeper's state from disk and display it + list List reachable FSM states from current state + check Check that every monitor FSM transition has a matching keeper edge + gv Output the FSM as a .gv program suitable for graphviz/dot + mermaid Output the FSM as Mermaid stateDiagram-v2 programs, split by phase for readability + + pg_autoctl inspect fsm mermaid + init Mermaid diagram: how a node comes into existence or rejoins + steady-state Mermaid diagram: normal operation, no failure + failover Mermaid diagram: primary failover/promotion, including multi-standby candidate election + maintenance Mermaid diagram: planned maintenance + removal Mermaid diagram: node removal/drop pg_autoctl inspect show ipaddr Print this node's IP address information From 1033f43576ec254a3c4c869d6b848cf184253abd Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 18:38:37 +0200 Subject: [PATCH 45/52] regress: regenerate stale PG19 expected-output overrides CI failed on PG19 only (build_test_image / PG19): 'monitor' and 'timeline_fork_detection' both diffed against src/monitor/expected/pg19/expected/*.out, the PG19-specific override directory (--expecteddir, used because PG19 changed pg_lsn display format; pg_regress falls back to the shared expected/ dir for every file not present there). Both overrides were stale, from before this branch's own additions: - monitor.out was missing the trailing blank line after last_events_by_formation_and_group_count_ok, added by this branch's own last_events() coverage. - timeline_fork_detection.out still had the old sequential node IDs (24-30); this branch's new candidate_count_gate test runs earlier in regress_schedule and registers additional nodes, shifting every later test's node_id sequence values (27-33 now). Reproduced locally via a manual pg19 pg_virtualenv + installcheck run against a pgaf-base:bookworm container (mirroring the Dockerfile's own build stage), confirmed the actual output matches CI's reported diff exactly, and regenerated both files from that actual output. Re-ran the same installcheck afterward: all 19 regress + 6 isolation tests pass. --- .../expected/pg19/expected/monitor.out | 1 + .../pg19/expected/timeline_fork_detection.out | 48 +++++++++---------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/monitor/expected/pg19/expected/monitor.out b/src/monitor/expected/pg19/expected/monitor.out index 7b8699fb3..a39d5767d 100644 --- a/src/monitor/expected/pg19/expected/monitor.out +++ b/src/monitor/expected/pg19/expected/monitor.out @@ -269,3 +269,4 @@ select count(*) >= 0 as last_events_by_formation_and_group_count_ok from pgautofailover.last_events('default', 0, 10); -[ RECORD 1 ]-------------------------------+-- last_events_by_formation_and_group_count_ok | t + diff --git a/src/monitor/expected/pg19/expected/timeline_fork_detection.out b/src/monitor/expected/pg19/expected/timeline_fork_detection.out index c68ad529e..baf10a384 100644 --- a/src/monitor/expected/pg19/expected/timeline_fork_detection.out +++ b/src/monitor/expected/pg19/expected/timeline_fork_detection.out @@ -21,7 +21,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-p', 5432, 'postgres', 'p', 1); -[ RECORD 1 ]---------------+------- -assigned_node_id | 24 +assigned_node_id | 27 assigned_group_id | 0 assigned_group_state | single assigned_candidate_priority | 100 @@ -34,7 +34,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-s1', 5432, 'postgres', 's1', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 25 +assigned_node_id | 28 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -47,7 +47,7 @@ SELECT * FROM pgautofailover.register_node('tlf_unit', 'tlfu-s2', 5432, 'postgres', 's2', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 26 +assigned_node_id | 29 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -88,27 +88,27 @@ SELECT nodeid, tli, parenttli, switchpoint_lsn FROM pgautofailover.node_timeline_history ORDER BY nodeid, tli; -[ RECORD 1 ]---+----------- -nodeid | 24 +nodeid | 27 tli | 1 parenttli | 0 switchpoint_lsn | 0/00000000 -[ RECORD 2 ]---+----------- -nodeid | 25 +nodeid | 28 tli | 1 parenttli | 0 switchpoint_lsn | 0/00000000 -[ RECORD 3 ]---+----------- -nodeid | 25 +nodeid | 28 tli | 2 parenttli | 1 switchpoint_lsn | 0/00006000 -[ RECORD 4 ]---+----------- -nodeid | 26 +nodeid | 29 tli | 1 parenttli | 0 switchpoint_lsn | 0/00000000 -[ RECORD 5 ]---+----------- -nodeid | 26 +nodeid | 29 tli | 3 parenttli | 1 switchpoint_lsn | 0/00006000 @@ -227,7 +227,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-p', 5432, 'postgres', 'p', 1); -[ RECORD 1 ]---------------+------- -assigned_node_id | 27 +assigned_node_id | 30 assigned_group_id | 0 assigned_group_state | single assigned_candidate_priority | 100 @@ -240,7 +240,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s1', 5432, 'postgres', 's1', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 28 +assigned_node_id | 31 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -253,7 +253,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s2', 5432, 'postgres', 's2', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 29 +assigned_node_id | 32 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -266,7 +266,7 @@ SELECT * FROM pgautofailover.register_node('tlf_election', 'tlfe-s3', 5432, 'postgres', 's3', 1); -[ RECORD 1 ]---------------+------------- -assigned_node_id | 30 +assigned_node_id | 33 assigned_group_id | 0 assigned_group_state | wait_standby assigned_candidate_priority | 100 @@ -571,13 +571,13 @@ reportedstate | single goalstate | single rule_pos | rule_section | -description | New state is reported by node 27 "p" (tlfe-p:5432): "single" +description | New state is reported by node 30 "p" (tlfe-p:5432): "single" -[ RECORD 3 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 28 "s1" (tlfe-s1:5432): "wait_standby" +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "wait_standby" -[ RECORD 4 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | single goalstate | wait_primary @@ -589,7 +589,7 @@ reportedstate | wait_primary goalstate | wait_primary rule_pos | rule_section | -description | New state is reported by node 27 "p" (tlfe-p:5432): "wait_primary" +description | New state is reported by node 30 "p" (tlfe-p:5432): "wait_primary" -[ RECORD 6 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup @@ -601,7 +601,7 @@ reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | -description | New state is reported by node 28 "s1" (tlfe-s1:5432): "catchingup" +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "catchingup" -[ RECORD 8 ]-+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | secondary @@ -613,7 +613,7 @@ reportedstate | secondary goalstate | secondary rule_pos | rule_section | -description | New state is reported by node 28 "s1" (tlfe-s1:5432): "secondary" +description | New state is reported by node 31 "s1" (tlfe-s1:5432): "secondary" -[ RECORD 10 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | primary @@ -625,13 +625,13 @@ reportedstate | primary goalstate | primary rule_pos | rule_section | -description | New state is reported by node 27 "p" (tlfe-p:5432): "primary" +description | New state is reported by node 30 "p" (tlfe-p:5432): "primary" -[ RECORD 12 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 29 "s2" (tlfe-s2:5432): "wait_standby" +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "wait_standby" -[ RECORD 13 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup @@ -649,31 +649,31 @@ reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | -description | New state is reported by node 29 "s2" (tlfe-s2:5432): "catchingup" +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "catchingup" -[ RECORD 16 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | catchingup rule_pos | rule_section | -description | New state is reported by node 29 "s2" (tlfe-s2:5432): "secondary" +description | New state is reported by node 32 "s2" (tlfe-s2:5432): "secondary" -[ RECORD 17 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 30 "s3" (tlfe-s3:5432): "wait_standby" +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "wait_standby" -[ RECORD 18 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 30 "s3" (tlfe-s3:5432): "catchingup" +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "catchingup" -[ RECORD 19 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | wait_standby rule_pos | rule_section | -description | New state is reported by node 30 "s3" (tlfe-s3:5432): "secondary" +description | New state is reported by node 33 "s3" (tlfe-s3:5432): "secondary" -[ RECORD 20 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- reportedstate | report_lsn goalstate | prepare_promotion From 975a233838566712d34659bbc8f4174ed42ba497 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 19:45:20 +0200 Subject: [PATCH 46/52] ci: split node-fsm-gaps out of node.sch (PG17-only), fixing CI timeouts node.sch's own header comment already measured ~30 min for its original 14 specs before this branch ever touched it, against the CI step's 20-minute timeout. This branch's own keeper/monitor FSM edge-gap work appended 14 more specs at the end, and CI run 83400372049 confirmed the result: all 6 'pgaftest / node' matrix jobs (PG14-19) timed out at 20 minutes, consistently stalling around spec #20-21/28 -- a cumulative time-budget overrun, not one stuck test. Split the 14 newly-added specs (keeper_fsm_gap_* and the two stop_replication specs) into a new node-fsm-gaps.sch, run PG17-only -- the same convention ci.yml already uses for multi-alternate/multi-misc/ multi-async/citus-1/citus-2 ("FSM logic, not version-specific code paths"). node.sch itself reverts to its original 14 specs, running on all 6 PG versions as before. --- .github/workflows/ci.yml | 4 ++-- tests/tap/schedules/node-fsm-gaps.sch | 23 +++++++++++++++++++++++ tests/tap/schedules/node.sch | 21 +++++---------------- 3 files changed, 30 insertions(+), 18 deletions(-) create mode 100644 tests/tap/schedules/node-fsm-gaps.sch diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2f24e7b7..81027169d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -315,8 +315,7 @@ jobs: # node: create_standby_with_pgdata, maintenance_and_drop, auth, # monitor_disabled, replace_monitor, extension_update, # debian_clusters, tablespaces, fsm_step_report_advance, - # replication_stall/demote_timeout/timeline_fork deadlocks, - # and the keeper/monitor FSM edge-gap specs (keeper_fsm_gap_*) + # replication_stall/demote_timeout/timeline_fork deadlocks # — see tests/tap/schedules/node.sch for the full list - { PGVERSION: 14, schedule: node } - { PGVERSION: 15, schedule: node } @@ -335,6 +334,7 @@ jobs: - { PGVERSION: 17, schedule: multi-alternate } - { PGVERSION: 17, schedule: multi-misc } - { PGVERSION: 17, schedule: multi-async } + - { PGVERSION: 17, schedule: node-fsm-gaps } - { PGVERSION: 17, schedule: citus-1 } - { PGVERSION: 17, schedule: citus-2 } # citus on PG18 (supported); allow failure until officially validated diff --git a/tests/tap/schedules/node-fsm-gaps.sch b/tests/tap/schedules/node-fsm-gaps.sch new file mode 100644 index 000000000..2dc601930 --- /dev/null +++ b/tests/tap/schedules/node-fsm-gaps.sch @@ -0,0 +1,23 @@ +# Keeper/monitor FSM edge-gap coverage, split out of node.sch (~14 min): +# node.sch's own comment already measured ~30 min for its original 14 specs +# against the CI step's 20-minute timeout before these were ever added, and +# adding these on top pushed every PG version over the limit (CI run +# 83400372049: all 6 node schedule jobs timed out at 20 minutes, stalling +# around spec #20/28 -- a cumulative time-budget overrun, not a single +# stuck test). Run on PG17 only, same rationale as multi-alternate/ +# multi-misc/multi-async/citus-1/citus-2 in ci.yml: this is FSM logic, not +# version-specific code paths. +keeper_fsm_gap_209_wait_maintenance +keeper_fsm_gap_211_wait_maintenance +keeper_fsm_gap_209_wait_standby +keeper_fsm_gap_211_wait_standby +keeper_fsm_gap_211_primary_priority_zero +keeper_fsm_gap_new_node_joins_report_lsn_group +keeper_fsm_gap_candidate_fast_forward_left_alone +keeper_fsm_gap_priority_zero_fast_forward_left_alone +keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff +keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion +keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion +keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff +keeper_fsm_gap_stop_replication_report_lsn_priority +keeper_fsm_gap_stop_replication_report_lsn_new_node diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index 0c879b3f0..1c9ed4fef 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -1,6 +1,9 @@ -# Node lifecycle, monitor operations, and Debian/tablespace layouts (~34 min). +# Node lifecycle, monitor operations, and Debian/tablespace layouts (~30 min). # Merged from former node, monitor, and node-extra schedules to reduce CI job -# count and GitHub Actions runner queue pressure. +# count and GitHub Actions runner queue pressure. The keeper/monitor FSM +# edge-gap specs that used to live here were split out to node-fsm-gaps.sch +# (PG17-only) once this schedule's own combined runtime started timing out +# the CI step on every PG version -- see that file's own header comment. create_standby_with_pgdata launch_deferred_set_metadata fsm_step_report_advance @@ -15,17 +18,3 @@ replication_stall_3dc demote_timeout_wait_primary_deadlock timeline_fork_report_lsn_deadlock timeline_fork_3node_auto_detect -keeper_fsm_gap_209_wait_maintenance -keeper_fsm_gap_211_wait_maintenance -keeper_fsm_gap_209_wait_standby -keeper_fsm_gap_211_wait_standby -keeper_fsm_gap_211_primary_priority_zero -keeper_fsm_gap_new_node_joins_report_lsn_group -keeper_fsm_gap_candidate_fast_forward_left_alone -keeper_fsm_gap_priority_zero_fast_forward_left_alone -keeper_fsm_gap_primary_left_alone_mid_maintenance_handoff -keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion -keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion -keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff -keeper_fsm_gap_stop_replication_report_lsn_priority -keeper_fsm_gap_stop_replication_report_lsn_new_node From 5ad901e36ecd51128a0926d0c83d564afdef89ab Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 19:45:41 +0200 Subject: [PATCH 47/52] upgrade: bootstrap ltree before ALTER EXTENSION UPDATE TO 2.3, port new FSM SQL objects to the 2.2->2.3 script CI run 83400372049's 'pgaftest / upgrade' job failed deterministically: ALTER EXTENSION "pgautofailover" UPDATE TO "2.3": ERROR: required extension "ltree" is not installed pgautofailover.control's requires line gained ltree (needed for pgautofailover.fsm's section_path::ltree column), but Postgres only auto-resolves 'requires' on a fresh CREATE EXTENSION, never on ALTER EXTENSION ... UPDATE -- and critically, it checks 'requires' against what's already installed *before* it ever runs the target version's upgrade script body. A 'CREATE EXTENSION IF NOT EXISTS ltree' placed inside pgautofailover--2.2--2.3.sql itself is therefore unreachable dead code: the ALTER EXTENSION statement already fails before that script line would ever execute. The real, working fix is client-side, in monitor_extension_update() (monitor.c): it already has this exact pattern for btree_gist, added years ago when 1.4 first required it ("It does not seem like Postgres knows how to handle changes in extension control requires, so let's do that manually here"). Added the same CREATE EXTENSION IF NOT EXISTS ltree bootstrap, gated the same way (targetVersionNum >= 203, matching the existing >= 104 convention). Verified end to end against a real v2.2 -> current upgrade (tests/upgrade/Makefile, which builds pgaf:current from the v2.2 git tag and pgaf:next from this branch): all 10 steps of upgrade.pgaf now pass, including test_006_verify_extension_version (confirms the extension actually reaches 2.3) and test_007_verify_data_intact. Also ported this branch's other new pgautofailover.sql objects into pgautofailover--2.2--2.3.sql, which had none of them (dump_fsm(), dump_fsm_edges(), check_fsm_reachability(), the pgautofailover.fsm view, pgautofailover.fsm_section enum, and event.rule_pos/rule_section) -- confirmed via 'git diff origin/main HEAD -- pgautofailover.sql', the exact 206-line diff this branch introduces there. Without this, even with the ltree fix, an in-place upgrade would have reached 2.3 missing this entire declarative-FSM SQL surface. --- src/bin/pg_autoctl/monitor.c | 31 +++ src/monitor/pgautofailover--2.2--2.3.sql | 232 +++++++++++++++++++++++ 2 files changed, 263 insertions(+) diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index 019193361..f24c34643 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -5342,6 +5342,37 @@ monitor_extension_update(Monitor *monitor, const char *targetVersion) } } + /* + * Same story as btree_gist above, this time for ltree: version 2.3 added + * it to control's "requires" (pgautofailover.fsm's section_path column + * is cast to ltree). ALTER EXTENSION ... UPDATE checks "requires" against + * what's already installed before ever running the upgrade script body, + * so putting a "CREATE EXTENSION IF NOT EXISTS ltree" inside + * pgautofailover--2.2--2.3.sql itself is too late: the ALTER EXTENSION + * statement already failed by the time that script would run. + */ + if (targetVersionNum >= 203) + { + char *ltreeExtName = "ltree"; + + if (!find_extension_control_file(monitor->config.pgSetup.pg_ctl, + ltreeExtName)) + { + log_warn("Failed to find extension control file for \"%s\"", + ltreeExtName); + log_info("You might have to install a PostgreSQL contrib package"); + } + + if (!pgsql_create_extension(pgsql, ltreeExtName)) + { + log_error("Failed to create extension \"%s\" " + "required by \"%s\" extension version 2.3", + ltreeExtName, + PG_AUTOCTL_MONITOR_EXTENSION_NAME); + return false; + } + } + return pgsql_alter_extension_update_to(pgsql, PG_AUTOCTL_MONITOR_EXTENSION_NAME, targetVersion); diff --git a/src/monitor/pgautofailover--2.2--2.3.sql b/src/monitor/pgautofailover--2.2--2.3.sql index 7b858f5a1..871e6b83a 100644 --- a/src/monitor/pgautofailover--2.2--2.3.sql +++ b/src/monitor/pgautofailover--2.2--2.3.sql @@ -457,3 +457,235 @@ $$; grant execute on function pgautofailover.get_most_advanced_standby(text,int,bigint) to autoctl_node; + +-- +-- Expose the monitor's declarative dispatch table (MonitorFSM[] in +-- group_state_machine.c) to SQL: pgautofailover.dump_fsm()/dump_fsm_edges()/ +-- pgautofailover.fsm/check_fsm_reachability(), plus rule_pos/rule_section +-- attribution on pgautofailover.event. See pgautofailover.sql's own +-- comments on each object below for the full rationale -- unchanged here, +-- this is the same DDL, just applied incrementally to an existing 2.2 +-- install instead of as part of a fresh CREATE EXTENSION. +-- + +-- pgautofailover.fsm's section_path column (below) is cast to ltree (a +-- plain SQL cast through Postgres's own type-casting machinery, not a +-- C-level dependency -- see that view's own comment). Note there is +-- deliberately no "CREATE EXTENSION IF NOT EXISTS ltree" in this script: +-- ALTER EXTENSION ... UPDATE checks every "requires" entry in the *target* +-- version's control file is already installed before it ever runs the +-- upgrade script body for any step on the path, so a CREATE EXTENSION +-- placed here would never get a chance to run -- the ALTER EXTENSION +-- statement itself already fails first ("required extension \"ltree\" is +-- not installed"). The actual fix is client-side, in +-- monitor_extension_update() (monitor.c), which now creates "ltree" before +-- issuing the ALTER EXTENSION statement -- mirroring the exact same +-- pre-existing pattern that function already uses for btree_gist. + +-- Mirrors group_state_machine.h's MonitorFSMSection: which of the three +-- real control-flow regions of the monitor's declarative dispatch table +-- (MonitorFSM[] in group_state_machine.c) a rule belongs to. See dump_fsm() +-- and pgautofailover.fsm below, and the rule_section column on +-- pgautofailover.event. +CREATE TYPE pgautofailover.fsm_section + AS ENUM + ( + 'api_triggered', + 'early_checks', + 'reporting_node', + 'primary_node' + ); + +-- Which MonitorFSM[] row this event is attributed to (see +-- CurrentMonitorFSMRulePos in notifications.h for the mechanism) -- see +-- pgautofailover.sql's own comment on the pgautofailover.event table +-- definition for the full rationale of what is and isn't attributed. +ALTER TABLE pgautofailover.event + ADD COLUMN IF NOT EXISTS rule_pos int, + ADD COLUMN IF NOT EXISTS rule_section pgautofailover.fsm_section; + +-- Exposes the monitor's declarative dispatch table (MonitorFSM[] in +-- group_state_machine.c) to SQL, one row per rule, in first-match-wins +-- order -- see dump_fsm()'s own C-side comment for exactly what this does +-- and doesn't cover. pgautofailover.event.rule_pos/rule_section (added +-- above) join back to this view's pos column to show which rule produced +-- a given event. section is plain text here (not pgautofailover.fsm_section): +-- an api_triggered row's section names its specific operator-triggered +-- entry point too, e.g. "api_triggered: remove_node". +CREATE FUNCTION pgautofailover.dump_fsm() +RETURNS TABLE + ( + pos int, + section text, + comment text, + active_node_current_state text, + other_node_current_state text, + candidate_node_current_state text, + active_node_conditions text, + other_node_conditions text, + candidate_node_conditions text, + group_conditions text, + active_node_assigned_state pgautofailover.replication_state, + other_node_assigned_state pgautofailover.replication_state, + has_extra_action bool, + section_path text + ) +LANGUAGE C SECURITY DEFINER +AS 'MODULE_PATHNAME', $$dump_fsm$$; + +grant execute on function pgautofailover.dump_fsm() to autoctl_node; + +-- section_path is cast to ltree here (a plain SQL cast, going through +-- Postgres's own ordinary type-casting machinery) rather than in the C +-- function itself: dump_fsm() only ever builds the dotted text, this view +-- is the sole place pgautofailover takes a dependency on ltree. +CREATE VIEW pgautofailover.fsm AS + SELECT pos, + section, + comment, + active_node_current_state, + other_node_current_state, + candidate_node_current_state, + active_node_conditions, + other_node_conditions, + candidate_node_conditions, + group_conditions, + active_node_assigned_state, + other_node_assigned_state, + has_extra_action, + section_path::ltree AS section_path + FROM pgautofailover.dump_fsm() + ORDER BY pos; + +-- Flat, fully-resolved (pos, current_state, assigned_state) edges derived +-- from MonitorFSM[] -- see dump_fsm_edges()'s own C-side comment. Never +-- queried directly by an operator; check_fsm_reachability() below is built +-- on top of it. +CREATE FUNCTION pgautofailover.dump_fsm_edges() +RETURNS TABLE + ( + pos int, + current_state pgautofailover.replication_state, + assigned_state pgautofailover.replication_state + ) +LANGUAGE C SECURITY DEFINER +AS 'MODULE_PATHNAME', $$dump_fsm_edges$$; + +grant execute on function pgautofailover.dump_fsm_edges() to autoctl_node; + +-- Compares the monitor's own declarative dispatch table against a keeper's +-- KeeperFSM[] edges (serialized to JSON by KeeperFSMToJSON(), +-- src/bin/pg_autoctl/fsm.c, and sent here by "pg_autoctl inspect fsm check") +-- and returns every monitor edge with no matching keeper entry -- an empty +-- result means every transition the monitor can ever assign has somewhere +-- for the keeper to go. keeper_edges is expected to be a jsonb array of +-- {"current": ..., "assigned": ...} objects, one per KeeperFSMTransition +-- row. "current" can be the literal string "any" (KeeperFSMToJSON()'s own +-- sentinel for a row whose real .current is ANY_STATE): matched against +-- every e.current_state without ever casting it to +-- pgautofailover.replication_state, via a CASE (not "k.current = 'any' OR +-- k.current::...= e.current_state", which does not reliably guarantee the +-- cast is skipped once the left side matches -- CASE WHEN/THEN is the only +-- construct Postgres guarantees short-circuits). Any other current value, +-- and "assigned" always, still go through the enum cast unconditionally -- +-- a keeper reporting a state name this enum doesn't recognize still fails +-- loudly, with a real cast error, rather than silently never matching. +CREATE FUNCTION pgautofailover.check_fsm_reachability(keeper_edges jsonb) +RETURNS TABLE + ( + pos int, + current_state pgautofailover.replication_state, + assigned_state pgautofailover.replication_state, + comment text + ) +LANGUAGE sql +AS $$ + SELECT e.pos, e.current_state, e.assigned_state, f.comment + FROM pgautofailover.dump_fsm_edges() e + JOIN pgautofailover.fsm f ON f.pos = e.pos + WHERE NOT EXISTS ( + SELECT 1 + FROM jsonb_to_recordset(keeper_edges) AS k(current text, assigned text) + WHERE k.assigned::pgautofailover.replication_state = e.assigned_state + AND CASE WHEN k.current = 'any' THEN true + ELSE k.current::pgautofailover.replication_state = e.current_state + END) + ORDER BY e.pos; +$$; + +grant execute on function pgautofailover.check_fsm_reachability(jsonb) to autoctl_node; + +-- Re-create last_events()'s three overloads to also select the new +-- rule_pos/rule_section columns added to pgautofailover.event above. +-- Return type is SETOF pgautofailover.event (the whole row type), which +-- already reflects the table's new columns automatically once ALTER TABLE +-- above has run -- only the function bodies' own column lists need +-- updating, via CREATE OR REPLACE (same signature, so no DROP needed). +CREATE OR REPLACE FUNCTION pgautofailover.last_events + ( + count int default 10 + ) +RETURNS SETOF pgautofailover.event LANGUAGE SQL STRICT +AS $$ +with last_events as +( + select eventid, eventtime, formationid, + nodeid, groupid, nodename, nodehost, nodeport, + reportedstate, goalstate, + reportedrepstate, reportedtli, reportedlsn, + candidatepriority, replicationquorum, description, + rule_pos, rule_section + from pgautofailover.event +order by eventid desc + limit count +) +select * from last_events order by eventtime, eventid; +$$; + +CREATE OR REPLACE FUNCTION pgautofailover.last_events + ( + formation_id text default 'default', + count int default 10 + ) +RETURNS SETOF pgautofailover.event LANGUAGE SQL STRICT +AS $$ +with last_events as +( + select eventid, eventtime, formationid, + nodeid, groupid, nodename, nodehost, nodeport, + reportedstate, goalstate, + reportedrepstate, reportedtli, reportedlsn, + candidatepriority, replicationquorum, description, + rule_pos, rule_section + from pgautofailover.event + where formationid = formation_id + order by eventid desc + limit count +) +select * from last_events order by eventtime, eventid; +$$; + +CREATE OR REPLACE FUNCTION pgautofailover.last_events + ( + formation_id text, + group_id int, + count int default 10 + ) +RETURNS SETOF pgautofailover.event LANGUAGE SQL STRICT +AS $$ +with last_events as +( + select eventid, eventtime, formationid, + nodeid, groupid, nodename, nodehost, nodeport, + reportedstate, goalstate, + reportedrepstate, reportedtli, reportedlsn, + candidatepriority, replicationquorum, description, + rule_pos, rule_section + from pgautofailover.event + where formationid = formation_id + and groupid = group_id + order by eventid desc + limit count +) +select * from last_events order by eventtime, eventid; +$$; From b8532f366e99d283dd26c9aafcd9bb9120cdf353 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 20:45:54 +0200 Subject: [PATCH 48/52] tests: fix stale no-autopilot keyword in 5 node-fsm-gaps specs commit b3a64ec renamed the pgaftest DSL modifier from no-autopilot to suspended (lexer/grammar/compose_gen), but 5 specs authored around the same time were never updated. Since "no-autopilot" is no longer a recognized token, the parser fell back to treating it as a second, independent node identifier on the same formation line -- producing a phantom extra container (and, when two lines both said "no-autopilot", a duplicate docker-compose.yml service key that failed to parse entirely). This is what CI run 30759600065 surfaced as spurious timeouts and a YAML parse error in the newly-split node-fsm-gaps schedule. Verified live (Docker): 4 of the 5 specs now pass in full -- keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion (3/3), keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion (6/6), keeper_fsm_gap_stop_replication_report_lsn_priority (4/4), and keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff (5/5). keeper_fsm_gap_stop_replication_report_lsn_new_node's first 3 steps now pass too, but its test_004 still fails for an unrelated, deeper reason: node4 never sees node3's replication reach Postgres's own "quorum"/"sync" sync_state, so it retries "wait_primary -> primary" forever. Not a step-count/timing issue (confirmed live: adding extra staggered "fsm step node3" calls during the wait did not help). Left for follow-up investigation. --- ..._priority_zero_candidate_left_alone_mid_promotion.pgaf | 4 ++-- ...rity_zero_losing_candidate_left_alone_mid_handoff.pgaf | 8 ++++---- ...gap_priority_zero_primary_left_alone_mid_demotion.pgaf | 6 +++--- ...eper_fsm_gap_stop_replication_report_lsn_new_node.pgaf | 4 ++-- ...eper_fsm_gap_stop_replication_report_lsn_priority.pgaf | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/tap/specs/keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf index 55953a69f..421982554 100644 --- a/tests/tap/specs/keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_candidate_left_alone_mid_promotion.pgaf @@ -42,7 +42,7 @@ cluster { monitor formation { node1 - node3 no-autopilot candidate-priority 50 + node3 suspended candidate-priority 50 } } @@ -66,7 +66,7 @@ step test_001_kill_primary_and_converge_locally_to_prepare_promotion { # way out and, as a side effect of processing *that* report, cascade a # fresh goal to node3 too) -- so nothing recalculates node3's own goal # state until node3 itself makes contact. node3 never ticks on its own - # (no-autopilot), so this explicit "fsm step node3" is what prompts + # (suspended), so this explicit "fsm step node3" is what prompts # that contact -- and, in the very same call, discovers the fresh # prepare_promotion assignment and performs the transition into it. # There's no way to observe the assignment without also applying it. diff --git a/tests/tap/specs/keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf index 37f15ad41..7107ec062 100644 --- a/tests/tap/specs/keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_losing_candidate_left_alone_mid_handoff.pgaf @@ -16,7 +16,7 @@ # the original primary; node2 and node3 both start as ordinary (nonzero # candidate-priority) standbys, so both are eligible and get sent through # report_lsn to have their LSNs compared once node1 is gone. Both are -# declared "no-autopilot" so the spec can pace the whole election by hand -- +# declared "suspended" so the spec can pace the whole election by hand -- # node2 is driven to actually win (report_lsn -> prepare_promotion), and # node3, the loser, is driven only as far as report_lsn and then frozen # there while node2 pulls ahead, exactly reproducing the real window in @@ -40,8 +40,8 @@ cluster { monitor formation { node1 - node2 no-autopilot candidate-priority 50 - node3 no-autopilot candidate-priority 50 + node2 suspended candidate-priority 50 + node3 suspended candidate-priority 50 } } @@ -69,7 +69,7 @@ step test_001_kill_primary_and_converge_both_locally_to_report_lsn { # A hard kill never gives node1 a chance to self-report (unlike a # graceful shutdown), so nothing recalculates either standby's own goal # state until each makes its own contact -- neither node2 nor node3 - # ticks on its own (no-autopilot), so it takes one explicit "fsm step" + # ticks on its own (suspended), so it takes one explicit "fsm step" # per node here, not a bare wait. Each call both discovers the new # report_lsn assignment *and* performs the transition into it, in the # same call -- there's no way to observe the assignment without also diff --git a/tests/tap/specs/keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf index 62db67867..b80ebad61 100644 --- a/tests/tap/specs/keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_primary_left_alone_mid_demotion.pgaf @@ -11,10 +11,10 @@ # }, # # Unlike the prepare_promotion sibling spec, here the node under test is the -# primary being demoted, not a standby being promoted. node3 (no-autopilot) +# primary being demoted, not a standby being promoted. node3 (suspended) # is declared *second*, not first: compose_gen only attaches a healthcheck # to the first data node (so that later nodes can depend_on it being -# healthy, making the initial election deterministic) -- a no-autopilot +# healthy, making the initial election deterministic) -- a suspended # node can never satisfy that healthcheck on its own (`pg_autoctl status` # requires Postgres to be running, and nothing restarts Postgres after the # one-shot node-init phase stops it until the very first "fsm step" call), @@ -32,7 +32,7 @@ cluster { monitor formation { node1 - node3 no-autopilot candidate-priority 50 + node3 suspended candidate-priority 50 } } diff --git a/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf index aa7306936..d0d2e6266 100644 --- a/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf +++ b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf @@ -42,7 +42,7 @@ cluster { monitor formation { node1 - node3 no-autopilot candidate-priority 50 + node3 suspended candidate-priority 50 node4 create and launch deferred } } @@ -95,7 +95,7 @@ step test_003_node3_converges_to_report_lsn { } step test_004_node4_joins_and_becomes_primary { - # node3 stays no-autopilot for this whole spec (step mode is a + # node3 stays suspended for this whole spec (step mode is a # startup-time setting, not something a running spec can toggle off # again once the earlier steps no longer need it), and that turns out # to matter here in a way it didn't in the earlier steps: diff --git a/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf index 136a2d228..4d9af3f6d 100644 --- a/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf +++ b/tests/tap/specs/keeper_fsm_gap_stop_replication_report_lsn_priority.pgaf @@ -37,7 +37,7 @@ cluster { monitor formation { node1 - node3 no-autopilot candidate-priority 50 + node3 suspended candidate-priority 50 } } From d1775745ebda173214302ba1139a0ebb8f25c35b Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 21:27:19 +0200 Subject: [PATCH 49/52] tests: rebuild both fast_forward gap specs on a suspended node, eliminating the assigned-state race CI run 30759600065 confirmed the previous design was still genuinely racy even after switching from reported-state to assigned-state polling: with a WAL gap this small (5000 rows), node3's own local fetch completes and reports back within about 2 seconds of the assignment landing, and the monitor's own cascade (driven by node2's regular ticking) advances assigned-state on to prepare_promotion before the external "wait until" poll can reliably observe fast_forward in between. Rebuilt both specs around a suspended (step-mode) node3 instead: nothing it reports to the monitor changes except in direct response to an explicit "fsm step node3" command, which removes the race structurally rather than narrowing it. Confirmed directly against FSM_REPORT_LSN_OR_FAST_FORWARD (group_state_machine.c): the cascade past fast_forward requires reportedState == goalState == FAST_FORWARD, so as long as node3 itself never contacts the monitor, the assigned goal simply cannot advance past whatever it currently is. Discovered live in the process that the very first assignment isn't fast_forward either: the monitor can't decide fast_forward vs. a direct promotion without first learning node3's own actual LSN, which needs a real contact from node3 -- so the first goal that lands (driven by node2's own regular ticking) is the ordinary standby fan-out to REPORT_LSN, and stays there until node3 itself reports in. Both specs now drive node3 through two explicit "fsm step" calls before removing peers (report_lsn assignment -> act on it -> report_lsn reported with real LSN -> fast_forward assignment -> act on it), matching what's actually observed live rather than the originally assumed single hop. Verified live in Docker, 4/4 runs each, both specs passing consistently with no timing variance. --- ...gap_candidate_fast_forward_left_alone.pgaf | 180 ++++++++++++------ ...priority_zero_fast_forward_left_alone.pgaf | 174 +++++++++++------ 2 files changed, 233 insertions(+), 121 deletions(-) diff --git a/tests/tap/specs/keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf b/tests/tap/specs/keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf index 78fe45177..53782d1e2 100644 --- a/tests/tap/specs/keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf +++ b/tests/tap/specs/keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf @@ -27,64 +27,92 @@ # tests/tap/specs/multi_ifdown.pgaf and # tests/tap/specs/debug_citus_worker_fast_forward.pgaf. # -# Getting the KeeperFSM[] row under test (rather than the pre-existing -# STOP_REPLICATION_STATE -> SINGLE_STATE row) to actually be the one that -# fires took an extra subtlety, found by adding a diagnostic -# pgautofailover.event/rule_pos dump to an earlier draft of this spec: once -# node3 (the candidate) reports fast_forward for the first time, the -# monitor's own MS-failover cascade advances it to prepare_promotion within -# the same ~1s cycle (node2, still alive, keeps re-triggering evaluation of -# node3's candidate status) -- so waiting for "node3 state is fast_forward" -# and only THEN removing node1/node2 is already too late; by the time that -# external wait observes fast_forward, the monitor may have already moved -# node3 on to prepare_promotion/stop_replication, and it would reach single -# via that pre-existing row instead of the one this spec means to exercise. +# First draft of this spec drove node3 as an ordinary (autonomous) node and +# tried to catch the narrow fast_forward *assigned*-state window by external +# polling before removing node1/node2 -- confirmed live (CI run 30759600065) +# that this is still genuinely racy: with a WAL gap this small (5000 rows), +# node3's own local fetch completes and reports back to the monitor within +# about 2 seconds of the assignment landing, and the monitor's own cascade +# (node2, still alive, keeps re-triggering evaluation of node3's candidate +# status) advances assigned-state on to prepare_promotion before the +# external "wait until" poll can reliably observe fast_forward in between -- +# even though it polls assigned-state, not reported-state. # -# The fix is to make the group "alone" (pos 209's own NodeStatePattern -# check, evaluated in the early_checks section, ahead of the -# reporting_node.ms_failover section that drives the ordinary cascade) -# BEFORE node3 ever gets to report fast_forward for the first time, not -# after. node1's and node2's rows are therefore dropped immediately once -# the failover is triggered, while node3 is still mid-fetch (its own -# fsm_fast_forward already established a direct Postgres-level connection -# to node2 to stream the missing WAL, independent of the monitor's own -# bookkeeping -- removing node2's *row* doesn't interrupt that already -# in-flight transfer, since node2's actual Postgres process and network -# path are left untouched here). node2 is still alive and connected at -# that point, so pgautofailover.remove_node() needs force=true (without -# it, remove_node() only marks goalState=dropped and waits on node2's own -# cooperative shutdown, which both takes an indeterminate amount of time -# and would kill the WAL source's Postgres mid-transfer). +# Fixed for real this time by declaring node3 "suspended" (step mode, +# PG_AUTOCTL_SUSPENDED -- src/bin/pg_autoctl/service_keeper.c): its +# node-active service never ticks on its own, so nothing it reports to the +# monitor ever changes except in direct response to an explicit "fsm step +# node3" command. This removes the race entirely, for a reason confirmed +# directly against FSM_REPORT_LSN_OR_FAST_FORWARD's own definition +# (src/monitor/group_state_machine.c): the monitor's cascade past +# fast_forward requires reportedState == goalState == FAST_FORWARD (a +# NODE_STATE_STABLE match) -- as long as node3's own reportedState column +# never updates (because node3 itself never contacts the monitor), the +# *assignment* of fast_forward can still land purely from node2's own +# regular ticking (which is what selects node3 as the candidate and points +# it at node2 as WAL source in the first place), but the cascade *past* it +# structurally cannot fire, however long node2 keeps re-triggering +# evaluation -- there is no more race window to lose. # -# 1. 3-node formation (node1 primary, node2 + node3 secondaries). -# node2's candidate-priority is set to 0 -- it stays a synchronous, -# always-caught-up standby that is never itself selected as a -# failover candidate, existing purely to be node3's WAL-fetch source -# later. node3 keeps its default (nonzero) candidate-priority -- -# pos 209's own candidate-eligible case. +# 1. 3-node formation (node1 primary, node2 + node3 secondaries). node3 is +# declared suspended -- it must not be the first node in the +# formation (compose_gen only attaches a healthcheck to the first data +# node so that later nodes can depend_on it being healthy, and a +# suspended node can never satisfy that healthcheck on its own, since +# nothing restarts Postgres after the one-shot node-init phase until +# the first explicit "fsm step"). node2's candidate-priority is set to +# 0 -- it stays a synchronous, always-caught-up standby that is never +# itself selected as a failover candidate, existing purely to be +# node3's WAL-fetch source later. node3 keeps its default (nonzero) +# candidate-priority -- pos 209's own candidate-eligible case. # 2. node3 is network-disconnected, then node1 (still primary) receives # real write traffic + a CHECKPOINT -- node2 (still connected, still # synchronous) replicates and catches up to the new LSN; node3, cut # off, is now genuinely behind. # 3. node1 is network-disconnected (killing the primary) and node3 is -# reconnected in the same step -- this forces an MS-failover election: +# reconnected in the same step -- this forces an MS-failover election. # node2 (candidate-priority 0) is excluded from candidacy entirely -# (GroupListCandidates), so node3 is the only candidate, but it is -# behind node2's LSN -- the monitor assigns it FAST_FORWARD_STATE, -# pointing at node2 as the WAL source, and node3 starts fetching. -# 4. Immediately (node3 is still mid-fetch, not yet reporting -# fast_forward): node1's and node2's rows are dropped directly via +# (GroupListCandidates), so node3 is the only candidate -- but the +# monitor can't yet tell it needs a fetch: that decision depends on +# comparing node3's own current LSN against node2's, and the monitor +# has no fresh LSN for node3 until node3 itself makes contact. So the +# first thing that lands, driven purely by node2's own regular +# ticking, is the ordinary standby fan-out: goalState = REPORT_LSN. +# This can be polled with a comfortable timeout and no race at all -- +# node3 itself (suspended) has made no contact yet, so nothing moves +# it off this goal until the explicit steps below. +# 4. A first explicit "fsm step node3" reports node3's own still-stale +# "secondary" (unreported since before this whole scenario started) +# and, in the same call, performs the local secondary -> report_lsn +# transition -- recording node3's own genuinely-behind LSN, still +# unreported to the monitor. +# 5. A second explicit "fsm step node3" reports "report_lsn" -- with +# node3's real LSN now visible to the monitor for the first time, +# BuildCandidateList can finally compare it against node2's LSN, +# discover node3 is behind, and assign FAST_FORWARD_STATE, pointing +# at node2 as the WAL source. node3 acts on that brand new goal in +# this very same call: a live, physical WAL fetch from node2's still- +# running Postgres instance -- still unreported. node1's and node2's +# rows still exist at both of these steps (node1 merely network- +# disconnected, not yet removed), so pos 209's "alone in group" check +# does not fire on either contact. +# 6. Immediately (the fetch above already completed synchronously, inside +# that one call): node1's and node2's rows are dropped directly via # pgautofailover.remove_node(..., true) -- "alone in group" is true -# from this point on, well before node3 finishes catching up. -# 5. Once node3's own local recovery genuinely completes and it reports -# "fast_forward" for the first time, the monitor evaluates pos 209 -# immediately on that exact report (alone=true, reported=fast_forward) -# and assigns SINGLE directly -- no cascade through prepare_promotion -# is possible, since node2's row is already gone. -# 6. node3's keeper now has a matching KeeperFSM[] row (the fix under -# test) and actually converges to single, instead of getting stuck -# logging "does not know how to reach state \"single\" from -# \"fast_forward\"". +# from this point on. node2 is still alive and connected, so +# force=true is required (without it, remove_node() only marks +# goalState=dropped and waits on node2's own cooperative shutdown, +# which both takes an indeterminate amount of time and would kill the +# WAL source's Postgres mid-transfer for anyone still relying on it). +# 7. A third explicit "fsm step node3" now reports "fast_forward" (the +# state the previous step already reached locally) -- with node1 and +# node2 both gone, pos 209's "alone in group, candidate-eligible" now +# matches this exact report and assigns SINGLE directly -- no cascade +# through prepare_promotion is possible, since node2's row is already +# gone. node3's keeper now has a matching KeeperFSM[] row (the fix +# under test) and actually converges to single via that same call. +# 8. A fourth explicit "fsm step node3" has node3 actually report +# "single" back to the monitor. cluster { monitor @@ -92,13 +120,20 @@ cluster { formation { node1 node2 - node3 + node3 suspended } } setup { wait until primary, secondary timeout 120s - wait until node3 state is secondary timeout 60s + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s promote node1 wait until node1 state is primary and node2 state is secondary @@ -130,13 +165,30 @@ step test_002_disconnect_node3_and_diverge { step test_003_failover_and_remove_peers_while_still_fetching { network disconnect node1 network connect node3 - # Confirm the monitor has already selected node3 and pointed it at - # node2 as its WAL source (assigned, not yet reported/converged) - # before removing node1/node2 -- this is what guarantees a real fetch - # is already underway (or about to start) using node2's still-live - # Postgres instance, rather than short-circuiting straight to single - # off of node3's own stale pre-failover data. - wait until node3 assigned-state = fast_forward timeout 60s + # node3 is suspended, so this can be polled with a comfortable timeout + # and no race at all: node3 itself has made no contact yet, and won't + # until the explicit "fsm step" calls below -- this assignment lands + # purely from node2's own regular ticking (the ordinary standby fan-out + # to REPORT_LSN once node1 is unhealthy), and then simply stays there. + # Confirmed live: the monitor cannot decide fast_forward vs. a direct + # promotion without first learning node3's own actual LSN, which needs + # a real contact from node3 itself -- so the goal genuinely stops here, + # not at fast_forward, until node3 makes contact. + wait until node3 assigned-state = report_lsn timeout 60s + # This call reports node3's still-stale "secondary" and performs the + # local secondary -> report_lsn transition (recording its own current, + # genuinely-behind LSN) -- still unreported. node1's row still exists + # (just network-disconnected, not yet removed) and node2 is still + # present, so pos 209's "alone in group" check does not fire on this + # contact. + fsm step node3 + # This call reports "report_lsn" (with node3's real LSN, now visible to + # the monitor for the first time) -- only now can BuildCandidateList + # compare it against node2's LSN, discover node3 is behind, and assign + # FAST_FORWARD_STATE pointing at node2. node3 acts on that brand new + # goal in this very same call: fsm_fast_forward, a live WAL fetch from + # node2's still-running Postgres instance -- still unreported. + fsm step node3 sql monitor { SELECT pgautofailover.remove_node(nodehost, nodeport, true) FROM pgautofailover.node WHERE nodename = 'node1'; @@ -148,8 +200,16 @@ step test_003_failover_and_remove_peers_while_still_fetching { } step test_004_node3_converges_straight_to_single { - wait until node3 assigned-state = single timeout 180s - wait until node3 state is single timeout 60s + # node3 is still suspended: this call reports "fast_forward" (the state + # the previous step already reached locally, but had not yet reported) + # -- with node1 and node2 both gone, pos 209's "alone in group, + # candidate-eligible" now fires on this exact report and assigns + # single, performing the fast_forward -> single transition under test + # in this very call. + fsm step node3 + # One more call to have node3 actually report "single" back. + fsm step node3 + wait until node3 state = single timeout 30s sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; } diff --git a/tests/tap/specs/keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf b/tests/tap/specs/keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf index 2bd7ecf5b..63795e277 100644 --- a/tests/tap/specs/keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf +++ b/tests/tap/specs/keeper_fsm_gap_priority_zero_fast_forward_left_alone.pgaf @@ -1,18 +1,47 @@ # Regression spec for MonitorFSM[] pos 211 ("alone in group, # candidatePriority zero -> report_lsn", src/monitor/group_state_machine.c), # for its fast_forward current_state -- the candidatePriority=0 sibling of -# keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf's own scenario. See that spec's header -# for the full story of how a real fast_forward state is forced (network -# disconnect + INSERT + CHECKPOINT to create genuine LSN divergence, the -# same recipe already established by tests/tap/specs/multi_ifdown.pgaf and -# tests/tap/specs/debug_citus_worker_fast_forward.pgaf) and, critically, why -# node1's and node2's rows must be dropped immediately once node3 is -# *assigned* fast_forward -- not after it *reports* fast_forward -- to -# reliably exercise the KeeperFSM[] row under test instead of racing against -# the monitor's own MS-failover cascade (which, left alone, advances -# fast_forward -> prepare_promotion -> stop_replication within about a -# second of node3 reporting convergence, converging to single via a -# different, pre-existing row instead). +# keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf's own scenario. See +# that spec's header for the shared setup (network disconnect + INSERT + +# CHECKPOINT to create genuine LSN divergence, the same recipe already +# established by tests/tap/specs/multi_ifdown.pgaf and +# tests/tap/specs/debug_citus_worker_fast_forward.pgaf), for why node3 is +# declared "suspended" (step mode), and for the two-step +# report_lsn-then-fast_forward mechanics this now relies on (confirmed live, +# both here and in the sibling spec): the monitor can't decide fast_forward +# vs. a direct promotion without first learning node3's own actual LSN, +# which needs a real contact from node3 itself, so the first assignment +# that lands is the ordinary standby fan-out to REPORT_LSN, not +# fast_forward directly. +# +# First draft of this spec (like its sibling) drove node3 as an ordinary +# (autonomous) node and tried to catch the narrow fast_forward +# *assigned*-state window by external polling before removing node1/node2, +# and even switched from `pg_autoctl set node candidate-priority` to a +# direct SQL call for zeroing node3's own priority once that CLI's own +# confirmation loop turned out to be slow enough to lose the race on its +# own. Confirmed live (CI run 30759600065) that even that tightened window +# was still genuinely racy: with a WAL gap this small (5000 rows), node3's +# own local fetch completes and reports back to the monitor within about 2 +# seconds of the assignment landing, and the monitor's own cascade (node2, +# still alive, keeps re-triggering evaluation of node3's candidate status) +# advances assigned-state on to prepare_promotion before the external "wait +# until" poll can reliably observe fast_forward in between. +# +# Fixed for real this time the same way as the sibling spec: node3 is +# declared "suspended" (step mode, PG_AUTOCTL_SUSPENDED -- +# src/bin/pg_autoctl/service_keeper.c), so nothing it reports to the +# monitor ever changes except in direct response to an explicit "fsm step +# node3" command. Confirmed directly against FSM_REPORT_LSN_OR_FAST_FORWARD +# (src/monitor/group_state_machine.c): the monitor's cascade past +# fast_forward requires reportedState == goalState == FAST_FORWARD (a +# NODE_STATE_STABLE match), so as long as node3 itself never contacts the +# monitor, there is no race window left to lose -- and since zeroing +# node3's own candidate-priority is now also driven by an explicit step +# (not a race against the CLI's own timing), the plain +# `pg_autoctl set node candidate-priority` CLI could be used again just as +# well; the direct SQL call is kept here anyway, simplest and just as +# correct. # # This spec asserts the candidatePriority=0 sibling row that was added # alongside pos 209's: @@ -35,44 +64,50 @@ # place -- GroupListCandidates excludes candidatePriority == 0 nodes from # selection entirely, so a node can only reach fast_forward while still # candidate-eligible. Its priority is only dropped to 0 afterwards, once it -# is already assigned fast_forward (confirmed via "assigned-state = -# fast_forward", the same checkpoint keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf -# uses before removing peers) -- and, same as the peer removal, done via -# the underlying pgautofailover.set_node_candidate_priority() SQL function -# directly rather than the `pg_autoctl set node candidate-priority` CLI: -# that CLI spawns a docker exec plus its own "wait for the settings to be -# applied to ... the primary node" confirmation loop, slow enough that the -# monitor's own MS-failover cascade (fast_forward -> prepare_promotion) -# already won the race by the time an earlier draft of this spec got around -# to removing node1/node2 below -- the plain SQL call is exactly as fast as -# those remove_node() calls, closing the window instead of losing the race -# to confirm against at that point: +# has already performed the fast_forward transition locally (confirmed via +# an explicit step, not a race). # -# 1. Identical setup to keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf: node2 -# (candidate-priority 0, WAL source) stays connected and caught up; -# node3 (default nonzero candidate-priority) is disconnected while -# node1 receives writes, then reconnected as node1 is disconnected -- -# forcing an MS-failover election that selects node3 (the only -# candidate) and assigns it FAST_FORWARD_STATE to fetch the missing -# WAL from node2. -# 2. Once node3 is confirmed *assigned* fast_forward (still fetching, -# not yet converged): its own candidate-priority is set to 0, and +# 1. Identical setup to keeper_fsm_gap_candidate_fast_forward_left_alone.pgaf: +# node2 (candidate-priority 0, WAL source) stays connected and caught +# up; node3 (default nonzero candidate-priority, suspended) is +# disconnected while node1 receives writes, then reconnected as node1 +# is disconnected -- forcing an MS-failover election that selects +# node3 (the only candidate) as the group's failover target. +# 2. This can't yet be resolved to fast_forward without node3's own real +# LSN, so the first thing that lands (driven purely by node2's own +# regular ticking) is the ordinary standby fan-out: goalState = +# REPORT_LSN. Polled with a comfortable timeout and no race at all -- +# node3 itself (suspended) has made no contact yet. +# 3. A first explicit "fsm step node3" reports node3's own still-stale +# "secondary" and, in the same call, performs the local secondary -> +# report_lsn transition -- recording node3's own genuinely-behind LSN, +# still unreported. +# 4. A second explicit "fsm step node3" reports "report_lsn" -- with +# node3's real LSN now visible, BuildCandidateList compares it against +# node2's, discovers node3 is behind, and assigns FAST_FORWARD_STATE, +# pointing at node2. node3 acts on that brand new goal in the very +# same call: a live, physical WAL fetch from node2's still-running +# Postgres instance -- still unreported. node1's and node2's rows +# still exist at both of these steps (node1 merely network- +# disconnected, not yet removed), so pos 211's "alone in group" check +# does not fire on either contact. +# 5. Immediately (the fetch above already completed synchronously, inside +# that one call): node3's own candidate-priority is set to 0, and # node1's and node2's rows are dropped directly via -# pgautofailover.remove_node(..., true) (node2 is still alive and -# connected, so force=true is required; its actual Postgres instance -# and the already-established direct fetch from node3 are -# unaffected by removing its monitor row). This makes "alone in -# group" true well before node3 finishes catching up. -# 3. Once node3's own local recovery genuinely completes and it reports -# "fast_forward" for the first time, the monitor evaluates pos 211 -# immediately on that exact report (alone=true, candidate-priority=0, -# reported=fast_forward) and assigns REPORT_LSN directly -- no -# cascade through prepare_promotion is possible, since node2's row is -# already gone. -# 4. node3's keeper now has a matching KeeperFSM[] row (the fix under -# test) and actually converges to report_lsn, instead of getting -# stuck logging "does not know how to reach state \"report_lsn\" -# from \"fast_forward\"". +# pgautofailover.remove_node(..., true) -- node2 is still alive and +# connected, so force=true is required (its actual Postgres instance +# and the already-completed fetch from node3 are unaffected by +# removing its monitor row). +# 6. A third explicit "fsm step node3" now reports "fast_forward" (the +# state the previous step already reached locally) -- with node1 and +# node2 both gone and candidate-priority 0, pos 211's "alone in group, +# candidatePriority zero" now matches this exact report and assigns +# REPORT_LSN directly -- no cascade through prepare_promotion is +# possible, since node2's row is already gone. node3's keeper now has +# a matching KeeperFSM[] row (the fix under test) and actually +# converges to report_lsn via that same call. +# 7. A fourth explicit "fsm step node3" has node3 actually report +# "report_lsn" back to the monitor. cluster { monitor @@ -80,13 +115,20 @@ cluster { formation { node1 node2 - node3 + node3 suspended } } setup { wait until primary, secondary timeout 120s - wait until node3 state is secondary timeout 60s + wait until node3 state = catchingup timeout 60s + fsm step node3 + fsm step node3 + fsm step node3 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s promote node1 wait until node1 state is primary and node2 state is secondary @@ -118,15 +160,19 @@ step test_002_disconnect_node3_and_diverge { step test_003_failover_zero_priority_and_remove_peers_while_still_fetching { network disconnect node1 network connect node3 - wait until node3 assigned-state = fast_forward timeout 60s - # Zero node3's own candidate-priority via the underlying SQL function - # (pgautofailover.set_node_candidate_priority), not the - # `pg_autoctl set node candidate-priority` CLI: that CLI spawns a new - # docker exec + its own "wait for settings to be applied" confirmation - # loop, slow enough that the monitor's own cascade (fast_forward -> - # prepare_promotion) already wins the race by the time it would get to - # removing node1/node2 below. A direct SQL call is just as fast as the - # remove_node() calls that follow it. + # node3 is suspended: nothing moves this off REPORT_LSN until the + # explicit steps below (see header for why the monitor lands here + # first, not directly at fast_forward). + wait until node3 assigned-state = report_lsn timeout 60s + # Reports node3's still-stale "secondary" and performs the local + # secondary -> report_lsn transition -- still unreported. + fsm step node3 + # Reports "report_lsn" (with node3's real LSN, now visible for the + # first time) -- the monitor discovers node3 is behind node2 and + # assigns FAST_FORWARD_STATE; node3 acts on it in this same call (a + # live WAL fetch from node2) -- still unreported. node1's and node2's + # rows still exist at this point, so pos 211 does not fire yet. + fsm step node3 sql monitor { SELECT pgautofailover.set_node_candidate_priority('default', 'node3', 0); } @@ -141,8 +187,14 @@ step test_003_failover_zero_priority_and_remove_peers_while_still_fetching { } step test_004_node3_converges_straight_to_report_lsn { - wait until node3 assigned-state = report_lsn timeout 180s - wait until node3 state is report_lsn timeout 60s + # Reports "fast_forward" (reached locally by the previous step) -- with + # node1 and node2 both gone and candidate-priority 0, pos 211 now fires + # and assigns report_lsn, performing the fast_forward -> report_lsn + # transition under test in this very call. + fsm step node3 + # One more call to have node3 actually report "report_lsn" back. + fsm step node3 + wait until node3 state = report_lsn timeout 30s sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'node3'; } From deae2b902ae53c349320ff2b5be775f14df89604 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Sun, 2 Aug 2026 23:21:36 +0200 Subject: [PATCH 50/52] keeper: repair groupId/replication-slot drift in step mode too Suspended (step-mode) nodes' node-active loop, keeper_suspended_loop(), calls keeper_fsm_step()/keeper_fsm_step_report() directly, bypassing service_keeper_node_active() -- which is where the only self-heal for groupId/replication_slot_name drift used to live inline. Autonomous nodes get this repair on every tick; suspended nodes never did. This stayed invisible for as long as a suspended node kept following the same primary it originally registered against. It surfaces the first time such a node is later assigned to follow a *different* primary (REPORT_LSN_STATE -> SECONDARY_STATE via fsm_follow_new_primary, e.g. the "a new node joins and takes over" recovery path out of a parked, candidate-priority-zero report_lsn state): with no self-heal having ever run for it, config->replication_slot_name can be empty by that point, so prepare_recovery_settings() writes no primary_slot_name at all, Postgres streams without a slot, and the new primary's own quorum-candidate query (pgsql_get_postgres_metadata, an inner join from pg_replication_slots to pg_stat_replication on active_pid) can never match a slot-less connection -- so it retries "wait_primary -> primary" forever, logging "we don't have a quorum candidate yet" (fsm_enable_sync_rep) even though a healthy, caught-up standby is right there. Root-caused live: confirmed via direct SQL that the standby's replication connection was genuinely active and caught up (pg_stat_replication.sync_state = quorum) but not attached to its own named slot (pg_replication_slots.active = false), and that SHOW primary_slot_name on the standby came back empty. Fixed by extracting the self-heal block into a new shared function, keeper_maybe_update_group_and_slot(), and calling it from keeper_fsm_step() and keeper_fsm_step_report() (fsm.c) -- the two functions a suspended node's every step actually goes through -- in addition to service_keeper_node_active() (service_keeper.c), whose own call site is now a straight extraction with identical behavior for the autonomous path. Verified live in Docker: keeper_fsm_gap_stop_replication_report_lsn_new_node.pgaf's previously-hanging test_004 now passes in ~22s instead of timing out at 90s+ (3/3 consecutive runs). Full node-fsm-gaps.sch schedule (all 14 specs) passes 14/14 with this change in place. --- src/bin/pg_autoctl/fsm.c | 23 ++++++++++ src/bin/pg_autoctl/keeper.c | 66 +++++++++++++++++++++++++++++ src/bin/pg_autoctl/keeper.h | 2 + src/bin/pg_autoctl/service_keeper.c | 34 +++------------ 4 files changed, 96 insertions(+), 29 deletions(-) diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index 587755882..fb66b060d 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -1398,6 +1398,22 @@ keeper_fsm_step(Keeper *keeper) return false; } + /* + * Repair our own groupId/replication slot name if the monitor's view of + * who we are has drifted from ours -- see + * keeper_maybe_update_group_and_slot's own comment for why this can't + * be skipped here: unlike the autonomous tick body + * (service_keeper_node_active), this is also the code path a suspended + * (step-mode) node's every "fsm step"/"fsm step report" goes through, + * and it's the only place such a node ever gets a fresh + * MonitorAssignedState to check against. + */ + if (!keeper_maybe_update_group_and_slot(keeper, &assignedState)) + { + /* errors have already been logged */ + return false; + } + /* * Assign the new state. We skip writing the state file here since we can * (and should) always get the assigned state from the monitor. @@ -1481,6 +1497,13 @@ keeper_fsm_step_report(Keeper *keeper) return false; } + /* see keeper_fsm_step's own identical call for why this is here too */ + if (!keeper_maybe_update_group_and_slot(keeper, &assignedState)) + { + /* errors have already been logged */ + return false; + } + keeperState->assigned_role = assignedState.state; if (!keeper_update_state(keeper, assignedState.nodeId, assignedState.groupId, diff --git a/src/bin/pg_autoctl/keeper.c b/src/bin/pg_autoctl/keeper.c index 887b9178a..aa378a7d0 100644 --- a/src/bin/pg_autoctl/keeper.c +++ b/src/bin/pg_autoctl/keeper.c @@ -863,6 +863,72 @@ keeper_create_self_signed_cert(Keeper *keeper) } +/* + * keeper_maybe_update_group_and_slot compares what the monitor just told us + * in assignedState (our own nodeId and groupId, as returned by this exact + * node_active() contact) against our own persisted groupId and replication + * slot name, and repairs the on-disk configuration plus the running + * Postgres settings when they've drifted. + * + * This must run on every contact with the monitor, autonomous or step-mode + * alike: keeper_suspended_loop() used to skip it entirely (it called + * keeper_fsm_step()/keeper_fsm_step_report() directly, bypassing + * service_keeper_node_active() where this check used to live inline), which + * left a suspended node permanently unable to notice or repair a + * replication-slot mismatch. In practice this stayed invisible for as long + * as a suspended node kept following the same primary it registered + * against -- the bug surfaces specifically the first time it's later + * assigned to follow a *different* primary (report_lsn -> secondary via + * fsm_follow_new_primary): with no self-heal ever having run for this node, + * its config's replication_slot_name can be empty by that point, so + * prepare_recovery_settings() writes no primary_slot_name at all, Postgres + * streams without a slot, and the new primary's own quorum-candidate query + * (pgsql_get_postgres_metadata, an inner join from pg_replication_slots to + * pg_stat_replication on active_pid) can never match a slot-less + * connection -- so it retries "wait_primary -> primary" forever, logging + * "we don't have a quorum candidate yet" (fsm_enable_sync_rep, + * fsm_transition.c) even though a healthy, caught-up standby is right + * there. + */ +bool +keeper_maybe_update_group_and_slot(Keeper *keeper, MonitorAssignedState *assignedState) +{ + KeeperConfig *config = &(keeper->config); + + char expectedSlotName[BUFSIZE] = { 0 }; + + (void) postgres_sprintf_replicationSlotName(assignedState->nodeId, + expectedSlotName, + sizeof(expectedSlotName)); + + if (assignedState->groupId != config->groupId || + strneq(config->replication_slot_name, expectedSlotName)) + { + bool postgresNotRunningIsOk = false; + + if (!keeper_config_update(config, + assignedState->nodeId, + assignedState->groupId)) + { + log_error("Failed to update the configuration file " + "with groupId %d and replication.slot \"%s\"", + assignedState->groupId, expectedSlotName); + return false; + } + + if (!keeper_ensure_configuration(keeper, postgresNotRunningIsOk)) + { + log_error("Failed to update our Postgres configuration " + "after a change of groupId or " + "replication slot name, see above for details"); + return false; + } + } + + return true; +} + + /* * keeper_ensure_configuration updates the Postgres settings to match the * pg_autoctl configuration file, if necessary. diff --git a/src/bin/pg_autoctl/keeper.h b/src/bin/pg_autoctl/keeper.h index 2139876b7..d136998e7 100644 --- a/src/bin/pg_autoctl/keeper.h +++ b/src/bin/pg_autoctl/keeper.h @@ -61,6 +61,8 @@ bool keeper_maintain_replication_slots(Keeper *keeper); bool keeper_ensure_current_state(Keeper *keeper); bool keeper_create_self_signed_cert(Keeper *keeper); bool keeper_ensure_configuration(Keeper *keeper, bool postgresNotRunningIsOk); +bool keeper_maybe_update_group_and_slot(Keeper *keeper, + MonitorAssignedState *assignedState); bool keeper_update_pg_state(Keeper *keeper, int logLevel); bool keeper_node_active(Keeper *keeper, bool doInit, MonitorAssignedState *assignedState); diff --git a/src/bin/pg_autoctl/service_keeper.c b/src/bin/pg_autoctl/service_keeper.c index 3b0569e81..f67a7d715 100644 --- a/src/bin/pg_autoctl/service_keeper.c +++ b/src/bin/pg_autoctl/service_keeper.c @@ -1305,7 +1305,6 @@ keeper_suspended_loop(Keeper *keeper, pid_t start_pid) static bool service_keeper_node_active(Keeper *keeper, bool doInit) { - KeeperConfig *config = &(keeper->config); KeeperStateData *keeperState = &(keeper->state); MonitorAssignedState assignedState = { 0 }; @@ -1362,36 +1361,13 @@ service_keeper_node_active(Keeper *keeper, bool doInit) /* * Also update the groupId and replication slot name in the - * configuration file. + * configuration file, if the monitor's own view of who we are has + * drifted from ours. */ - char expectedSlotName[BUFSIZE] = { 0 }; - - (void) postgres_sprintf_replicationSlotName(assignedState.nodeId, - expectedSlotName, - sizeof(expectedSlotName)); - - if (assignedState.groupId != config->groupId || - strneq(config->replication_slot_name, expectedSlotName)) + if (!keeper_maybe_update_group_and_slot(keeper, &assignedState)) { - bool postgresNotRunningIsOk = false; - - if (!keeper_config_update(config, - assignedState.nodeId, - assignedState.groupId)) - { - log_error("Failed to update the configuration file " - "with groupId %d and replication.slot \"%s\"", - assignedState.groupId, expectedSlotName); - return false; - } - - if (!keeper_ensure_configuration(keeper, postgresNotRunningIsOk)) - { - log_error("Failed to update our Postgres configuration " - "after a change of groupId or " - "replication slot name, see above for details"); - return false; - } + /* errors have already been logged */ + return false; } return true; From 286a543d7d7ca83f1fa4d8e7a7d0cabd642a5f7e Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Mon, 3 Aug 2026 01:34:21 +0200 Subject: [PATCH 51/52] keeper: narrow group/slot self-heal to a config-file update only keeper_maybe_update_group_and_slot() (added in deae2b9) called the full keeper_ensure_configuration() to repair a drifted groupId/replication slot name. That function does more than write the config file: when state->current_role is CATCHINGUP/SECONDARY/MAINTENANCE, it also reconfigures live standby settings (primary_conninfo) by calling keeper_get_primary(). That's unsafe from this self-heal, because it can run *before* the FSM transition that would bring current_role up to date -- in particular keeper_fsm_step_report() reports without transitioning. A node that just lost its primary can still have current_role reflecting the old, now-removed primary at the moment the self-heal fires, so keeper_ensure_configuration() tries to reconnect Postgres to a stale/nonexistent primary and fails outright. This was pre-existing latent behavior on the autonomous path too (service_keeper_node_active's original inline block made the same call, unconditionally on postgresNotRunningIsOk=false) -- deae2b9's extraction just made the self-heal fire from more call sites, giving this pre-existing gap more chances to trigger. All the original replication-slot bug needed was a correct config->replication_slot_name the next time fsm_follow_new_primary() reads it, which a plain keeper_config_update() already provides. Drop the keeper_ensure_configuration() call and its now-unneeded postgresNotRunningIsOk handling entirely. Verified locally before pushing (CI run 83427068405 regressed on the previous version of this fix across node.sch and several multi/citus pytest suites): - fsm_step_report_advance.pgaf: 4/4, 3 consecutive runs - maintenance_and_drop.pgaf: 8/8 - node-fsm-gaps.sch (14 specs, including the original hanging test_004_node4_joins_and_becomes_primary): 14/14 - pytest multi group: 101/101 - pytest test_citus_force_failover.py (including the previously failing test_005_drop_primary_worker): 7/7 - citus_indent, ci/banned.h.sh: clean --- src/bin/pg_autoctl/keeper.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/bin/pg_autoctl/keeper.c b/src/bin/pg_autoctl/keeper.c index aa378a7d0..c4461f894 100644 --- a/src/bin/pg_autoctl/keeper.c +++ b/src/bin/pg_autoctl/keeper.c @@ -867,8 +867,8 @@ keeper_create_self_signed_cert(Keeper *keeper) * keeper_maybe_update_group_and_slot compares what the monitor just told us * in assignedState (our own nodeId and groupId, as returned by this exact * node_active() contact) against our own persisted groupId and replication - * slot name, and repairs the on-disk configuration plus the running - * Postgres settings when they've drifted. + * slot name, and repairs the on-disk configuration file when they've + * drifted. * * This must run on every contact with the monitor, autonomous or step-mode * alike: keeper_suspended_loop() used to skip it entirely (it called @@ -889,6 +889,20 @@ keeper_create_self_signed_cert(Keeper *keeper) * "we don't have a quorum candidate yet" (fsm_enable_sync_rep, * fsm_transition.c) even though a healthy, caught-up standby is right * there. + * + * We deliberately only call keeper_config_update() here, not the heavier + * keeper_ensure_configuration(): that function also reconfigures live + * standby settings (primary_conninfo) based on state->current_role, via + * keeper_get_primary(). Calling it from this drift check is unsafe, because + * this check can run *before* the FSM transition that would bring + * current_role up to date -- e.g. keeper_fsm_step_report() reports without + * transitioning, so a node that just lost its primary can still have + * current_role reflecting the old, now-gone primary, and + * keeper_ensure_configuration() would then try to reconnect Postgres to + * that stale/nonexistent primary. All that fsm_follow_new_primary() and + * friends actually need from this self-heal is a correct + * config->replication_slot_name the next time they read it, which a plain + * config-file update already provides. */ bool keeper_maybe_update_group_and_slot(Keeper *keeper, MonitorAssignedState *assignedState) @@ -904,8 +918,6 @@ keeper_maybe_update_group_and_slot(Keeper *keeper, MonitorAssignedState *assigne if (assignedState->groupId != config->groupId || strneq(config->replication_slot_name, expectedSlotName)) { - bool postgresNotRunningIsOk = false; - if (!keeper_config_update(config, assignedState->nodeId, assignedState->groupId)) @@ -915,14 +927,6 @@ keeper_maybe_update_group_and_slot(Keeper *keeper, MonitorAssignedState *assigne assignedState->groupId, expectedSlotName); return false; } - - if (!keeper_ensure_configuration(keeper, postgresNotRunningIsOk)) - { - log_error("Failed to update our Postgres configuration " - "after a change of groupId or " - "replication slot name, see above for details"); - return false; - } } return true; From 92836a24a92d538ed1d20a1917a07abd069a1899 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Mon, 3 Aug 2026 16:37:30 +0200 Subject: [PATCH 52/52] monitor: rewrite group_state_machine.c comments to describe current code Full pass over every comment in this file. Two problems, fixed together: - Several comments referenced an external design doc used to plan this refactor -- fine as working notes, but not part of this PR, so citing it left a reader with no way to check the claim. Rewrote each to state the fact directly instead of deferring to that document. - Comments were written as a migration narrative: "the original if-chain", "the real source", "an earlier version of this file had X and it broke", stale self-referential line numbers pointing at code this refactor already removed, dead symbol names (MonitorFSM_EarlyChecksStart and friends, ActionCatchupUnhealthySecondaries) left over from before the sectionPath mechanism replaced them, and cross-file line-number citations to node_active_protocol.c/ formation_metadata.c that had already drifted 100-250 lines out of date. Rewrote all of this in present tense, describing what the current table/functions do and why, the way a docstring should read -- not a commit-message account of how it got here. That history belongs in git log, not in the comments. No behavior change. Verified: clean compile, citus_indent, banned.h.sh. --- src/monitor/group_state_machine.c | 898 +++++++++++++++--------------- 1 file changed, 435 insertions(+), 463 deletions(-) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 8c9d6f5e0..33310e12d 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -82,23 +82,25 @@ static void AssertMonitorFSMWellFormed(void); /* * --------------------------------------------------------------------- - * Declarative dispatch for ProceedGroupStateFromContext(): its own - * sequential if-chain, and the sequential if-chain that used to be a - * separate ProceedGroupStateForPrimaryNode() function, are both replaced by - * one table of MonitorFSMTransition rows (MonitorFSM[] below), matched - * first-match-wins by RuleMatches(). ProceedGroupStateForMSFailover() and - * everything it calls (BuildCandidateList, SelectFailoverCandidateNode, - * PromoteSelectedNode, ProceedWithMSFailover, WalSourceNodesAreAllUnhealthy) - * stays hand-written C exactly as before, reached from the table via - * extraAction -- the candidate-selection algorithm itself (priority sort, - * LSN comparison, WAL-fetch orchestration) doesn't reduce to declarative - * conditions any more cleanly than it did before this change. Only the - * plain AssignGoalState calls at the tail end of that algorithm -- - * BuildCandidateList's own fan-out to REPORT_LSN, and PromoteSelectedNode's - * own PREPARE_PROMOTION/FAST_FORWARD choice -- are dispatched through the - * table too (via TryFanOutReportLsnRow/DispatchMonitorFSMRuleByPos, see - * their own comments), each falling back to the original hand-written call - * on no match. + * FSM dispatch is table-driven: MonitorFSM[] (below) holds an ordered list + * of MonitorFSMTransition rows, and RuleMatches() picks the first row whose + * conditions hold for the current NodeActiveContext. ProceedGroupStateFrom + * Context() and the primary-role dispatch path both work this way -- see + * the "WHY THESE CONSTANTS EXIST AT ALL" comment above the + * MonitorFSMTransition typedef for how each entry point bounds its search + * to the right rows. + * + * The MS-failover candidate-selection algorithm -- BuildCandidateList, + * SelectFailoverCandidateNode, PromoteSelectedNode, ProceedWithMSFailover, + * WalSourceNodesAreAllUnhealthy -- is hand-written C, reached from the + * table via a row's own extraAction: priority sorting, LSN comparison, and + * WAL-fetch orchestration aren't expressible as a plain set of matched + * conditions. Only the plain goal-state assignments at that algorithm's own + * tail end -- BuildCandidateList's own fan-out to REPORT_LSN, and + * PromoteSelectedNode's own PREPARE_PROMOTION/FAST_FORWARD choice -- are + * dispatched through the table too, via TryFanOutReportLsnRow/ + * DispatchMonitorFSMRuleByPos (see their own comments), each falling back + * to a plain AssignGoalState call when no row matches. * --------------------------------------------------------------------- */ @@ -272,10 +274,7 @@ MatchStateSet(ReplicationState actual, ReplicationStateSet declared) #define FSM_STATE(x) \ { .kind = NODE_STATE_STABLE, .reportedStates = STATES(x) } -/* - * group_state_machine.c:504-523/1059-1106 -- three IsCurrentState(primaryNode, - * X) ORed - */ +/* WAIT_PRIMARY, JOIN_PRIMARY, or PRIMARY -- the "primary is up in some form" set */ static const NodeStatePattern FSM_PRIMARY_OR_WAIT_OR_JOIN = { .kind = NODE_STATE_STABLE, .reportedStates = STATES(REPLICATION_STATE_WAIT_PRIMARY, @@ -294,9 +293,8 @@ static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY = { }; /* - * the "primary role" states MONITOR_FSM_SECTION_PRIMARY_NODE's own rows match - * against (the declarative replacement for the old, now-removed - * ProceedGroupStateForPrimaryNode()) -- a different three-element set from + * the "primary role" states MONITOR_FSM_SECTION_PRIMARY_NODE's own rows + * match against -- a different three-element set from * FSM_PRIMARY_OR_WAIT_OR_JOIN above (no JOIN_PRIMARY, has APPLY_SETTINGS) */ static const NodeStatePattern FSM_PRIMARY_ROLE_STATES = { @@ -524,8 +522,9 @@ BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *stat if (node == NULL) { /* - * NodeIsUnhealthy(NULL, ctx) returns true -- a nonexistent node being - * "unhealthy" is exactly the semantics the original if-chain relies on. + * NodeIsUnhealthy(NULL, ctx) returns true: a nonexistent node is + * "unhealthy" by definition, so every row gated on isUnhealthy stays + * correct when there's no node to check. */ status->isUnhealthy = true; return; @@ -549,11 +548,10 @@ BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *stat * reassign the very node being matched (e.g. DRAINING on primaryNode) in * the SAME node_active() call, before dispatch continues to a later row -- * exactly like NodeStateMatchesPattern below, which reads status->node's - * fields live for the same reason. Caching these as snapshot booleans (an - * earlier version of this code did) left later rows in the same call - * matching against a stale "still in primary state" fact even after - * primaryNode had just been moved to DRAINING -- confirmed by - * concurrent_health_check_and_report, which requires the "secondary -> + * fields live for the same reason. Caching these as snapshot booleans would + * leave later rows in the same call matching against a stale "still in + * primary state" fact even after primaryNode had just been moved to + * DRAINING -- concurrent_health_check_and_report requires the "secondary -> * prepare_promotion" row to correctly stop matching once primaryNode is no * longer IsInPrimaryState(). * @@ -585,12 +583,12 @@ BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *stat * before pos 209/211's own early_checks evaluation ever runs on that node's * next heartbeat -- a goalState-dependent exclusion (e.g. NOT_STABLE) would * see reported != goal at that point and treat wait_standby as "no longer - * stable", reviving a match it was supposed to permanently exclude. - * Confirmed live: an earlier NOT_STABLE-based version of this exclusion - * looked correct in dump_fsm_edges()'s own static analysis (which never - * sees pos 101's cross-row goalState write) but still let pos 209/211 fire - * for a real wait_standby node in a live pgaftest run, exactly because of - * this. Matching on reportedState alone sidesteps it entirely. + * stable", reviving a match it was supposed to permanently exclude. A + * NOT_STABLE-based exclusion here would look correct in dump_fsm_edges()'s + * own static analysis (which never sees pos 101's cross-row goalState + * write) but would still let pos 209/211 fire for a real wait_standby node, + * exactly because of this -- confirmed live in a pgaftest run. Matching on + * reportedState alone sidesteps it entirely. * * reportedIsJoinSecondary, same reported-only shape as reportedIsWaitStandby * just above, excludes JOIN_SECONDARY_STATE from pos 209's own "alone in @@ -687,9 +685,8 @@ NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) * Every builder in this file (BuildFromContextNodeActiveContext, * BuildForPrimaryNodeNodeActiveContext) memset()s its NodeActiveContext to * zero before filling it in, so apiFunction defaults to API_FUNCTION_NONE - * unless a caller explicitly sets it -- exactly the design doc's "zero - * changes" guarantee: no row written before this mechanism existed changes - * meaning just because the field now exists. + * unless a caller explicitly sets it: no row that omits .conditions.apiTrigger + * changes meaning just because this field exists. */ typedef enum ApiTriggerKind @@ -754,17 +751,13 @@ typedef struct NodeActiveContext * in this transition" and "the group's primary" are always the same * node, so otherNode.node == primaryNode.node everywhere. Kept as its * own field rather than reusing .primaryNode directly so that role - * stays conceptually activeNode/otherNode, matching the design doc's - * own framing and this array's own dispatch semantics ("assign - * activeNodeAssignedState to activeNode, otherNodeAssignedState to - * otherNode") independently of whichever node the monitor's own - * domain concepts (primary, candidate, ...) say it happens to be. A - * future otherNodesFn-resolved row (see the design doc's "MS-failover - * / candidate-selection cluster" section) could populate this from - * some other resolution entirely -- e.g. a dynamically selected - * failover candidate, not the primary -- without disturbing every - * existing row's own primaryNode-shaped conditions, which keep reading - * .primaryNode exactly as before. + * stays conceptually activeNode/otherNode independently of whichever + * node the monitor's own domain concepts (primary, candidate, ...) say + * it happens to be. A row can instead resolve a dynamically-sized + * target list via otherNodesFn (see MonitorOtherNodesResolverFunction + * below) -- e.g. remove_node()'s fan-out to every surviving standby -- + * without disturbing every other row's own primaryNode-shaped + * conditions, which simply read .primaryNode directly. */ NodeStatus otherNode; @@ -809,9 +802,9 @@ typedef struct NodeActiveContext * remaining healthy synchronous standby and formation->number_sync_ * standbys is zero, meaning putting it into maintenance would otherwise * block writes on the primary until the monitor separately assigns it - * wait_primary -- node_active_protocol.c:1973-1986's own real condition, - * verbatim. Left false (the memset default) for every other dispatch - * pass; not a general-purpose fact reused elsewhere. + * wait_primary -- mirrors node_active_protocol.c's start_maintenance() + * own condition verbatim. Left false (the memset default) for every + * other dispatch pass; not a general-purpose fact reused elsewhere. */ bool lastHealthySyncStandbyGoingToMaintenance; @@ -836,8 +829,7 @@ typedef struct NodeActiveContext * != NULL" branch will actually drive that candidate via * ProceedWithMSFailover this round, rather than falling through to * BuildCandidateList/selection instead -- see this file's own - * ActionRunMultiStandbyFailoverCascade comment and the design doc's - * identically-named fact for the full derivation. + * ActionRunMultiStandbyFailoverCascade comment for the full derivation. */ bool candidatePromotionInProgress; @@ -860,19 +852,20 @@ typedef struct NodeActiveContext * Set unconditionally true by BuildMSFailoverNodeActiveContext, false * (the memset default) everywhere else -- the MS-failover cluster's own * "am I actually being dispatched from inside the MS-failover cluster's - * own bounded nested search" marker. Every row from - * MonitorFSM_MSFailoverStart onwards that doesn't already carry a - * condition guaranteed false outside that nested search (as pos 363/365 - * do, via activeNodeAllWalSourcesUnhealthy/candidatePromotionInProgress) - * must require this true: the top-level driver's own ordinary lookup - * (ProceedGroupStateFromContext) shares MonitorFSM_PrimaryNodeSectionStart - * as its own upper bound, so it scans straight through this whole - * cluster too -- without this guard, pos 367-373's bare activeNode-state - * patterns (no distinguishing condition of their own) would fire for - * any ordinary, non-MS-failover heartbeat whose reported/goal states - * happened to line up, hijacking normal secondary/catchingup/maintenance - * convergence. Confirmed by node_active_protocol.out/guard_data_loss.out/ - * etc. regressing exactly this way before this field existed. + * own bounded nested search" marker. Every row under SectionMSFailover + * that doesn't already carry a condition guaranteed false outside that + * nested search (as pos 363/365 do, via + * activeNodeAllWalSourcesUnhealthy/candidatePromotionInProgress) must + * require this true: the top-level driver's own ordinary lookup + * (ProceedGroupStateFromContext) scans under SectionReportingNode, which + * -- since every MS-failover row's sectionPath[0] is also + * REPORTING_NODE -- scans straight through this whole cluster too. + * Without this guard, pos 367-373's bare activeNode-state patterns (no + * distinguishing condition of their own) would fire for any ordinary, + * non-MS-failover heartbeat whose reported/goal states happened to line + * up, hijacking normal secondary/catchingup/maintenance convergence -- + * node_active_protocol.out/guard_data_loss.out/ etc. depend on this + * guard to keep that from happening. */ bool inMSFailoverCluster; @@ -980,34 +973,32 @@ typedef struct GoalStateAssignment * A row's otherNodesFn, when set, resolves a dynamically-sized list of * nodes -- rather than the single nac->otherNode.node target -- that its * own otherNodeAssignedState gets assigned to. DispatchMonitorFSMRule loops - * over the resolved list and calls AssignDeclaredGoalState once per node, - * so each of those assignments gets exactly the same rule_pos/rule_section + * over the resolved list and calls AssignDeclaredGoalState once per node, so + * each of those assignments gets exactly the same rule_pos/rule_section * attribution any other row's own single-target assignment already gets -- - * closing the gap the design doc's own "otherNodesFn" concept was meant to - * close (see otherNode's own comment above, and - * ActionFanOutReportLsnOnPrimaryRemoval's -- both flagged this as a future - * mechanism before it existed). A row sets at most one of "otherNodesFn - * resolves the target list" (this) or "nac->otherNode.node is the single, - * already-resolved target" (omitted, the original mechanism) -- never both. + * e.g. OtherNodesNotInMaintenance (pos 101's own remove_node() fan-out) and + * OtherNodesDueForCatchingUp (pos 419's own catchup fan-out). A row sets at + * most one of "otherNodesFn resolves the target list" (this) or + * "nac->otherNode.node is the single, already-resolved target" (omitted, + * the more common case) -- never both. */ typedef List *(*MonitorOtherNodesResolverFunction) (GroupStateContext *ctx, NodeActiveContext *nac); /* * A row's extraAction runs before its own activeNodeAssignedState/ - * otherNodeAssignedState are applied (matching the original if-chain's - * order). When a row's real-source counterpart falls through to more of the - * function after its own assignment (the MS-failover cascade, the - * join_secondary -> nested primary pass), the action itself performs one - * bounded, explicitly-named nested search+dispatch over MonitorFSM[] -- - * see ActionRunMultiStandbyFailoverCascade and ActionRunPrimaryNodeTransition + * otherNodeAssignedState are applied. When a transition needs to keep + * going after its own assignment -- the MS-failover cascade, the + * join_secondary -> nested primary pass -- the action itself performs one + * bounded, explicitly-named nested search+dispatch over MonitorFSM[] -- see + * ActionRunMultiStandbyFailoverCascade and ActionRunPrimaryNodeTransition * below -- rather than signaling the top-level driver to keep scanning. - * There is deliberately no generic "continue dispatch" flag here: an earlier - * version of this file had extraAction return bool for exactly that purpose, - * and it reproduced a real bug (a sibling row in the same "family" matching - * a second time after the intended row declined -- see - * ActionRunMultiStandbyFailoverCascade's comment) that a bounded, named jump - * cannot have, because it can only ever land on one specific row family, not + * There is deliberately no generic "continue dispatch" flag here: if + * extraAction could return a bool meaning "keep scanning from here", a + * sibling row in the same "family" could match a second time after the + * intended row declined (see ActionRunMultiStandbyFailoverCascade's + * comment). A bounded, named jump cannot have that problem, because it can + * only ever land on one specific row family, not * wander into whichever row happens to be next. */ typedef void (*MonitorExtraActionFunction) (GroupStateContext *ctx, @@ -1024,14 +1015,16 @@ typedef void (*MonitorExtraActionFunction) (GroupStateContext *ctx, * * MonitorFSMSectionPath is a small, fixed-depth array of MonitorFSMSection * values -- a row's own place in the section hierarchy (e.g. { REPORTING_NODE, - * MS_FAILOVER, MS_FAILOVER_RETRY_RESET }), replacing the hand-maintained - * array-index-range boundary constants this design used to rely on (see - * the design doc's own "Open items": those "need to stay in sync with the - * table by hand as rows are added, removed, or reordered"). Trailing unused - * slots default to MONITOR_FSM_SECTION_NONE via ordinary aggregate - * initialization -- a plain array member needs no compound-literal cast any - * more than ReplicationStateSet's own states[4] does (see STATES()'s own - * comment on why that stays bare-brace too). + * MS_FAILOVER, MS_FAILOVER_RETRY_RESET }). A search bounds itself to every + * row whose sectionPath is under a given prefix (SectionPathIsUnderPrefix, + * below) instead of an array-index range, so a row's section membership is a + * fact carried on the row itself, not an implication of where it happens to + * sit in the array -- inserting, removing, or reordering rows within a + * section needs no separate bookkeeping. Trailing unused slots default to + * MONITOR_FSM_SECTION_NONE via ordinary aggregate initialization -- a plain + * array member needs no compound-literal cast any more than + * ReplicationStateSet's own states[4] does (see STATES()'s own comment on + * why that stays bare-brace too). */ #define MONITOR_FSM_SECTION_PATH_MAX_DEPTH 4 typedef MonitorFSMSection MonitorFSMSectionPath[MONITOR_FSM_SECTION_PATH_MAX_DEPTH]; @@ -1147,31 +1140,33 @@ typedef struct MonitorFSMTransition } MonitorFSMTransition; /* - * MonitorFSM[] is one array, not several: see its own definition far below - * for why ("One array, not three" in the design doc this table implements). - * Forward-declared here so the extraActions defined above it (which each - * perform one bounded, named nested search over it) can reference it by - * name; the section-path constants below are forward-declared the same way, - * for the same reason -- both those actions and the top-level driver need - * them to bound their searches. + * MonitorFSM[] is one array holding every FSM transition: operator-triggered + * rows, heartbeat rows for a reporting non-primary node, the MS-failover + * cluster, and primary-role rows all live here, disambiguated by + * .sectionPath and reached via bounded searches (see "WHY THESE CONSTANTS + * EXIST AT ALL" below). Forward-declared here so the extraActions defined + * above it (which each perform one bounded, named nested search over it) + * can reference it by name; the section-path constants below are + * forward-declared the same way, for the same reason -- both those actions + * and the top-level driver need them to bound their searches. * - * WHY THESE CONSTANTS EXIST AT ALL: the original if-chain has real, load- - * bearing structure that a single flat "first match wins over the whole - * array" search would destroy. Two different things are true about - * activeNode/primaryNode depending on WHERE in the original control flow a - * row came from (whether .activeNode means "the reporting node" or "the - * primary node substituted in"), and one specific fallback (the MS-failover - * cascade declining) needs to resume scanning from a specific *later* point, - * not from the top. Each named MonitorFSMSectionPath constant below is the - * section a search should be bounded to, so it only ever considers rows - * semantically valid for the situation at hand -- replacing what used to be - * six hand-maintained array-index constants (recomputed by hand whenever a - * row was added, removed, or moved across a boundary; the design doc's own - * "Open items" flagged exactly this as fragile). A row's membership is now - * a fact carried on the row itself (.sectionPath, see MonitorFSMTransition + * WHY THESE CONSTANTS EXIST AT ALL: a single flat "first match wins over + * the whole array" search can't work here, for two reasons. First, + * .activeNode means something different depending on which section a row + * belongs to: in the reporting-node sections it's the node that just called + * node_active(), while in SectionPrimaryNode it's the group's primary, + * substituted into the activeNode role (see BuildForPrimaryNodeNodeActive + * Context) -- a row from one section matched against the wrong section's + * NodeActiveContext would test the wrong node entirely. Second, one + * fallback (the MS-failover cascade declining) needs to resume scanning + * from a specific *later* point, not from the top. Each named + * MonitorFSMSectionPath constant below is the section a search should be + * bounded to, so it only ever considers rows semantically valid for the + * situation at hand. A row's section membership is a fact carried on the + * row itself (.sectionPath, see MonitorFSMTransition * above) rather than an implication of where it happens to sit in the - * array, so inserting, removing, or reordering rows within a section no - * longer requires touching any constant at all: + * array, so inserting, removing, or reordering rows within a section + * requires touching no constant at all: * * SectionApiTriggered = { MONITOR_FSM_SECTION_API_TRIGGERED } * Rows whose sectionPath[0] is API_TRIGGERED (pos 101-1xx) -- the @@ -1206,16 +1201,16 @@ typedef struct MonitorFSMTransition * used only by ActionRunMultiStandbyFailoverCascade -- the pos of the * merged nodesCount>2-unhealthy-primary row itself ("nodesCount>2, * primary unhealthy -> draining/maintenance + MS-failover cascade"). - * When ProceedGroupStateForMSFailover() declines, the real source falls - * through to whatever if-statement is textually next, and this is where - * that "next" starts in this table. Section-path containment alone - * can't express "resume after this specific row" (it answers "is this - * row under X?", not "in array order, after row Y") -- pos is already - * the row's own stable, human-facing identity, and - * AssertMonitorFSMWellFormed() confirms pos is strictly increasing == - * array order, so "pos > afterPos" is exactly the resume semantics - * needed. Named similarly to (but NOT the same concept as) the design - * doc's MonitorFSM_MSFailoverClusterStart -- see + * When ProceedGroupStateForMSFailover() declines, dispatch needs to + * keep evaluating the reporting-node rows that come after this point in + * the table, for the same activeNode, in the same node_active() call. + * Section-path containment alone can't express "resume after this + * specific row" (it answers "is this row under X?", not "in array + * order, after row Y") -- pos is already the row's own stable, + * human-facing identity, and AssertMonitorFSMWellFormed() confirms pos + * is strictly increasing == array order, so "pos > afterPos" is exactly + * the resume semantics needed. Conceptually similar to, but NOT the + * same bound as, SectionMSFailover below -- see * ActionRunMultiStandbyFailoverCascade's own comment for the * distinction. * @@ -1231,9 +1226,12 @@ typedef struct MonitorFSMTransition * one per node the fan-out loop touches; two (PromoteSelectedNode's own * two outcomes) through DispatchMonitorFSMRuleByPos, since first-match- * wins can't distinguish between them (see their own comment); three - * (the counting gates ProceedGroupStateForMSFailover's own hand-written - * ifs used to be, see BuildMSFailoverCandidateGateNodeActiveContext) and - * one more (the "still gathering candidates" catch-all) purely for + * for the counting gates' own outcomes -- ProceedGroupStateForMSFailover + * still tests missingNodesCount/candidateCount/quorumCandidateCount as + * plain hand-written ifs (see BuildMSFailoverCandidateGateNodeActiveContext), + * dispatching through one of these rows only for the resulting message + * and rule_pos attribution -- and one more (the "still gathering + * candidates" catch-all) purely for * dump_fsm() completeness/fallback, not reached through any other * bounded search. None reached through the ordinary top-level driver. * @@ -1532,8 +1530,8 @@ DispatchMonitorFSMRule(GroupStateContext *ctx, NodeActiveContext *nac, * match, if any -- the one building block every call site in this file * needs: the top-level driver's own two straight-line lookups (early checks, * then either the primary-role section or the rest of - * ProceedGroupStateFromContext()'s rows -- see its comment for why two, not - * the design doc's one), plus the two extraActions that each perform one + * ProceedGroupStateFromContext()'s rows -- see its comment for why two + * separate lookups), plus the two extraActions that each perform one * further, separate bounded nested search of their own when their row's own * cascade declines: ActionRunMultiStandbyFailoverCascade and * ActionRunPrimaryNodeTransition below. Returns whether a row matched, so @@ -1633,11 +1631,11 @@ ProceedGroupState(AutoFailoverNode *activeNode) /* * OtherNodeIsDueForCatchingUp is shared between the count computation in - * BuildForPrimaryNodeNodeActiveContext() and the fan-out assignment in - * ActionCatchupUnhealthySecondaries() below, so the two can never drift - * apart on which nodes they mean by "unhealthy secondary" -- both need to - * agree, since the counts drive which row matches and the fan-out is that - * row's own side effect. + * BuildForPrimaryNodeNodeActiveContext() and the fan-out resolution in + * OtherNodesDueForCatchingUp() below, so the two can never drift apart on + * which nodes they mean by "unhealthy secondary" -- both need to agree, + * since the counts drive which row matches and the fan-out is that row's + * own side effect. */ static bool OtherNodeIsDueForCatchingUp(GroupStateContext *ctx, AutoFailoverNode *otherNode) @@ -1658,63 +1656,63 @@ ActionRemoveDroppedNode(GroupStateContext *ctx, NodeActiveContext *nac, char *me /* * ActionRunMultiStandbyFailoverCascade implements the whole - * nodesCount>2-unhealthy-primary block as a single extraAction: the DRAINING/ - * MAINTENANCE/nothing if/else-if decision, followed unconditionally by - * ProceedGroupStateForMSFailover(). The DRAINING/MAINTENANCE decision itself - * is dispatched through MonitorFSM[]'s own pos 381/383 rows first (see their - * own comment) -- the hand-written if/else-if below only runs as a fallback if + * nodesCount>2-unhealthy-primary transition as a single extraAction: it makes + * the DRAINING/MAINTENANCE/nothing decision, then unconditionally calls + * ProceedGroupStateForMSFailover(). The DRAINING/MAINTENANCE decision is + * dispatched through MonitorFSM[]'s own pos 381/383 rows first (see their own + * comment) -- the hand-written if/else-if below only runs as a fallback if * neither row's own conditions somehow line up with the ones just checked - * (should never happen). The real source never `return`s after assigning + * (should never happen). This function never returns after assigning * DRAINING/MAINTENANCE to the primary -- it always falls through to try - * ProceedGroupStateForMSFailover next, in the SAME outer if-block, and if THAT - * declines (returns false), falls through further still to the rest of the - * original source's own if-chain inside ProceedGroupStateFromContext (now the - * report_lsn/prepare_promotion/stop_replication/... rows further down - * MonitorFSM[]'s REPORTING_NODE section, for this SAME activeNode). + * ProceedGroupStateForMSFailover next, and if THAT declines (returns false), + * falls through further still to a bounded resume search over the rest of + * MonitorFSM[]'s REPORTING_NODE section (the report_lsn/prepare_promotion/ + * stop_replication/... rows further down, for this SAME activeNode). * * This has to be ONE row/action, not three separate declarative rows sharing - * this action (as an earlier version of this file had it): once dispatch - * continues past a declined row, it keeps scanning forward and a later, broader - * row matching the same outer "nodesCount>2, primary unhealthy" condition (the - * catch-all "neither DRAINING nor MAINTENANCE applies" case) would match too - * and re-invoke ProceedGroupStateForMSFailover a *second* time in the same - * node_active() call -- something the original single-pass if/else-if structure - * never does. Confirmed by concurrent_second_primary_ death_report and - * concurrent_health_check_and_report, which got stuck (the former) or produced - * a spurious second cascade invocation changing the outcome (the latter) until - * this was folded into a single row/action pair. + * this action: splitting the DRAINING/MAINTENANCE/catch-all outcomes into + * three rows would let dispatch, after a declined row, keep scanning forward + * and match a later, broader row against the same outer "nodesCount>2, + * primary unhealthy" condition (the catch-all "neither DRAINING nor + * MAINTENANCE applies" case) -- re-invoking ProceedGroupStateForMSFailover a + * *second* time in the same node_active() call. concurrent_second_primary_ + * death_report and concurrent_health_check_and_report both depend on this not + * happening: splitting the action would get the former stuck and make the + * latter produce a spurious second cascade invocation that changes the + * outcome. * * When ProceedGroupStateForMSFailover() declines, the fallthrough to "the rest - * of ProceedGroupStateFromContext" is a single bounded nested search from - * MonitorFSM_FromContextResumeStart, not a flag back to the top-level driver: - * FindAndDispatchMonitorFSMRule's own internal loop already finds whichever row - * is the correct next match, however many rows down that is, in one call -- no - * repeated re-dispatch needed to walk past intervening non-matches. + * of ProceedGroupStateFromContext" is a single bounded nested search over + * SectionReportingNode starting after + * MonitorFSM_MultiStandbyCascadeResumeAfterPos, not a flag back to the + * top-level driver: FindAndDispatchMonitorFSMRuleUnderPath's own internal loop + * already finds whichever row is the correct next match, however many rows + * down that is, in one call -- no repeated re-dispatch needed to walk past + * intervening non-matches. * - * NOTE on naming: MonitorFSM_FromContextResumeStart is NOT - * MonitorFSM_MSFailoverStart, despite both marking a conceptually similar - * "resume point" -- they bound two different things. - * MonitorFSM_FromContextResumeStart (used here) just marks "resume scanning - * ordinary REPORTING_NODE rows after ProceedGroupStateForMSFailover declines" - * -- ProceedGroupStateForMSFailover() itself, and the - * BuildCandidateList/SelectFailoverCandidateNode/ PromoteSelectedNode functions - * it calls, stay hand-written C, called wholesale from here exactly as before - * this refactor (the candidate-selection algorithm itself doesn't reduce to - * declarative conditions any more cleanly than it did before -- see this file's - * own top-of-file design comment). MonitorFSM_MSFailoverStart, by contrast, - * bounds the *separate* eleven-row MS-failover cluster (pos 363-383, - * "MS-failover / candidate-selection cluster" section below) that those same - * hand-written functions now reach *into*, at their own tail end, via + * NOTE on naming: this resume search is NOT the same bound as + * SectionMSFailover, despite both marking a conceptually similar "resume + * point" -- they bound two different things. The resume search (used here) + * just marks "resume scanning ordinary REPORTING_NODE rows after + * ProceedGroupStateForMSFailover declines" -- ProceedGroupStateForMSFailover() + * itself, and the BuildCandidateList/SelectFailoverCandidateNode/ + * PromoteSelectedNode functions it calls, stay hand-written C, called + * wholesale from here (the candidate-selection algorithm itself doesn't + * reduce to declarative conditions any more cleanly than a table row can + * express). SectionMSFailover, by contrast, bounds the *separate* MS-failover + * cluster (pos 363-383, "MS-failover / candidate-selection cluster" section + * below) that those same hand-written functions reach *into*, at their own + * tail end, via * TryMSFailoverDeclarativeRow/ * TryFanOutReportLsnRow/DispatchMonitorFSMRuleByPos -- covering the plain * per-node goal assignments (retry-reset, join_secondary, BuildCandidateList's * own report_lsn fan-out, PromoteSelectedNode's prepare_promotion/fast_forward - * choice) that those functions used to make via a raw AssignGoalState call, - * with the original call kept as an unconditional fallback on no match. The - * last two rows in that same cluster (pos 381/383) are a fourth, unrelated - * caller reusing the same bounded range: ActionRunMultiStandbyFailoverCascade's - * own DRAINING/MAINTENANCE outcomes (see that function's own comment) -- not - * part of the candidate-selection machinery at all, just sharing the same "safe + * choice), with a raw AssignGoalState call kept as an unconditional fallback + * on no match. The last two rows in that same cluster (pos 381/383) are a + * fourth, unrelated caller reusing the same bounded range: + * ActionRunMultiStandbyFailoverCascade's own DRAINING/MAINTENANCE outcomes + * (see that function's own comment) -- not part of the candidate-selection + * machinery at all, just sharing the same "safe * to scan with the ordinary nac" bound. The candidate-selection algorithm's own * logic (priority sort, LSN comparison, WAL-fetch orchestration) is not part of * either bounded range. @@ -1746,10 +1744,10 @@ ActionRunMultiStandbyFailoverCascade(GroupStateContext *ctx, NodeActiveContext * /* * ActionRunPlainMSFailoverCascade is the "continue an already-started - * failover" call site: activeNode itself is REPORT_LSN or FAST_FORWARD, and - * the real source just `return`s ProceedGroupStateForMSFailover()'s result - * directly, with no DRAINING/MAINTENANCE decision attached and no further - * fallthrough either way -- so its return value is simply discarded here. + * failover" call site: activeNode itself is REPORT_LSN or FAST_FORWARD, so + * there's no DRAINING/MAINTENANCE decision to make and nothing to fall + * through to afterward -- just call ProceedGroupStateForMSFailover() and + * discard its return value. */ static void ActionRunPlainMSFailoverCascade(GroupStateContext *ctx, NodeActiveContext *nac, @@ -1813,10 +1811,10 @@ OtherNodesDueForCatchingUp(GroupStateContext *ctx, NodeActiveContext *nac) /* - * BuildFromContextNodeActiveContext computes every fact MonitorFSM_FromContext - * needs, mirroring exactly what the original ProceedGroupStateFromContext() - * if-chain read inline. primaryNode may be NULL (failover already in - * progress, primary removed). + * BuildFromContextNodeActiveContext computes every fact the + * reporting_node.from_context rows (sectionPath[1] == + * MONITOR_FSM_SECTION_FROM_CONTEXT) need. primaryNode may be NULL (failover + * already in progress, primary removed). */ static void BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *primaryNode, @@ -1831,9 +1829,9 @@ BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *prim nac->otherNode = nac->primaryNode; /* see NodeActiveContext's own comment on .otherNode */ /* - * isComparableToReferenceTli defaults to true (row :328 doesn't fire) -- a - * node that hasn't reported a timeline yet (reportedTLI == 0) has nothing - * to check, same as the original. + * isComparableToReferenceTli defaults to true -- a node that hasn't + * reported a timeline yet (reportedTLI == 0) has nothing to check, so + * the timeline-fork rows below don't fire for it. */ nac->activeNode.isComparableToReferenceTli = true; if (activeNode->reportedTLI > 0) @@ -1896,12 +1894,13 @@ BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *prim /* - * BuildForPrimaryNodeNodeActiveContext computes every fact the ForPrimaryNode - * section of MonitorFSM[] (from MonitorFSM_PrimaryNodeSectionStart onward) - * needs, mirroring the counting loop that used to be inline at the top of the - * old, now-folded-in ProceedGroupStateForPrimaryNode() (the same loop - * OtherNodeIsDueForCatchingUp's condition drives the fan-out assignment for, - * in ActionCatchupUnhealthySecondaries above). + * BuildForPrimaryNodeNodeActiveContext computes every fact SectionPrimaryNode + * (MonitorFSM[]'s pos 401-421 rows) needs: it loops over every other node in + * the primary's group, using the same OtherNodeIsDueForCatchingUp() test + * OtherNodesDueForCatchingUp() (above) uses for its own fan-out, to derive + * the group-level counts (replicationQuorumCount, secondaryNodesCount, + * secondaryQuorumNodesCount) and the anyOtherNodeWaitingStandby flag those + * rows match against. */ static void BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, @@ -2007,9 +2006,7 @@ OtherNodesNotInMaintenance(GroupStateContext *ctx, NodeActiveContext *nac) * BuildApiTriggerNodeActiveContext computes every fact the API_TRIGGERED * section of MonitorFSM[] needs, for one particular apiFunction. * - * activeNode is whichever single node the call is most fundamentally about - * -- the design doc's own reframing of "activeNode" for operator-triggered - * rows (see "Operator-triggered transitions belong in this table too"): + * activeNode is whichever single node the call is most fundamentally about: * the standby being promoted for perform_failover's 2-node row, the primary * itself for perform_failover's >2-node row and every * set_node_candidate_priority/set_node_replication_quorum/ @@ -2069,22 +2066,20 @@ BuildApiTriggerNodeActiveContext(GroupStateContext *ctx, MonitorApiFunction apiF /* * ProceedGroupStateForApiTrigger dispatches a single operator-triggered - * transition through MonitorFSM[]'s API_TRIGGERED section (pos 101-1xx -- - * see "Operator-triggered transitions belong in this table too" in the - * design doc). Every SQL-callable wrapper in node_active_protocol.c/ - * formation_metadata.c that assigns a goal state as a direct consequence of - * an operator call -- perform_failover, remove_node, start/stop_ - * maintenance, set_node_candidate_priority, set_node_replication_quorum, + * transition through MonitorFSM[]'s API_TRIGGERED section (pos 101-1xx). + * Every SQL-callable wrapper in node_active_protocol.c/formation_metadata.c + * that assigns a goal state as a direct consequence of an operator call -- + * perform_failover, remove_node, start/stop_maintenance, + * set_node_candidate_priority, set_node_replication_quorum, * set_formation_number_sync_standbys -- keeps its own imperative shape - * exactly as before (argument parsing, locking, resolving which node(s) are - * involved, and every existing validation ereport(ERROR)/WARNING/NOTICE, - * all preserved unchanged so their exact message text stays test-stable), - * and calls this once in the middle to make the actual state assignment(s), - * then continues with whatever POST side effect the real source still does - * (a continuation ProceedGroupState() call, a candidatePriority trick, - * number_sync_standbys bookkeeping) as further hand-written code -- none of - * that imperative surrounding code becomes a row, matching the design doc's - * own "pre/post side effects stay hand-written C" principle. + * (argument parsing, locking, resolving which node(s) are involved, and + * every existing validation ereport(ERROR)/WARNING/NOTICE, whose exact + * message text stays test-stable), and calls this once in the middle to + * make the actual state assignment(s), then continues with whatever POST + * side effect the wrapper still needs (a continuation ProceedGroupState() + * call, a candidatePriority trick, number_sync_standbys bookkeeping) as + * further hand-written code -- none of that surrounding code becomes a row; + * only the state assignment itself is declarative. * * Unlike the heartbeat side (ProceedGroupStateFromContext, which silently * no-ops on no match -- see its own comment for why that's the right call @@ -2128,50 +2123,48 @@ ProceedGroupStateForApiTrigger(MonitorApiFunction apiFunction, /* - * MonitorFSM[]: one array, not several -- see the boundary-constant comment - * above the MonitorFSMTransition typedef for the section layout and why it's - * a single ordered list rather than one array per real C function. Rows are - * kept in the exact order the original if-chain(s) checked them in: - * first-match-wins over this array is a straight extraction, not a - * behaviour change, exactly as it was over the three separate arrays this - * replaces. + * MonitorFSM[]: one array, not several -- see the "WHY THESE CONSTANTS EXIST + * AT ALL" comment above the MonitorFSMTransition typedef for the section + * layout and why a search is bounded by section rather than scanning the + * whole array unconditionally. Row order matters within a bounded search: + * RuleMatches() picks the first matching row it finds scanning in array + * order, so a row earlier in the array takes precedence over a later, + * broader row that would otherwise also match. * - * --- [0, MonitorFSM_EarlyChecksStart): MONITOR_FSM_SECTION_API_TRIGGERED, - * pos 101-1xx. Reached only via ProceedGroupStateForApiTrigger(), never via - * the ordinary node_active() heartbeat path -- every row here requires a - * specific non-NONE .conditions.apiTrigger, which an ordinary heartbeat - * call's implicit API_FUNCTION_NONE can never match (MatchApiTrigger). See - * that function's own comment (just above this array) for the operator - * side's dispatch shape and why a no-match there is always ereport(ERROR), - * unlike the heartbeat side below. + * --- SectionApiTriggered: pos 101-1xx. Reached only via + * ProceedGroupStateForApiTrigger(), never via the ordinary node_active() + * heartbeat path -- every row here requires a specific non-NONE + * .conditions.apiTrigger, which an ordinary heartbeat call's implicit + * API_FUNCTION_NONE can never match (MatchApiTrigger). See that function's + * own comment (just above this array) for the operator side's dispatch + * shape and why a no-match there is always ereport(ERROR), unlike the + * heartbeat side below. * - * --- [MonitorFSM_EarlyChecksStart, MonitorFSM_FromContextStart): the six - * checks the real if-chain runs BEFORE the IsInPrimaryState(activeNode) - * early return (group_state_machine.c:284) -- DROPPED, goal==DROPPED, - * MAINTENANCE, the demote_timeout self-fence, and both "alone in group" - * rows. These fire regardless of whether activeNode is currently the - * primary (a primary that just lost its only standby must still reach - * SINGLE here, before ever redirecting into the - * ProceedGroupStateForPrimaryNode section) -- confirmed by the drop_node - * regression test, which failed the first time this table put the - * primary-state redirect ahead of these six checks instead of after them. - * None of these six rows reference .primaryNode at all, so they can be - * matched against a NodeActiveContext built with primaryNode == NULL, before - * primaryNode is even resolved. + * --- SectionEarlyChecks: pos 201-211, the six checks + * ProceedGroupStateFromContext() runs BEFORE its IsInPrimaryState(activeNode) + * branch point -- DROPPED, goal==DROPPED, MAINTENANCE, the demote_timeout + * self-fence, and both "alone in group" rows. These fire regardless of + * whether activeNode is currently the primary (a primary that just lost its + * only standby must still reach SINGLE here, before ever redirecting into + * SectionPrimaryNode) -- the drop_node regression test depends on this + * ordering: it fails if the primary-state redirect is checked ahead of + * these six checks instead of after them. None of these six rows reference + * .primaryNode at all, so they can be matched against a NodeActiveContext + * built with primaryNode == NULL, before primaryNode is even resolved. */ static const MonitorFSMTransition MonitorFSM[] = { /* - * remove_node(), node_active_protocol.c:1163-1270 (RemoveNode) -- + * remove_node(), node_active_protocol.c's RemoveNode() -- * primary being removed: fan out report_lsn to every surviving, * non-maintenance standby (extraAction, runs first), then mark the - * removed node itself dropped (activeNodeAssignedState, runs after -- - * matches the real source's own order). canTakeWrites, not - * isInPrimaryState: RemoveNode's own guard is CanTakeWritesInState( - * currentNode->goalState) with no requirement that reportedState has - * converged -- see canTakeWrites's own comment on NodeMatchesPattern - * above. Must come before the catchall row below: first-match-wins - * dispatch means whichever row is tried first wins when both could - * apply, and only a node that canTakeWrites should get the fan-out. + * removed node itself dropped (activeNodeAssignedState, runs after). + * canTakeWrites, not isInPrimaryState: RemoveNode's own guard is + * CanTakeWritesInState(currentNode->goalState) with no requirement that + * reportedState has converged -- see canTakeWrites's own comment on + * NodeMatchesPattern above. Must come before the catchall row below: + * first-match-wins dispatch means whichever row is tried first wins + * when both could apply, and only a node that canTakeWrites should get + * the fan-out. */ { .pos = 101, .sectionPath = { @@ -2186,19 +2179,15 @@ static const MonitorFSMTransition MonitorFSM[] = { "every surviving non-maintenance standby joins report_lsn" }, /* - * remove_node(), the removed node itself when it's NOT the primary -- - * unconditional at this point in the real source (the "already - * DROPPED" idempotency case returns earlier, before dispatch is ever - * called; see RemoveNode()'s own pre-checks, kept hand-written). Note - * this is doc-corrected from an earlier draft of this table, which - * modeled the fan-out row above and this row as two competing - * alternatives under first-match-wins -- that would have skipped - * assigning DROPPED to a removed *primary* entirely, since the row - * above would already have matched and stopped dispatch. The real - * source does both unconditionally in sequence (fan out, THEN mark - * dropped), not as alternatives -- reflected here by having the row - * above do both itself, and this row only needing to cover the - * non-primary case that never matched the row above at all. + * remove_node(), the removed node itself when it's NOT the primary. + * RemoveNode()'s own pre-checks (kept hand-written) already reject the + * "already DROPPED" idempotency case before dispatch is ever called, so + * this row is unconditional here. It must stay a separate row from the + * fan-out row above, not a shared alternative for the DROPPED + * assignment: removing a primary needs BOTH the fan-out AND the DROPPED + * assignment in the same call, so the row above does both itself under + * first-match-wins; this row only needs to cover the non-primary case, + * which the row above's own canTakeWrites condition never matches. */ { .pos = 103, .sectionPath = { @@ -2209,8 +2198,9 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "remove_node, removed node cannot take writes -> dropped" }, /* - * perform_failover(), 2-node group -- node_active_protocol.c:1456-1554. - * The SQL wrapper resolves the sole standby and validates + * perform_failover(), 2-node group -- node_active_protocol.c's + * perform_failover(), 2-node branch. The SQL wrapper resolves the sole + * standby and validates * candidatePriority != 0 and both nodes converged BEFORE dispatch (an * operator-facing ereport(ERROR) on failure, not a table row -- see * ProceedGroupStateForApiTrigger's own comment on pre/post side @@ -2230,10 +2220,11 @@ static const MonitorFSMTransition MonitorFSM[] = { "standby prepare_promotion, primary draining" }, /* - * perform_failover(), >2-node group -- node_active_protocol.c:1555-1601. - * No standby is named at this point in the real source at all -- there's - * nothing else for activeNode to be here but the primary itself, the - * one node this specific call is fundamentally about. The + * perform_failover(), >2-node group -- node_active_protocol.c's + * perform_failover(), >2-node branch. No standby is named at this + * point at all -- there's nothing else for activeNode to be here but + * the primary itself, the one node this specific call is fundamentally + * about. The * candidatePriority trick and the ProceedGroupState(firstStandbyNode) * continuation are POST side effects, hand-written in perform_failover() * itself after this row's own DRAINING assignment has committed -- not @@ -2253,8 +2244,9 @@ static const MonitorFSMTransition MonitorFSM[] = { "via the heartbeat-driven MS-failover cluster rows" }, /* - * start_maintenance(), primary, 2-node group -- - * node_active_protocol.c:1901-1934. The WARNING about blocking writes, + * start_maintenance(), primary, 2-node group -- node_active_protocol.c's + * start_maintenance(), primary branch, 2-node case. The WARNING about + * blocking writes, * the candidatesCount<1 guard, and the already-in-maintenance * idempotency check all stay hand-written pre-dispatch, exactly where * they already are. The firstStandbyNode -> prepare_promotion @@ -2275,8 +2267,8 @@ static const MonitorFSMTransition MonitorFSM[] = { "(standby separately assigned prepare_promotion)" }, /* - * start_maintenance(), primary, >2-node group -- - * node_active_protocol.c:1936-1950. The ProceedGroupState( + * start_maintenance(), primary, >2-node group -- node_active_protocol.c's + * start_maintenance(), primary branch, >2-node case. The ProceedGroupState( * firstStandbyNode) continuation is a POST side effect, hand-written in * start_maintenance() itself, called after this row's own * prepare_maintenance assignment has committed (it re-fetches fresh @@ -2295,11 +2287,11 @@ static const MonitorFSMTransition MonitorFSM[] = { /* * start_maintenance(), secondary, last healthy sync standby -- - * node_active_protocol.c:1973-1986. + * node_active_protocol.c's start_maintenance(), secondary branch. * lastHealthySyncStandbyGoingToMaintenance is computed by * BuildApiTriggerNodeActiveContext (see its own comment) only for this - * apiFunction, mirroring the real source's own number_sync_standbys==0 && - * secondaryNodesCount==1 && IsHealthySyncStandby(currentNode) check + * apiFunction, mirroring start_maintenance()'s own number_sync_standbys==0 + * && secondaryNodesCount==1 && IsHealthySyncStandby(currentNode) check * verbatim. Must come before the ordinary-secondary row below: both match * the same activeNode/ primaryNode state pattern, and only the more * specific condition should win. @@ -2322,8 +2314,8 @@ static const MonitorFSMTransition MonitorFSM[] = { "wait_maintenance, primary wait_primary (disables sync rep)" }, /* - * start_maintenance(), secondary, ordinary case -- - * node_active_protocol.c:1987-1996. + * start_maintenance(), secondary, ordinary case -- node_active_protocol.c's + * start_maintenance(), secondary branch's own final case. */ { .pos = 115, .sectionPath = { @@ -2340,16 +2332,16 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "start_maintenance, secondary, ordinary case -> maintenance" }, /* - * stop_maintenance(), no primary at all -- node_active_protocol.c: - * 2090-2102. totalNodesCount==1 (skip dispatch, direct + * stop_maintenance(), no primary at all. node_active_protocol.c's + * stop_maintenance() handles totalNodesCount==1 (skip dispatch, direct * ProceedGroupState(currentNode)) and primaryNode==NULL&&totalNodesCount - * ==2 (ereport(ERROR)) both stay hand-written pre-dispatch branches in - * stop_maintenance() itself -- by the time this row's own dispatch call - * runs, primaryNode==NULL only happens with totalNodesCount>2 (the real - * source's own condition, `(primaryNode == NULL || IsDemotedPrimary( - * primaryNode)) && totalNodesCount > 2`, but the >2 half of that - * disjunct is redundant here since the 2-node&&NULL case never reaches - * dispatch at all, per the guard above). + * ==2 (ereport(ERROR)) as hand-written pre-dispatch branches -- by the + * time this row's own dispatch call runs, primaryNode==NULL only + * happens with totalNodesCount>2, so this row's plain .primaryNode.exists + * = BOOL_FALSE condition is enough; the four outcomes below (this row and + * the next three) are otherwise entirely disambiguated by their own + * declarative conditions, dispatched through a single unconditional + * ProceedGroupStateForApiTrigger() call in stop_maintenance() itself. */ { .pos = 117, .sectionPath = { @@ -2361,12 +2353,8 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "stop_maintenance, no primary -> report_lsn" }, /* - * stop_maintenance(), primary fully demoted -- node_active_protocol.c: - * 2103-2125 (both the >2-node and ==2-node demoted-primary branches: - * they assign the identical report_lsn outcome, differing only in log - * message text, so this one row covers both -- isDemotedPrimary alone, - * with no node-count condition, is exactly their shared real - * condition). + * stop_maintenance(), primary fully demoted -> report_lsn, regardless of + * node count: isDemotedPrimary alone is the whole condition. */ { .pos = 119, .sectionPath = { @@ -2378,11 +2366,10 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "stop_maintenance, primary demoted -> report_lsn" }, /* - * stop_maintenance(), failover in progress -- node_active_protocol.c: - * 2133-2142. The real source's own LogAndNotifyMessage text says - * "catchingup" here, but the actual SetNodeGoalState call assigns - * REPORT_LSN -- a real, pre-existing message/behavior mismatch in the - * source, not a modeling error in this table. + * stop_maintenance(), failover in progress -> report_lsn. This row's own + * .comment intentionally logs "catchingup" while actually assigning + * REPORT_LSN -- a real message/behavior mismatch, not a modeling error + * in this table. */ { .pos = 121, .sectionPath = { @@ -2396,10 +2383,8 @@ static const MonitorFSMTransition MonitorFSM[] = { "REPORT_LSN -- see this row's own comment above)" }, /* - * stop_maintenance(), ordinary case -- node_active_protocol.c:2143-2152. - * Catchall: reached only once primaryNode exists, isn't demoted, and no - * failover is in progress -- exactly the real source's own final - * "else" branch. + * stop_maintenance(), ordinary case -- catchall: reached only once + * primaryNode exists, isn't demoted, and no failover is in progress. */ { .pos = 123, .sectionPath = { @@ -2410,15 +2395,15 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "stop_maintenance, ordinary case -> catchingup" }, /* - * set_node_candidate_priority(), node_active_protocol.c:2282-2296. - * activeNode IS the primary here (mirrors + * set_node_candidate_priority(), node_active_protocol.c's + * set_node_candidate_priority(). activeNode IS the primary here (mirrors * BuildForPrimaryNodeNodeActiveContext's own "primaryNode IS activeNode" * convention): the row's only real target is the primary's own * apply_settings transition. The "primary is already apply_settings" - * ereport(ERROR) and the "no primary yet, just proceed" no-op both stay - * hand-written pre-dispatch in set_node_candidate_priority() itself, - * exactly where they already are -- this row is only reached once the - * wrapper has confirmed a primary exists and isn't already apply_settings. + * ereport(ERROR) and the "no primary yet, just proceed" no-op are + * hand-written pre-dispatch checks in set_node_candidate_priority() + * itself -- this row is only reached once the wrapper has confirmed a + * primary exists and isn't already apply_settings. */ { .pos = 125, .sectionPath = { @@ -2435,8 +2420,9 @@ static const MonitorFSMTransition MonitorFSM[] = { "apply_settings" }, /* - * set_node_replication_quorum(), node_active_protocol.c:2427-2441. Same - * shape as set_node_candidate_priority above. + * set_node_replication_quorum(), node_active_protocol.c's + * set_node_replication_quorum(). Same shape as + * set_node_candidate_priority above. */ { .pos = 127, .sectionPath = { @@ -2453,8 +2439,9 @@ static const MonitorFSMTransition MonitorFSM[] = { "apply_settings" }, /* - * set_formation_number_sync_standbys(), formation_metadata.c:591-606,639. - * The "primary not in primary/wait_primary state" ereport(ERROR) stays + * set_formation_number_sync_standbys(), formation_metadata.c's + * set_formation_number_sync_standbys(). The "primary not in + * primary/wait_primary state" ereport(ERROR) stays * hand-written pre-dispatch, exactly where it already is -- kept as this * row's own condition too (belt and suspenders: dump_fsm() should show * what's actually valid, and a future drift between the two fails loudly @@ -2679,11 +2666,11 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "alone in group, candidatePriority zero -> report_lsn" }, /* - * --- [MonitorFSM_FromContextStart, MonitorFSM_PrimaryNodeSectionStart): - * the rest of ProceedGroupStateFromContext()'s own sequential if-chain -- - * everything from the timeline-fork check (group_state_machine.c:328, right - * after the IsInPrimaryState(activeNode) early return) onward. Reached only - * when activeNode is NOT currently primary-role. + * --- sectionPath[1] == MONITOR_FSM_SECTION_FROM_CONTEXT, under + * SectionReportingNode: the rest of ProceedGroupStateFromContext()'s own + * sequential if-chain -- everything from the timeline-fork check, right + * after the IsInPrimaryState(activeNode) early return, onward. Reached + * only when activeNode is NOT currently primary-role. */ /* @@ -3163,19 +3150,18 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "join_secondary, primary converged primary -> secondary" }, /* - * --- [MonitorFSM_MSFailoverStart, MonitorFSM_PrimaryNodeSectionStart): - * the MS-failover / candidate-selection cluster's two genuinely - * declarative transitions (see the design doc's "The MS-failover / - * candidate-selection cluster" section, and TryMSFailoverDeclarativeRow's - * own comment below): BuildCandidateList/SelectFailoverCandidateNode/ - * PromoteSelectedNode themselves stay hand-written C, called from - * ProceedGroupStateForMSFailover exactly as before -- these two rows - * only cover the pair of assignments that were already expressible as - * plain per-node facts, gated by the exact same hand-written condition - * that already decided whether to make them (so a mismatch here can - * only widen the row's own no-op fallback, never change real behavior). - * Appended after the ordinary REPORTING_NODE rows rather than - * renumbered into them, so nothing else shifts. + * --- SectionMSFailover (sectionPath { REPORTING_NODE, MS_FAILOVER }): + * the MS-failover / candidate-selection cluster's own declarative rows + * (pos 363 onward, see TryMSFailoverDeclarativeRow's own comment below). + * BuildCandidateList/SelectFailoverCandidateNode/PromoteSelectedNode + * themselves stay hand-written C, called from + * ProceedGroupStateForMSFailover -- these rows only cover the + * assignments that are expressible as plain per-node facts, gated by + * the exact same hand-written condition that already decides whether to + * make them (so a mismatch here can only widen the row's own no-op + * fallback, never change real behavior). Appended after the ordinary + * REPORTING_NODE rows rather than renumbered into them, so nothing else + * shifts. */ /* @@ -3219,29 +3205,29 @@ static const MonitorFSMTransition MonitorFSM[] = { "WAL -> join_secondary" }, /* - * MS-failover: BuildCandidateList's own fan-out (group_state_machine.c, - * "Nodes in SECONDARY or CATCHINGUP states are candidates due to report - * their LSN..."). Every AssignGoalState call in that loop is dispatched + * MS-failover: BuildCandidateList's own fan-out loop ("Nodes in + * SECONDARY or CATCHINGUP states are candidates due to report their + * LSN..."). Every AssignGoalState call in that loop is dispatched * through TryFanOutReportLsnRow (see its own comment) against exactly - * these four rows -- one per distinct fromState shape the real if-chain - * checks, since no single NodeStatePattern covers all three disjuncts - * (SECONDARY/CATCHINGUP transitioning, MAINTENANCE->CATCHINGUP, + * these four rows -- one per distinct fromState shape the loop's own + * if-branches check, since no single NodeStatePattern covers all three + * disjuncts (SECONDARY/CATCHINGUP transitioning, MAINTENANCE->CATCHINGUP, * DRAINING/DEMOTED stable or DEMOTED->CATCHINGUP transitioning) at once. * The loop's own skip conditions (old/new primary unless draining or - * demoted, unhealthy-and-not-reporting) stay exactly where they are, - * hand-written, ahead of these rows -- by the time one of these four is - * reached, the loop has already established the node is a legitimate - * fan-out target; these rows only need to name which state it's coming - * from, not re-derive that eligibility. + * demoted, unhealthy-and-not-reporting) are hand-written, evaluated + * ahead of these rows -- by the time one of these four is reached, the + * loop has already established the node is a legitimate fan-out target; + * these rows only need to name which state it's coming from, not + * re-derive that eligibility. * * Each carries .conditions.inMSFailoverCluster = BOOL_TRUE: unlike pos * 363/365, none of these four has an activeNode-state pattern narrow * enough on its own to stay clear of ProceedGroupStateFromContext's own - * ordinary top-level lookup, which shares this same section's upper - * bound (MonitorFSM_PrimaryNodeSectionStart) and would otherwise match - * them against any ordinary, non-MS-failover heartbeat whose reported/ - * goal states happened to line up -- see inMSFailoverCluster's own - * comment on NodeActiveContext. + * ordinary top-level lookup, which scans under SectionReportingNode -- + * a prefix this cluster's own sectionPath is also under -- and would + * otherwise match them against any ordinary, non-MS-failover heartbeat + * whose reported/goal states happened to line up -- see + * inMSFailoverCluster's own comment on NodeActiveContext. */ { .pos = 367, .sectionPath = { @@ -3325,11 +3311,9 @@ static const MonitorFSMTransition MonitorFSM[] = { * first-match-wins can never distinguish them on its own. The real * choice -- selectedNode->reportedLSN == candidateList-> * mostAdvancedReportedLSN, an internal LSN comparison no BoolPattern - * can express -- is made in PromoteSelectedNode itself, exactly as - * before; both rows exist so dump_fsm() shows both reachable outcomes, - * matching the design doc's own resolution of this exact ambiguity - * ("kept as 2 rows anyway, for dump_fsm() edge visibility... this pair - * genuinely isn't disambiguated by this table's own dispatch model"). + * can express -- is made in PromoteSelectedNode itself; both rows exist + * so dump_fsm() shows both reachable outcomes, even though this table's + * own dispatch model can't disambiguate between them on its own. */ { .pos = 375, .sectionPath = { @@ -3407,9 +3391,10 @@ static const MonitorFSMTransition MonitorFSM[] = { /* * MS-failover: zero candidates have reported their LSN yet -- a hard, - * silent decline (the original code never logged here either). Never itself - * dispatched, listed for dump_fsm() completeness only, same as the - * no_candidate_yet row below. + * silent decline (ProceedGroupStateForMSFailover's own + * candidateCount==0 gate never logs here). Never itself dispatched, + * listed for dump_fsm() completeness only, same as the no_candidate_yet + * row below. */ { .pos = 383, .sectionPath = { @@ -3483,7 +3468,7 @@ static const MonitorFSMTransition MonitorFSM[] = { /* * ActionRunMultiStandbyFailoverCascade's own two outcomes (pos 305's - * extraAction, group_state_machine.c). Both rows below share pos 305's own + * extraAction). Both rows below share pos 305's own * gating conditions (primaryNode.isUnhealthy, groupHasMoreThanTwoNodes) * verbatim -- not a new marker field -- which is what keeps them safe from * the ordinary top-level lookup: that lookup can only ever reach these two @@ -3491,13 +3476,13 @@ static const MonitorFSMTransition MonitorFSM[] = { * activeNode-state restriction of its own), which unconditionally * dispatches through here via its own extraAction before the ordinary scan * could ever resume past it in the same call. atLeastOneHealthyCandidate is - * the exact same fact (same AutoFailoverOtherNodesListInState + - * CountHealthyCandidates computation, same isUnhealthy/groupNodeCount>2 - * gate) ActionRunMultiStandbyFailoverCascade used to compute locally -- - * reused here instead of duplicated. ResolveAcceptedTimeline-style side - * effects don't apply to either row (there are none here); only the plain - * AssignGoalState calls these replace, each falling back to the original - * hand-written condition on no match. + * computed once by BuildFromContextNodeActiveContext (same + * AutoFailoverOtherNodesListInState + CountHealthyCandidates computation, + * same isUnhealthy/groupNodeCount>2 gate) and reused here rather than + * recomputed. ResolveAcceptedTimeline-style side effects don't apply to + * either row (there are none here); only the plain AssignGoalState calls + * these rows dispatch, each falling back to a hand-written condition on + * no match. */ { .pos = 391, .sectionPath = { @@ -3728,7 +3713,7 @@ static const MonitorFSMTransition MonitorFSM[] = { * added over time at whatever free position is nearest where they belong, * not at a specific computed slot. * - * Every row's .sectionPath[0] must be one of the original four top-level + * Every row's .sectionPath[0] must be one of the four top-level * values -- this is what makes it safe for MonitorFSMSectionGetEnum/ * MonitorFSMSectionGetName (see their own comments) to only ever be called * with .sectionPath[0], never a deeper path element. @@ -3743,19 +3728,15 @@ AssertMonitorFSMWellFormed(void) /* * MonitorFSM[] ends with a terminator row (.pos left at its zero - * default) rather than a separately maintained count -- a hand- - * maintained MonitorFSM_SIZE #define used to serve this purpose, and - * adding a row without also bumping it once silently dropped pos 421 - * (the actual last row at the time) out of every loop bounded by it, - * including dispatch itself -- caught only by chance, cross-checking - * dump_fsm_edges() output by hand against expected keeper edges, not by - * anything in this file. A terminator can't go stale the same way: it's - * part of the array's own literal initializer, so any row added before - * it is automatically in scope for every loop below. The iteration cap - * here is just a sanity net against the terminator itself ever being - * removed or a new row accidentally inserted after it, which would - * otherwise turn every one of these loops into an unbounded scan past - * the end of the array. + * default) rather than a separately maintained count: a terminator + * can't go stale the way a hand-maintained size constant would, since + * it's part of the array's own literal initializer, so any row added + * before it is automatically in scope for every loop below -- no + * separate constant to remember to update when a row is added. The + * iteration cap here is just a sanity net against the terminator itself + * ever being removed or a new row accidentally inserted after it, which + * would otherwise turn every one of these loops into an unbounded scan + * past the end of the array. */ for (int i = 0; MonitorFSM[i].pos != 0; i++) { @@ -3942,7 +3923,7 @@ MonitorFSMTransitionSectionText(const MonitorFSMTransition *rule) * MonitorFSMLeafSectionName returns the (enum) name of any MonitorFSMSection * value, including the fine-grained leaves appended after PRIMARY_NODE -- * unlike MonitorFSMSectionGetName (SQL-facing, and only ever called with one - * of the original 4 top-level values, see that function's own comment), this + * of the 4 top-level values, see that function's own comment), this * is used purely to render a row's own full section_path text below, one * path element at a time. */ @@ -4425,12 +4406,12 @@ PG_FUNCTION_INFO_V1(dump_fsm); /* * dump_fsm exposes MonitorFSM[] to SQL, one row per rule, in table order * (first-match-wins order -- the same order RuleMatches() itself scans in). - * This is the cross-check surface the design doc's dump_fsm()/ - * check_fsm_reachability() proposal calls for: enough to correlate a - * pgautofailover.event row's rule_pos/rule_section (see notifications.h) back - * to the exact rule that produced it, and enough for an operator or a future - * keeper-side reachability check to see every transition the monitor's table - * can produce, without reading the C source. The active_node_current_state/ + * This is the cross-check surface pgautofailover.check_fsm_reachability() + * (below) is built on: enough to correlate a pgautofailover.event row's + * rule_pos/rule_section (see notifications.h) back to the exact rule that + * produced it, and enough for an operator or a keeper-side reachability + * check to see every transition the monitor's table can produce, without + * reading the C source. The active_node_current_state/ * other_node_current_state/candidate_node_current_state columns (rendered via * NodeStatePatternReportedStatesText, see its own * comment) plus the assigned-state columns already here give a keeper- @@ -4854,7 +4835,7 @@ NodeStatusPatternSurvivesIsInPrimaryState(const NodeStatusPattern *pattern, * either has either not yet started streaming, or already stopped Postgres * as the OLD primary mid-handoff -- promoting either straight to a writable * role is exactly the split-brain/data-loss risk pos 209's own comment - * describes). Confirmed by exhaustive grep as of this writing: every row + * describes). Every row * matching activeNode against WAIT_STANDBY (pos 315/317/319) or * JOIN_SECONDARY (pos 359/361) assigns only CATCHINGUP/SECONDARY, never one * of the 5 writable states -- so no row anywhere ever gives @@ -4990,25 +4971,23 @@ NodeStatusPatternSurvivesReportedIsPrepareMaintenance(const NodeStatusPattern *p * reportedState irrelevant", ASSIGNED kind) would otherwise look like it * resolves to (and therefore unconditionally matches) every one of the 21 * states, when it actually still requires a completely separate, real fact - * (goalState == DROPPED) that has nothing to do with reportedState at all -- - * confirmed by dump_fsm_edges()'s own regression: an early, buggy version of - * this shadowing check treated pos 203 as unconditional and wrongly - * suppressed several of pos 209's genuinely reachable fanned-out states - * (wait_standby, prepare_maintenance, wait_maintenance, fast_forward, - * join_secondary) that have nothing to do with the node's goal being - * DROPPED. + * (goalState == DROPPED) that has nothing to do with reportedState at all. + * Treating ASSIGNED/NOT_ASSIGNED rows as eligible for unconditional-match + * shadowing here would wrongly suppress several of pos 209's genuinely + * reachable fanned-out states (wait_standby, prepare_maintenance, + * wait_maintenance, fast_forward, join_secondary), none of which have + * anything to do with the node's goal being DROPPED. * * STABLE is kept eligible despite its own "reported == goal" wrinkle: for * every row actually written using it (a bare FSM_STATE(x), no other * condition), the codebase's own edge-source resolution * (NodeStatePatternResolveFromStates) already treats STABLE identically to - * REPORTED, and confirmed live (this file's own comment on pos 205/pos 209) - * that treatment is what makes real shadowing detection work at all -- - * requiring the stricter, fully rigorous "and goal really does equal - * reported in every possible calling context" would need modeling whether - * some OTHER row's otherNodeAssignedState could have changed this exact - * node's own goalState moments earlier, out of scope for a static, - * per-table check like this one. + * REPORTED (see this file's own comment on pos 205/pos 209), which is what + * makes real shadowing detection work at all -- requiring the stricter, + * fully rigorous "and goal really does equal reported in every possible + * calling context" would need modeling whether some OTHER row's + * otherNodeAssignedState could have changed this exact node's own goalState + * moments earlier, out of scope for a static, per-table check like this one. */ static bool NodeStatePatternKindIsReportedStateOnly(NodeStatePatternKind kind) @@ -5235,9 +5214,9 @@ PG_FUNCTION_INFO_V1(dump_fsm_edges); * (pos, current_state, assigned_state) edges -- one row per actual * (reportedState, assignedState) pair a row can produce, fully resolved (never * NULL, unlike dump_fsm()'s own pattern-summary columns, which stay unresolved - * for human display). This is the keeper-cross-check surface the design doc's - * check_fsm_reachability() proposal needs: pgautofailover. - * check_fsm_reachability(jsonb) anti-joins this against a keeper's own + * for human display). This is the keeper-cross-check surface + * pgautofailover.check_fsm_reachability(jsonb) needs: it anti-joins this + * against a keeper's own * KeeperFSM[] edges (see KeeperFSMToJSON(), src/bin/pg_autoctl/fsm.c) to find * any monitor transition with no matching keeper edge -- exactly the bug class * issue #774 was. @@ -5247,9 +5226,8 @@ PG_FUNCTION_INFO_V1(dump_fsm_edges); * and primaryNode's own (statePattern -> otherNodeAssignedState) when it does. * candidateNode never has an assignment slot of its own (see * MonitorFSMTransition's own comment), so it never contributes an edge. pgKind - * is deliberately not modeled here -- see this function's own header comment in - * the design doc discussion; every Citus-specific KeeperFSM[] edge already has - * a NODE_KIND_ANY counterpart with the same (current, assigned) shape + * is deliberately not modeled here: every Citus-specific KeeperFSM[] edge + * already has a NODE_KIND_ANY counterpart with the same (current, assigned) shape * (fsm_mermaid.c's own comment establishes this as an invariant this codebase * already relies on elsewhere), so a flat, pgKind-blind edge set on both sides * is already correct. @@ -5284,21 +5262,22 @@ PG_FUNCTION_INFO_V1(dump_fsm_edges); * kind of false gap for every one of these ten rows, all confirmed by that same * live run. * - * A third category is filtered the same way, for a different reason: this - * function used to resolve each row's own edges entirely independently, - * never considering any OTHER row, so it could report an edge for a + * A third category is filtered the same way, for a different reason: a + * row's own edges can't be resolved entirely independently, without + * considering any OTHER row, because that would report an edge for a * current_state that an EARLIER row (lower array index, matching * unconditionally -- e.g. a bare no-op like pos 205's "converged to - * maintenance -> no-op, frozen until stop_maintenance()") would actually - * intercept first in real first-match-wins dispatch, making that edge - * practically unreachable. Confirmed concretely via a live pgaftest run + * maintenance -> no-op, frozen until stop_maintenance()") actually + * intercepts first in real first-match-wins dispatch, making that edge + * practically unreachable. A live pgaftest run * (tests/tap/specs/keeper_fsm_gap_211_primary_priority_zero.pgaf's own - * investigation): pos 209's "maintenance" edge looked like a real gap here, - * but pos 205 comes first in array order and intercepts every node_active() - * call from a node in MAINTENANCE_STATE regardless of group size, so pos - * 209 can never actually fire for that state at all. + * investigation) confirmed exactly this: pos 209's "maintenance" edge looks + * like a real gap in isolation, but pos 205 comes first in array order and + * intercepts every node_active() call from a node in MAINTENANCE_STATE + * regardless of group size, so pos 209 can never actually fire for that + * state at all. * - * EdgeIsShadowedByEarlierRule (below) now detects exactly this: for each + * EdgeIsShadowedByEarlierRule (below) detects exactly this: for each * candidate edge, it scans every earlier row sharing the same top-level * section (every section is its own independent top-level scan, so a row * outside it is never reachable in the same dispatch pass regardless of @@ -5526,23 +5505,21 @@ dump_fsm_edges(PG_FUNCTION_ARGS) * This separation lets test code inject a synthetic context and exercise the * FSM without a live database connection. * - * Single-shot, two straight-line lookups at most -- matching the design - * doc's own top-level driver, not a loop: the cascades that need more than - * one row's worth of assignment in a single call (the MS-failover cascade, - * the join_secondary -> nested primary pass) get there via a bounded, named - * nested search inside their own extraAction (see ActionRunMultiStandby - * FailoverCascade and ActionRunPrimaryNodeTransition), not by this driver - * looping. + * Single-shot, two straight-line lookups at most, not a loop: the cascades + * that need more than one row's worth of assignment in a single call (the + * MS-failover cascade, the join_secondary -> nested primary pass) get there + * via a bounded, named nested search inside their own extraAction (see + * ActionRunMultiStandbyFailoverCascade and ActionRunPrimaryNodeTransition), + * not by this driver looping. * - * Two lookups, not the design doc's one ("startIndex = isInPrimaryState ? - * MonitorFSM_PrimaryNodeSectionStart : 0"): the six early-check rows must - * always be tried first, regardless of whether activeNode is already - * primary-role -- a primary that just lost its only standby must still - * reach SINGLE via those checks, not get redirected to the - * ProceedGroupStateForPrimaryNode section first. Jumping straight past them - * whenever activeNode is already primary-role would skip that case - * entirely; confirmed by the drop_node regression test, which failed - * exactly this way the first time this table's ordering got this wrong. + * Two lookups, not one straight to SectionPrimaryNode when activeNode is + * already primary-role: the six early-check rows (SectionEarlyChecks) must + * always be tried first regardless -- a primary that just lost its only + * standby must still reach SINGLE via those checks, not get redirected to + * the primary-role section first. Jumping straight past them whenever + * activeNode is already primary-role would skip that case entirely; + * confirmed by the drop_node regression test, which failed exactly this way + * the first time this table's ordering got this wrong. */ bool ProceedGroupStateFromContext(GroupStateContext *ctx) @@ -5675,9 +5652,9 @@ WalSourceNodesAreAllUnhealthy(GroupStateContext *ctx, /* - * BuildMSFailoverNodeActiveContext computes the facts the MS-failover cluster's - * own declarative rows (MonitorFSM_MSFailoverStart onwards) need. candidateNode - * is NULL at exactly one call site (TryFanOutReportLsnRow, wrapping + * BuildMSFailoverNodeActiveContext computes the facts the MS-failover + * cluster's own declarative rows (SectionMSFailover, pos 363 onward) need. + * candidateNode is NULL at exactly one call site (TryFanOutReportLsnRow, wrapping * BuildCandidateList's own fan-out loop, which hasn't selected a candidate yet) * -- everywhere else (both call sites TryMSFailoverDeclarativeRow wraps) it's * non-NULL, called from within ProceedGroupStateForMSFailover's own @@ -5747,18 +5724,15 @@ BuildMSFailoverCandidateGateNodeActiveContext(GroupStateContext *ctx, /* * ActionLogMSFailoverMissingNodesDecline/Continue and ActionLogMSFailover - * QuorumDecline/Continue reproduce, verbatim, the LogAndNotifyMessage text - * ProceedGroupStateForMSFailover used to build inline for its own - * missingNodesCount/quorumCandidateCount gates -- moved here so the - * declarative rows that now match these same conditions (see the - * missing_nodes_gate/quorum_candidate_gate rows in MonitorFSM[]) are the - * single source of truth for the message, not a hand-written duplicate of - * it. Neither gate assigns a goal state either way (the original code - * never called AssignGoalState in either branch), so none of these four - * actions do either -- the control-flow decision itself (decline vs. - * continue) stays exactly the hand-written `if (GuardDataLoss)` in - * ProceedGroupStateForMSFailover, unchanged; only the message text is - * delegated here. + * QuorumDecline/Continue build the LogAndNotifyMessage text for + * ProceedGroupStateForMSFailover's own missingNodesCount/quorumCandidateCount + * gates, run as the extraAction of the declarative rows that match these same + * conditions (see the missing_nodes_gate/quorum_candidate_gate rows in + * MonitorFSM[]) -- those rows are the single source of truth for the + * message text. Neither gate assigns a goal state either way, so none of + * these four actions do either -- the control-flow decision itself (decline + * vs. continue) is the hand-written `if (GuardDataLoss)` in + * ProceedGroupStateForMSFailover; only the message text is delegated here. */ static void ActionLogMSFailoverMissingNodesDecline(GroupStateContext *ctx, NodeActiveContext *nac, @@ -6102,11 +6076,10 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, * candidateCount/quorumCandidateCount/sufficientQuorumCandidates) for * MonitorFSM[]'s own declarative rows under reporting_node.ms_failover. * promotion_outcome.*_gate -- see BuildMSFailoverCandidateGateNodeActive - * Context's own comment. Each gate below still makes its own decline-vs- - * continue decision in plain C, exactly as before this refactor; only - * the message text each branch logs is now delegated to the matching - * row's own extraAction, so the table stays the single source of truth - * for what gets logged and why. + * Context's own comment. Each gate below makes its own decline-vs- + * continue decision in plain C; only the message text each branch logs + * is delegated to the matching row's own extraAction, so the table + * stays the single source of truth for what gets logged and why. */ NodeActiveContext gateNac; @@ -6150,10 +6123,9 @@ ProceedGroupStateForMSFailover(GroupStateContext *ctx, int minCandidates = ctx->formation->number_sync_standbys + 1; /* - * no candidates is a hard pass -- see MonitorFSM[]'s own - * candidate_count_gate row for this same fact, matched declaratively but - * never itself dispatched (a silent decline, same as the original code: - * no log here either). + * no candidates is a hard pass, with no log message -- see MonitorFSM[]'s + * own candidate_count_gate row for this same fact, matched declaratively + * but never itself dispatched, purely for dump_fsm() completeness. */ if (candidateList.candidateCount == 0) {