diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81027169d..413d08f53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -323,6 +323,16 @@ jobs: - { PGVERSION: 17, schedule: node } - { PGVERSION: 18, schedule: node } - { PGVERSION: 19, schedule: node } + # archiver: WAL capture, base backups, rebuild-from-archiver — all + # PG versions, since pg_walsender's wire-protocol + # correctness is version-sensitive (see + # tests/tap/schedules/archiver.sch's own header comment) + - { PGVERSION: 14, schedule: archiver } + - { PGVERSION: 15, schedule: archiver } + - { PGVERSION: 16, schedule: archiver } + - { PGVERSION: 17, schedule: archiver } + - { PGVERSION: 18, schedule: archiver } + - { PGVERSION: 19, schedule: archiver } # ssl: enable_ssl, ssl_self_signed, ssl_cert - { PGVERSION: 14, schedule: ssl } - { PGVERSION: 15, schedule: ssl } @@ -335,6 +345,7 @@ jobs: - { PGVERSION: 17, schedule: multi-misc } - { PGVERSION: 17, schedule: multi-async } - { PGVERSION: 17, schedule: node-fsm-gaps } + - { PGVERSION: 17, schedule: archiver-multi } - { PGVERSION: 17, schedule: citus-1 } - { PGVERSION: 17, schedule: citus-2 } # citus on PG18 (supported); allow failure until officially validated diff --git a/.gitignore b/.gitignore index 83e10cec8..d260eb535 100644 --- a/.gitignore +++ b/.gitignore @@ -56,5 +56,6 @@ valgrind/ src/bin/pgaftest/test_spec_parse.tab.* src/bin/pgaftest/test_spec_parse.output src/bin/pgaftest/pgaftest +src/bin/pg_walsender/pg_walsender run-test.sh tests/tablespaces/__pycache__/ diff --git a/Dockerfile b/Dockerfile index 116d00961..223c1c605 100644 --- a/Dockerfile +++ b/Dockerfile @@ -120,6 +120,12 @@ COPY --from=build /usr/lib/postgresql/${PGVERSION}/lib/pgautofailover.so \ COPY --from=build /usr/share/postgresql/${PGVERSION}/extension/pgautofailover* \ /usr/share/postgresql/${PGVERSION}/extension/ COPY --from=build /usr/local/bin/pg_autoctl /usr/local/bin/ +# Bracket-glob makes this an optional copy: BuildKit treats [r] as a glob, +# and an empty glob match is not an error for COPY (unlike a literal missing +# path). This lets tests/upgrade build the "current" Dockerfile against an +# old release's source tree, which predates pg_walsender and has no binary +# to copy. Stopgap only -- revisit after the release with a cleaner fix. +COPY --from=build /usr/local/bin/pg_walsende[r] /usr/local/bin/ RUN mkdir -p /var/lib/postgres \ && chown -R docker /var/lib/postgres diff --git a/docs/_static/css/zoom.css b/docs/_static/css/zoom.css new file mode 100644 index 000000000..dddae2e15 --- /dev/null +++ b/docs/_static/css/zoom.css @@ -0,0 +1,77 @@ +/* Click-to-zoom overlay for docs figures -- see js/zoom.js */ + +figure img.pgaf-zoomable-img { + cursor: zoom-in; +} + +html.pgaf-zoom-locked, +html.pgaf-zoom-locked body { + overflow: hidden; +} + +.pgaf-zoom-overlay { + display: none; + position: fixed; + inset: 0; + z-index: 10000; + background: rgba(20, 20, 20, 0.92); +} + +.pgaf-zoom-overlay.pgaf-zoom-open { + display: flex; + flex-direction: column; +} + +.pgaf-zoom-viewport { + flex: 1 1 auto; + overflow: hidden; + display: flex; + align-items: center; + justify-content: center; + cursor: grab; +} + +.pgaf-zoom-viewport.pgaf-zoom-dragging { + cursor: grabbing; +} + +.pgaf-zoom-img { + max-width: 92vw; + max-height: 82vh; + will-change: transform; + transition: transform 0.05s linear; + user-select: none; + -webkit-user-drag: none; + background: #fff; + border-radius: 4px; + box-shadow: 0 10px 40px rgba(0, 0, 0, 0.5); +} + +.pgaf-zoom-close { + position: absolute; + top: 1.25rem; + right: 1.5rem; + width: 2.5rem; + height: 2.5rem; + border-radius: 50%; + border: none; + background: rgba(255, 255, 255, 0.12); + color: #fff; + font-size: 1.75rem; + line-height: 1; + cursor: pointer; + z-index: 1; +} + +.pgaf-zoom-close:hover, +.pgaf-zoom-close:focus { + background: rgba(255, 255, 255, 0.25); +} + +.pgaf-zoom-hint { + flex: 0 0 auto; + text-align: center; + color: rgba(255, 255, 255, 0.7); + font-size: 0.85rem; + padding: 0.5rem 1rem 1.25rem; +} diff --git a/docs/_static/js/zoom.js b/docs/_static/js/zoom.js new file mode 100644 index 000000000..936004ff4 --- /dev/null +++ b/docs/_static/js/zoom.js @@ -0,0 +1,174 @@ +/* + * Click-to-zoom for the docs' own figures (architecture/sequence/FSM + * diagrams rendered from tikz), generalizing the pan/scroll-to-zoom + * already available on Mermaid diagrams (mermaid_d3_zoom, conf.py) to + * every other the docs embed via `.. figure::`. + * + * Mermaid diagrams render as inline , not , and already ship + * their own in-place zoom -- this script only ever wires plain + * elements, so the two never compete for the same wheel/drag events. + */ +(function () { + "use strict"; + + function buildOverlay() { + var overlay = document.createElement("div"); + overlay.className = "pgaf-zoom-overlay"; + overlay.setAttribute("role", "dialog"); + overlay.setAttribute("aria-modal", "true"); + overlay.innerHTML = + '' + + '
' + + '' + + "
" + + '
scroll to zoom · drag to pan · ' + + "double-click to reset · Esc to close
"; + document.body.appendChild(overlay); + + var viewport = overlay.querySelector(".pgaf-zoom-viewport"); + var img = overlay.querySelector(".pgaf-zoom-img"); + var closeBtn = overlay.querySelector(".pgaf-zoom-close"); + + var scale = 1; + var panX = 0; + var panY = 0; + var dragging = false; + var startX = 0; + var startY = 0; + var startPanX = 0; + var startPanY = 0; + var lastFocused = null; + + function applyTransform() { + img.style.transform = + "translate(" + panX + "px, " + panY + "px) scale(" + scale + ")"; + } + + function reset() { + scale = 1; + panX = 0; + panY = 0; + applyTransform(); + } + + function isOpen() { + return overlay.classList.contains("pgaf-zoom-open"); + } + + function open(src, alt) { + lastFocused = document.activeElement; + img.src = src; + img.alt = alt || ""; + reset(); + overlay.classList.add("pgaf-zoom-open"); + document.documentElement.classList.add("pgaf-zoom-locked"); + closeBtn.focus(); + } + + function close() { + overlay.classList.remove("pgaf-zoom-open"); + document.documentElement.classList.remove("pgaf-zoom-locked"); + img.removeAttribute("src"); + if (lastFocused && typeof lastFocused.focus === "function") { + lastFocused.focus(); + } + } + + closeBtn.addEventListener("click", close); + + overlay.addEventListener("click", function (event) { + if (event.target === overlay) { + close(); + } + }); + + document.addEventListener("keydown", function (event) { + if (event.key === "Escape" && isOpen()) { + close(); + } + }); + + viewport.addEventListener( + "wheel", + function (event) { + if (!isOpen()) { + return; + } + event.preventDefault(); + var factor = event.deltaY < 0 ? 1.15 : 1 / 1.15; + scale = Math.min(8, Math.max(0.5, scale * factor)); + applyTransform(); + }, + { passive: false } + ); + + viewport.addEventListener("dblclick", function (event) { + event.preventDefault(); + reset(); + }); + + viewport.addEventListener("mousedown", function (event) { + dragging = true; + startX = event.clientX; + startY = event.clientY; + startPanX = panX; + startPanY = panY; + viewport.classList.add("pgaf-zoom-dragging"); + event.preventDefault(); + }); + + window.addEventListener("mousemove", function (event) { + if (!dragging) { + return; + } + panX = startPanX + (event.clientX - startX); + panY = startPanY + (event.clientY - startY); + applyTransform(); + }); + + window.addEventListener("mouseup", function () { + dragging = false; + viewport.classList.remove("pgaf-zoom-dragging"); + }); + + return { open: open, close: close }; + } + + function isZoomable(img) { + if (img.closest(".pgaf-zoom-overlay")) { + return false; + } + if (img.classList.contains("no-zoom")) { + return false; + } + return !!img.closest("figure"); + } + + function wireImages(zoom) { + var images = document.querySelectorAll("figure img"); + images.forEach(function (img) { + if (!isZoomable(img) || img.dataset.pgafZoomWired) { + return; + } + img.dataset.pgafZoomWired = "1"; + img.classList.add("pgaf-zoomable-img"); + img.tabIndex = 0; + img.setAttribute("role", "button"); + img.setAttribute("aria-label", "Click to zoom: " + (img.alt || "image")); + img.addEventListener("click", function () { + zoom.open(img.currentSrc || img.src, img.alt); + }); + img.addEventListener("keydown", function (event) { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + zoom.open(img.currentSrc || img.src, img.alt); + } + }); + }); + } + + document.addEventListener("DOMContentLoaded", function () { + var zoom = buildOverlay(); + wireImages(zoom); + }); +})(); diff --git a/docs/architecture.rst b/docs/architecture.rst index 0df322890..1c30248d7 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -31,6 +31,69 @@ PostgreSQL service to accept writes when there's a single server available, and opens the service for potential data loss if the primary server were also to fail. +High Availability +------------------ + +pg_auto_failover treats "High Availability" as two related but distinct +guarantees, rather than one. Most of what follows on this page -- the +Monitor, the keeper, synchronous replication, node recovery -- is in service +of the first of the two; :ref:`archiving_architecture` and the pages it +links to are in service of the second: + +- **Service Availability**: the Postgres *service* itself stays reachable + and able to accept reads and writes, with as little downtime as + possible when a node is lost. +- **Disaster Recovery**: the *data* survives even in scenarios Service + Availability alone can't help with -- an operator mistake, a bad + deployment, or every node that ever held the data being lost at once. + +Most Postgres setups reach for two separate, independently-operated +products for these -- an HA tool for the first, a backup tool for the +second. pg_auto_failover treats them as one system instead; see +:ref:`ha_dr_backups` for the full comparison. + +Service Availability (failover) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is what the rest of this page, and :ref:`failover_state_machine` / +:ref:`fault_tolerance` in detail, describe: a primary and one or more +secondary nodes, a Monitor orchestrating automated failover when the +primary is lost, and synchronous replication (`Synchronous vs. +asynchronous replication`_ below) guaranteeing no committed write is lost +in the process. This is the guarantee that answers "the primary just +died -- who serves the next query?". + +Service Availability can be setup to obtain Business Continuity in the face +of production incidents with a basic setup of two Postgres nodes, and +Postgres High Availability starting with a setup of three Postgres nodes. + +Given integrated archiving support, a trade-off or *budget* architecture can +be easily deployed with two Postgres nodes and an archiver to obtain an HA +setup that complies with many production needs. + +Disaster Recovery +^^^^^^^^^^^^^^^^^^ + +Service Availability only makes sense for a database system when there is a +compliant setup for durability, or data safety. When using PostgreSQL, that +means a proper archiving implementation that allows *Point in Time Recovery* +operations. + +With PITR it's possible to mitigate data loss operations such as a missing +WHERE clause in a DELETE or a DROP TABLE done in production instead of the +development environment, also known as human errors. + +Disaster Recovery is handled by a physically distinct kind of node, the +**archiver**, added on top of any of the architectures on this page: it +continuously captures WAL from the group's current primary and periodically +produces base backups. + +See :ref:`archiving_and_disaster_recovery` for where an archiver fits +alongside the architectures below, :ref:`archiving_architecture` for exactly +how WAL capture and base-backup generation work, and +:ref:`archiving_fault_tolerance` for how this changes what a total loss of +the rest of the formation actually means. + The pg_auto_failover Monitor ---------------------------- @@ -152,6 +215,25 @@ As a result, refrain from naming your nodes with the role you intend for them. Their roles can change. If they didn't, your system wouldn't need pg_auto_failover! +Archiver +^^^^^^^^ + +An archiver is a server (virtual or physical) that runs PostgreSQL archiving +storage for one or many formations. The archiver hosts any number of +*archiving nodes* and schedules *base backups* in order to be able to +implement Postgres `Point in Time Recovery`__ which is the foundations for +implementing Disaster Recovery. + +__ https://www.postgresql.org/docs/current/continuous-archiving.html + +Archiving Node +^^^^^^^^^^^^^^ + +A process managed in an archiver instance that reports to the monitor as a +node in a group and that runs ``pg_receivewal``. An archiving node as no +PGDATA, it can participate in the replication quorum but can not be a +failover candidate: its ``candidate_priority`` is always zero. + State ^^^^^ diff --git a/docs/archiving-details.rst b/docs/archiving-details.rst new file mode 100644 index 000000000..94cedbd0e --- /dev/null +++ b/docs/archiving-details.rst @@ -0,0 +1,341 @@ +.. _archiving_architecture: + +Archiving in Detail +===================== + +:ref:`archiving_and_disaster_recovery` introduces the archiver at a glance, +and :ref:`archiving_operations` walks through the day-to-day commands for +registering one and attaching a base-backup policy. This page goes one +level deeper: what actually moves over the network and onto disk while an +archiver is running, and what processes are involved -- the level of +detail worth having before sizing storage, deciding where an archiver +should sit on your network, or reasoning about how a single archiver +covers a whole topology (every group of a Citus formation, or several +independent formations at once). + +Data flow +--------- + +An archiver does three things, and none of them ever route through the +monitor -- WAL and base backups always flow directly between the archiver +and whichever node it's talking to, with the monitor only ever seeing +small status reports (what's been captured, what's been backed up, how +much disk is left), never the data itself: + +1. **It streams WAL continuously** from whichever node is currently the + primary, over an ordinary PostgreSQL physical replication connection -- + the same kind of connection a standby uses, protected by its own + dedicated replication slot so that nothing already captured is ever + lost, even across a connection that drops and stays down for a while. + If the primary changes, the archiver notices and reconnects to the new + one on its own; no operator action is needed. An archiver attached to + several groups (see `Process model`_ below) runs one of these streams + per group, entirely independently -- one group's primary changing, or + its stream stalling, has no effect on any other group's. +2. **It produces base backups on a schedule**, either as a real + ``pg_basebackup`` taken directly from a live node, or entirely on its + own: replaying already-captured WAL against a local copy of the last + base backup until that copy reaches a consistent, promotable state, + and backing up that instead. The second mode never touches the + primary or any standby at all -- useful when you want frequent base + backups without adding load to production. +3. **It hands both back out** on request: a real ``pg_basebackup`` + command, a real standby's own ``primary_conninfo``, or this project's + own restore tooling can all connect to an archiver directly and get + what they ask for, with no special client needed -- see `What you can + point at an archiver`_ below. + +Storage +------- + +Everything an archiver holds lives under one local directory -- the path +given as ``--pgdata`` when the archiver was created. Despite the flag's +name, this is never a real Postgres data directory (there is no +``initdb``, nothing ever starts Postgres against it directly); it's a +cache root. + +A single archiver can be attached to more than one (formation, group) at +once -- every group of a Citus formation, or several independent +formations altogether (see `Process model`_ below). Each such membership +gets its own subdirectory, one level under the archiver's own root, named +after the formation and group it belongs to, so that two memberships' +WAL and base backups never collide even though they share one archiver +identity and one root directory:: + + /var/lib/pgaf/archiver1/ + ├── archiver-routes.ini + ├── default/ + │ └── 0/ + │ ├── 000000010000000000000041 + │ ├── 000000010000000000000042 + │ ├── 000000010000000000000043.partial + │ ├── archiver-position + │ └── basebackups/ + │ ├── basebackup-20260803T020000Z/ + │ ├── basebackup-20260804T020000Z/ + │ └── basebackup-20260805T020000Z/ + └── billing/ + └── 0/ + ├── 000000010000000000000012 + ├── archiver-position + └── basebackups/ + └── basebackup-20260805T030000Z/ + +- WAL segments sit directly under their own ``//`` + subdirectory, named exactly the way Postgres itself names them. The + most recently-started one carries a ``.partial`` suffix until it's + complete -- archiving doesn't wait for a segment to fill up before it + counts: whatever has already been flushed into that ``.partial`` file + is captured too. +- Each retained base backup is its own subdirectory under that + membership's own ``basebackups/``, in the same layout an ordinary + ``pg_basebackup`` run by hand would produce. You could point + ``postgres -D`` straight at one of them and it would start -- that's + exactly what disaster recovery relies on. +- Each membership has its own ``archiver-position`` file, tracking that + group's own captured LSN. ``archiver-routes.ini`` sits at the archiver's + own root instead, one section per membership -- see `Keeping the + routes file current`_ below for exactly when and why it gets rewritten. + All of these are small internal bookkeeping files -- coordinates and + status, never a copy of any actual data. Safe to ignore day to day, and + not something that needs backing up itself -- all of them are + regenerated automatically. + +A single-membership archiver (the common case: one formation, one group) +looks the same, just with only one ``//`` subdirectory +under its root. + +Sizing disk for one membership comes down to two mostly-independent +numbers: + +- **Base backups**: roughly the policy's ``maxcount`` times the size of + one backup, since retention prunes anything beyond that count (or + older than ``maxage``, whichever comes first) right after each new one + lands. See :ref:`archiving_operations` for how to set these. +- **WAL**: however much WAL has accumulated since your *oldest + still-retained* base backup -- once a base backup is pruned, the WAL + segments only it still needed are pruned right along with it. A longer + retention window keeps more history recoverable, at the cost of more + WAL kept around to cover it. + +An archiver attached to several groups needs the sum of this across every +membership -- each has its own base-backup policy and its own WAL +retention, sized independently. + +Network exposure +----------------- + +An archiver listens on a TCP port (``6543`` by default) speaking a subset +of the PostgreSQL replication protocol, authenticated the same trust-based +way every node's own replication connections already are in a +pg_auto_failover cluster (there is no password or TLS on this connection +in the current release). Treat it the same way you'd treat any other +node's own replication port: reachable from wherever you expect to run +``pg_basebackup``, point a standby's ``primary_conninfo`` at it, or run a +restore from, and firewalled off from everywhere else. + +Process model +-------------- + +Once started (``pg_autoctl archiver run``, or ``pg_autoctl node run`` +against a ``kind = archiver`` node specification), an archiver supervises +exactly two long-running processes: ``serve``, and a ``reconciler`` that +in turn keeps one WAL-capture child running per (formation, group) +membership this archiver currently holds -- added and removed on its own +as the archiver is attached to or detached from a formation, no restart +of the archiver itself required. They hand off small files (`Storage`_ +above) and nothing else: + +.. figure:: ./tikz/arch-archiver-internals.svg + :alt: pg_autoctl archiver run supervises two processes, reconciler and serve; reconciler forks one capture child per membership, each running pg_receivewal and writing its own archiver-position; serve writes archiver-routes.ini (one section per membership) and runs pg_walsender, which reads the WAL cache and routes file and serves pg_basebackup, streaming standbys, and restore_command fetches + + Two supervised top-level processes per archiver; the reconciler forks + one WAL-capture child per membership underneath it + +:: + + pg_autoctl archiver run + ├── reconciler -- keeps the set of running captures in sync with the + │ │ monitor's own membership list for this archiver + │ ├── capture (default/0) -- one per membership, reports its own + │ │ └── pg_receivewal progress to the monitor independently + │ └── capture (billing/0) + │ └── pg_receivewal + └── serve -- keeps the archiver reachable over the network, + └── pg_walsender --port 6543 --routes archiver-routes.ini + (serves every membership through the one process) + +If any child stops unexpectedly, its supervisor notices on its next tick +and restarts it -- an archiver recovering from a crashed +``pg_receivewal`` or ``pg_walsender`` needs no operator action, the same +way a keeper recovers a crashed Postgres. A crash of the reconciler +itself is likewise just restarted by the top-level supervisor; on +restart it re-discovers its current memberships from the monitor and +resumes capturing all of them -- a replication slot keeps the WAL a +capture needs regardless of how many times its own consumer reconnects, +so this costs nothing. + +Keeping the routes file current +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``pg_walsender`` never queries the monitor itself, on purpose: an +archiver exists to keep serving already-captured data even when the +monitor it would otherwise depend on is unreachable, and staying free of +that dependency also keeps ``pg_walsender`` a small, standalone binary +with nothing to mock or stand up just to test it. ``archiver-routes.ini`` +is the decoupling point -- ``serve`` is the one process that actually +talks to the monitor, resolving each membership's current WAL-cache +directory and latest complete base backup and writing them here; every +``pg_walsender`` connection just reads this one local file straight off +disk, fresh, with no monitor round trip on its own hot path. One section +per membership:: + + [default/0] + walcache = /var/lib/pgaf/archiver1/default/0 + position = 0/0 + basebackup = /var/lib/pgaf/archiver1/default/0/basebackups/basebackup-20260806T132954Z + timeline = 1 + systemid = 7670908901798703128 + +The file is always rewritten as a whole -- one full pass over every +membership this archiver currently holds, written to a temporary file +and atomically renamed into place -- never patched in place. A +connection arriving mid-refresh always sees either the complete previous +version or the complete new one, never a torn write; nothing here needs +a lock. ``serve`` triggers a rewrite: + +- once at startup, before ``pg_walsender`` is even started; +- every 30 seconds, as a periodic catch-all -- covers anything not + otherwise signaled, such as a membership having just been attached; +- immediately, the moment a base backup finishes and is reported + complete -- the process that just produced it signals ``serve`` + directly, rather than leaving a freshly-completed backup unservable + for up to that 30-second window; and +- on ``SIGHUP``, the same reload signal every other pg_autoctl process + already understands. + +Each membership generates its own base backups independently (its own +schedule, its own retention), so more than one can genuinely be in +progress at once on a multi-membership archiver -- there's no archiver- +wide lock serializing them. + +More or fewer standby nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A membership's own capture process doesn't change shape based on how many +standby nodes are in its group. WAL capture always talks to whichever +node is currently primary, never to a standby directly, so a two-node +group and a five-node group look identical from the archiver's side. The +only place standby count matters at all is when a base backup is sourced +live: with more healthy standbys available, there are more candidates to +pick from before falling back to the primary -- everything else about +the archiver is unaffected. + +Several formations +^^^^^^^^^^^^^^^^^^^ + +One archiver can be attached to several formations at once -- each with +its own group of two or three standby nodes, say -- with no need to run a +separate archiver process per formation (see :ref:`archiving_operations` +for the repeated ``--formation`` this takes at creation time). Each +formation attached this way is one more membership, which shows up as one +more ``capture`` child under the reconciler and one more section in +``archiver-routes.ini``; nothing about the archiver's own identity, port, +or ``--pgdata`` root changes: + +:: + + pg_autoctl archiver run + ├── reconciler + │ ├── capture (default/0) -> pg_receivewal + │ └── capture (billing/0) -> pg_receivewal + └── serve -> pg_walsender (serves both memberships) + +Each membership's own capture is entirely independent -- separate storage +subdirectory, separate WAL stream, separate base-backup schedule, no +shared state with any other membership. Losing one (its capture process +crashing, say) has no effect on the others; the reconciler restarts just +that one. Running one archiver per formation instead, on separate hosts, +is still a perfectly reasonable choice -- for isolating blast radius, or +spreading load across machines -- just no longer a requirement. + +A Citus formation +^^^^^^^^^^^^^^^^^^ + +A Citus formation is really several node groups under one name: the +coordinator's own group, plus one group per worker. Attaching an archiver +to a Citus formation attaches it to every group that already exists in +that formation at the time -- the coordinator's and every worker's -- +each becoming its own membership with its own capture process, exactly +like several independent formations would. A worker group added to the +formation *afterwards* is not picked up on its own: the reconciler only +ever starts capture for memberships the monitor already knows about, and +nothing today re-attaches an archiver to a formation automatically when +that formation grows a new group. Re-running the attach for that +formation covers the new group too (existing memberships are left alone), +and the reconciler picks it up on its own next periodic check, no +archiver restart required. + +What you can point at an archiver +------------------------------------ + +An archiver's serving side understands enough of the real PostgreSQL +replication protocol that ordinary, unmodified tools can talk to it +directly -- nothing here needs a custom client. The commands below are +what those tools actually send; useful to know if you're connecting by +hand with ``psql "... replication=database"`` to check on an archiver, or +deciding what else could talk to one. + +``IDENTIFY_SYSTEM`` + + The first thing any of these tools asks: which system and timeline the + archiver is tracking, and how far it's captured so far. + +``BASE_BACKUP`` + + Streams the archiver's most recent base backup, in the same plain tar + format a real ``pg_basebackup --format=plain`` produces. Point a real, + unmodified ``pg_basebackup`` at an archiver and it works exactly as it + would against a live node -- this is what ``pg_autoctl create postgres + --from-archiver`` uses to bootstrap a brand new node straight from an + archiver's cache instead of a live primary or secondary. + +``START_REPLICATION`` + + Streams WAL from a given position onward, the same way a live primary + would. This is what lets a real standby's own ``primary_conninfo`` + point at an archiver instead of a live node, and what a multi-standby + failover election falls back on to fetch WAL a promoted candidate is + still missing, straight from the archiver's own cache, when no live + node has it anymore. + +``TIMELINE_HISTORY`` + + Returns the timeline history for a given timeline -- needed by any + streaming client following a timeline change, such as after a + failover. + +``CREATE_REPLICATION_SLOT`` / ``READ_REPLICATION_SLOT`` + + Basic physical replication slot support, for tools that expect to + manage their own slot against whatever they're streaming from. + +Fetching a single WAL file + + A small side channel used by this project's own ``restore_command`` + tooling: ask for one file by name, get its exact bytes back. This is + what makes an archiver usable as a ``restore_command`` target on its + own, without needing a full streaming connection just to recover one + missing segment. + +See also +-------- + +- :ref:`archiving_and_disaster_recovery` -- what an archiver is and + where it fits among the other architectures +- :ref:`archiving_operations` -- registering an archiver, attaching a + base-backup policy, rebuilding a node from one +- :ref:`archiving_fault_tolerance` -- what changes about fault tolerance + once an archiver is in the picture +- :ref:`failover_state_machine` -- the ``archiving`` state's own + transitions diff --git a/docs/archiving.rst b/docs/archiving.rst new file mode 100644 index 000000000..4b3de6535 --- /dev/null +++ b/docs/archiving.rst @@ -0,0 +1,194 @@ +.. _archiving_operations: + +Archiving +========= + +This page covers the operational side of running an **archiver** node: +registering one, attaching a base-backup policy to control how it produces +and prunes base backups, watching what it's captured, and rebuilding a +node from its cache when disaster recovery is what's needed. For the +architecture and the reasoning behind archiving nodes, see +:ref:`archiving_architecture` and :ref:`archiving_fault_tolerance`; for the +``archiving`` state's exact transitions in the keeper's state machine, see +:ref:`failover_state_machine`. + +Registering an archiver +------------------------ + +An archiver is created the same way as any other node kind, with its own +dedicated verb:: + + $ pg_autoctl create archiver \ + --pgdata /var/lib/pgaf/archiver1 \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --hostname archiver1.example.com \ + --formation default \ + --run + +Unlike ``pg_autoctl create postgres``, this does not initialize a +PostgreSQL data directory: ``--pgdata`` here names the archiver's local +cache directory for captured WAL segments and base backups. Once +registered, the archiver starts one ``pg_receivewal`` per group of the +formation it's attached to (every worker of a Citus formation included, +not just the coordinator) against each group's current primary, following +it across any later promotion, and reports its progress to the monitor +the same way a standby reports replication state -- see +:ref:`archiving_architecture` for the full process model. + +``--formation`` may be given more than once, to attach the same archiver +to several formations right from the start:: + + $ pg_autoctl create archiver \ + --pgdata /var/lib/pgaf/archiver1 \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --hostname archiver1.example.com \ + --formation default \ + --formation billing \ + --run + +There is currently no command to attach an already-running archiver to a +further formation later on -- covering an additional formation, or a +worker group added to an already-attached Citus formation, requires +specifying every formation up front with a repeated ``--formation``. + +``--region`` labels which data-centre or availability zone this archiver +runs in -- purely informational, shown by ``pg_autoctl watch``. More than +one archiver can be attached to the very same formation at once (each +gets its own independent capture and its own replication slot against +that formation's primary), so registering a second archiver in a +different region against the same formation is how geographically +redundant disaster-recovery coverage is set up:: + + $ pg_autoctl create archiver \ + --pgdata /var/lib/pgaf/archiver-eu \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --hostname archiver-eu.example.com \ + --formation default \ + --region eu-west \ + --run + +The full set of options:: + + --pgdata path to the archiver's local data/cache directory + --pgctl path to pg_ctl (used to locate pg_receivewal) + --monitor pg_auto_failover Monitor Postgres URL + --hostname hostname to advertise for this archiver + --formation formation to attach to, may be repeated + (default: "default") + --region data-centre or availability-zone label for this + archiver (default: "default") + --basebackup-policy base-backup production/retention policy to attach + (default: "default") + --run create node then run pg_autoctl service + +Base-backup policies +---------------------- + +Every archiver produces full base backups on a schedule, and prunes older +ones, according to a **base-backup policy** attached to its formation (or +overridden per group). A formation that never attaches one of its own +falls back to the schema's built-in ``default`` policy: a base backup +every 24 hours, keeping the 3 most recent, none older than 3 days. + +Create a policy from a JSON document:: + + $ cat > /tmp/nightly.json <<'EOF' + { + "source": "replay", + "replaymode": "volatile", + "frequency": "6h", + "maxcount": 3, + "maxage": "7d", + "onpromotion": true + } + EOF + + $ pg_autoctl create basebackup-policy \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --name nightly --config /tmp/nightly.json + +Attach it to an archiver at creation time with +``--basebackup-policy nightly`` (see above), or to an already-running +archiver's formation with :ref:`pg_autoctl_set`:: + + $ pg_autoctl set basebackup-policy \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --name nightly --config /tmp/nightly.json + +Every archiver whose formation resolves to a changed policy picks up the +change on its own next tick, no restart needed. Read a policy back with:: + + $ pg_autoctl show basebackup-policy \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --name nightly --json + +Two fields are worth calling out: + + - ``source`` chooses whether the *next* base backup is taken ``live`` + (a real ``pg_basebackup`` against a running node) or ``replay`` + (replayed locally from already-captured WAL, at no cost to the live + primary or any standby). An archiver's very first base backup is + always taken live, regardless of policy, since a replay needs an + existing backup to start from. + - ``onpromotion``, when true, forces an extra base backup right after a + failover or switchover, independent of ``frequency`` -- useful when a + fresh backup taken on the new primary's timeline is worth more than + waiting out the rest of the schedule. + +See :ref:`pg_autoctl_create_basebackup_policy` for the full field +reference, and :ref:`pg_autoctl_show_basebackup_policy` / +:ref:`pg_autoctl_set_basebackup_policy` for the read and update commands. + +Watching an archiver +---------------------- + +An archiver reports state through the same node-active protocol as every +other node, so it shows up in the usual commands:: + + $ pg_autoctl show state + $ pg_autoctl watch + +alongside its captured WAL position, replication lag, and current disk +usage on its cache volume -- the same signals an operator already checks +for a standby, applied to an archiver's own job of holding onto WAL and +base backups rather than serving traffic. + +Rebuilding a node from an archiver +------------------------------------- + +When a node needs a fresh copy of the data -- provisioning a new standby +without adding load to the live primary, or rebuilding after every other +node in the formation was lost -- point ``pg_autoctl create postgres`` at +the archiver instead of a live node:: + + $ pg_autoctl create postgres \ + --pgdata /var/lib/postgresql/data \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --formation default \ + --from-archiver + +This bootstraps from the archiver's latest base backup and then catches +up using its cached WAL, the same recovery machinery ``pg_rewind``/ +``pg_basebackup`` fallback already uses elsewhere in pg_auto_failover -- +just sourced from the archiver's cache instead of a running node. Once +caught up, the new node joins the formation and is assigned a role by the +monitor the ordinary way. + +This is also the disaster-recovery path described in +:ref:`archiving_fault_tolerance`: if the primary and every standby are +lost at once, a single surviving archiver is enough to rebuild a new +primary from scratch with ``--from-archiver``, and re-grow standbys from +there. + +See also +-------- + +- :ref:`archiving_architecture` -- what an archiver is and where it fits + among the other architectures +- :ref:`archiving_fault_tolerance` -- WAL capture independent of any + standby, and rebuilding after every other node is lost +- :ref:`failover_state_machine` -- the ``archiving`` state's own + transitions +- :ref:`pg_autoctl_create_basebackup_policy`, + :ref:`pg_autoctl_show_basebackup_policy`, + :ref:`pg_autoctl_set_basebackup_policy` diff --git a/docs/conf.py b/docs/conf.py index 9c678d65e..a2454decd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -112,11 +112,18 @@ def __init__(self, **options): # html_theme_options = {} -# Add our custom CSS +# Add our custom CSS and JS def setup(app): if hasattr(app, "add_css_file"): app.add_css_file("css/citus.css") app.add_css_file("css/pygments.css") + app.add_css_file("css/zoom.css") + if hasattr(app, "add_js_file"): + # Click-to-zoom for our own figures (tikz-rendered diagrams), + # generalizing the pan/scroll-to-zoom already available on + # Mermaid diagrams (mermaid_d3_zoom, above) to every other + # image the docs embed via `.. figure::`. + app.add_js_file("js/zoom.js") # Add any paths that contain custom static files (such as style sheets) here, @@ -464,6 +471,34 @@ def setup(app): [author], 1, ), + ( + "ref/pg_autoctl_create_archiver", + "pg_autoctl create archiver", + "pg_autoctl create archiver", + [author], + 1, + ), + ( + "ref/pg_autoctl_create_basebackup_policy", + "pg_autoctl create basebackup-policy", + "pg_autoctl create basebackup-policy", + [author], + 1, + ), + ( + "ref/pg_autoctl_show_basebackup_policy", + "pg_autoctl show basebackup-policy", + "pg_autoctl show basebackup-policy", + [author], + 1, + ), + ( + "ref/pg_autoctl_set_basebackup_policy", + "pg_autoctl set basebackup-policy", + "pg_autoctl set basebackup-policy", + [author], + 1, + ), ( "ref/pg_autoctl_activate", "pg_autoctl activate", diff --git a/docs/failover-state-machine.rst b/docs/failover-state-machine.rst index da3130bc2..cd4d59412 100644 --- a/docs/failover-state-machine.rst +++ b/docs/failover-state-machine.rst @@ -326,6 +326,42 @@ Missing WAL bytes are fetched from one of the most advanced standby nodes by using Postgres cascading replication features: it is possible to use any standby node in the ``primary_conninfo``. +.. _archiving_state: + +Archiving +^^^^^^^^^ + +The archiving state is assigned to an **archiving node** — a physically +distinct kind of cluster member added with ``pg_autoctl create archiver``, +never a candidate for promotion or failover, since it holds no ``PGDATA`` +of its own to promote (see :ref:`archiving_architecture`). An archiving +node's own state machine only ever visits three states, mirroring just +enough of the ordinary standby lifecycle to participate safely in an +election without ever competing to win one: + +- ``wait_standby`` → ``archiving``, once the group's primary has + authorized the archiver's connection — the same bootstrap step an + ordinary standby goes through, up to this point. +- ``archiving`` → ``report_lsn``, when the group's primary becomes + unreachable and a failover starts: the archiver stops + ``pg_receivewal`` against the now-untrustworthy primary, the same way + an ordinary standby's own `Report_LSN`_ transition detaches it from a + dying upstream. +- ``report_lsn`` → ``archiving``, once a new primary is confirmed: the + archiver re-points ``pg_receivewal`` at it and resumes capturing WAL. + +An archiving node reaching ``report_lsn`` is never itself considered as a +promotion candidate — its own ``haspgdata`` flag (there is no real +Postgres instance to promote) excludes it from candidacy, and since it +never competes for the primary role, it also never counts toward +``number_sync_standbys`` or the quorum a failover election needs to +proceed. What it does contribute during an election is exactly what its +name promises: its own already-captured WAL becomes a real, +`Fast_forward`_-eligible source for whichever candidate did win, if that +candidate turns out to be behind the most advanced standby — including +when every ordinary standby has been lost and the archiver is the only +node left with the data. + Dropped ^^^^^^^ @@ -341,12 +377,12 @@ command, and then the node entry is removed from the monitor. pg_auto_failover keeper's State Machine --------------------------------------- -The full keeper FSM is 20 states and 77 transitions -- legible as a reference +The full keeper FSM is 21 states and 102 transitions -- legible as a reference table, but too dense to read at a glance as a single diagram. ``pg_autoctl inspect fsm mermaid`` renders it instead as five smaller diagrams, one per phase of a node's life, generated directly from ``KeeperFSM[]`` (``src/bin/pg_autoctl/fsm.c``) so they can never drift out of sync with the -actual state machine the way a hand-maintained image can. The 68 edges shown +actual state machine the way a hand-maintained image can. The edges shown below exclude ``join_primary``, a deprecated state (see `Join_primary`_ above) no longer assigned to any node -- ``KeeperFSM[]`` still carries its 9 transitions for backward compatibility with on-disk state from old @@ -392,16 +428,19 @@ restarted. dropped --> report_lsn : This node is being reinitialized after having been dropped single --> wait_primary : A new secondary was added wait_standby --> catchingup : The primary is now ready to accept a standby + wait_standby --> archiving : wait_standby to archiving init --> wait_standby : Start following a primary dropped --> wait_standby : Start following a primary init --> report_lsn : Creating a new node from a standby node that is not a candidate. - note right of single : also appears in Node removal / drop + note right of init : also appears in Failover / promotion + note right of single : also appears in Failover / promotion, Node removal / drop note right of dropped : also appears in Node removal / drop note right of report_lsn : also appears in Failover / promotion, Maintenance, Node removal / drop note right of wait_primary : also appears in Steady-state / config changes, Failover / promotion, Node removal / drop note right of wait_standby : also appears in Steady-state / config changes note right of catchingup : also appears in Steady-state / config changes, Failover / promotion, Maintenance, Node removal / drop + note right of archiving : also appears in Failover / promotion classDef metaState fill:#e0e0e0,stroke:#888888,color:#333333 classDef primaryState fill:#cfe2ff,stroke:#3b6fb6,color:#1a1a1a @@ -414,6 +453,7 @@ restarted. class wait_primary primaryState class wait_standby secondaryState class catchingup secondaryState + class archiving electionState Steady-state / config changes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -455,8 +495,11 @@ Failover / promotion The primary going away and a candidate taking over, including the multi-standby candidate-election machinery (``report_lsn``, -``fast_forward``, ``join_secondary``) -- this is the largest of the five, -still less than half the size of the full graph. Citus coordinator/worker +``fast_forward``, ``join_secondary``) -- this is by far the largest of the +five, over half the size of the full graph on its own, since it is also +where every other phase's states end up if a failover interrupts them: +most of its edges are the "wherever you were, you're being demoted now" +fan-out into ``demoted``/``demote_timeout``. Citus coordinator/worker transitions are not shown separately: every Citus-specific transition in ``KeeperFSM[]`` reuses an edge that already exists here, just with a different underlying implementation, so a separate "Citus diagram" would @@ -472,10 +515,35 @@ be identical in shape to this one. apply_settings --> draining : A failover occurred, stopping writes apply_settings --> demoted : A failover occurred, no longer primary apply_settings --> demote_timeout : A failover occurred, no longer primary - draining --> demote_timeout : Secondary confirms it is receiving no more writes + draining --> demote_timeout : Secondary confirms it's receiving no more writes demote_timeout --> demoted : Demote timeout expired wait_primary --> demoted : A failover occurred, no longer primary + init --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + single --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + catchingup --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + secondary --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + prepare_promotion --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + stop_replication --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + maintenance --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + prepare_maintenance --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + wait_maintenance --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + report_lsn --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + fast_forward --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + init --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + single --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + demoted --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + catchingup --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + secondary --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + prepare_promotion --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + stop_replication --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + maintenance --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + prepare_maintenance --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + wait_maintenance --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + report_lsn --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + fast_forward --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running demote_timeout --> primary : Detected a network partition, but monitor didn't do failover + archiving --> report_lsn : archiving to report_lsn + report_lsn --> archiving : report_lsn to archiving demoted --> catchingup : A new primary is available. First, try to rewind. If that fails, do a pg_basebackup. secondary --> prepare_promotion : Stop traffic to primary, wait for it to finish draining. catchingup --> prepare_promotion : Stop traffic to primary, wait for it to finish draining. @@ -484,6 +552,11 @@ be identical in shape to this one. prepare_promotion --> wait_primary : Promoting a Citus Worker standby after having blocked writes from the coordinator. secondary --> report_lsn : Reporting the last write-ahead log location received catchingup --> report_lsn : Reporting the last write-ahead log location received + fast_forward --> report_lsn : Reporting the last write-ahead log location received + prepare_promotion --> report_lsn : Reporting the last write-ahead log location received + stop_replication --> report_lsn : Reporting the last write-ahead log location received + demote_timeout --> report_lsn : Reporting the last write-ahead log location received + join_secondary --> report_lsn : Reporting the last write-ahead log location received report_lsn --> prepare_promotion : Stop traffic to primary, wait for it to finish draining. report_lsn --> fast_forward : Fetching missing WAL bits from another standby before promotion fast_forward --> prepare_promotion : Got the missing WAL bytes, promoted @@ -499,15 +572,24 @@ be identical in shape to this one. note right of demote_timeout : also appears in Node removal / drop note right of apply_settings : also appears in Steady-state / config changes, Node removal / drop note right of wait_primary : also appears in Node init / join, Steady-state / config changes, Node removal / drop + note right of init : also appears in Node init / join + note right of single : also appears in Node init / join, Node removal / drop note right of catchingup : also appears in Node init / join, Steady-state / config changes, Maintenance, Node removal / drop note right of secondary : also appears in Steady-state / config changes, Maintenance, Node removal / drop note right of prepare_promotion : also appears in Node removal / drop note right of stop_replication : also appears in Node removal / drop + note right of maintenance : also appears in Maintenance + note right of prepare_maintenance : also appears in Maintenance + note right of wait_maintenance : also appears in Maintenance, Node removal / drop note right of report_lsn : also appears in Node init / join, Maintenance, Node removal / drop + note right of fast_forward : also appears in Node removal / drop + note right of archiving : also appears in Node init / join + classDef metaState fill:#e0e0e0,stroke:#888888,color:#333333 classDef primaryState fill:#cfe2ff,stroke:#3b6fb6,color:#1a1a1a classDef secondaryState fill:#d4edda,stroke:#4c9a5b,color:#1a1a1a classDef demotingState fill:#f8d7da,stroke:#c0392b,color:#1a1a1a + classDef maintenanceState fill:#e8dff5,stroke:#8e6bb0,color:#1a1a1a classDef electionState fill:#fff3cd,stroke:#c99a1e,color:#1a1a1a class primary primaryState class draining demotingState @@ -515,12 +597,18 @@ be identical in shape to this one. class demote_timeout demotingState class apply_settings primaryState class wait_primary primaryState + class init metaState + class single metaState class catchingup secondaryState class secondary secondaryState class prepare_promotion electionState class stop_replication electionState + class maintenance maintenanceState + class prepare_maintenance maintenanceState + class wait_maintenance maintenanceState class report_lsn electionState class fast_forward electionState + class archiving electionState class join_secondary electionState Maintenance @@ -543,9 +631,13 @@ Planned maintenance on either a secondary or the primary. prepare_maintenance --> catchingup : Restarting standby after manual maintenance is done. maintenance --> report_lsn : Reporting the last write-ahead log location received prepare_maintenance --> report_lsn : Reporting the last write-ahead log location received + wait_maintenance --> report_lsn : Reporting the last write-ahead log location received note right of primary : also appears in Steady-state / config changes, Failover / promotion, Node removal / drop + note right of prepare_maintenance : also appears in Failover / promotion + note right of maintenance : also appears in Failover / promotion note right of secondary : also appears in Steady-state / config changes, Failover / promotion, Node removal / drop + note right of wait_maintenance : also appears in Failover / promotion, Node removal / drop note right of catchingup : also appears in Node init / join, Steady-state / config changes, Failover / promotion, Node removal / drop note right of report_lsn : also appears in Node init / join, Failover / promotion, Node removal / drop @@ -580,11 +672,13 @@ node reacting to the other side of that removal. prepare_promotion --> single : Primary was forcibly removed stop_replication --> single : Went down to force the primary to time out, but then it was removed report_lsn --> single : There is no other node anymore, promote this node + wait_maintenance --> single : Was waiting to be sent to maintenance, but the primary vanished, promote this node + fast_forward --> single : Was fetching missing WAL from another standby, but every other node vanished, promote this node with whatever it has apply_settings --> single : Other node was forcibly removed, now single any_state --> dropped : This node is being dropped from the monitor note right of primary : also appears in Steady-state / config changes, Failover / promotion, Maintenance - note right of single : also appears in Node init / join + note right of single : also appears in Node init / join, Failover / promotion note right of wait_primary : also appears in Node init / join, Steady-state / config changes, Failover / promotion note right of demoted : also appears in Failover / promotion note right of demote_timeout : also appears in Failover / promotion @@ -594,6 +688,8 @@ node reacting to the other side of that removal. note right of prepare_promotion : also appears in Failover / promotion note right of stop_replication : also appears in Failover / promotion note right of report_lsn : also appears in Node init / join, Failover / promotion, Maintenance + note right of wait_maintenance : also appears in Failover / promotion, Maintenance + note right of fast_forward : also appears in Failover / promotion note right of apply_settings : also appears in Steady-state / config changes, Failover / promotion note right of dropped : also appears in Node init / join @@ -601,6 +697,7 @@ node reacting to the other side of that removal. classDef primaryState fill:#cfe2ff,stroke:#3b6fb6,color:#1a1a1a classDef secondaryState fill:#d4edda,stroke:#4c9a5b,color:#1a1a1a classDef demotingState fill:#f8d7da,stroke:#c0392b,color:#1a1a1a + classDef maintenanceState fill:#e8dff5,stroke:#8e6bb0,color:#1a1a1a classDef electionState fill:#fff3cd,stroke:#c99a1e,color:#1a1a1a class primary primaryState class single metaState @@ -613,6 +710,8 @@ node reacting to the other side of that removal. class prepare_promotion electionState class stop_replication electionState class report_lsn electionState + class wait_maintenance maintenanceState + class fast_forward electionState class apply_settings primaryState class any_state metaState class dropped metaState @@ -622,7 +721,7 @@ node reacting to the other side of that removal. This replaces the previous single Graphviz diagram (``pg_autoctl inspect fsm gv | dot -Tsvg``, rendered from a checked-in ``fsm.png`` last regenerated by hand in 2021). The five diagrams above cover every - currently-reachable transition the old single diagram did -- 68 edges, + currently-reachable transition the old single diagram did -- 102 edges, split by phase rather than shown at once -- deliberately excluding only the 9 transitions involving the deprecated ``join_primary`` state, so ``fsm.png`` is no longer needed as documentation. The ``pg_autoctl diff --git a/docs/fault-tolerance.rst b/docs/fault-tolerance.rst index ac9da4f87..c981e8465 100644 --- a/docs/fault-tolerance.rst +++ b/docs/fault-tolerance.rst @@ -1,3 +1,5 @@ +.. _fault_tolerance: + Failover and Fault Tolerance ============================ @@ -255,6 +257,84 @@ walkthrough. A standby forks out-of-band; once the mismatch is visible to the monitor, it is pushed to catchingup and rewound within about a second +.. _archiving_fault_tolerance: + +Archiving Nodes and Disaster Recovery +-------------------------------------- + +On-top of the Service Availability a database system needs Data +Availability, and it is expected to survive some data loss scenarios that +are not covered with the previous sections about fault tolerance. + +Typically, an erroneous ``DELETE`` without a ``WHERE`` clause, or a ``DROP +TABLE`` that happened on the wrong server, by mistake or because of a +security exploit of some sorts. + +.. note:: + + Always make sure to have a separate role for the normal application + activities that is separate from the database owner, and use yet another + specific role for database schema upgrade, or migrations. + + This alone avoids most of the security risk surface. + +While the previous sections concerns keeping the PostgreSQL *service* +available thanks to being able to failover from a primary node to its +secondary within seconds of a failure, an **archiver** addresses a different +failure mode entirely: either the loss of multiple (all) nodes at the same +time, or a data loss that happens while the service is running fine. + +See also :ref:`archiving_architecture` for more details about the archiving +support in pg_auto_failover. + +When an archiver is enabled on a pg_auto_failover architecture in +production, the following operations are covered: + + - Point in Time Recovery can be driven on transient nodes created from the + archives. + + - Disaster Recovery can be implemented by copying the data recovered in a + transient PITR node up to the current primary, a manual operation, or by + reifying the transient PITR node into its own new group in the + formation, allowing to redeploy a new cluster from a selected position + in the WAL history. + + - Archiving nodes may paritipate in the replication quorum, and as they + only implement ``pg_receivewal`` without maintaining a full PGDATA + directory, there is no crash recovery happening on the WAL stream -- it + is often the case that an archiving node would be the first to report + LSN progress. + + - Taking base backup happens on the primary node by default (a live source + setting) and can also be setup as a replay source, meaning that a new + node is created from the latest base backup and instructed to replay all + the WAL that have been archived since this base backup, up to the + current moment in time. The replay source can in turn be setup as a + volatile or a persistent node. + + - It is possible to maintain standby servers that only connect to the + archive, because we have added a way to serve the archives using the + Postgres protocol replication. Such a standby would be named a WARM + standby, even though it can be using WAL streaming, with a cascading hop + in the archives. + +How archiving nodes participate in failover +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Archiving nodes are health-checked and report state through the node-active +protocol exactly like a primary or secondary, and the monitor tracks their +``archiving`` state the same way it tracks ``primary``/``secondary`` -- +but they are never assigned a ``candidate-priority``-driven role and never +considered for promotion, since there is no data directory to promote. +When the group's primary changes -- whether through an ordinary failover or +an operator-driven switchover -- an archiver notices its `pg_receivewal` +connection has gone stale, stops it, and re-points at the new primary +automatically; see the ``Archiving`` state's transitions in +:ref:`failover_state_machine` for the exact FSM edges involved. From the +perspective of the rest of this page's failover sequences, an archiver is +simply along for the ride: it never blocks a promotion, and it never needs +one of its own. + Failure handling and network partition detection ------------------------------------------------ diff --git a/docs/index.rst b/docs/index.rst index 5f7f8440e..cdf92dee1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -55,6 +55,7 @@ __ https://github.com/hapostgres/pg_auto_failover architecture-multi-standby failover-state-machine fault-tolerance + archiving-details security .. toctree:: @@ -76,6 +77,7 @@ __ https://github.com/hapostgres/pg_auto_failover :caption: Operations operations + archiving testing reporting-bugs faq diff --git a/docs/intro.rst b/docs/intro.rst index 15d772baa..84c13676c 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -1,9 +1,82 @@ Introduction to pg_auto_failover ================================ -pg_auto_failover is an extension for PostgreSQL that monitors and manages -failover for postgres clusters. It is optimised for simplicity and -correctness. +pg_auto_failover is a complete system for operating PostgreSQL in +production. Its ``pg_autoctl`` process may runs `pid 1` or init in a +container based environment and supervises the Postgres ``postmaster`` +underneath it. A dedicated monitor node coordinates state across every node +in the cluster: the monitor is a Postgres instance with the +``pgautofailover`` extension installed to implement our inter-node +communication protocol. + +Together they provide full cluster management with a dynamic topology: nodes +can be added, removed, and reconfigured while the cluster keeps serving +production traffic, whether driven by an operator's own commands or +automatically by the monitor's own health checks. + +Automated failover and full high availability can both be implemented and a +production cluster can evolve from simple failover capabilities to enhanced +data protection settings. + +Two modes of operation are available side by side: the traditional +command-driven CLI (``pg_autoctl create ...``, ``pg_autoctl set ...``), and +a specification- file-driven mode, where a single ``node.ini`` file +describes a node's own desired configuration and :ref:`pg_autoctl_node_run` +continuously reconciles reality to match it. + +.. _ha_dr_backups: + +High Availability and Disaster Recovery: One System +------------------------------------------------------ + +.. figure:: ./tikz/arch-ha-dr-typical.svg + :alt: Typical setup, High Availability from Patroni or repmgr, Disaster Recovery and Backups from pgBackRest or pgBarman, two entirely separate boxes + + A typical setup reaches for a product per box: Patroni or repmgr for + High Availability, pgBackRest or pgBarman for Disaster Recovery and + Backups + +.. figure:: ./tikz/arch-ha-dr-pgautofailover.svg + :alt: With pg_auto_failover, High Availability and Disaster Recovery collapse into a single box, with Backups (pgBackRest or pgBarman) as the one remaining separate concern + + With pg_auto_failover, High Availability and Disaster Recovery collapse + into one system; Backups remains its own concern + +With RDBMS such as PostgreSQL the concept of High Availability applies to +the service and also the data. Where most PostgreSQL setups treat these as +two separate problems, solved by two separate products, pg_auto_failover +addresses both HA aspects into a single deployment. + +Postgres backup systems need to be able Point in Time Recovery, which +requires an archiving implemnentation when using Postgres. Also, Disaster +Recovery is built on-top of PITR. As a consequence, most systems are +implementing Disaster Recovery with their backup software solution, not +their High Availability solution. + +Running both solutions together means trusting two different failure +domains, and, very often, discovering only during a real incident that they +were never actually exercised together. + +pg_auto_failover starts from a different question: how to make things so +simple to setup and test that they just work once shipped in production? + +High Availability of the Postgres service and Disaster Recovery of its data +set are two sides of the same problem, best solved by one system designed +around it rather than by gluing together two tools each designed in +isolation. + +The same monitor that orchestrates failover also tracks every archiver's +captured WAL and base backups; the same WAL stream and base backups a +failover election already depends on to guarantee no data loss are what +disaster recovery, including point-in-time recovery, is built on. + +High Availability and Disaster Recovery come from a single package, with a +single control plane, rather than from two independently-operated systems +that are only put to the test when a production incident happens. + +Backups — in the narrower sense of long-term retention, cataloguing, and +cloud storage tiers — remain their own concern, typically still handled by a +dedicated tool like pgBackRest or pgBarman. Single Standby Architecture --------------------------- @@ -36,6 +109,38 @@ setting on the *primary* node. Until the *secondary* is back to being monitored healthy, failover and switchover operations are not allowed, preventing data loss. +.. _archiving_and_disaster_recovery: + +Archiving & Disaster Recovery Architecture +------------------------------------------- + +.. figure:: ./tikz/arch-archiver.svg + :alt: pg_auto_failover Architecture with a primary, a standby, and an archiver + + pg_auto_failover architecture with a primary, a standby, and an archiver + +An **archiver** is a separate node, added on top of any of the architectures +on this page — it applies just as well to the single-standby setup above as +it does to a multi-standby fleet, since it addresses a different concern: +disaster recovery, independent of how many nodes currently participate in +the failover quorum. + +An archiver then register archiving nodes to groups on formations managed by +the monitor it reports to. An archiving node is running ``pg_receivewal`` to +maintain the Postgres PITR archive storage, and schedules regular base +backup activity using ``pg_basebackup``. The *archiving node* reports to the +pg_auto_failover Monitor and participates in a group Finite State Machine: +it reports its WAL position and can be used in the replication quorum, and +other nodes in the same group can fetch WAL from an *archiving node* (see +REPORT_LSN and FORWARD_LSN states in the :ref:`failover_state_machine`:. + +For that, pg_auto_failover implements its own server-side implementation of +the PostgreSQL replication protocol, a ``pg_walsender`` process that knows +how to serve the data from the archive local on-disk location (or remote +Cloud Object Storage) to the PostgreSQL client replication tools already +listed: ``pg_basebackup`` and `pg_receivewal``, as described in more details +in :ref:`archiving_architecture`. + Multiple Standby Architecture ----------------------------- diff --git a/docs/operations.rst b/docs/operations.rst index 928f3ef62..309efb47f 100644 --- a/docs/operations.rst +++ b/docs/operations.rst @@ -583,19 +583,23 @@ A single command then creates the node if absent and starts the supervisor:: This makes ``pg_autoctl node run`` a natural ``CMD`` (Docker) or ``command:`` (Kubernetes) entry-point for every node type. The same image -works for monitor, primary, standby, coordinator, and worker nodes — per-node -differences live entirely in the bind-mounted ini file. +works for monitor, primary, standby, coordinator, worker, and +:ref:`archiving_architecture` archiver nodes — per-node differences live +entirely in the bind-mounted ini file. **Live reconfiguration** — the supervisor watches the ini file. Editing ``candidate_priority``, ``replication_quorum``, ``ssl`` settings, or ``monitor.pguri`` and saving the file is sufficient to converge the running node; no restart is required. -**Ordered startup** — add ``[launch] mode = deferred`` to any node that -should wait for an external signal before initialising. Call -``pg_autoctl node start `` from a sidecar or init container to release -it. This replaces external orchestration for the common case where data -nodes must wait until the monitor is ready. +**Ordered startup** — add ``[launch] create = deferred`` and/or ``run = +deferred`` to any node that should wait for an external signal before +initialising. Call ``pg_autoctl node start `` from a sidecar or +init container to release it (clears both flags). This replaces external +orchestration for the common case where data nodes must wait until the +monitor is ready — or, for an archiver, until its target formation (every +group of it, for a Citus formation) is already registered, since +``pg_autoctl create archiver`` has no retry-until-ready loop of its own. For the full property reference and mutability table see :ref:`pg_autoctl_node`. diff --git a/docs/ref/pg_autoctl_create.rst b/docs/ref/pg_autoctl_create.rst index 117b96001..0e0c10157 100644 --- a/docs/ref/pg_autoctl_create.rst +++ b/docs/ref/pg_autoctl_create.rst @@ -12,4 +12,6 @@ pg_autoctl create - Create a pg_auto_failover node, or formation pg_autoctl_create_postgres pg_autoctl_create_coordinator pg_autoctl_create_worker + pg_autoctl_create_archiver pg_autoctl_create_formation + pg_autoctl_create_basebackup_policy diff --git a/docs/ref/pg_autoctl_create_archiver.rst b/docs/ref/pg_autoctl_create_archiver.rst new file mode 100644 index 000000000..4633a034d --- /dev/null +++ b/docs/ref/pg_autoctl_create_archiver.rst @@ -0,0 +1,143 @@ +.. _pg_autoctl_create_archiver: + +pg_autoctl create archiver +=========================== + +pg_autoctl create archiver - Initialize a pg_auto_failover archiver node + +Synopsis +-------- + +The command ``pg_autoctl create archiver`` registers a new **Archiver** +identity on the monitor and attaches it to one or more formations for +Archiving & Disaster Recovery. See :ref:`archiving_architecture` for what +an archiver actually does once running, and :ref:`archiving_operations` +for the operational side of this command. + +:: + + usage: pg_autoctl create archiver + + --pgdata path to the archiver's local data/cache directory + --pgctl path to pg_ctl (used to locate pg_receivewal) + --monitor pg_auto_failover Monitor Postgres URL + --hostname hostname to advertise for this archiver + --name archiver name (default: derived from hostname) + --formation formation to attach to, may be repeated + (default: "default") + --region data-centre or availability-zone label for this + archiver (default: "default") + --basebackup-policy base-backup production/retention policy to attach + (default: "default") + --run create node then run pg_autoctl service + +Description +----------- + +Unlike ``pg_autoctl create postgres`` and the other node kinds, this +command does not initialize a PostgreSQL data directory: ``--pgdata`` +here names the archiver's local cache directory for captured WAL segments +and base backups, and no ``initdb`` ever runs against it. Once +registered, the archiver starts one WAL-capture process per group of +every formation it is attached to (every worker of a Citus formation +included, not just the coordinator), each following its own group's +current primary and reconnecting on its own across any later promotion, +and reports its progress to the monitor the same way an ordinary standby +reports replication state. + +``--formation`` may be given more than once, to attach the same archiver +to several formations from the start -- there is currently no separate +command to attach an already-running archiver to a further formation +later on, so every formation (and, for a Citus formation gaining a new +worker group afterwards, that new group too) needs to be covered by a +repeated ``--formation`` up front. See :ref:`archiving_architecture`'s own +"Several formations" and "A Citus formation" sections for the process +model this produces. + +``--basebackup-policy`` attaches a named base-backup production/retention +policy (see :ref:`pg_autoctl_create_basebackup_policy`) to every formation +given, formation-wide. A formation that never gets a policy of its own +this way, or via :ref:`pg_autoctl_set_basebackup_policy`, falls back to +the schema's own built-in ``default`` policy. + +Options +------- + +The following options are available to ``pg_autoctl create archiver``: + +--pgdata + + Path to the archiver's local cache directory for captured WAL segments + and base backups. Despite the flag's name shared with every other node + kind, this is never a real Postgres data directory. Defaults to the + environment variable ``PGDATA``. + +--pgctl + + Path to the ``pg_ctl`` tool, used only to locate the ``pg_receivewal`` + binary the archiver runs alongside it. Same discovery rules as + :ref:`pg_autoctl_create_postgres`'s own ``--pgctl``. + +--monitor + + Postgres URI used to connect to the monitor. Must use the + ``autoctl_node`` username and target the ``pg_auto_failover`` database + name. It is possible to show the Postgres URI from the monitor node + using the command :ref:`pg_autoctl_show_uri`. + +--hostname + + Hostname or IP address other nodes and clients use to reach this + archiver -- in particular, what a standby's ``primary_conninfo`` or a + ``restore_command`` would point at when using this archiver as a + disaster-recovery source. Same discovery rules as + :ref:`pg_autoctl_create_postgres`'s own ``--hostname`` when not + provided. + +--name + + Archiver name used on the monitor. Defaults to ``--hostname`` when not + provided. + +--formation + + Formation to attach this archiver to. May be repeated to attach the + same archiver to several formations at once; defaults to the + ``default`` formation when not given at all. + +--region + + Free-form label identifying the data-centre or availability zone this + archiver runs in. Purely informational, same convention as + :ref:`pg_autoctl_create_postgres`'s own ``--region``: displayed by + ``pg_autoctl watch``'s archivers section, does not affect any placement + or quorum decision on its own. More than one archiver can be attached + to the very same formation at once -- distinct regions across them is + the intended shape for geographically-redundant disaster-recovery + coverage of one formation. + +--basebackup-policy + + Name of an existing base-backup policy (see + :ref:`pg_autoctl_create_basebackup_policy`) to attach to every + ``--formation`` given, formation-wide. + +--run + + Immediately run the ``pg_autoctl`` archiver service after having + created this node, instead of requiring a separate ``pg_autoctl run`` + invocation afterwards. + +See Also +-------- + +:ref:`pg_autoctl_node_run` provides a declarative alternative to this +command: describe the node once in a ``pg_autoctl_node.ini`` file and run +``pg_autoctl node run`` — it creates the archiver if absent and starts +the supervisor in one step. See :ref:`pg_autoctl_node` for the full +reference. + +:ref:`archiving_architecture` covers what runs once an archiver is +started, and :ref:`archiving_operations` covers the rest of the +day-to-day commands (attaching a policy after the fact, watching what's +captured, rebuilding a node from an archiver's cache). diff --git a/docs/ref/pg_autoctl_create_basebackup_policy.rst b/docs/ref/pg_autoctl_create_basebackup_policy.rst new file mode 100644 index 000000000..ebbaeb7fd --- /dev/null +++ b/docs/ref/pg_autoctl_create_basebackup_policy.rst @@ -0,0 +1,85 @@ +.. _pg_autoctl_create_basebackup_policy: + +pg_autoctl create basebackup-policy +==================================== + +pg_autoctl create basebackup-policy - Create a named base-backup +production/retention policy + +Synopsis +-------- + +This command registers a new base-backup production/retention policy on +the monitor. An archiver's own scheduling (when to take the next base +backup) and retention (which older ones to prune) are driven entirely by +whichever policy applies to its (formation, group) -- attach a policy to +an archiver's own formation with ``pg_autoctl create archiver +--basebackup-policy ``:: + + usage: pg_autoctl create basebackup-policy --monitor --name --config + + --monitor pg_auto_failover Monitor Postgres URL + --name policy name + --config path to a JSON document with the policy body + +Description +----------- + +A base-backup policy controls three independent things for whichever +archiver(s) it applies to: + + - **when** to produce the next base backup (``frequency``, and + ``onpromotion`` to force one immediately after a failover regardless + of ``frequency``), + - **how** to produce it (``source``: ``live``, straight from a running + node, or ``replay``, replayed locally from already-captured WAL -- + and ``replaymode`` when ``source`` is ``replay``), + - **how many to keep** (``maxcount``, ``maxage``: whichever fires first + prunes a given backup -- the directory is removed and the base + backup's own history row is marked deleted, which in turn prunes any + WAL segments no remaining backup still needs). + +A policy is a standalone, independently-referenceable row: the same one +can be shared by every archiver in a fleet, or kept private to a single +(formation, group) via :ref:`pg_autoctl_set` ``archiver-policy``-style +group overrides. A formation that never creates or attaches a policy of +its own uses this schema's own ``default`` policy (nightly-equivalent: +``frequency`` 24 hours, ``maxcount`` 3, ``maxage`` 3 days). + +The ``--config`` document is a flat JSON object with any subset of the +following keys -- any key left out keeps its own default (on ``create``) +or its current value (on :ref:`pg_autoctl_set_basebackup_policy`):: + + { + "source": "replay", + "replaymode": "volatile", + "cache": "local", + "frequency": "6h", + "maxcount": 3, + "maxage": "7d", + "onpromotion": true, + "concurrency": 1 + } + +``frequency`` and ``maxage`` accept any text Postgres itself parses as an +``interval`` (``"6h"``, ``"3 days"``, ``"90 minutes"``, ...). + +Options +------- + +The following options are available to ``pg_autoctl create basebackup-policy``: + +--monitor + + Postgres URI used to connect to the monitor. Must use the ``autoctl_node`` + username and target the ``pg_auto_failover`` database name. It is possible + to show the Postgres URI from the monitor node using the command + :ref:`pg_autoctl_show_uri`. + +--name + + Name of the policy to create. + +--config + + Path to a JSON document with the policy body, as described above. diff --git a/docs/ref/pg_autoctl_node.rst b/docs/ref/pg_autoctl_node.rst index ad4b05840..51e177489 100644 --- a/docs/ref/pg_autoctl_node.rst +++ b/docs/ref/pg_autoctl_node.rst @@ -35,8 +35,8 @@ Description and Kubernetes deployments. The complete node description lives in one ini file that can be version-controlled, templated, and bind-mounted into a container. The same image and the same entry-point work for every node type -(monitor, primary, standby, Citus coordinator, Citus worker); per-node -differences live entirely in the mounted ini file. +(monitor, primary, standby, Citus coordinator, Citus worker, archiver); +per-node differences live entirely in the mounted ini file. The ``pg_autoctl_node.ini`` File -------------------------------- @@ -94,8 +94,9 @@ node is created or started from scratch. ``kind`` - Node role. One of ``postgres``, ``monitor``, ``coordinator``, or - ``worker``. Required; immutable. + Node role. One of ``postgres``, ``monitor``, ``coordinator``, + ``worker``, or ``archiver`` (see :ref:`archiving_architecture`). + Required; immutable. ``name`` @@ -147,7 +148,19 @@ node is created or started from scratch. ``group`` Citus group identifier. ``0`` means coordinator. Defaults to ``0``. - Immutable. + Immutable. Not meaningful for ``kind = archiver`` -- see below. + +For ``kind = archiver``, this section works the same way but with one +real difference in behavior worth knowing: an ordinary node's own +registration retries until its target formation exists (and, once it +does, applies immediately), while ``pg_autoctl create archiver`` (which +this section's ``name`` ultimately drives, once through ``[launch]`` +below) has no such retry -- it attaches to whichever groups already exist +in that formation at the exact moment it runs, and never re-attaches to +groups added afterwards on its own. If the target formation (or, for a +Citus formation, all of its groups) might not exist yet when this node's +own container would otherwise start, use ``[launch]`` below to hold it +back until an operator or orchestrator confirms the formation is ready. ``[settings]`` ^^^^^^^^^^^^^^ @@ -215,11 +228,30 @@ SSL live via ``pg_autoctl enable ssl``. ``[launch]`` ^^^^^^^^^^^^ -``mode`` +Two independent gates, both defaulting to ``immediate``. ``pg_autoctl node +run`` checks ``create`` first (holding back node creation entirely), then +-- once the node exists, whether this run just created it or it already +existed -- checks ``run`` (holding back actually starting Postgres and the +supervisor). Setting only one of the two is meaningful: ``create = +immediate`` with ``run = deferred`` creates the node right away but leaves +it stopped; ``create = deferred`` with ``run = immediate`` (the common +case, usually set together as ``create = deferred`` / ``run = deferred``) +waits before doing anything at all. + +``create`` + + When set to ``deferred``, ``pg_autoctl node run`` polls this file every + 0.5s and waits instead of running ``pg_autoctl create --run`` + immediately. Call ``pg_autoctl node start`` to release it (clears both + ``create`` and ``run`` together). Defaults to ``immediate``. See + :ref:`pg_autoctl_node_start`. + +``run`` - When set to ``deferred``, the node starts a polling loop and waits instead - of creating or starting Postgres immediately. Call ``pg_autoctl node - start`` to release it. Defaults to ``immediate``. See + When set to ``deferred``, ``pg_autoctl node run`` polls this file every + 0.5s and waits (after any pending ``create`` has already resolved) + instead of exec'ing into ``pg_autoctl run`` immediately. Call + ``pg_autoctl node start`` to release it. Defaults to ``immediate``. See :ref:`pg_autoctl_node_start`. ``[formation ]`` @@ -275,25 +307,31 @@ Changing an **immutable** field (``kind``, ``pgdata``, ``hostname``, ``port``, ``auth``, ``pg_hba_lan``) while the node is running is logged as a warning; the value takes effect the next time the node is started. -The ``launch = deferred`` Pattern ----------------------------------- +The Deferred-Launch Pattern +---------------------------- :: [launch] - mode = deferred - -A node configured with ``mode = deferred`` starts a polling loop and waits. -A sidecar container or init script then calls:: - - pg_autoctl node start /etc/pgaf/node.ini - -which rewrites the ini file with ``mode = immediate``. The waiting node -detects the change within the poll interval and proceeds to create or run. -This enables ordered startup without an external orchestrator: the monitor -container can be given ``mode = immediate`` while all data nodes start with -``mode = deferred``, and each data node is released with ``node start`` only -after the monitor is confirmed ready. + create = deferred + run = deferred + +A node configured this way still runs the ordinary ``pg_autoctl node run`` +command, but it starts a polling loop and waits rather than creating or +starting Postgres. A sidecar container or init script then calls:: + + pg_autoctl node start + +which clears both flags (rewriting the ini file with ``create = immediate`` +/ ``run = immediate``). The waiting node detects the change within the +poll interval and proceeds. This enables ordered startup without an +external orchestrator: the monitor container can be given the defaults +(``immediate``) while all data nodes start deferred, and each data node is +released with ``node start`` only after the monitor is confirmed ready -- +or, for an :ref:`archiving_architecture` archiver that needs every group of +its target formation to already exist (a Citus formation's several worker +groups, in particular), only after every one of those groups is confirmed +registered. See Also -------- diff --git a/docs/ref/pg_autoctl_node_run.rst b/docs/ref/pg_autoctl_node_run.rst index c57ab76d9..90f8a6a08 100644 --- a/docs/ref/pg_autoctl_node_run.rst +++ b/docs/ref/pg_autoctl_node_run.rst @@ -22,48 +22,63 @@ Description deployments. Given a ``pg_autoctl_node.ini`` file it: 1. Reads and validates the ini file. -2. If ``[launch] mode = deferred``, polls the file until the section is - removed or changed to ``mode = immediate`` (see ``pg_autoctl node start``). +2. If ``[launch] create = deferred``, polls the file every 0.5s until it's + changed to ``create = immediate`` (see ``pg_autoctl node start``). 3. Checks whether the node already exists (looks for ``pg_autoctl.cfg`` inside ``pgdata``). - - **First start** — builds the ``pg_autoctl create [flags] --run`` - argument list from the ini file and exec's into it, which creates Postgres - and starts the supervisor in one step. - - **Subsequent starts** — applies any mutable setting changes found in the - ini file, then exec's into ``pg_autoctl run --pgdata ``. + - **First start** — runs ``pg_autoctl create [flags]`` (built + from the ini file, *without* ``--run``) to create the node. + - **Subsequent starts** — applies any mutable setting changes found in + the ini file to the already-existing node. -4. Sets the ``PG_AUTOCTL_NODESPEC`` environment variable to the ini file +4. If ``[launch] run = deferred``, polls the file every 0.5s until it's + changed to ``run = immediate``. +5. Exec's into ``pg_autoctl run --pgdata ``, which starts Postgres (if + applicable for this node kind) and the supervisor. +6. Sets the ``PG_AUTOCTL_NODESPEC`` environment variable to the ini file path before exec'ing, so the supervisor can watch the file for live changes to ``[settings]``. +``create`` and ``run`` are independent gates: creating the node (step 3) +and starting it (step 5) can each be deferred on their own. Setting only +``create = deferred`` (leaving ``run`` at its default) creates the node +immediately once released and starts it right away in the same +invocation; setting only ``run = deferred`` creates the node immediately +but leaves it stopped until separately released. + Because the command uses ``execv()``, the pg_autoctl supervisor becomes the direct child process (PID 1 in a container), preserving the standard Unix signal contract — ``SIGTERM`` stops the supervisor cleanly, ``SIGHUP`` reloads configuration. See :ref:`pg_autoctl_stop` for what a graceful ``SIGTERM`` actually does before the node stops. -The ``launch = deferred`` pattern ----------------------------------- +The deferred-launch pattern +---------------------------- The ``[launch]`` section enables ordered startup without an external orchestrator:: [launch] - mode = deferred + create = deferred + run = deferred -A node with ``mode = deferred`` starts the polling loop and waits. A -second container, sidecar, or init script calls:: +A node configured this way starts the polling loop and waits. A second +container, sidecar, or init script calls:: - pg_autoctl node start /etc/pgaf/node.ini + pg_autoctl node start -which rewrites the file with ``mode = immediate``. The waiting node detects -the change and proceeds. This is useful when you need to ensure the monitor -is fully up before any data node attempts registration, or when bringing up -Citus workers in a specific order. +which clears both flags. The waiting node detects the change and +proceeds. This is useful when you need to ensure the monitor is fully up +before any data node attempts registration, when bringing up Citus +workers in a specific order, or -- for an :ref:`archiving_architecture` +archiver -- when its target formation (or, for a Citus formation, every +one of its groups) might not exist yet: ``pg_autoctl create archiver`` +has no retry-until-ready loop of its own the way an ordinary node's +registration does, so it must not run before the formation is ready. See Also -------- :ref:`pg_autoctl_node`, :ref:`pg_autoctl_create_postgres`, -:ref:`pg_autoctl_run` +:ref:`pg_autoctl_create_archiver`, :ref:`pg_autoctl_run` diff --git a/docs/ref/pg_autoctl_node_start.rst b/docs/ref/pg_autoctl_node_start.rst index c70718658..30af6d705 100644 --- a/docs/ref/pg_autoctl_node_start.rst +++ b/docs/ref/pg_autoctl_node_start.rst @@ -3,7 +3,7 @@ pg_autoctl node start ===================== -pg_autoctl node start - Release a node waiting in launch=deferred mode +pg_autoctl node start - Release a node waiting in a deferred launch Synopsis -------- @@ -18,30 +18,38 @@ Synopsis Description ----------- -``pg_autoctl node start`` releases a node that is waiting in -``[launch] mode = deferred``. It rewrites the ini file with -``mode = immediate``; the waiting node detects the change within the poll -interval and proceeds to create or run. +``pg_autoctl node start`` releases a node that is waiting on either or +both of ``[launch] create = deferred`` / ``run = deferred``. It clears +both flags in the ini file (rewriting them to ``immediate``); the waiting +node detects the change within the poll interval and proceeds. This command is idempotent: calling it on a node that is already running -(or has ``mode = immediate``) is a no-op. +(both flags already ``immediate``) is a no-op. -The ``launch = deferred`` Pattern ----------------------------------- +The Deferred-Launch Pattern +---------------------------- -A node configured with ``[launch] mode = deferred`` starts a polling loop -and waits instead of immediately creating or starting Postgres. This -enables ordered startup without an external orchestrator:: +A node configured with ``[launch] create = deferred`` and/or ``run = +deferred`` starts a polling loop and waits instead of immediately +creating or starting Postgres. This enables ordered startup without an +external orchestrator:: # In the ini file for each data node: [launch] - mode = deferred + create = deferred + run = deferred -The monitor can be given ``mode = immediate`` (the default), while data nodes -start with ``mode = deferred``. Once the monitor is confirmed ready, release -each data node:: +The monitor can be left at the defaults (``immediate``), while data nodes +start deferred. Once the monitor is confirmed ready, release each data +node:: - pg_autoctl node start /etc/pgaf/node.ini + pg_autoctl node start + +An :ref:`archiving_architecture` archiver whose target formation (or, for +a Citus formation, one or more of its groups) might not exist yet at +container-start time follows the same pattern -- see +:ref:`pg_autoctl_node_run`'s own note on why an archiver specifically +needs this, unlike an ordinary node. See Also -------- diff --git a/docs/ref/pg_autoctl_set.rst b/docs/ref/pg_autoctl_set.rst index 466bb5d9a..d311582ee 100644 --- a/docs/ref/pg_autoctl_set.rst +++ b/docs/ref/pg_autoctl_set.rst @@ -12,3 +12,4 @@ pg_autoctl set - Set a pg_auto_failover node, or formation setting pg_autoctl_set_node_replication_quorum pg_autoctl_set_node_candidate_priority pg_autoctl_set_node_region + pg_autoctl_set_basebackup_policy diff --git a/docs/ref/pg_autoctl_set_basebackup_policy.rst b/docs/ref/pg_autoctl_set_basebackup_policy.rst new file mode 100644 index 000000000..33d14ec92 --- /dev/null +++ b/docs/ref/pg_autoctl_set_basebackup_policy.rst @@ -0,0 +1,51 @@ +.. _pg_autoctl_set_basebackup_policy: + +pg_autoctl set basebackup-policy +================================== + +pg_autoctl set basebackup-policy - Update a named base-backup +production/retention policy + +Synopsis +-------- + +This command updates a base-backup production/retention policy that +already exists on the monitor:: + + usage: pg_autoctl set basebackup-policy --monitor --name --config + + --monitor pg_auto_failover Monitor Postgres URL + --name policy name + --config path to a JSON document with the fields to change + +Description +----------- + +Only the fields present in the ``--config`` document change; any field +left out keeps its current value. See :ref:`pg_autoctl_create_basebackup_policy` +for the full set of fields and what each one controls -- the document +shape is identical, just with only the fields you want to change. + +Every archiver whose (formation, group) resolves to this policy (directly, +or through its formation's own default) picks up the change on its next +tick -- there is no need to restart anything. + +Options +------- + +The following options are available to ``pg_autoctl set basebackup-policy``: + +--monitor + + Postgres URI used to connect to the monitor. Must use the ``autoctl_node`` + username and target the ``pg_auto_failover`` database name. It is possible + to show the Postgres URI from the monitor node using the command + :ref:`pg_autoctl_show_uri`. + +--name + + Name of the policy to update. + +--config + + Path to a JSON document with the fields to change. diff --git a/docs/ref/pg_autoctl_show.rst b/docs/ref/pg_autoctl_show.rst index 896b94342..a6301236d 100644 --- a/docs/ref/pg_autoctl_show.rst +++ b/docs/ref/pg_autoctl_show.rst @@ -16,3 +16,4 @@ pg_autoctl show - Show pg_auto_failover information pg_autoctl_show_timeline pg_autoctl_show_file pg_autoctl_show_systemd + pg_autoctl_show_basebackup_policy diff --git a/docs/ref/pg_autoctl_show_basebackup_policy.rst b/docs/ref/pg_autoctl_show_basebackup_policy.rst new file mode 100644 index 000000000..1f07f1821 --- /dev/null +++ b/docs/ref/pg_autoctl_show_basebackup_policy.rst @@ -0,0 +1,47 @@ +.. _pg_autoctl_show_basebackup_policy: + +pg_autoctl show basebackup-policy +=================================== + +pg_autoctl show basebackup-policy - Show a named base-backup +production/retention policy + +Synopsis +-------- + +This command fetches a base-backup production/retention policy by name +from the monitor and prints it:: + + usage: pg_autoctl show basebackup-policy --monitor --name [ --json ] + + --monitor pg_auto_failover Monitor Postgres URL + --name policy name + --json output data in the JSON format + +Description +----------- + +Prints every field of the named policy: ``source``, ``replaymode``, +``cache``, ``frequency``, ``maxcount``, ``maxage``, ``onpromotion``, and +``concurrency`` -- see :ref:`pg_autoctl_create_basebackup_policy` for what +each one controls. + +Options +------- + +The following options are available to ``pg_autoctl show basebackup-policy``: + +--monitor + + Postgres URI used to connect to the monitor. Must use the ``autoctl_node`` + username and target the ``pg_auto_failover`` database name. It is possible + to show the Postgres URI from the monitor node using the command + :ref:`pg_autoctl_show_uri`. + +--name + + Name of the policy to show. + +--json + + Output data in the JSON format. diff --git a/docs/ref/pgaftest.rst b/docs/ref/pgaftest.rst index a4889d511..88c6f6f82 100644 --- a/docs/ref/pgaftest.rst +++ b/docs/ref/pgaftest.rst @@ -438,13 +438,28 @@ Node modifiers: ``candidate-priority `` Failover priority 0–100 (default: 50) ``region `` Data-centre / availability-zone label (``--region``; default: ``default``) -``launch deferred`` Container starts with ``sleep infinity``; - use ``exec node pg_autoctl node start`` +``create deferred`` Container still runs the ordinary + ``pg_autoctl node run `` command, but + the ini's own ``[launch] create = deferred`` + makes it poll and wait rather than actually + registering; release with + ``exec node pg_autoctl node start`` +``launch deferred`` Same mechanism, gating only the final + "start Postgres and the supervisor" step + (``[launch] run = deferred``) -- the node is + still created, just not started yet +``create and launch deferred`` Both gates at once -- the common case, + matching how ``pg_autoctl create + --run`` bundles create+run for an + immediate node ``suspended`` The node-active service never transitions on its own; drive it explicitly with the ``fsm step `` DSL command (see `Suspended nodes`_ below) ``coordinator`` / ``worker group `` Citus role +``archiver`` Archiving & Disaster Recovery node + (see `Top-level archiver nodes`_ below for + the more common declaration form) ``no-monitor`` Standalone node (no monitor) ``listen`` Bind all interfaces (``--listen 0.0.0.0``) ``auth `` Per-node auth override @@ -452,6 +467,60 @@ Node modifiers: ``volume `` Mount a named Docker volume at ```` ============================================ ============================================= +Top-level archiver nodes +~~~~~~~~~~~~~~~~~~~~~~~~~ + +An archiver may also be declared directly inside ``cluster { }``, as its own +``archiver { }`` block -- a sibling of ``monitor``/``formation``, not +nested inside either. This matches the real data model +(``pgautofailover.archiver`` has no formation column at all; it attaches to +one or more formations by name, it isn't a member of any one of them), and +is the recommended form over declaring an ``archiver`` node inline inside a +``formation { }`` block: + +.. code-block:: text + + cluster { + monitor + formation { + node1 + node2 + } + archiver archiver1 { + formation default # required; exactly one + region eu-west # optional; default "default" + create and launch deferred # optional -- see below + } + } + +Internally this is folded into an ordinary node entry in the named +formation's own node list right after parsing, so it launches immediately +by default and supports every modifier above (``region``, the deferred +forms, ...) exactly the same way an ordinary node does -- there is nothing +archiver-specific about the mechanism, only about where it's declared. + +Only one ``formation `` is accepted: ``pg_autoctl create archiver``'s +own ini-driven bootstrap has no notion of attaching to more than one +formation at create time (unlike the CLI's own repeatable ``--formation`` +flag). To cover a second formation, attach it dynamically once the +archiver is already running -- a direct ``sql monitor { SELECT +pgautofailover.archiver_add_formation(...) }`` step is the idiom used by +the ``archiver_multi_formation.pgaf`` spec in this test suite. + +Immediate (the default) launch is only safe when the target formation's +group already exists by the time the archiver's own container starts -- +guaranteed for a formation whose other nodes it already ``depends_on`` +(the ordinary node-ordering rules apply the same way here), but *not* +guaranteed across independent formations or a Citus formation's several +groups, since ``pg_autoctl create archiver`` has no retry-until-ready loop +the way ordinary nodes' registration does. Use ``create and launch +deferred`` plus an explicit ``exec pg_autoctl node start`` step +once every target group is confirmed to exist whenever that ordering +isn't otherwise guaranteed -- see the ``citus_basic_operation.pgaf`` +spec's own archiver step for a worked example (a Citus formation's worker +groups must all be registered before the archiver attaches, so it can +cover every one of them in a single call). + Node registration order ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -561,6 +630,49 @@ propagated. expect { } expect error [] +**SQL-condition waits** + +.. code-block:: text + + wait until sql { SELECT ... } is { } [timeout s] + + wait until wal segment "" archived in / [timeout s] + wait until archiver state is in [/] [timeout s] + wait until basebackup is in / [timeout s] + +The generic form polls an arbitrary scalar SQL expression every second +until its (substring-matched, same semantics as ``expect``) result contains +````, or the timeout elapses — the primitive to reach for when a +condition can't be expressed as a node-state wait and none of the sugar +forms below fit. It exists specifically to replace ``sleep s`` followed +by a single ``sql``/``expect`` pair: a fixed sleep either wastes time +waiting past a condition that was already true, or — under CI load — isn't +long enough and produces a flaky failure; polling adapts to how long the +condition actually takes. + +The three sugar forms below are just this primitive with a pre-built SQL +query, covering the checks archiver specs need most: + +- ``wait until wal segment "" archived in /`` + polls ``pgautofailover.wal_archived()``. The segment name must be quoted + (it's all digits, which would otherwise be lexed as an integer and + overflow). +- ``wait until archiver state is in [/]`` polls + an archiver's own ``reportedstate``, matching on + ``nodename LIKE 'archiver-%'`` and ``formationid`` (and ``groupid`` when + given) rather than a plain node name: an ``ARCHIVING`` row's ``nodename`` + is always synthesized by ``archiver_add_formation()`` as + ``archiver--``, never the plain ``--name`` given at + ``create archiver`` time, so the ordinary ``wait until state is + `` form can't see these rows at all, let alone disambiguate more + than one membership sharing the same archiver. Omit the group when the + formation has exactly one archiver membership; give it to disambiguate a + multi-group Citus formation. +- ``wait until basebackup is in + /`` polls ``pgautofailover.get_latest_basebackup()``'s + 2-argument form. For the 3-argument ``preferred_source`` overload, or any + other ``pgautofailover.*`` function, use the generic form directly. + **Network** .. code-block:: text diff --git a/docs/tikz/arch-archiver-internals.svg b/docs/tikz/arch-archiver-internals.svg new file mode 100644 index 000000000..1c6209957 --- /dev/null +++ b/docs/tikz/arch-archiver-internals.svg @@ -0,0 +1,1048 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/tikz/arch-archiver-internals.tex b/docs/tikz/arch-archiver-internals.tex new file mode 100644 index 000000000..d0264dba8 --- /dev/null +++ b/docs/tikz/arch-archiver-internals.tex @@ -0,0 +1,76 @@ +% Fix for: https://tex.stackexchange.com/a/315027/43228 +\RequirePackage{luatex85} +\documentclass[border=10pt,17pt]{standalone} + +\usepackage{cfr-lm} +\usepackage{pgf} +\usepackage{tikz} +\usetikzlibrary{arrows,shapes,snakes,automata,backgrounds,petri} +\usetikzlibrary{shapes.multipart} + +\begin{document} + +%% sans-serif fonts, large by default, and bold too +\sffamily +\sbweight +\bfseries +\large + +\begin{tikzpicture}[>=stealth',bend angle=45,auto,rounded corners] + + \input{common.tex} + + %% \draw [help lines] (-13,0) grid (13,20); + + \tikzstyle{proc}=[rectangle,rounded corners=5pt,very thick,draw=stxt, + fill=async,text=stxt,minimum width=6.4cm,minimum height=1.6cm,align=center] + \tikzstyle{child}=[rectangle,rounded corners=5pt,thick,draw=abox, + fill=abox!10,text=stxt,minimum width=4.6cm,minimum height=1.3cm,align=center] + \tikzstyle{file}=[rectangle,rounded corners=3pt,thick,draw=pbox,dashed, + fill=pbox!6,text=stxt,minimum width=5cm,minimum height=1.3cm,align=center, + font=\ttfamily\small] + \tikzstyle{consumers}=[rectangle,rounded corners=5pt,very thick,draw=mbox, + fill=mbox!15,text=stxt,minimum width=6.6cm,minimum height=2.4cm,align=center] + + \node (super) at (0,18.6) [proc] {\normalsize \texttt{pg\_autoctl archiver run}\\[2pt]\small two supervised services}; + + \node (recon) at (-6.4,14.6) [proc] {\normalsize reconciler\\[2pt]\small \texttt{service\_archiver\_reconciler\_loop()}}; + \node (serve) at (6.4,14.6) [proc] {\normalsize serve\\[2pt]\small \texttt{service\_archiver\_serve\_loop()}}; + + \path (super.west) edge[->,thick,out=200,in=90] node[left,pos=0.5] {\small fork} (recon.north) + (super.east) edge[->,thick,out=-20,in=90] node[right,pos=0.5] {\small fork} (serve.north); + + \node (cap) at (-6.4,11.0) [child] {\ttfamily\small capture}; + \path (recon.south) edge[->,thick] node[left,pos=0.5,align=left] {\small fork,\\[-2pt]\small one per\\[-2pt]\small membership} (cap.north); + + \node (recv) at (-9.4,7.4) [child] {\ttfamily\small pg\_receivewal}; + \path (cap.south) edge[->,thick,out=230,in=90] node[left,pos=0.55,align=left] {\small fork + exec\\[-2pt]\scriptsize \texttt{-S }} (recv.north); + + \node (walcache) at (-9.4,3.9) [file] {WAL cache / basebackups}; + \path (recv.south) edge[->,thick] node[right] {\small writes} (walcache.north); + + \node (pos) at (-3.1,7.4) [file] {archiver-position}; + \path (cap.south) edge[->,dashed,color=abox,out=-70,in=110] node[below,sloped] {\small writes} (pos.north); + + \node (routes) at (3.1,10.4) [file] {archiver-routes.ini}; + \path (serve.south) edge[->,dashed,color=abox,out=250,in=70] node[below,sloped] {\small writes} (routes.north); + + \path (pos.east) edge[->,dashed,color=pbox,out=-20,in=200] node[below,pos=0.5] {\small reads} (routes.west); + + \node (wsend) at (9.4,10.4) [child] {\ttfamily\small pg\_walsender}; + \path (serve.south) edge[->,thick,out=-50,in=90] node[right,pos=0.55,align=left] {\small fork + exec\\[-2pt]\scriptsize \texttt{--routes ...}} (wsend.north); + \path (routes.east) edge[->,thick,out=0,in=160] node[above,pos=0.5] {\small reads} (wsend.west); + + \node (consumers) at (9.4,3.6) [consumers] + {\normalsize Consumers\\[4pt] + \small \texttt{pg\_basebackup}\\ + \small streaming standby (\texttt{primary\_conninfo})\\ + \small \texttt{restore\_command} fetch}; + + \path (wsend.south) edge[->,very thick,color=mbox] node[right,align=left] + {\small \texttt{BASE\_BACKUP}\\[-2pt] \texttt{START\_REPLICATION}\\[-2pt] \texttt{FETCH\_FILE}, ...} + (consumers.north); + +\end{tikzpicture} + +\end{document} diff --git a/docs/tikz/arch-archiver.svg b/docs/tikz/arch-archiver.svg new file mode 100644 index 000000000..d54c3476b --- /dev/null +++ b/docs/tikz/arch-archiver.svg @@ -0,0 +1,529 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/tikz/arch-archiver.tex b/docs/tikz/arch-archiver.tex new file mode 100644 index 000000000..ad9790957 --- /dev/null +++ b/docs/tikz/arch-archiver.tex @@ -0,0 +1,54 @@ +% Fix for: https://tex.stackexchange.com/a/315027/43228 +\RequirePackage{luatex85} +\documentclass[border=10pt,17pt]{standalone} + +\usepackage{cfr-lm} +\usepackage{pgf} +\usepackage{tikz} +\usetikzlibrary{arrows,shapes,snakes,automata,backgrounds,petri} +\usetikzlibrary{shapes.multipart} + +\begin{document} + +%% sans-serif fonts, large by default, and bold too +\sffamily +\sbweight +\bfseries +\large + +\begin{tikzpicture}[>=stealth',bend angle=45,auto,rounded corners] + + \input{common.tex} + + %% \draw [help lines] (-11,0) grid (11,24); + + \node (p) at (0,20) [primary] + {\textbf{\Large Primary}}; + \node (s) at (0,12) [standby] + {\textbf{\Large Secondary}}; + \node (app) at (-8,16) [app] {\textbf{Application}}; + + \node (a) at (9,20) [archiver] + {\textbf{\normalsize Archiver} + \nodepart{second} + \textbf{\large ARCHIVING} + \nodepart[align=left]{third} + \texttt{WAL cache} \\ + \texttt{Base backups} + }; + + \node (m) at (9,12) [monitor] {\textbf{Monitor}}; + + \path (app.north east) edge [sql,out=90,in=180] node {SQL} (p) + (app.south east) edge [sqlf,out=-90,in=180] node[below] {SQL (fallback)} (s) + (p) edge [sr] + node[left] {Streaming} + node [right] {Replication} (s) + (p.east) edge [wal] node[above,pos=0.38] {WAL streaming} + node[below,pos=0.38] {(\texttt{pg\_receivewal})} (a.west) + (a.south) edge [hc] node[right] {WAL reports} (m.north) + (m.west) edge [hc,out=180,in=-70] node[below,sloped] {Health checks} (s.east) + (m.east) edge [hc,out=20,in=-20,looseness=1.6] node[right] {Health checks} (p.east); +\end{tikzpicture} + +\end{document} diff --git a/docs/tikz/arch-ha-dr-pgautofailover.svg b/docs/tikz/arch-ha-dr-pgautofailover.svg new file mode 100644 index 000000000..dc4e3cbf9 --- /dev/null +++ b/docs/tikz/arch-ha-dr-pgautofailover.svg @@ -0,0 +1,294 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/tikz/arch-ha-dr-pgautofailover.tex b/docs/tikz/arch-ha-dr-pgautofailover.tex new file mode 100644 index 000000000..51adb8ae4 --- /dev/null +++ b/docs/tikz/arch-ha-dr-pgautofailover.tex @@ -0,0 +1,81 @@ +% Fix for: https://tex.stackexchange.com/a/315027/43228 +\RequirePackage{luatex85} +\documentclass[border=10pt,17pt]{standalone} + +\usepackage{cfr-lm} +\usepackage{pgf} +\usepackage{tikz} +\usetikzlibrary{arrows,shapes,snakes,automata,backgrounds,petri} +\usetikzlibrary{shapes.multipart} + +\begin{document} + +%% sans-serif fonts, large by default, and bold too +\sffamily +\sbweight +\bfseries +\large + +\begin{tikzpicture}[>=stealth',bend angle=45,auto,rounded corners] + + \input{common.tex} + + %% High Availability and Disaster Recovery keep the exact same colors + %% they have on arch-ha-dr-typical.tex, so the two diagrams read as + %% the same three boxes, just regrouped. + \definecolor{haTxt}{HTML}{2F5C9B} + \definecolor{haBorder}{HTML}{7FA0CC} + + \definecolor{bkTxt}{HTML}{8A5A12} + \definecolor{bkBorder}{HTML}{D3A257} + + \definecolor{pgafTxt}{HTML}{2E6B4F} + \definecolor{pgafBorder}{HTML}{7FBF9C} + + %% \draw [help lines] (-13,0) grid (13,7); + + \newcommand{\boundarybox}[5]{ + \draw[rounded corners=9pt,draw=#5,very thick,dashed,fill=#5!12] + (#1,#2) rectangle (#3,#4); + } + \newcommand{\boundaryheader}[4]{ + \node[anchor=north west,text=#3,font=\bfseries\Large,inner sep=0pt] + at (#1,#2) {#4}; + } + \newcommand{\servicebox}[5]{ + \node[rectangle,rounded corners=5pt,draw=#3,thick,fill=white, + text=stxt,minimum width=#4,minimum height=1.35cm,align=center] + at (#1,#2) {\normalsize #5}; + } + + %% three boxes on one horizontal line, same left-to-right order as + %% arch-ha-dr-typical.tex -- High Availability, Disaster Recovery, + %% Backups -- only which pair is wrapped changes: here High + %% Availability and Disaster Recovery are the pair nested in one + %% outer box (pg_auto_failover, one product covering both), on the + %% left; Backups stands alone on the right, same as it does on the + %% other diagram. + + \boundarybox{-13.0}{0.4}{4.4}{6.6}{pgafBorder} + \boundaryheader{-12.5}{6.2}{pgafTxt}{pg\_auto\_failover} + \draw[draw=pgafBorder,line width=0.6pt] (-12.5,5.5) -- (3.9,5.5); + + \boundarybox{-12.5}{0.9}{-4.55}{5.1}{haBorder} + \boundaryheader{-12.0}{4.7}{haTxt}{High Availability} + \draw[draw=haBorder,line width=0.6pt] (-12.0,4.0) -- (-5.05,4.0); + \servicebox{-8.525}{1.95}{haBorder}{6.2cm}{pg\_auto\_failover} + + \boundarybox{-4.05}{0.9}{3.9}{5.1}{bkBorder} + \boundaryheader{-3.55}{4.7}{bkTxt}{Disaster Recovery} + \draw[draw=bkBorder,line width=0.6pt] (-3.55,4.0) -- (3.4,4.0); + \servicebox{-0.075}{1.95}{bkBorder}{6.2cm}{pg\_auto\_failover} + + \boundarybox{5.3}{0.4}{13.0}{6.6}{bkBorder} + \boundaryheader{5.8}{6.2}{bkTxt}{Backups} + \draw[draw=bkBorder,line width=0.6pt] (5.8,5.5) -- (12.5,5.5); + \servicebox{9.15}{4.3}{bkBorder}{6.2cm}{pgBackRest} + \servicebox{9.15}{1.7}{bkBorder}{6.2cm}{pgBarman} + +\end{tikzpicture} + +\end{document} diff --git a/docs/tikz/arch-ha-dr-typical.svg b/docs/tikz/arch-ha-dr-typical.svg new file mode 100644 index 000000000..7705cb1e0 --- /dev/null +++ b/docs/tikz/arch-ha-dr-typical.svg @@ -0,0 +1,266 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/tikz/arch-ha-dr-typical.tex b/docs/tikz/arch-ha-dr-typical.tex new file mode 100644 index 000000000..16b6fb360 --- /dev/null +++ b/docs/tikz/arch-ha-dr-typical.tex @@ -0,0 +1,74 @@ +% Fix for: https://tex.stackexchange.com/a/315027/43228 +\RequirePackage{luatex85} +\documentclass[border=10pt,17pt]{standalone} + +\usepackage{cfr-lm} +\usepackage{pgf} +\usepackage{tikz} +\usetikzlibrary{arrows,shapes,snakes,automata,backgrounds,petri} +\usetikzlibrary{shapes.multipart} + +\begin{document} + +%% sans-serif fonts, large by default, and bold too +\sffamily +\sbweight +\bfseries +\large + +\begin{tikzpicture}[>=stealth',bend angle=45,auto,rounded corners] + + \input{common.tex} + + \definecolor{haTxt}{HTML}{2F5C9B} + \definecolor{haBorder}{HTML}{7FA0CC} + + \definecolor{bkTxt}{HTML}{8A5A12} + \definecolor{bkBorder}{HTML}{D3A257} + + \definecolor{neutralBorder}{HTML}{ABABAB} + + %% \draw [help lines] (-13,0) grid (13,7); + + \newcommand{\boundarybox}[5]{ + \draw[rounded corners=9pt,draw=#5,very thick,dashed,fill=#5!12] + (#1,#2) rectangle (#3,#4); + } + \newcommand{\boundaryheader}[4]{ + \node[anchor=north west,text=#3,font=\bfseries\Large,inner sep=0pt] + at (#1,#2) {#4}; + } + \newcommand{\servicebox}[5]{ + \node[rectangle,rounded corners=5pt,draw=#3,thick,fill=white, + text=stxt,minimum width=#4,minimum height=1.35cm,align=center] + at (#1,#2) {\normalsize #5}; + } + + %% three boxes on one horizontal line: High Availability, Disaster + %% Recovery, Backups -- Disaster Recovery and Backups wrapped in one + %% outer box (same two products cover both, in a typical setup), + %% High Availability standing alone outside it. + + \boundarybox{-13.0}{0.4}{-5.3}{6.6}{haBorder} + \boundaryheader{-12.5}{6.2}{haTxt}{High Availability} + \draw[draw=haBorder,line width=0.6pt] (-12.5,5.5) -- (-5.8,5.5); + \servicebox{-9.15}{4.3}{haBorder}{6.2cm}{Patroni} + \servicebox{-9.15}{1.7}{haBorder}{6.2cm}{repmgr} + + \boundarybox{-4.4}{0.4}{13.0}{6.6}{neutralBorder} + + \boundarybox{-3.9}{0.9}{4.05}{6.1}{bkBorder} + \boundaryheader{-3.4}{5.7}{bkTxt}{Disaster Recovery} + \draw[draw=bkBorder,line width=0.6pt] (-3.4,5.0) -- (3.55,5.0); + \servicebox{0.075}{3.7}{bkBorder}{6.2cm}{pgBackRest} + \servicebox{0.075}{1.4}{bkBorder}{6.2cm}{pgBarman} + + \boundarybox{4.55}{0.9}{12.5}{6.1}{bkBorder} + \boundaryheader{5.05}{5.7}{bkTxt}{Backups} + \draw[draw=bkBorder,line width=0.6pt] (5.05,5.0) -- (12.0,5.0); + \servicebox{8.525}{3.7}{bkBorder}{6.2cm}{pgBackRest} + \servicebox{8.525}{1.4}{bkBorder}{6.2cm}{pgBarman} + +\end{tikzpicture} + +\end{document} diff --git a/docs/tikz/common.tex b/docs/tikz/common.tex index c9c9624d1..fd414cb2d 100644 --- a/docs/tikz/common.tex +++ b/docs/tikz/common.tex @@ -12,6 +12,9 @@ \definecolor{async}{HTML}{EBEFF5} % very light grey +\definecolor{abox}{HTML}{FFB900} % MS amber -- cold storage, distinct from primary/standby +\definecolor{atxt}{HTML}{2F2F2F} % off-black + \tikzstyle{app}=[circle,thick, text=aptxt,draw=apbox,fill=white, line width=0.25em,minimum size=4cm] @@ -29,6 +32,9 @@ \tikzstyle{standby}=[mpnode,text=stxt,draw=white, rectangle split part fill={sbox,sbox,white}] +\tikzstyle{archiver}=[mpnode,text=atxt,draw=white, + rectangle split part fill={abox,abox,white}] + \tikzstyle{monitor}=[node,text=mtxt,draw=mbox,fill=mbox] \tikzstyle{citusnode}=[rectangle split,rectangle split parts=2, @@ -45,6 +51,7 @@ \tikzstyle{sql}=[->,color=pbox,text=stxt,line width=0.15em] \tikzstyle{sqlf}=[->,color=sbox,text=stxt,line width=0.15em,loosely dashed] \tikzstyle{sr}=[>->,color=stxt,text=stxt,line width=0.15em] +\tikzstyle{wal}=[>->,color=abox,text=atxt,line width=0.15em,densely dashed] \tikzstyle{hc}=[<->,color=mbox,text=mtxt,line width=0.15em,dotted] \tikzstyle{hcmid}=[color=mbox,text=mtxt,line width=0.15em,dotted] \tikzstyle{cw}=[<->,color=stxt,text=stxt,line width=0.1em] diff --git a/src/bin/Makefile b/src/bin/Makefile index 734eb561f..542b49701 100644 --- a/src/bin/Makefile +++ b/src/bin/Makefile @@ -3,12 +3,13 @@ COMMON_LIB = common/libpgaf_common.a -all: pg_autoctl pgaftest ; +all: pg_autoctl pgaftest pg_walsender ; -# Build the shared archive once, serially, before the two binaries run in -# parallel. Both pg_autoctl and pgaftest include Makefile.common which -# defines compile rules for common/*.c; without this serialisation a -# parallel make -j would race to write the same .o files simultaneously. +# Build the shared archive once, serially, before the binaries run in +# parallel. pg_autoctl, pgaftest, and pg_walsender all include +# Makefile.common which defines compile rules for common/*.c; without this +# serialisation a parallel make -j would race to write the same .o files +# simultaneously. $(COMMON_LIB): $(MAKE) -C common @@ -18,13 +19,18 @@ pg_autoctl: $(COMMON_LIB) pgaftest: $(COMMON_LIB) $(MAKE) -C pgaftest pgaftest +pg_walsender: $(COMMON_LIB) + $(MAKE) -C pg_walsender pg_walsender + clean: $(MAKE) -C common clean $(MAKE) -C pg_autoctl clean $(MAKE) -C pgaftest clean + $(MAKE) -C pg_walsender clean -install: pg_autoctl pgaftest +install: pg_autoctl pgaftest pg_walsender $(MAKE) -C pg_autoctl install $(MAKE) -C pgaftest install + $(MAKE) -C pg_walsender install -.PHONY: all pg_autoctl pgaftest install clean +.PHONY: all pg_autoctl pgaftest pg_walsender install clean diff --git a/src/bin/common/pgctl.c b/src/bin/common/pgctl.c index a8e5cad28..372ee1ee6 100644 --- a/src/bin/common/pgctl.c +++ b/src/bin/common/pgctl.c @@ -1264,7 +1264,8 @@ pg_basebackup(const char *pgdata, NodeAddress *primaryNode = &(replicationSource->primaryNode); char primaryConnInfo[MAXCONNINFO] = { 0 }; - char *args[18]; /* enough for all pg_basebackup flags incl. --checkpoint=fast */ + char *args[20]; /* enough for all pg_basebackup flags incl. --checkpoint=fast + * and --no-manifest */ int argsIndex = 0; char command[BUFSIZE]; @@ -1339,6 +1340,12 @@ pg_basebackup(const char *pgdata, args[argsIndex++] = replicationSource->slotName; } + /* see ReplicationSource.noManifest's own comment, pgsql.h */ + if (replicationSource->noManifest) + { + args[argsIndex++] = "--no-manifest"; + } + args[argsIndex] = NULL; /* @@ -2743,12 +2750,28 @@ pgctl_identify_system(ReplicationSource *replicationSource) char primaryConnInfoReplication[MAXCONNINFO] = { 0 }; PGSQL replicationClient = { 0 }; + /* + * Real Postgres ignores dbname for a replication=true connection (see + * libpqrcv_connect's own comment, libpqwalreceiver.c: "The database + * name is ignored by the server in replication mode, but specify + * 'replication' for .pgpass lookup"), so this is a no-op against a real + * primary. It is NOT a no-op against pg_walsender: unlike real + * walreceiver/pg_basebackup, which both default an unset dbname to the + * literal "replication" themselves (walreceiver hardcodes it; + * pg_basebackup's own GetConnection() does too), this is our own raw + * libpq connection with no such default applied for us -- leaving + * dbname unset here falls through to plain libpq's *own* default + * instead (dbname = the connection's user name, fe-connect.c), which + * pg_walsender's routes file was never going to have an entry for. + * Passing it explicitly matches what every other replication client + * already sends on the wire. + */ if (!prepare_primary_conninfo(primaryConnInfo, MAXCONNINFO, primaryNode->host, primaryNode->port, replicationSource->userName, - NULL, /* no database */ + "replication", replicationSource->password, replicationSource->applicationName, replicationSource->sslOptions, diff --git a/src/bin/common/pgsetup.c b/src/bin/common/pgsetup.c index 175f02072..486e94c44 100644 --- a/src/bin/common/pgsetup.c +++ b/src/bin/common/pgsetup.c @@ -1496,10 +1496,11 @@ nodeKindFromString(const char *nodeKind) NODE_KIND_UNKNOWN, NODE_KIND_STANDALONE, NODE_KIND_CITUS_COORDINATOR, - NODE_KIND_CITUS_WORKER + NODE_KIND_CITUS_WORKER, + NODE_KIND_ARCHIVER }; char *kindList[] = { - "", "unknown", "standalone", "coordinator", "worker", NULL + "", "unknown", "standalone", "coordinator", "worker", "archiver", NULL }; for (int listIndex = 0; kindList[listIndex] != NULL; listIndex++) @@ -1546,6 +1547,11 @@ nodeKindToString(PgInstanceKind kind) return "worker"; } + case NODE_KIND_ARCHIVER: + { + return "archiver"; + } + default: { log_fatal("nodeKindToString: unknown node kind %d", kind); diff --git a/src/bin/common/pgsetup.h b/src/bin/common/pgsetup.h index f8afcfa18..242946f7f 100644 --- a/src/bin/common/pgsetup.h +++ b/src/bin/common/pgsetup.h @@ -131,6 +131,7 @@ typedef enum PgInstanceKind NODE_KIND_STANDALONE = 1, NODE_KIND_CITUS_COORDINATOR = 2, NODE_KIND_CITUS_WORKER = 4, + NODE_KIND_ARCHIVER = 8, NODE_KIND_ANY = 0xff } PgInstanceKind; diff --git a/src/bin/common/pgsql.h b/src/bin/common/pgsql.h index f27c6f6e0..5edc42fcc 100644 --- a/src/bin/common/pgsql.h +++ b/src/bin/common/pgsql.h @@ -267,6 +267,17 @@ typedef struct ReplicationSource * for themselves when to promote, should leave this false. */ bool pauseAtRecoveryTarget; + + /* + * pg_walsender's BASE_BACKUP doesn't implement backup manifests yet + * (~/dev/temp/archiving-disaster-recovery.md's own documented scope for + * this milestone), which a real pg_basebackup requests by default from + * PG13+ -- set for an archiver-sourced base backup (create postgres + * --from-archiver) so pg_basebackup() knows to pass --no-manifest; + * false (the default) for a real primary/standby upstream, which does + * support manifests and should keep getting one. + */ + bool noManifest; SSLOptions sslOptions; IdentifySystem system; } ReplicationSource; diff --git a/src/bin/common/signals.c b/src/bin/common/signals.c index 58687da49..90b1041da 100644 --- a/src/bin/common/signals.c +++ b/src/bin/common/signals.c @@ -25,6 +25,7 @@ volatile sig_atomic_t asked_to_stop = 0; /* SIGTERM */ volatile sig_atomic_t asked_to_stop_fast = 0; /* SIGINT */ volatile sig_atomic_t asked_to_reload = 0; /* SIGHUP */ volatile sig_atomic_t asked_to_quit = 0; /* SIGQUIT */ +volatile sig_atomic_t asked_to_refresh_routes = 0; /* SIGUSR1 */ /* * set_signal_handlers sets our signal handlers for the 4 signals that we @@ -39,6 +40,7 @@ set_signal_handlers(bool exitOnQuit) pqsignal(SIGHUP, catch_reload); pqsignal(SIGINT, catch_int); pqsignal(SIGTERM, catch_term); + pqsignal(SIGUSR1, catch_refresh_routes); if (exitOnQuit) { @@ -59,7 +61,7 @@ set_signal_handlers(bool exitOnQuit) bool block_signals(sigset_t *mask, sigset_t *orig_mask) { - int signals[] = { SIGHUP, SIGINT, SIGTERM, SIGQUIT, -1 }; + int signals[] = { SIGHUP, SIGINT, SIGTERM, SIGQUIT, SIGUSR1, -1 }; if (sigemptyset(mask) == -1) { @@ -128,6 +130,19 @@ catch_reload(SIGNAL_ARGS) } +/* + * catch_refresh_routes receives the SIGUSR1 signal. + */ +void +catch_refresh_routes(SIGNAL_ARGS) +{ + int sig = postgres_signal_arg; + + asked_to_refresh_routes = 1; + pqsignal(sig, catch_refresh_routes); +} + + /* * catch_int receives the SIGINT signal. */ diff --git a/src/bin/common/signals.h b/src/bin/common/signals.h index f82b7a9ce..b4542c0a8 100644 --- a/src/bin/common/signals.h +++ b/src/bin/common/signals.h @@ -20,6 +20,17 @@ extern volatile sig_atomic_t asked_to_stop_fast; /* SIGINT */ extern volatile sig_atomic_t asked_to_reload; /* SIGHUP */ extern volatile sig_atomic_t asked_to_quit; /* SIGQUIT */ +/* + * Prompts service_archiver_serve_loop() to refresh its routes file on its + * next iteration instead of waiting for the next periodic tick -- see + * service_archiver_maybe_generate_basebackup()'s own comment (service_ + * archiver_basebackup.c) on why a freshly-completed base backup needs + * this. Harmless in every other process: nothing else checks it, same as + * asked_to_reload is already installed everywhere regardless of whether a + * given service body reacts to it. + */ +extern volatile sig_atomic_t asked_to_refresh_routes; /* SIGUSR1 */ + #define CHECK_FOR_FAST_SHUTDOWN { if (asked_to_stop_fast) { break; } \ } @@ -31,6 +42,7 @@ void catch_int(SIGNAL_ARGS); void catch_term(SIGNAL_ARGS); void catch_quit(SIGNAL_ARGS); void catch_quit_and_exit(SIGNAL_ARGS); +void catch_refresh_routes(SIGNAL_ARGS); int get_current_signal(int defaultSignal); int pick_stronger_signal(int sig1, int sig2); diff --git a/src/bin/pg_autoctl/cli_archiver.c b/src/bin/pg_autoctl/cli_archiver.c new file mode 100644 index 000000000..638673ba4 --- /dev/null +++ b/src/bin/pg_autoctl/cli_archiver.c @@ -0,0 +1,245 @@ +/* + * src/bin/pg_autoctl/cli_archiver.c + * See cli_archiver.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "postgres_fe.h" + +#include "cli_archiver.h" +#include "cli_common.h" +#include "commandline.h" +#include "defaults.h" +#include "file_utils.h" +#include "keeper.h" +#include "keeper_config.h" +#include "log.h" +#include "monitor.h" +#include "pidfile.h" +#include "service_archiver_serve.h" +#include "signals.h" +#include "string_utils.h" + +static int cli_archiver_serve_getopts(int argc, char **argv); +static void cli_archiver_serve(int argc, char **argv); + +/* set by --port; 0 means "use PG_AUTOCTL_ARCHIVER_SERVE_PORT" */ +static int archiverServePortOption = 0; + + +static int +cli_archiver_serve_getopts(int argc, char **argv) +{ + KeeperConfig options = { 0 }; + int c, option_index = 0; + int verboseCount = 0; + + static struct option long_options[] = { + { "pgdata", required_argument, NULL, 'D' }, + { "port", required_argument, NULL, 'p' }, + { "version", no_argument, NULL, 'V' }, + { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, 'q' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + optind = 0; + + while ((c = getopt_long(argc, argv, "D:p:Vvqh", + long_options, &option_index)) != -1) + { + switch (c) + { + case 'D': + { + strlcpy(options.pgSetup.pgdata, optarg, MAXPGPATH); + log_trace("--pgdata %s", options.pgSetup.pgdata); + break; + } + + case 'p': + { + if (!stringToInt(optarg, &archiverServePortOption) || + archiverServePortOption <= 0 || + archiverServePortOption > 65535) + { + log_fatal("Failed to parse --port value \"%s\"", optarg); + exit(EXIT_CODE_BAD_ARGS); + } + break; + } + + case 'V': + { + keeper_cli_print_version(argc, argv); + break; + } + + case 'v': + { + ++verboseCount; + switch (verboseCount) + { + case 1: + { + log_set_level(LOG_INFO); + break; + } + + case 2: + { + log_set_level(LOG_DEBUG); + break; + } + + default: + { + log_set_level(LOG_TRACE); + break; + } + } + break; + } + + case 'q': + { + log_set_level(LOG_ERROR); + break; + } + + case 'h': + { + commandline_help(stderr); + exit(EXIT_CODE_QUIT); + break; + } + + default: + { + commandline_help(stderr); + exit(EXIT_CODE_BAD_ARGS); + break; + } + } + } + + (void) prepare_keeper_options(&options); + + keeperOptions = options; + + return optind; +} + + +/* + * cli_archiver_serve implements `pg_autoctl archiver serve`: loads the + * archiver's own config/state (already written by `pg_autoctl create + * archiver`), connects to the monitor, and runs + * service_archiver_serve_loop() -- exec'ing pg_walsender and keeping its + * routes file current. See service_archiver_serve.h. + */ +static void +cli_archiver_serve(int argc, char **argv) +{ + Keeper keeper = { 0 }; + + keeper.config = keeperOptions; + + /* + * An archiver's pgdata is its local WAL-cache root, never a real + * Postgres instance (see service_archiver.c's own header comment) -- + * both flags must tolerate that, matching cli_create_archiver's own + * choice not to run pg_setup_init's real-instance checks at all. + */ + bool missingPgdataIsOk = true; + bool pgIsNotRunningIsOk = true; + bool monitorDisabledIsOk = false; + + if (!keeper_config_read_file(&(keeper.config), + missingPgdataIsOk, + pgIsNotRunningIsOk, + monitorDisabledIsOk)) + { + log_fatal("Failed to read the archiver configuration file \"%s\", " + "see above for details", keeper.config.pathnames.config); + exit(EXIT_CODE_BAD_CONFIG); + } + + if (strcmp(keeper.config.nodeKind, "archiver") != 0) + { + log_fatal("\"%s\" is not an archiver's configuration file " + "(pg_autoctl.nodekind is \"%s\", expected \"archiver\")", + keeper.config.pathnames.config, keeper.config.nodeKind); + exit(EXIT_CODE_BAD_CONFIG); + } + + if (keeper.config.archiverId <= 0) + { + log_fatal("This archiver's configuration file has no archiver_id " + "recorded -- it may predate `pg_autoctl archiver serve` " + "support; re-create the archiver with `pg_autoctl create " + "archiver` to pick it up"); + exit(EXIT_CODE_BAD_CONFIG); + } + + if (!keeper_load_state(&keeper)) + { + log_fatal("Failed to read the archiver state file \"%s\", " + "see above for details", keeper.config.pathnames.state); + exit(EXIT_CODE_BAD_STATE); + } + + if (!monitor_init(&(keeper.monitor), keeper.config.monitor_pguri)) + { + log_fatal("Failed to contact the monitor, see above for details"); + exit(EXIT_CODE_MONITOR); + } + + if (archiverServePortOption > 0) + { + service_archiver_serve_set_port(archiverServePortOption); + } + + (void) set_signal_handlers(false); + (void) set_ps_title("pg_autoctl: archiver serve"); + + if (!create_pidfile(keeper.config.pathnames.pid, getpid())) + { + log_fatal("Failed to write archiver pid file \"%s\"", + keeper.config.pathnames.pid); + exit(EXIT_CODE_BAD_STATE); + } + + if (!service_archiver_serve_loop(&keeper)) + { + exit(EXIT_CODE_INTERNAL_ERROR); + } +} + + +CommandLine archiver_serve_command = + make_command( + "serve", + "Start serving this archiver's captured WAL and base backups", + " [ --pgdata --port ] ", + " --pgdata path to the archiver's local data/cache directory\n" + " --port port for pg_walsender to listen on " + "(default: 6543)\n", + cli_archiver_serve_getopts, + cli_archiver_serve); + +CommandLine *archiver_subcommands[] = { + &archiver_serve_command, + NULL +}; + +CommandLine archiver_commands = + make_command_set("archiver", + "Manage a pg_auto_failover archiver node", NULL, NULL, + NULL, archiver_subcommands); diff --git a/src/bin/pg_autoctl/cli_archiver.h b/src/bin/pg_autoctl/cli_archiver.h new file mode 100644 index 000000000..0db95b09c --- /dev/null +++ b/src/bin/pg_autoctl/cli_archiver.h @@ -0,0 +1,23 @@ +/* + * src/bin/pg_autoctl/cli_archiver.h + * pg_autoctl archiver -- the archiver's own command group. Only `serve` + * is implemented this milestone; the other CLI-reference subverbs + * (add-storage, remove-storage, backup, prefetch, ...) belong to later + * milestones and stay unregistered until then, per + * ~/dev/temp/archiving-disaster-recovery.md's Build order. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef CLI_ARCHIVER_H +#define CLI_ARCHIVER_H + +#include "commandline.h" + +extern CommandLine archiver_serve_command; +extern CommandLine *archiver_subcommands[]; +extern CommandLine archiver_commands; + +#endif /* CLI_ARCHIVER_H */ diff --git a/src/bin/pg_autoctl/cli_basebackup_policy.c b/src/bin/pg_autoctl/cli_basebackup_policy.c new file mode 100644 index 000000000..4f000b052 --- /dev/null +++ b/src/bin/pg_autoctl/cli_basebackup_policy.c @@ -0,0 +1,436 @@ +/* + * src/bin/pg_autoctl/cli_basebackup_policy.c + * See cli_basebackup_policy.h. + * + * A basebackup_policy row is a monitor-side object, not tied to any one + * node's local pgdata (unlike `pg_autoctl create archiver`'s own --pgdata- + * rooted config), so these commands connect straight to --monitor, the + * same self-contained shape create_archiver_command already uses, rather + * than resolving a monitor URL through an existing node's config file the + * way the `get`/`set` property commands (cli_get_set_properties.c) do. + * + * --config is a JSON document read from disk and passed straight + * through, as text, to the monitor's own create_basebackup_policy()/set_ + * basebackup_policy() (pgautofailover.sql) -- their own jsonb cast and + * per-field coalesce-to-default/coalesce-to-current-value logic is the one + * and only place this document actually gets validated and applied, so + * there is nothing to duplicate client-side. The document is the flat + * policy body itself (source/replaymode/cache/frequency/maxcount/maxage/ + * onpromotion/concurrency, whichever subset is being set) -- not wrapped + * in the design doc's own illustrative "pgaf-archiver"/"basebackup-policy" + * namespace, since the monitor-side functions this calls don't unwrap one. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include + +#include "postgres_fe.h" + +#include "parson.h" + +#include "cli_basebackup_policy.h" +#include "cli_common.h" +#include "commandline.h" +#include "defaults.h" +#include "file_utils.h" +#include "log.h" +#include "monitor.h" +#include "string_utils.h" + +typedef struct BasebackupPolicyCLIOptions +{ + char monitorPguri[MAXCONNINFO]; + char policyName[NAMEDATALEN]; + char configFilePath[MAXPGPATH]; +} BasebackupPolicyCLIOptions; + +static BasebackupPolicyCLIOptions basebackupPolicyOptions = { 0 }; + +static int cli_basebackup_policy_getopts(int argc, char **argv, + bool requireConfig); +static int cli_create_basebackup_policy_getopts(int argc, char **argv); +static int cli_show_basebackup_policy_getopts(int argc, char **argv); +static int cli_set_basebackup_policy_getopts(int argc, char **argv); + +static void cli_create_basebackup_policy(int argc, char **argv); +static void cli_show_basebackup_policy(int argc, char **argv); +static void cli_set_basebackup_policy(int argc, char **argv); + +static bool read_json_config_file(const char *path, char *jsonOut, + size_t jsonOutSize); +static void print_basebackup_policy(BasebackupPolicy *policy); + + +/* + * cli_basebackup_policy_getopts parses the option set shared by all three + * commands (--monitor --name --json, plus --config for create/set). Kept + * as one function with a requireConfig switch rather than three near- + * duplicates, matching cli_create_archiver_getopts's own minimal, hand- + * rolled style for this milestone's own archiver-adjacent commands + * (rather than the ordinary-node cli_create_node_getopts, which assumes a + * real PostgresSetup none of these commands have any use for). + */ +static int +cli_basebackup_policy_getopts(int argc, char **argv, bool requireConfig) +{ + int c, option_index = 0, errors = 0; + + static struct option long_options[] = { + { "monitor", required_argument, NULL, 'm' }, + { "name", required_argument, NULL, 'a' }, + { "config", required_argument, NULL, 'c' }, + { "json", no_argument, NULL, 'J' }, + { "version", no_argument, NULL, 'V' }, + { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, 'q' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + optind = 0; + + while ((c = getopt_long(argc, argv, "m:a:c:JVvqh", + long_options, &option_index)) != -1) + { + switch (c) + { + case 'm': + { + if (!validate_connection_string(optarg)) + { + log_fatal("Failed to parse --monitor connection string, " + "see above for details."); + exit(EXIT_CODE_BAD_ARGS); + } + strlcpy(basebackupPolicyOptions.monitorPguri, optarg, + MAXCONNINFO); + log_trace("--monitor %s", basebackupPolicyOptions.monitorPguri); + break; + } + + case 'a': + { + strlcpy(basebackupPolicyOptions.policyName, optarg, + NAMEDATALEN); + log_trace("--name %s", basebackupPolicyOptions.policyName); + break; + } + + case 'c': + { + strlcpy(basebackupPolicyOptions.configFilePath, optarg, + MAXPGPATH); + log_trace("--config %s", basebackupPolicyOptions.configFilePath); + break; + } + + case 'J': + { + outputJSON = true; + break; + } + + case 'V': + { + keeper_cli_print_version(argc, argv); + exit(EXIT_CODE_QUIT); + } + + case 'v': + { + log_set_level(LOG_INFO); + break; + } + + case 'q': + { + log_set_level(LOG_ERROR); + break; + } + + case 'h': + { + commandline_help(stderr); + exit(EXIT_CODE_QUIT); + } + + default: + { + ++errors; + break; + } + } + } + + if (errors > 0) + { + commandline_help(stderr); + exit(EXIT_CODE_BAD_ARGS); + } + + if (IS_EMPTY_STRING_BUFFER(basebackupPolicyOptions.monitorPguri)) + { + log_fatal("Failed to get value for --monitor"); + exit(EXIT_CODE_BAD_ARGS); + } + + if (IS_EMPTY_STRING_BUFFER(basebackupPolicyOptions.policyName)) + { + log_fatal("Failed to get value for --name"); + exit(EXIT_CODE_BAD_ARGS); + } + + if (requireConfig && IS_EMPTY_STRING_BUFFER(basebackupPolicyOptions.configFilePath)) + { + log_fatal("Failed to get value for --config"); + exit(EXIT_CODE_BAD_ARGS); + } + + return optind; +} + + +static int +cli_create_basebackup_policy_getopts(int argc, char **argv) +{ + return cli_basebackup_policy_getopts(argc, argv, true); +} + + +static int +cli_set_basebackup_policy_getopts(int argc, char **argv) +{ + return cli_basebackup_policy_getopts(argc, argv, true); +} + + +static int +cli_show_basebackup_policy_getopts(int argc, char **argv) +{ + return cli_basebackup_policy_getopts(argc, argv, false); +} + + +/* + * read_json_config_file reads path's whole contents into jsonOut, for + * pass-through to the monitor's own ::jsonb cast -- no client-side JSON + * parsing/validation, see this file's own header comment on why. + */ +static bool +read_json_config_file(const char *path, char *jsonOut, size_t jsonOutSize) +{ + char *contents = NULL; + long fileSize = 0; + + if (!read_file(path, &contents, &fileSize)) + { + log_error("Failed to read base-backup policy config file \"%s\"", + path); + return false; + } + + strlcpy(jsonOut, contents, jsonOutSize); + free(contents); + + return true; +} + + +/* + * print_basebackup_policy prints a resolved policy either as plain text + * (one "field: value" line each) or, with --json, the same fields as a + * JSON object -- matching cli_get_set_properties.c's own established + * plain/--json duality for monitor-resolved properties. + */ +static void +print_basebackup_policy(BasebackupPolicy *policy) +{ + if (outputJSON) + { + JSON_Value *js = json_value_init_object(); + JSON_Object *jsObj = json_value_get_object(js); + + json_object_set_string(jsObj, "name", policy->policyName); + json_object_set_string(jsObj, "source", policy->source); + json_object_set_string(jsObj, "replaymode", policy->replayMode); + json_object_set_string(jsObj, "cache", policy->cache); + json_object_set_number(jsObj, "frequency-seconds", + (double) policy->frequencySeconds); + json_object_set_number(jsObj, "maxcount", (double) policy->maxCount); + json_object_set_number(jsObj, "maxage-seconds", + (double) policy->maxAgeSeconds); + json_object_set_boolean(jsObj, "onpromotion", policy->onPromotion); + json_object_set_number(jsObj, "concurrency", + (double) policy->concurrency); + + (void) cli_pprint_json(js); + + return; + } + + fformat(stdout, "%12s: %s\n", "name", policy->policyName); + fformat(stdout, "%12s: %s\n", "source", policy->source); + fformat(stdout, "%12s: %s\n", "replaymode", + IS_EMPTY_STRING_BUFFER(policy->replayMode) ? "-" : policy->replayMode); + fformat(stdout, "%12s: %s\n", "cache", policy->cache); + fformat(stdout, "%12s: %d\n", "frequency", policy->frequencySeconds); + fformat(stdout, "%12s: %d\n", "maxcount", policy->maxCount); + fformat(stdout, "%12s: %d\n", "maxage", policy->maxAgeSeconds); + fformat(stdout, "%12s: %s\n", "onpromotion", + policy->onPromotion ? "true" : "false"); + fformat(stdout, "%12s: %d\n", "concurrency", policy->concurrency); +} + + +/* + * cli_create_basebackup_policy implements `pg_autoctl create basebackup- + * policy`. + */ +static void +cli_create_basebackup_policy(int argc, char **argv) +{ + char jsonSpec[BUFSIZE] = { 0 }; + + if (!read_json_config_file(basebackupPolicyOptions.configFilePath, + jsonSpec, sizeof(jsonSpec))) + { + /* errors already logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, basebackupPolicyOptions.monitorPguri)) + { + /* errors already logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + int64_t basebackupPolicyId = 0; + + if (!monitor_create_basebackup_policy(&monitor, + basebackupPolicyOptions.policyName, + jsonSpec, &basebackupPolicyId)) + { + log_fatal("Failed to create base-backup policy \"%s\", see above " + "for details", basebackupPolicyOptions.policyName); + exit(EXIT_CODE_MONITOR); + } + + log_info("Created base-backup policy \"%s\" (id %" PRId64 ")", + basebackupPolicyOptions.policyName, basebackupPolicyId); +} + + +/* + * cli_show_basebackup_policy implements `pg_autoctl show basebackup- + * policy`. + */ +static void +cli_show_basebackup_policy(int argc, char **argv) +{ + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, basebackupPolicyOptions.monitorPguri)) + { + /* errors already logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + BasebackupPolicy policy = { 0 }; + bool found = false; + + if (!monitor_get_basebackup_policy(&monitor, + basebackupPolicyOptions.policyName, + &policy, &found)) + { + log_fatal("Failed to get base-backup policy \"%s\", see above for " + "details", basebackupPolicyOptions.policyName); + exit(EXIT_CODE_MONITOR); + } + + if (!found) + { + log_fatal("Base-backup policy \"%s\" does not exist", + basebackupPolicyOptions.policyName); + exit(EXIT_CODE_BAD_ARGS); + } + + print_basebackup_policy(&policy); +} + + +/* + * cli_set_basebackup_policy implements `pg_autoctl set basebackup- + * policy`. + */ +static void +cli_set_basebackup_policy(int argc, char **argv) +{ + char jsonSpec[BUFSIZE] = { 0 }; + + if (!read_json_config_file(basebackupPolicyOptions.configFilePath, + jsonSpec, sizeof(jsonSpec))) + { + /* errors already logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, basebackupPolicyOptions.monitorPguri)) + { + /* errors already logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + if (!monitor_set_basebackup_policy(&monitor, + basebackupPolicyOptions.policyName, + jsonSpec)) + { + log_fatal("Failed to set base-backup policy \"%s\", see above for " + "details", basebackupPolicyOptions.policyName); + exit(EXIT_CODE_MONITOR); + } + + log_info("Updated base-backup policy \"%s\"", + basebackupPolicyOptions.policyName); +} + + +CommandLine create_basebackup_policy_command = + make_command( + "basebackup-policy", + "Create a named base-backup production/retention policy", + " --monitor --name --config ", + " --monitor pg_auto_failover Monitor Postgres URL\n" + " --name policy name\n" + " --config path to a JSON document with the policy body\n", + cli_create_basebackup_policy_getopts, + cli_create_basebackup_policy); + +CommandLine show_basebackup_policy_command = + make_command( + "basebackup-policy", + "Show a named base-backup production/retention policy", + " --monitor --name [ --json ] ", + " --monitor pg_auto_failover Monitor Postgres URL\n" + " --name policy name\n" + " --json output data in the JSON format\n", + cli_show_basebackup_policy_getopts, + cli_show_basebackup_policy); + +CommandLine set_basebackup_policy_command = + make_command( + "basebackup-policy", + "Update a named base-backup production/retention policy", + " --monitor --name --config ", + " --monitor pg_auto_failover Monitor Postgres URL\n" + " --name policy name\n" + " --config path to a JSON document with the fields to change\n", + cli_set_basebackup_policy_getopts, + cli_set_basebackup_policy); diff --git a/src/bin/pg_autoctl/cli_basebackup_policy.h b/src/bin/pg_autoctl/cli_basebackup_policy.h new file mode 100644 index 000000000..73150a9c3 --- /dev/null +++ b/src/bin/pg_autoctl/cli_basebackup_policy.h @@ -0,0 +1,20 @@ +/* + * src/bin/pg_autoctl/cli_basebackup_policy.h + * CLI for pgautofailover.basebackup_policy: create/show/set a named + * base-backup production/retention policy on the monitor. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef CLI_BASEBACKUP_POLICY_H +#define CLI_BASEBACKUP_POLICY_H + +#include "commandline.h" + +extern CommandLine create_basebackup_policy_command; +extern CommandLine show_basebackup_policy_command; +extern CommandLine set_basebackup_policy_command; + +#endif /* CLI_BASEBACKUP_POLICY_H */ diff --git a/src/bin/pg_autoctl/cli_common.c b/src/bin/pg_autoctl/cli_common.c index 709504b7c..360c47f56 100644 --- a/src/bin/pg_autoctl/cli_common.c +++ b/src/bin/pg_autoctl/cli_common.c @@ -99,6 +99,7 @@ cli_common_keeper_getopts(int argc, char **argv, /* force some non-zero default values */ LocalOptionConfig.monitorDisabled = false; + LocalOptionConfig.fromArchiver = false; LocalOptionConfig.groupId = -1; LocalOptionConfig.network_partition_timeout = -1; LocalOptionConfig.prepare_promotion_catchup = -1; @@ -471,6 +472,14 @@ cli_common_keeper_getopts(int argc, char **argv, break; } + case 'K': + { + /* { "from-archiver", no_argument, NULL, 'K' }, */ + LocalOptionConfig.fromArchiver = true; + log_trace("--from-archiver"); + break; + } + case 's': { /* { "ssl-self-signed", no_argument, NULL, 's' }, */ diff --git a/src/bin/pg_autoctl/cli_common.h b/src/bin/pg_autoctl/cli_common.h index 45bfca490..348b17cb1 100644 --- a/src/bin/pg_autoctl/cli_common.h +++ b/src/bin/pg_autoctl/cli_common.h @@ -92,6 +92,7 @@ extern CommandLine create_monitor_command; extern CommandLine create_postgres_command; extern CommandLine create_coordinator_command; extern CommandLine create_worker_command; +extern CommandLine create_archiver_command; extern CommandLine activate_node_command; /* cli_drop_node.c */ diff --git a/src/bin/pg_autoctl/cli_create_node.c b/src/bin/pg_autoctl/cli_create_node.c index 39b3f4aab..9e4585629 100644 --- a/src/bin/pg_autoctl/cli_create_node.c +++ b/src/bin/pg_autoctl/cli_create_node.c @@ -36,10 +36,13 @@ #include "pghba.h" #include "pidfile.h" #include "primary_standby.h" +#include "service_archiver.h" +#include "service_archiver_run.h" #include "service_keeper.h" #include "service_keeper_init.h" #include "service_monitor.h" #include "service_monitor_init.h" +#include "signals.h" #include "string_utils.h" /* @@ -62,6 +65,32 @@ static void cli_activate_node(int argc, char **argv); static int cli_create_monitor_getopts(int argc, char **argv); static void cli_create_monitor(int argc, char **argv); +static int cli_create_archiver_getopts(int argc, char **argv); +static void cli_create_archiver(int argc, char **argv); + +/* --basebackup-policy on `create archiver`: a policy name to resolve and + * attach via set_archiver_policy(), not part of KeeperConfig/keeperOptions + * -- it's applied once at creation time, never persisted to the archiver's + * own config file (see cli_create_archiver()'s own use of this). */ +static char archiverBasebackupPolicyName[NAMEDATALEN] = { 0 }; + +/* + * --formation is repeatable on `create archiver`: a single archiver can + * hold a membership in more than one formation at once (service_archiver_ + * reconciler.c runs one WAL-capture child per (formation, group), and + * service_archiver_serve.c serves all of them through one shared + * pg_walsender). Collected here rather than into KeeperConfig/keeperOptions + * (whose own .formation field is a single NAMEDATALEN buffer): archiverFormations[0] + * is still mirrored into options.formation, so the legacy single-membership + * fallback paths (this archiver's own persisted config file, + * service_archiver_serve.c's own no-memberships-yet fallback, and the still- + * single-membership `create archiver --run` path) keep behaving exactly as + * before for the common single-formation case. + */ +#define CLI_CREATE_ARCHIVER_MAX_FORMATIONS 64 +static char archiverFormations[CLI_CREATE_ARCHIVER_MAX_FORMATIONS][NAMEDATALEN]; +static int archiverFormationsCount = 0; + static void check_hostname(const char *hostname); CommandLine create_monitor_command = @@ -102,7 +131,8 @@ CommandLine create_postgres_command = KEEPER_CLI_SSL_OPTIONS " --candidate-priority priority of the node to be promoted to become primary\n" " --replication-quorum true if node participates in write quorum\n" - " --maximum-backup-rate maximum transfer rate of data transferred from the server during initial sync\n", + " --maximum-backup-rate maximum transfer rate of data transferred from the server during initial sync\n" + " --from-archiver bootstrap from a registered archiver's base backup and WAL cache\n", cli_create_postgres_getopts, cli_create_postgres); @@ -352,12 +382,13 @@ cli_create_postgres_getopts(int argc, char **argv) { "ssl-crl-file", required_argument, &ssl_flag, SSL_CRL_FILE_FLAG }, { "server-cert", required_argument, &ssl_flag, SSL_SERVER_CRT_FLAG }, { "server-key", required_argument, &ssl_flag, SSL_SERVER_KEY_FLAG }, + { "from-archiver", no_argument, NULL, 'K' }, { NULL, 0, NULL, 0 } }; int optind = cli_create_node_getopts(argc, argv, long_options, - "C:D:H:p:l:U:A:SLd:a:n:f:m:MI:W:w:RGVvqhP:r:xsN", + "C:D:H:p:l:U:A:SLd:a:n:f:m:MI:W:w:RGVvqhP:r:xsNK", &options); /* publish our option parsing in the global variable */ @@ -1294,6 +1325,472 @@ cli_create_monitor(int argc, char **argv) } +/* + * cli_create_archiver_getopts parses `pg_autoctl create archiver`'s own + * command line options -- deliberately not cli_create_node_getopts (used by + * every ordinary node kind): that shared parser and the KeeperConfig + * defaults it applies assume a real PostgresSetup (pgport, pghost, a real + * PGDATA to validate), none of which apply to an archiver (see haspgdata's + * own design comment, pgautofailover.sql). Milestone 2's own minimal flag + * set, matching the design doc's own Quickstart: --pgdata --monitor + * --hostname --name --formation --run. + */ +static int +cli_create_archiver_getopts(int argc, char **argv) +{ + KeeperConfig options = { 0 }; + int c, option_index = 0, errors = 0; + + static struct option long_options[] = { + { "pgdata", required_argument, NULL, 'D' }, + { "pgctl", required_argument, NULL, 'C' }, + { "monitor", required_argument, NULL, 'm' }, + { "hostname", required_argument, NULL, 'n' }, + { "name", required_argument, NULL, 'a' }, + { "formation", required_argument, NULL, 'f' }, + { "basebackup-policy", required_argument, NULL, 'P' }, + { "region", required_argument, NULL, 'G' }, + { "run", no_argument, NULL, 'x' }, + { "version", no_argument, NULL, 'V' }, + { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, 'q' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + optind = 0; + + while ((c = getopt_long(argc, argv, "D:C:m:n:a:f:P:G:xVvqh", + long_options, &option_index)) != -1) + { + switch (c) + { + case 'D': + { + strlcpy(options.pgSetup.pgdata, optarg, MAXPGPATH); + log_trace("--pgdata %s", options.pgSetup.pgdata); + break; + } + + case 'C': + { + strlcpy(options.pgSetup.pg_ctl, optarg, MAXPGPATH); + log_trace("--pgctl %s", options.pgSetup.pg_ctl); + break; + } + + case 'm': + { + if (!validate_connection_string(optarg)) + { + log_fatal("Failed to parse --monitor connection string, " + "see above for details."); + exit(EXIT_CODE_BAD_ARGS); + } + strlcpy(options.monitor_pguri, optarg, MAXCONNINFO); + log_trace("--monitor %s", options.monitor_pguri); + break; + } + + case 'n': + { + strlcpy(options.hostname, optarg, _POSIX_HOST_NAME_MAX); + log_trace("--hostname %s", options.hostname); + break; + } + + case 'a': + { + strlcpy(options.name, optarg, _POSIX_HOST_NAME_MAX); + log_trace("--name %s", options.name); + break; + } + + case 'f': + { + /* --formation (may be repeated) */ + if (archiverFormationsCount >= CLI_CREATE_ARCHIVER_MAX_FORMATIONS) + { + log_fatal("pg_autoctl create archiver only supports up " + "to %d --formation options", + CLI_CREATE_ARCHIVER_MAX_FORMATIONS); + exit(EXIT_CODE_BAD_ARGS); + } + + strlcpy(archiverFormations[archiverFormationsCount], optarg, + NAMEDATALEN); + ++archiverFormationsCount; + log_trace("--formation %s", optarg); + break; + } + + case 'P': + { + strlcpy(archiverBasebackupPolicyName, optarg, NAMEDATALEN); + log_trace("--basebackup-policy %s", archiverBasebackupPolicyName); + break; + } + + case 'G': + { + /* same field ordinary nodes' own --region uses + * (cli_common.c's cli_common_keeper_getopts), not reused + * here since an archiver has its own, much smaller getopts + * -- but the underlying PostgresSetup.settings.region + * field is still there on every KeeperConfig regardless of + * node kind. */ + strlcpy(options.pgSetup.settings.region, optarg, NAMEDATALEN); + log_trace("--region %s", options.pgSetup.settings.region); + break; + } + + case 'x': + { + createAndRun = true; + log_trace("--run"); + break; + } + + case 'V': + { + keeper_cli_print_version(argc, argv); + exit(EXIT_CODE_QUIT); + } + + case 'v': + { + log_set_level(LOG_INFO); + break; + } + + case 'q': + { + log_set_level(LOG_ERROR); + break; + } + + case 'h': + { + commandline_help(stderr); + exit(EXIT_CODE_QUIT); + } + + default: + { + ++errors; + break; + } + } + } + + if (errors > 0) + { + commandline_help(stderr); + exit(EXIT_CODE_BAD_ARGS); + } + + if (IS_EMPTY_STRING_BUFFER(options.pgSetup.pgdata)) + { + log_fatal("Failed to get value for --pgdata"); + exit(EXIT_CODE_BAD_ARGS); + } + + if (IS_EMPTY_STRING_BUFFER(options.monitor_pguri)) + { + log_fatal("Failed to get value for --monitor"); + exit(EXIT_CODE_BAD_ARGS); + } + + if (archiverFormationsCount == 0) + { + strlcpy(archiverFormations[0], "default", NAMEDATALEN); + archiverFormationsCount = 1; + } + + /* + * options.formation (persisted to the archiver's own config file) keeps + * mirroring the first --formation given -- see this file's own comment + * on archiverFormations above for why. + */ + strlcpy(options.formation, archiverFormations[0], NAMEDATALEN); + + options.pgSetup.pgKind = NODE_KIND_ARCHIVER; + strlcpy(options.nodeKind, "archiver", NAMEDATALEN); + + keeperOptions = options; + + return optind; +} + + +/* + * cli_create_archiver implements `pg_autoctl create archiver`: registers a + * new Archiver identity and attaches it to a formation via M1's own + * register_archiver()/archiver_add_formation() plpgsql functions (not the + * ordinary C register_node() RPC every other node kind goes through -- an + * Archiver is a process identity, not a (formation, group) membership by + * itself, see pgautofailover.sql's own comment on that function), writes a + * KeeperConfig + initial state file, and with --run hands off to + * service_archiver_loop() (service_archiver.c) -- deliberately not + * service_keeper_init()/keeper_node_active_loop(), which assume a real + * Postgres instance an ARCHIVING node never has. + */ +static void +cli_create_archiver(int argc, char **argv) +{ + pid_t pid = 0; + Keeper keeper = { 0 }; + KeeperConfig *config = &(keeper.config); + + keeper.config = keeperOptions; + + if (!check_or_discover_hostname(config)) + { + /* errors have already been logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + if (!keeper_config_set_pathnames_from_pgdata(&config->pathnames, + config->pgSetup.pgdata)) + { + /* errors have already been logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + if (read_pidfile(config->pathnames.pid, &pid)) + { + log_fatal("pg_autoctl is already running with pid %d", pid); + exit(EXIT_CODE_BAD_STATE); + } + + if (IS_EMPTY_STRING_BUFFER(config->pgSetup.pg_ctl) && + !config_find_pg_ctl(&(config->pgSetup))) + { + /* errors have already been logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + if (!directory_exists(config->pgSetup.pgdata)) + { + if (pg_mkdir_p(config->pgSetup.pgdata, 0700) != 0) + { + log_fatal("Failed to create archiver directory \"%s\": %m", + config->pgSetup.pgdata); + exit(EXIT_CODE_BAD_ARGS); + } + } + + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, config->monitor_pguri)) + { + /* errors have already been logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + keeper.monitor = monitor; + + int64_t archiverId = 0; + int64_t archiverNodeId = 0; + + char *archiverName = + IS_EMPTY_STRING_BUFFER(config->name) ? config->hostname : config->name; + + if (!monitor_register_archiver(&monitor, archiverName, config->hostname, + config->pgSetup.settings.region, + &archiverId)) + { + log_fatal("Failed to register archiver \"%s\" on the monitor, " + "see above for details", archiverName); + exit(EXIT_CODE_MONITOR); + } + + /* + * --basebackup-policy resolves a name to its basebackuppolicyid once, + * then gets attached to every --formation given below via set_archiver_ + * policy() -- a formation/group-level setting (archiver_policy), not + * per-archiver, matching get_archiver_policy()'s own resolution scope: + * any other archiver later added to the same (formation, group) + * inherits it too. archiverQuorum=1, replicationQuorumEligible=false + * are this schema's own hardcoded defaults (get_archiver_policy()'s + * final fallback tier) -- passed through explicitly here since set_ + * archiver_policy() only coaleses NULL to "keep existing" for a row + * that already exists, and this may be the first policy ever set for + * this (formation, group). + */ + bool hasBasebackupPolicy = !IS_EMPTY_STRING_BUFFER(archiverBasebackupPolicyName); + BasebackupPolicy basebackupPolicy = { 0 }; + + if (hasBasebackupPolicy) + { + bool foundPolicy = false; + + if (!monitor_get_basebackup_policy(&monitor, archiverBasebackupPolicyName, + &basebackupPolicy, &foundPolicy)) + { + log_fatal("Failed to resolve base-backup policy \"%s\", see " + "above for details", archiverBasebackupPolicyName); + exit(EXIT_CODE_MONITOR); + } + + if (!foundPolicy) + { + log_fatal("Base-backup policy \"%s\" does not exist", + archiverBasebackupPolicyName); + exit(EXIT_CODE_BAD_ARGS); + } + } + + /* + * Attach this archiver to every --formation given (at least one: either + * what was passed on the command line, or "default" -- see cli_create_ + * archiver_getopts()). archiver_add_formation() itself attaches one + * ARCHIVING node per group already in that formation, so a Citus + * formation with several worker groups is fully covered by a single + * call here. + */ + for (int i = 0; i < archiverFormationsCount; i++) + { + char *formation = archiverFormations[i]; + int64_t thisArchiverNodeId = 0; + + if (!monitor_archiver_add_formation(&monitor, archiverId, + formation, &thisArchiverNodeId)) + { + log_fatal("Failed to attach archiver \"%s\" to formation \"%s\", " + "see above for details", archiverName, formation); + exit(EXIT_CODE_MONITOR); + } + + log_info("Registered archiver \"%s\" (id %" PRId64 ") for formation " + "\"%s\", ARCHIVING node id %" + PRId64, + archiverName, archiverId, formation, thisArchiverNodeId); + + /* + * archiverNodeId (used below for this process's own local state + * file) tracks only the first --formation's own returned node id -- + * that local state is only ever consulted by the legacy single- + * membership `create archiver --run` path (service_archiver_loop() + * called directly, see below); the reconciler-based path discovers + * every membership fresh from the monitor instead and never reads + * it. + */ + if (i == 0) + { + archiverNodeId = thisArchiverNodeId; + } + + if (hasBasebackupPolicy) + { + if (!monitor_set_archiver_policy(&monitor, formation, + -1, /* formation-wide, not one group */ + 1, /* archiverQuorum */ + basebackupPolicy.basebackupPolicyId, + false /* replicationQuorumEligible */)) + { + log_fatal("Failed to attach base-backup policy \"%s\" to " + "formation \"%s\", see above for details", + archiverBasebackupPolicyName, formation); + exit(EXIT_CODE_MONITOR); + } + + log_info("Attached base-backup policy \"%s\" to formation \"%s\"", + archiverBasebackupPolicyName, formation); + } + } + + strlcpy(config->role, KEEPER_ROLE, sizeof(config->role)); + config->groupId = 0; + config->network_partition_timeout = NETWORK_PARTITION_TIMEOUT; + config->listen_notifications_timeout = PG_AUTOCTL_LISTEN_NOTIFICATIONS_TIMEOUT; + + /* + * Persist the archiver's own archiverid (distinct from archiverNodeId + * below, which is this specific ARCHIVING membership's nodeid) so that + * `pg_autoctl archiver serve`'s supervisor loop can identify itself to + * the monitor on a later, separate invocation -- see keeper_config.h's + * own comment on archiverIdStr/archiverId. + */ + config->archiverId = archiverId; + sformat(config->archiverIdStr, sizeof(config->archiverIdStr), + "%" PRId64, archiverId); + + if (!keeper_config_write_file(config)) + { + log_fatal("Failed to write archiver configuration file \"%s\", " + "see above for details", config->pathnames.config); + exit(EXIT_CODE_BAD_CONFIG); + } + + /* + * The ARCHIVING node row starts at (goalstate, reportedstate) = + * (wait_standby, wait_standby) -- see archiver_add_formation()'s own + * comment, pgautofailover.sql -- so our own local state mirrors that + * starting point exactly, same as an ordinary node's INIT_STATE. + */ + keeper_state_init(&(keeper.state)); + keeper.state.current_node_id = archiverNodeId; + keeper.state.current_group = 0; + keeper.state.current_role = WAIT_STANDBY_STATE; + keeper.state.assigned_role = WAIT_STANDBY_STATE; + + if (!keeper_store_state(&keeper)) + { + log_fatal("Failed to write archiver state file \"%s\", " + "see above for details", config->pathnames.state); + exit(EXIT_CODE_BAD_STATE); + } + + if (createAndRun) + { + /* + * Go through start_archiver() (service_archiver_run.c), exactly + * like `pg_autoctl node run` does for an archiver node + * (cli_service.c) -- not service_archiver_loop() directly, which + * only ever ran the WAL-capture half and skipped "serve" (pg_ + * walsender) entirely. start_archiver() owns the pidfile itself + * (via its own supervisor_start_with_callback() call), so this + * process must not create one first -- doing so would make its + * own startup check see a pidfile already populated with this + * same pid and fail as "already running". + * + * We don't keep this connection open in the long-lived supervisor + * process either, matching cli_service.c's own cli_keeper_run(): + * every service re-connects independently once started. + */ + pgsql_finish(&(keeper.monitor.pgsql)); + + if (!start_archiver(&keeper)) + { + log_fatal("Failed to start pg_autoctl archiver service, " + "see above for details"); + exit(EXIT_CODE_INTERNAL_ERROR); + } + } +} + + +CommandLine create_archiver_command = + make_command( + "archiver", + "Initialize a pg_auto_failover archiver node", + " [ --pgdata --pgctl --monitor --hostname --name --formation --region --basebackup-policy ] ", + " --pgdata path to the archiver's local data/cache directory\n" + " --pgctl path to pg_ctl (used to locate pg_receivewal)\n" + " --monitor pg_auto_failover Monitor Postgres URL\n" + " --hostname hostname by which the archiver is reachable\n" + " --name archiver name (default: derived from hostname)\n" + " --formation formation to attach to, may be repeated (default: \"default\")\n" + " --region data-centre or availability-zone label for this " + "archiver (default: \"default\")\n" + " --basebackup-policy base-backup production/retention policy to attach " + "(default: \"default\")\n" + " --run create node then run pg_autoctl service\n", + cli_create_archiver_getopts, + cli_create_archiver); + + /* * check_or_discover_hostname checks given --hostname or attempt to discover a * suitable default value for the current node when it's not been provided on diff --git a/src/bin/pg_autoctl/cli_get_set_properties.c b/src/bin/pg_autoctl/cli_get_set_properties.c index f080cf55e..ae152ff4d 100644 --- a/src/bin/pg_autoctl/cli_get_set_properties.c +++ b/src/bin/pg_autoctl/cli_get_set_properties.c @@ -9,6 +9,7 @@ */ #include "parson.h" +#include "cli_basebackup_policy.h" #include "cli_common.h" #include "parsing.h" #include "string_utils.h" @@ -212,6 +213,7 @@ static CommandLine set_formation_command = static CommandLine *set_subcommands[] = { &set_node_command, &set_formation_command, + &set_basebackup_policy_command, NULL }; diff --git a/src/bin/pg_autoctl/cli_root.c b/src/bin/pg_autoctl/cli_root.c index 211c64aab..5293a47fa 100644 --- a/src/bin/pg_autoctl/cli_root.c +++ b/src/bin/pg_autoctl/cli_root.c @@ -8,6 +8,8 @@ * */ +#include "cli_archiver.h" +#include "cli_basebackup_policy.h" #include "cli_common.h" #include "cli_do_root.h" #include "cli_inspect.h" @@ -31,6 +33,8 @@ CommandLine *create_subcommands[] = { &create_postgres_command, &create_coordinator_command, &create_worker_command, + &create_archiver_command, + &create_basebackup_policy_command, &create_formation_command, NULL }; @@ -48,6 +52,7 @@ CommandLine *show_subcommands_with_debug[] = { &show_standby_names_command, &show_timeline_command, &show_file_command, + &show_basebackup_policy_command, &systemd_cat_service_file_command, NULL }; @@ -65,6 +70,7 @@ CommandLine *show_subcommands[] = { &show_standby_names_command, &show_timeline_command, &show_file_command, + &show_basebackup_policy_command, &systemd_cat_service_file_command, NULL }; @@ -108,6 +114,7 @@ CommandLine *root_subcommands[] = { &internal_commands, &do_compat_commands, &node_commands, + &archiver_commands, &service_run_command, &watch_command, &service_stop_command, diff --git a/src/bin/pg_autoctl/cli_service.c b/src/bin/pg_autoctl/cli_service.c index 33a95ce8f..b50962a46 100644 --- a/src/bin/pg_autoctl/cli_service.c +++ b/src/bin/pg_autoctl/cli_service.c @@ -26,6 +26,7 @@ #include "monitor.h" #include "monitor_config.h" #include "pidfile.h" +#include "service_archiver_run.h" #include "service_keeper.h" #include "service_monitor.h" #include "signals.h" @@ -203,6 +204,25 @@ cli_keeper_run(int argc, char **argv) pgsql_finish(&(monitor->pgsql)); } + /* + * An archiver has no real Postgres instance of its own (see + * service_archiver.c's own comment on config->pgSetup.pgdata's reused + * meaning for an ARCHIVING node) -- local_postgres_init()/start_keeper() + * both assume one, so branch to start_archiver() instead, milestone 3's + * own `pg_autoctl run` support (service_archiver_run.c). + */ + if (strcmp(config->nodeKind, "archiver") == 0) + { + if (!start_archiver(&keeper)) + { + log_fatal("Failed to start pg_autoctl archiver service, " + "see above for details"); + exit(EXIT_CODE_INTERNAL_ERROR); + } + + return; + } + /* initialize our local Postgres instance representation */ (void) local_postgres_init(postgres, pgSetup); diff --git a/src/bin/pg_autoctl/defaults.h b/src/bin/pg_autoctl/defaults.h index 80448667b..a473661f6 100644 --- a/src/bin/pg_autoctl/defaults.h +++ b/src/bin/pg_autoctl/defaults.h @@ -226,6 +226,15 @@ #define PG_AUTOCTL_HEALTH_PASSWORD "pgautofailover_monitor" #define PG_AUTOCTL_REPLICA_USERNAME "pgautofailover_replicator" +/* default port pg_walsender listens on, started via `pg_autoctl archiver + * serve` -- matches src/bin/pg_walsender/defaults.h's own WS_DEFAULT_PORT */ +#define PG_AUTOCTL_ARCHIVER_SERVE_PORT 6543 + +/* port the archiver's own throwaway replay-mode staging Postgres instance + * listens on, loopback only -- see service_archiver_basebackup.c's own + * replay/volatile implementation */ +#define PG_AUTOCTL_ARCHIVER_REPLAY_PORT 6899 + #define PG_AUTOCTL_MONITOR_DBNAME "pg_auto_failover" #define PG_AUTOCTL_MONITOR_EXTENSION_NAME "pgautofailover" #define PG_AUTOCTL_MONITOR_DBOWNER "autoctl" diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index fb66b060d..45a5fd47b 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -824,6 +824,36 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_INIT }, + /* + * Archiving & Disaster Recovery (milestone 2): the ARCHIVING mirror of + * the ordinary standby-init/failover-participation/rejoin rows just + * above and further below (SECONDARY/CATCHINGUP <-> REPORT_LSN) -- an + * ARCHIVING node is only ever assigned these three transitions by the + * monitor (MonitorFSM[] pos 367/396-398, group_state_machine.c), so + * NODE_KIND_ANY carries no ambiguity here despite being the same + * bitmask every ordinary row uses. + */ + { + WAIT_STANDBY_STATE, ARCHIVING_STATE, NODE_KIND_ANY, + "wait_standby to archiving", + &fsm_init_archiver, + FSM_PHASE_INIT + }, + + { + ARCHIVING_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + "archiving to report_lsn", + &fsm_archiver_report_lsn, + FSM_PHASE_FAILOVER + }, + + { + REPORT_LSN_STATE, ARCHIVING_STATE, NODE_KIND_ANY, + "report_lsn to archiving", + &fsm_archiver_follow_new_primary, + FSM_PHASE_FAILOVER + }, + { DEMOTED_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, COMMENT_DEMOTED_TO_CATCHINGUP, diff --git a/src/bin/pg_autoctl/fsm.h b/src/bin/pg_autoctl/fsm.h index 2ab182fc1..c84f82bcd 100644 --- a/src/bin/pg_autoctl/fsm.h +++ b/src/bin/pg_autoctl/fsm.h @@ -95,6 +95,10 @@ bool fsm_prepare_cascade(Keeper *keeper); bool fsm_follow_new_primary(Keeper *keeper); bool fsm_cleanup_as_primary(Keeper *keeper); +bool fsm_init_archiver(Keeper *keeper); +bool fsm_archiver_report_lsn(Keeper *keeper); +bool fsm_archiver_follow_new_primary(Keeper *keeper); + bool fsm_init_from_standby(Keeper *keeper); bool fsm_drop_node(Keeper *keeper); diff --git a/src/bin/pg_autoctl/fsm_transition.c b/src/bin/pg_autoctl/fsm_transition.c index b0579adfa..58bb16864 100644 --- a/src/bin/pg_autoctl/fsm_transition.c +++ b/src/bin/pg_autoctl/fsm_transition.c @@ -39,6 +39,7 @@ #include "parson.h" #include "pghba.h" #include "primary_standby.h" +#include "service_archiver.h" #include "state.h" #include "timeline_history.h" @@ -924,20 +925,64 @@ fsm_init_standby(Keeper *keeper) NodeAddress *primaryNode = NULL; + /* + * `pg_autoctl create postgres --from-archiver`: bootstrap from a + * registered archiver's base backup + WAL cache instead of the group's + * live primary -- the disaster-recovery case this flag exists for. The + * archiver serves the same real replication protocol a live primary + * does (pg_walsender), so standby_init_replication_source/ + * standby_init_database below don't need to know the difference, except + * for one: pg_walsender has no slot-based retention in this milestone + * (see cmd_start_replication.c's own header comment), so we mustn't ask + * standby_init_database to first verify a replication slot exists on + * the archiver -- it never will. Passing an empty slot name here + * matches standby_init_database's own existing "initialising from + * another standby, no primary yet" precedent (see that function's + * comment on needsReplicationSlot). + */ + const char *slotName = config->replication_slot_name; - /* get the primary node to follow */ - if (!keeper_get_primary(keeper, &(postgres->replicationSource.primaryNode))) + if (config->fromArchiver) { - log_error("Failed to initialize standby for lack of a primary node, " - "see above for details"); - return false; + NodeAddress archiverNode = { 0 }; + bool found = false; + + if (!keeper_get_archiver_node(keeper, &archiverNode, &found)) + { + log_error("Failed to initialize standby from an archiver, " + "see above for details"); + return false; + } + + if (!found) + { + log_error("Failed to initialize standby from an archiver: " + "no archiver is registered for formation \"%s\" " + "group %d", config->formation, + keeper->state.current_group); + return false; + } + + postgres->replicationSource.primaryNode = archiverNode; + postgres->replicationSource.noManifest = true; + slotName = ""; + } + else + { + /* get the primary node to follow */ + if (!keeper_get_primary(keeper, &(postgres->replicationSource.primaryNode))) + { + log_error("Failed to initialize standby for lack of a primary node, " + "see above for details"); + return false; + } } if (!standby_init_replication_source(postgres, primaryNode, PG_AUTOCTL_REPLICA_USERNAME, config->replication_password, - config->replication_slot_name, + slotName, config->maximum_backup_rate, config->backupDirectory, NULL, /* no targetLSN */ @@ -1696,3 +1741,73 @@ fsm_drop_node(Keeper *keeper) return unlink_file(config->pathnames.init); } + + +/* + * fsm_init_archiver starts pg_receivewal against the group's current + * primary. Reached from WAIT_STANDBY_STATE once the monitor has assigned + * ARCHIVING as the goal state instead of fsm_init_standby's own + * CATCHINGUP target -- the monitor makes that choice based on this node's + * own haspgdata = false row, see MonitorFSM[] pos 396-398 + * (group_state_machine.c). Unlike fsm_init_standby, there is no local + * Postgres instance to configure as a standby: pg_receivewal is a real, + * unmodified Postgres client that streams straight from the primary's own + * walsender, so no new wire protocol is involved on this node's side + * either (see archiving-disaster-recovery.md's own milestone 2(a) scope). + */ +bool +fsm_init_archiver(Keeper *keeper) +{ + NodeAddress primaryNode = { 0 }; + + /* get the primary node to stream WAL from */ + if (!keeper_get_primary(keeper, &primaryNode)) + { + log_error("Failed to initialize archiver for lack of a primary node, " + "see above for details"); + return false; + } + + return service_archiver_start_pgreceivewal(keeper, &primaryNode); +} + + +/* + * fsm_archiver_report_lsn stops pg_receivewal: the group's primary is + * presumed gone (an election is starting), so the upstream this archiver + * was streaming from is no longer trustworthy to keep querying. Mirrors + * fsm_report_lsn's own "disconnect from current source" half without any + * of its real-Postgres recovery-config/restart machinery, which doesn't + * apply here -- an ARCHIVING row has no PGDATA to reconfigure (see + * haspgdata's own design comment, pgautofailover.sql). + */ +bool +fsm_archiver_report_lsn(Keeper *keeper) +{ + return service_archiver_stop_pgreceivewal(); +} + + +/* + * fsm_archiver_follow_new_primary re-points pg_receivewal at the group's + * newly elected primary -- the archiver's own mirror of + * fsm_follow_new_primary, without that function's live-Postgres-standby + * machinery: pg_receivewal has no "replaying, not caught up yet" + * continuum to wait out (see haspgdata's own design comment), just a + * fresh connection to make. + */ +bool +fsm_archiver_follow_new_primary(Keeper *keeper) +{ + NodeAddress primaryNode = { 0 }; + + /* get the newly elected primary node to stream WAL from */ + if (!keeper_get_primary(keeper, &primaryNode)) + { + log_error("Failed to follow new primary for lack of a primary node, " + "see above for details"); + return false; + } + + return service_archiver_start_pgreceivewal(keeper, &primaryNode); +} diff --git a/src/bin/pg_autoctl/keeper.c b/src/bin/pg_autoctl/keeper.c index c4461f894..ddcdd4ee5 100644 --- a/src/bin/pg_autoctl/keeper.c +++ b/src/bin/pg_autoctl/keeper.c @@ -3327,6 +3327,23 @@ keeper_get_most_advanced_standby(Keeper *keeper, NodeAddress *upstreamNode, return false; } + /* + * port == 0 is the ARCHIVING row sentinel documented in + * pgautofailover.sql ("an ARCHIVING row has no postmaster of its + * own") -- get_most_advanced_standby() returns it verbatim from + * pgautofailover.node, which has no column for an archiver's real + * pg_walsender serve port (archiver-host-local information the + * monitor is never told, matching service_archiver_serve.c's own + * routes-file rationale). This milestone's own scope is one + * archiver on the well-known default serve port, so resolving it + * here is enough; a configurable-port archiver is a follow-up that + * would need the monitor to actually track it. + */ + if (*found && upstreamNode->port == 0) + { + upstreamNode->port = PG_AUTOCTL_ARCHIVER_SERVE_PORT; + } + return true; } else @@ -3378,6 +3395,52 @@ keeper_get_most_advanced_standby(Keeper *keeper, NodeAddress *upstreamNode, } +/* + * keeper_get_archiver_node fetches the ARCHIVING node registered for our + * (formation, group), for `create postgres --from-archiver` to bootstrap + * from -- deliberately not keeper_get_most_advanced_standby's election + * machinery (see monitor_get_archiver_node's own comment for why that + * function can't find an archiver outside of an election). Monitor-only: + * a brand new node discovering an archiver to rebuild from is exactly the + * disaster-recovery case --disable-monitor's manually-populated otherNodes + * list isn't meant to serve. + */ +bool +keeper_get_archiver_node(Keeper *keeper, NodeAddress *archiverNode, bool *found) +{ + KeeperConfig *config = &(keeper->config); + int groupId = keeper->state.current_group; + + if (config->monitorDisabled) + { + log_error("Failed to find an archiver to bootstrap from: " + "--from-archiver requires a monitor"); + return false; + } + + Monitor *monitor = &(keeper->monitor); + + if (!monitor_get_archiver_node(monitor, + config->formation, + groupId, + archiverNode, + found)) + { + log_error("Failed to get the archiver node from the monitor, " + "see above for details"); + return false; + } + + /* see keeper_get_most_advanced_standby's own comment on this sentinel */ + if (*found && archiverNode->port == 0) + { + archiverNode->port = PG_AUTOCTL_ARCHIVER_SERVE_PORT; + } + + return true; +} + + /* * keeper_pg_autoctl_get_version_from_disk calls pg_autoctl version --json and * parses the output to fill-in the keeper version. diff --git a/src/bin/pg_autoctl/keeper.h b/src/bin/pg_autoctl/keeper.h index d136998e7..2fdcd92d2 100644 --- a/src/bin/pg_autoctl/keeper.h +++ b/src/bin/pg_autoctl/keeper.h @@ -125,6 +125,8 @@ bool keeper_read_nodes_from_file(Keeper *keeper, NodeAddressArray *nodesArray); bool keeper_get_primary(Keeper *keeper, NodeAddress *primaryNode); bool keeper_get_most_advanced_standby(Keeper *keeper, NodeAddress *primaryNode, bool *found); +bool keeper_get_archiver_node(Keeper *keeper, NodeAddress *archiverNode, + bool *found); bool keeper_pg_autoctl_get_version_from_disk(Keeper *keeper, diff --git a/src/bin/pg_autoctl/keeper_config.c b/src/bin/pg_autoctl/keeper_config.c index 7621ff530..9b97aed69 100644 --- a/src/bin/pg_autoctl/keeper_config.c +++ b/src/bin/pg_autoctl/keeper_config.c @@ -61,6 +61,11 @@ make_strbuf_option("pg_autoctl", "nodekind", NULL, false, NAMEDATALEN, \ config->nodeKind) +#define OPTION_AUTOCTL_ARCHIVER_ID(config) \ + make_strbuf_option_default("pg_autoctl", "archiver_id", NULL, false, \ + INTSTRING_MAX_DIGITS, \ + config->archiverIdStr, "") + #define OPTION_POSTGRESQL_PGDATA(config) \ make_strbuf_option("postgresql", "pgdata", "pgdata", true, MAXPGPATH, \ config->pgSetup.pgdata) @@ -227,6 +232,7 @@ OPTION_AUTOCTL_HOSTNAME(config), \ OPTION_AUTOCTL_NODENAME(config), \ OPTION_AUTOCTL_NODEKIND(config), \ + OPTION_AUTOCTL_ARCHIVER_ID(config), \ OPTION_POSTGRESQL_PGDATA(config), \ OPTION_POSTGRESQL_PG_CTL(config), \ OPTION_POSTGRESQL_USERNAME(config), \ @@ -517,6 +523,18 @@ keeper_config_read_file_skip_pgsetup(KeeperConfig *config, return false; } + /* parse archiverIdStr (see keeper_config.h's own comment) into archiverId */ + if (IS_EMPTY_STRING_BUFFER(config->archiverIdStr)) + { + config->archiverId = 0; + } + else if (!stringToInt64(config->archiverIdStr, &(config->archiverId))) + { + log_error("Failed to parse pg_autoctl.archiver_id \"%s\" as a number", + config->archiverIdStr); + return false; + } + return true; } diff --git a/src/bin/pg_autoctl/keeper_config.h b/src/bin/pg_autoctl/keeper_config.h index 0c71a65e0..131277aa0 100644 --- a/src/bin/pg_autoctl/keeper_config.h +++ b/src/bin/pg_autoctl/keeper_config.h @@ -17,6 +17,7 @@ #include "defaults.h" #include "pgctl.h" #include "pgsql.h" +#include "string_utils.h" /* * We support "primary" and "secondary" roles in Citus, when Citus support is @@ -47,6 +48,40 @@ typedef struct KeeperConfig char hostname[_POSIX_HOST_NAME_MAX]; char nodeKind[NAMEDATALEN]; + /* + * The archiver's own archiverid (distinct from keeper.state. + * current_node_id, which holds the ARCHIVING membership row's nodeid -- + * see cli_create_archiver's own comment). Only meaningful when nodeKind + * is "archiver"; 0 otherwise. archiverIdStr is the ini-persisted form + * (ini_file.c's INI_INT_T only supports a plain int, too narrow for a + * bigserial id -- same string-plus-parsed-value pattern citusRoleStr/ + * citusRole already use in this struct), archiverId is parsed from it + * once at config-read time. + */ + char archiverIdStr[INTSTRING_MAX_DIGITS]; + int64_t archiverId; + + /* + * The archiver-level (not per-membership) supervisor's own pidfile + * path, stashed by service_archiver_reconciler.c's build_membership_ + * keeper() from the template keeper's pathnames.pid before they get + * overwritten with this membership's own per-(formation, group) + * paths. This is the *shared* pidfile every one of this archiver's + * supervised services (archiver-serve, archiver-reconciler, each + * archiver-capture--) has one line in -- not a + * dedicated pidfile of its own -- so a reader must look up a specific + * service's own pid by name (supervisor_find_service_pid(), + * SERVICE_NAME_ARCHIVER_SERVE), not just read the first line. + * + * A capture child that just finished generating a base backup + * (service_archiver_basebackup.c) uses this to find archiver-serve's + * pid and signal it (SIGUSR1) to prompt an immediate routes refresh, + * rather than leaving pg_walsender to serve a stale route for up to + * ARCHIVER_SERVE_ROUTES_REFRESH_TICKS more ticks. Only meaningful for + * a per-membership keeper built that way; empty otherwise. + */ + char archiverPidFilePath[MAXPGPATH]; + /* PostgreSQL setup */ PostgresSetup pgSetup; @@ -75,6 +110,17 @@ typedef struct KeeperConfig /* allow data loss during a perform failover operation */ bool allowDataLoss; + + /* + * `pg_autoctl create postgres --from-archiver`: bootstrap this standby + * from a registered archiver's base backup + WAL cache instead of from + * the group's live primary -- the disaster-recovery case where no live + * standby (or even primary) is left to clone from. Runtime-only, same + * as createAndRun (cli_common.c): only meaningful for the single + * in-process reach_initial_state() call `create postgres` itself makes, + * never persisted to the ini file. + */ + bool fromArchiver; } KeeperConfig; #define PG_AUTOCTL_MONITOR_IS_DISABLED(config) \ diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index f24c34643..ccc801df3 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -848,6 +848,80 @@ monitor_get_most_advanced_standby(Monitor *monitor, } +/* + * monitor_get_archiver_node finds the ARCHIVING node for (formation, group), + * for a client-side bootstrap (create postgres --from-archiver) rather than + * an election: unlike monitor_get_most_advanced_standby, this doesn't filter + * on reportedstate = 'report_lsn' (a transient election-only state), since + * an archiver sits in its normal 'archiving' state outside of elections. + */ +bool +monitor_get_archiver_node(Monitor *monitor, + char *formation, int groupId, + NodeAddress *node, bool *found) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT * FROM pgautofailover.get_archiver_node($1, $2)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, INT4OID }; + const char *paramValues[2]; + + /* we expect zero or one entry */ + NodeAddressArray nodeArray = { 0 }; + NodeAddressArrayParseContext parseContext = { { 0 }, &nodeArray, false }; + + IntString groupIdString = intToString(groupId); + + paramValues[0] = formation; + paramValues[1] = groupIdString.strValue; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &parseContext, parseNodeArray)) + { + log_error( + "Failed to get the archiver node in the HA group " + "from the monitor while running \"%s\" with " + "formation \"%s\" and group ID %d", + sql, formation, groupId); + return false; + } + + if (!parseContext.parsedOK) + { + log_error( + "Failed to get the archiver node from the monitor " + "while running \"%s\" with formation \"%s\" and group ID %d " + "because it returned an unexpected result. " + "See previous line for details.", + sql, formation, groupId); + return false; + } + + if (nodeArray.count == 0) + { + *found = false; + return true; + } + + /* copy the node we retrieved in the expected place */ + node->nodeId = nodeArray.nodes[0].nodeId; + strlcpy(node->name, nodeArray.nodes[0].name, _POSIX_HOST_NAME_MAX); + strlcpy(node->host, nodeArray.nodes[0].host, _POSIX_HOST_NAME_MAX); + node->port = nodeArray.nodes[0].port; + strlcpy(node->lsn, nodeArray.nodes[0].lsn, PG_LSN_MAXLENGTH); + node->isPrimary = nodeArray.nodes[0].isPrimary; + + log_debug("The archiver node for %s/%d is node " NODE_FORMAT, + formation, groupId, node->nodeId, node->name, + node->host, node->port); + + *found = true; + return true; +} + + /* * monitor_register_node performs the initial registration of a node with the * monitor in the given formation. @@ -868,6 +942,1177 @@ monitor_get_most_advanced_standby(Monitor *monitor, * The node ID and group ID selected by the monitor, as well as the goal * state, are set in assignedState, which must not be NULL. */ + + +/* + * monitor_register_archiver calls pgautofailover.register_archiver() on the + * monitor -- the Archiving & Disaster Recovery schema's own registration + * entry point (see ~/dev/temp/archiving-disaster-recovery.md), a plain + * plpgsql function rather than the C register_node() RPC every ordinary + * node kind goes through: an Archiver is a process identity, not a + * (formation, group) membership by itself (see that function's own comment, + * pgautofailover.sql). + */ +bool +monitor_register_archiver(Monitor *monitor, char *name, char *hostname, + char *region, int64_t *archiverId) +{ + PGSQL *pgsql = &monitor->pgsql; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BIGINT, false }; + + /* + * region is passed by name ("region => $3") to skip over register_ + * archiver()'s own storagepath/basebackuppolicyid/autoregister/ + * maxresidentreplay/rcloneconfigname parameters, which stay at their + * own SQL-level defaults here -- Postgres allows mixing positional and + * named arguments as long as every positional one comes first. + */ + const char *sql = + "SELECT * FROM pgautofailover.register_archiver($1, $2, region => $3)"; + int paramCount = 3; + Oid paramTypes[3] = { TEXTOID, TEXTOID, TEXTOID }; + const char *paramValues[3] = { + name, hostname, + IS_EMPTY_STRING_BUFFER(region) ? "default" : region + }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseSingleValueResult)) + { + log_error("Failed to register archiver \"%s\" on the monitor", + name); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to register archiver \"%s\" on the monitor " + "because it returned an unexpected result, " + "see previous lines for details", name); + return false; + } + + *archiverId = context.bigint; + + return true; +} + + +/* + * monitor_archiver_add_formation calls pgautofailover.archiver_add_formation() + * on the monitor, attaching an already-registered archiver to every group of + * the given formation -- a multi-group formation gets one ARCHIVING + * membership row per group, all created in this one call. Only the first + * returned nodeid is kept, for a log message at attach time; it is not + * how an archiver process itself discovers its full membership list at + * runtime (that's monitor_list_archiver_memberships(), which returns + * every membership across every attached formation, called by the + * archiver's own reconciler instead of relying on this one-shot result). + */ +bool +monitor_archiver_add_formation(Monitor *monitor, int64_t archiverId, + char *formation, int64_t *archiverNodeId) +{ + PGSQL *pgsql = &monitor->pgsql; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BIGINT, false }; + + const char *sql = + "SELECT * FROM pgautofailover.archiver_add_formation($1, $2) LIMIT 1"; + int paramCount = 2; + Oid paramTypes[2] = { INT8OID, TEXTOID }; + IntString archiverIdString = intToString(archiverId); + const char *paramValues[2] = { archiverIdString.strValue, formation }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseSingleValueResult)) + { + log_error("Failed to attach archiver %" PRId64 " to formation \"%s\" " + "on the monitor", archiverId, + formation); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to attach archiver %" PRId64 " to formation \"%s\" " + "on the monitor because it returned an unexpected result, " + "see previous lines for details", + archiverId, formation); + return false; + } + + *archiverNodeId = context.bigint; + + return true; +} + + +/* + * monitor_report_archiver_storage calls pgautofailover.report_archiver_ + * storage() to record this archiver's own disk usage and free space, + * alongside a fresh lastreporttime -- the same periodic heartbeat + * service_archiver_loop() already uses to report captured-WAL LSN. + */ +bool +monitor_report_archiver_storage(Monitor *monitor, int64_t archiverId, + uint64_t usedBytes, uint64_t freeBytes) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.report_archiver_storage($1, $2, $3)"; + int paramCount = 3; + Oid paramTypes[3] = { INT8OID, INT8OID, INT8OID }; + IntString archiverIdString = intToString(archiverId); + IntString usedBytesString = intToString((int64_t) usedBytes); + IntString freeBytesString = intToString((int64_t) freeBytes); + const char *paramValues[3] = { + archiverIdString.strValue, + usedBytesString.strValue, + freeBytesString.strValue + }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to report storage usage for archiver %" PRId64, + archiverId); + return false; + } + + return true; +} + + +typedef struct ArchiverInfoArrayParseContext +{ + char sqlstate[SQLSTATE_LENGTH]; + ArchiverInfoArray *archiversArray; + bool parsedOK; +} ArchiverInfoArrayParseContext; + + +/* + * parseArchiverInfo parses one row of pgautofailover.get_archivers()'s + * result: archiver_id, archiver_name, hostname, region, used_bytes, + * free_bytes, last_report_time, node_id, reported_state, goal_state. + * usedbytes/freebytes (NULL until the archiver's first storage report) and + * the node_id/reportedState/goalState side of the LEFT JOIN (NULL if this + * milestone's one-'wal-receiver'-row-per-group assumption isn't met yet, + * see get_archivers()'s own comment) are both optional -- everything else, + * including region (NOT NULL, defaults to "default"), is not. + */ +static bool +parseArchiverInfo(PGresult *result, int rowNumber, ArchiverInfo *archiver) +{ + if (PQgetisnull(result, rowNumber, 0) || + PQgetisnull(result, rowNumber, 1) || + PQgetisnull(result, rowNumber, 2) || + PQgetisnull(result, rowNumber, 3)) + { + log_error("archiver_id, archiver_name, hostname or region returned " + "by the monitor is NULL"); + return false; + } + + char *value = PQgetvalue(result, rowNumber, 0); + + archiver->archiverId = strtol(value, NULL, 0); + + value = PQgetvalue(result, rowNumber, 1); + strlcpy(archiver->archiverName, value, _POSIX_HOST_NAME_MAX); + + value = PQgetvalue(result, rowNumber, 2); + strlcpy(archiver->hostname, value, _POSIX_HOST_NAME_MAX); + + value = PQgetvalue(result, rowNumber, 3); + strlcpy(archiver->region, value, sizeof(archiver->region)); + + archiver->hasStorageStats = + !PQgetisnull(result, rowNumber, 4) && !PQgetisnull(result, rowNumber, 5); + + if (archiver->hasStorageStats) + { + value = PQgetvalue(result, rowNumber, 4); + archiver->usedBytes = strtoull(value, NULL, 0); + + value = PQgetvalue(result, rowNumber, 5); + archiver->freeBytes = strtoull(value, NULL, 0); + } + + archiver->hasNode = !PQgetisnull(result, rowNumber, 7); + + if (archiver->hasNode) + { + value = PQgetvalue(result, rowNumber, 7); + archiver->nodeId = strtol(value, NULL, 0); + + value = PQgetvalue(result, rowNumber, 8); + archiver->reportedState = NodeStateFromString(value); + + value = PQgetvalue(result, rowNumber, 9); + archiver->goalState = NodeStateFromString(value); + } + + return true; +} + + +static void +parseArchiverInfoArray(void *ctx, PGresult *result) +{ + ArchiverInfoArrayParseContext *context = (ArchiverInfoArrayParseContext *) ctx; + bool parsedOk = true; + + if (PQntuples(result) > ARCHIVER_ARRAY_MAX_COUNT) + { + log_error("Query returned %d rows, pg_auto_failover supports only " + "up to %d archivers at the moment", + PQntuples(result), ARCHIVER_ARRAY_MAX_COUNT); + context->parsedOK = false; + return; + } + + context->archiversArray->count = PQntuples(result); + + for (int rowNumber = 0; rowNumber < PQntuples(result); rowNumber++) + { + ArchiverInfo *archiver = &(context->archiversArray->archivers[rowNumber]); + + parsedOk = parsedOk && parseArchiverInfo(result, rowNumber, archiver); + } + + context->parsedOK = parsedOk; +} + + +/* + * monitor_get_archivers calls pgautofailover.get_archivers(formation) and + * returns every archiver attached to that formation, with its storage + * stats and FSM state -- used by `pg_autoctl watch`'s own archivers + * section. + */ +bool +monitor_get_archivers(Monitor *monitor, const char *formation, + ArchiverInfoArray *archiversArray) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = "SELECT * FROM pgautofailover.get_archivers($1)"; + int paramCount = 1; + Oid paramTypes[1] = { TEXTOID }; + const char *paramValues[1] = { formation }; + ArchiverInfoArrayParseContext parseContext = { { 0 }, archiversArray, false }; + + archiversArray->count = 0; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &parseContext, &parseArchiverInfoArray)) + { + log_error("Failed to get the list of archivers from the monitor " + "for formation \"%s\"", formation); + return false; + } + + if (!parseContext.parsedOK) + { + log_error("Failed to parse the list of archivers returned by the " + "monitor for formation \"%s\", see previous lines for " + "details", formation); + return false; + } + + return true; +} + + +typedef struct ArchiverMembershipArrayParseContext +{ + char sqlstate[SQLSTATE_LENGTH]; + ArchiverMembershipArray *membershipsArray; + bool parsedOK; +} ArchiverMembershipArrayParseContext; + + +/* + * parseArchiverMembership parses one row of pgautofailover. + * list_archiver_memberships()'s result: formation_id, group_id, node_id, + * reported_state, goal_state. The underlying query is an inner join (see + * that function's own comment), so none of these are ever NULL in + * practice -- checked anyway, matching this file's own defensive + * convention for every other row parser. + */ +static bool +parseArchiverMembership(PGresult *result, int rowNumber, + ArchiverMembership *membership) +{ + if (PQgetisnull(result, rowNumber, 0) || + PQgetisnull(result, rowNumber, 1) || + PQgetisnull(result, rowNumber, 2) || + PQgetisnull(result, rowNumber, 3) || + PQgetisnull(result, rowNumber, 4)) + { + log_error("formation_id, group_id, node_id, reported_state, or " + "goal_state returned by the monitor is NULL"); + return false; + } + + char *value = PQgetvalue(result, rowNumber, 0); + + strlcpy(membership->formation, value, NAMEDATALEN); + + value = PQgetvalue(result, rowNumber, 1); + membership->groupId = strtol(value, NULL, 0); + + value = PQgetvalue(result, rowNumber, 2); + membership->nodeId = strtoll(value, NULL, 0); + + value = PQgetvalue(result, rowNumber, 3); + membership->reportedState = NodeStateFromString(value); + + value = PQgetvalue(result, rowNumber, 4); + membership->goalState = NodeStateFromString(value); + + return true; +} + + +static void +parseArchiverMembershipArray(void *ctx, PGresult *result) +{ + ArchiverMembershipArrayParseContext *context = + (ArchiverMembershipArrayParseContext *) ctx; + bool parsedOk = true; + + if (PQntuples(result) > ARCHIVER_MEMBERSHIP_ARRAY_MAX_COUNT) + { + log_error("Query returned %d rows, pg_auto_failover supports only " + "up to %d memberships per archiver at the moment", + PQntuples(result), ARCHIVER_MEMBERSHIP_ARRAY_MAX_COUNT); + context->parsedOK = false; + return; + } + + context->membershipsArray->count = PQntuples(result); + + for (int rowNumber = 0; rowNumber < PQntuples(result); rowNumber++) + { + ArchiverMembership *membership = + &(context->membershipsArray->memberships[rowNumber]); + + parsedOk = parsedOk && parseArchiverMembership(result, rowNumber, membership); + } + + context->parsedOK = parsedOk; +} + + +/* + * monitor_list_archiver_memberships calls pgautofailover. + * list_archiver_memberships(archiverid) and returns every (formation, + * group) membership this archiver currently holds a 'wal-receiver' row + * in, across every formation it is attached to -- what the archiver's + * own reconciler calls, at startup and periodically thereafter, to + * discover the full set of WAL streams and base-backup schedules it is + * responsible for running (service_archiver_reconciler.c). + */ +bool +monitor_list_archiver_memberships(Monitor *monitor, int64_t archiverId, + ArchiverMembershipArray *membershipsArray) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT * FROM pgautofailover.list_archiver_memberships($1)"; + int paramCount = 1; + Oid paramTypes[1] = { INT8OID }; + IntString archiverIdString = intToString(archiverId); + const char *paramValues[1] = { archiverIdString.strValue }; + ArchiverMembershipArrayParseContext parseContext = + { { 0 }, membershipsArray, false }; + + membershipsArray->count = 0; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &parseContext, &parseArchiverMembershipArray)) + { + log_error("Failed to list memberships for archiver %" PRId64 + " from the monitor", archiverId); + return false; + } + + if (!parseContext.parsedOK) + { + log_error("Failed to parse the list of memberships returned by " + "the monitor for archiver %" PRId64 ", see previous " + "lines for details", archiverId); + return false; + } + + return true; +} + + +/* + * BasebackupInfoParseContext/parseBasebackupInfo parse the two columns + * monitor_get_latest_basebackup_info() needs out of a single-row result -- + * SingleValueResultContext only carries one column, not enough here. + */ +typedef struct BasebackupInfoParseContext +{ + char sqlstate[SQLSTATE_LENGTH]; + bool parsedOk; + int ntuples; + char *storageLocation; + char *source; + int timeline; +} BasebackupInfoParseContext; + + +static void +parseBasebackupInfo(void *ctx, PGresult *result) +{ + BasebackupInfoParseContext *context = (BasebackupInfoParseContext *) ctx; + + context->ntuples = PQntuples(result); + + if (context->ntuples != 1) + { + /* zero rows is a valid "no backup yet" signal, not a parse error */ + context->parsedOk = (context->ntuples == 0); + return; + } + + char *storageLocation = PQgetvalue(result, 0, 0); + char *source = PQgetvalue(result, 0, 1); + char *timeline = PQgetvalue(result, 0, 2); + + context->storageLocation = strdup(storageLocation); + context->source = strdup(source); + context->timeline = strtol(timeline, NULL, 10); + + context->parsedOk = + context->storageLocation != NULL && context->source != NULL; + + if (!context->parsedOk) + { + log_error(ALLOCATION_FAILED_ERROR); + } +} + + +/* + * monitor_get_latest_basebackup_info calls + * pgautofailover.get_latest_basebackup(formationId, groupId) and returns + * its storagelocation, source, and timeline columns. *found is set to false + * (not an error) when the archiver hasn't taken a base backup for this + * group yet -- every caller must already tolerate that. + * + * timeline matters beyond metadata: pg_walsender's BASE_BACKUP response + * reads it straight out of the served backup_label (cmd_base_backup.c's + * own read_backup_label), and a real pg_basebackup's own background WAL + * streaming then requests exactly that timeline back via START_REPLICATION + * -- for a "replay" base backup (basebackup_replay_mode), which promotes a + * throwaway extracted copy to make it self-consistent, that's genuinely a + * *later* timeline than what the archiver's own captured WAL cache holds + * (which only ever advances on the real primary's timeline). Passing it + * through into the routes file's own "timeline" key (already parsed by + * routes.c, previously never written by anyone) is what lets pg_walsender + * serve a START_REPLICATION request consistent with whichever backup it + * just described, instead of always defaulting to timeline 1. + */ +bool +monitor_get_latest_basebackup_info(Monitor *monitor, + const char *formationId, int groupId, + const char *preferredSource, + char *storageLocation, size_t storageLocationSize, + char *source, size_t sourceSize, + int *timeline, + bool *found) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + + /* + * get_latest_basebackup() is not SETOF: called with no matching + * backup, it still produces one row, with every output column + * (including storagelocation) NULL -- not zero rows. Filtering on + * "IS NOT NULL" here, rather than trying to detect that NULL + * composite downstream, is what makes context.ntuples == 0 below + * an accurate "no backup yet" signal. + * + * preferredSource is passed as text plus an explicit cast (matching + * this file's own established pattern for enum parameters, e.g. + * monitor_register_node's use of ::pgautofailover.replication_ + * state below) rather than a raw enum OID -- NULL means "any", the + * function's own default. + */ + "SELECT storagelocation, source::text, timeline " + " FROM pgautofailover.get_latest_basebackup(" + " $1, $2, $3::pgautofailover.basebackup_source) " + " WHERE storagelocation IS NOT NULL"; + int paramCount = 3; + Oid paramTypes[3] = { TEXTOID, INT4OID, TEXTOID }; + IntString groupIdString = intToString(groupId); + const char *paramValues[3] = { formationId, groupIdString.strValue, preferredSource }; + BasebackupInfoParseContext context = { { 0 }, false, 0, NULL, NULL, 0 }; + + *found = false; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseBasebackupInfo)) + { + log_error("Failed to get the latest base backup info from the " + "monitor for \"%s\"/%d", formationId, groupId); + return false; + } + + if (context.ntuples == 0) + { + /* no base backup taken yet for this group -- not an error */ + return true; + } + + if (!context.parsedOk) + { + log_error("Failed to parse the latest base backup info returned " + "by the monitor for \"%s\"/%d, see above for details", + formationId, groupId); + return false; + } + + strlcpy(storageLocation, context.storageLocation, storageLocationSize); + strlcpy(source, context.source, sourceSize); + *timeline = context.timeline; + free(context.storageLocation); + free(context.source); + *found = true; + + return true; +} + + +/* + * monitor_get_group_system_identifier calls + * pgautofailover.get_group_system_identifier(formationId, groupId) -- + * needed by an archiving node (which has no real Postgres instance of its + * own to report one) to serve a correct IDENTIFY_SYSTEM response, so a real + * standby streaming from it doesn't reject the connection with "database + * system identifier differs". *found is false (not an error) when no other + * node in the group has reported one yet. + */ +bool +monitor_get_group_system_identifier(Monitor *monitor, + const char *formationId, int groupId, + uint64_t *systemIdentifier, bool *found) +{ + PGSQL *pgsql = &monitor->pgsql; + + /* + * COALESCE to 0, the same "unset" sentinel this column already uses + * elsewhere (node_metadata.c/pgautofailover.sql): the SQL function + * itself is a plain scalar, not SETOF, so a no-match query still + * produces one row with a NULL value rather than zero rows -- + * PGSQL_RESULT_BIGINT's own parser treats a NULL value as a parse + * failure, which "not reported yet" is not. + */ + const char *sql = + "SELECT coalesce(" + "pgautofailover.get_group_system_identifier($1, $2), 0)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, INT4OID }; + IntString groupIdString = intToString(groupId); + const char *paramValues[2] = { formationId, groupIdString.strValue }; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BIGINT, false }; + + *found = false; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseSingleValueResult)) + { + log_error("Failed to get the system identifier for \"%s\"/%d " + "from the monitor", formationId, groupId); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to parse the system identifier returned by the " + "monitor for \"%s\"/%d, see above for details", + formationId, groupId); + return false; + } + + if (context.bigint == 0) + { + /* no node in the group has reported one yet -- not an error */ + return true; + } + + *systemIdentifier = context.bigint; + *found = true; + + return true; +} + + +/* + * monitor_report_wal_received calls pgautofailover.report_wal_received() + * to record that nodeId (the ARCHIVING membership's own nodeid, not the + * archiver's archiverid) has durably captured walFileName up to lsn. + * Idempotent on the monitor side (ON CONFLICT DO NOTHING), so callers are + * free to re-report an already-known segment without checking first -- + * see service_archiver.c's own use of this. + */ +bool +monitor_report_wal_received(Monitor *monitor, int64_t nodeId, + const char *walFileName, const char *lsn) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.report_wal_received($1, $2, $3)"; + int paramCount = 3; + Oid paramTypes[3] = { INT8OID, TEXTOID, LSNOID }; + IntString nodeIdString = intToString(nodeId); + const char *paramValues[3] = { nodeIdString.strValue, walFileName, lsn }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to report WAL file \"%s\" received for node %" + PRId64, walFileName, nodeId); + return false; + } + + return true; +} + + +/* + * monitor_report_basebackup_started calls + * pgautofailover.report_basebackup_started() to record the start of a new + * base-backup production job and returns its basebackupid, needed by the + * matching monitor_report_basebackup_completed() call once the backup + * finishes. source is one of "live"/"replay" (basebackup_source's own + * labels); replaymode is required when source is "replay" ("volatile"/ + * "persistent"), NULL otherwise -- pass NULL for a "live" backup. + */ +bool +monitor_report_basebackup_started(Monitor *monitor, int64_t archiverId, + const char *formationId, int groupId, + const char *label, int timeline, + const char *startLsn, + const char *source, const char *replaymode, + int64_t *basebackupId) +{ + PGSQL *pgsql = &monitor->pgsql; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BIGINT, false }; + const char *sql = + "SELECT pgautofailover.report_basebackup_started(" + "$1, $2, $3, $4, $5, $6, " + "$7::pgautofailover.basebackup_source, " + "$8::pgautofailover.basebackup_replay_mode)"; + int paramCount = 8; + Oid paramTypes[8] = { + INT8OID, TEXTOID, INT4OID, TEXTOID, INT4OID, LSNOID, TEXTOID, TEXTOID + }; + IntString archiverIdString = intToString(archiverId); + IntString groupIdString = intToString(groupId); + IntString timelineString = intToString(timeline); + const char *paramValues[8] = { + archiverIdString.strValue, formationId, groupIdString.strValue, + label, timelineString.strValue, startLsn, source, replaymode + }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseSingleValueResult)) + { + log_error("Failed to report the start of base backup \"%s\" to " + "the monitor", label); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to report the start of base backup \"%s\" to " + "the monitor because it returned an unexpected result, " + "see previous lines for details", label); + return false; + } + + *basebackupId = context.bigint; + + return true; +} + + +/* + * monitor_report_basebackup_completed calls + * pgautofailover.report_basebackup_completed() to record the successful + * completion of a base-backup production job previously created with + * monitor_report_basebackup_started(). + */ +bool +monitor_report_basebackup_completed(Monitor *monitor, int64_t basebackupId, + const char *endLsn, int64_t sizeBytes, + const char *storageLocation) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.report_basebackup_completed($1, $2, $3, $4)"; + int paramCount = 4; + Oid paramTypes[4] = { INT8OID, LSNOID, INT8OID, TEXTOID }; + IntString basebackupIdString = intToString(basebackupId); + IntString sizeBytesString = intToString(sizeBytes); + const char *paramValues[4] = { + basebackupIdString.strValue, endLsn, sizeBytesString.strValue, + storageLocation + }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to report base backup %" PRId64 " as completed " + "to the monitor", basebackupId); + return false; + } + + return true; +} + + +/* + * monitor_report_basebackup_deleted calls + * pgautofailover.report_basebackup_deleted() to mark a base backup deleted + * (retaining its history row) once service_archiver_basebackup.c's own + * retention pass has actually removed the directory on disk. Cascades on + * the monitor side to prune any archiver_wal rows no remaining backup + * needs anymore (prune_archiver_wal(), that function's own comment). + */ +bool +monitor_report_basebackup_deleted(Monitor *monitor, int64_t basebackupId) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = "SELECT pgautofailover.report_basebackup_deleted($1)"; + int paramCount = 1; + Oid paramTypes[1] = { INT8OID }; + IntString basebackupIdString = intToString(basebackupId); + const char *paramValues[1] = { basebackupIdString.strValue }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to report base backup %" PRId64 " as deleted " + "to the monitor", basebackupId); + return false; + } + + return true; +} + + +typedef struct BasebackupInfoArrayParseContext +{ + char sqlstate[SQLSTATE_LENGTH]; + BasebackupInfoArray *backupsArray; + bool parsedOK; +} BasebackupInfoArrayParseContext; + + +static bool +parseBasebackupInfoRow(PGresult *result, int rowNumber, BasebackupInfo *backup) +{ + if (PQgetisnull(result, rowNumber, 0) || + PQgetisnull(result, rowNumber, 2) || + PQgetisnull(result, rowNumber, 3)) + { + log_error("basebackupid, storagelocation, or startedat_epoch " + "returned by the monitor is NULL"); + return false; + } + + char *value = PQgetvalue(result, rowNumber, 0); + + backup->basebackupId = strtoll(value, NULL, 0); + + value = PQgetvalue(result, rowNumber, 1); + strlcpy(backup->label, value, NAMEDATALEN); + + value = PQgetvalue(result, rowNumber, 2); + strlcpy(backup->storageLocation, value, MAXPGPATH); + + value = PQgetvalue(result, rowNumber, 3); + backup->startedAtEpoch = strtoll(value, NULL, 0); + + return true; +} + + +static void +parseBasebackupInfoArray(void *ctx, PGresult *result) +{ + BasebackupInfoArrayParseContext *context = + (BasebackupInfoArrayParseContext *) ctx; + bool parsedOk = true; + + if (PQntuples(result) > BASEBACKUP_ARRAY_MAX_COUNT) + { + log_error("Query returned %d rows, pg_auto_failover supports only " + "up to %d base backups per group at the moment", + PQntuples(result), BASEBACKUP_ARRAY_MAX_COUNT); + context->parsedOK = false; + return; + } + + context->backupsArray->count = PQntuples(result); + + for (int rowNumber = 0; rowNumber < PQntuples(result); rowNumber++) + { + BasebackupInfo *backup = &(context->backupsArray->backups[rowNumber]); + + parsedOk = parsedOk && parseBasebackupInfoRow(result, rowNumber, backup); + } + + context->parsedOK = parsedOk; +} + + +/* + * monitor_list_basebackups calls pgautofailover.list_basebackups(formation, + * group) and returns every complete base backup for that group, newest + * first -- what service_archiver_basebackup.c's own retention pass walks + * to decide what survives maxcount/maxage. + */ +bool +monitor_list_basebackups(Monitor *monitor, + const char *formationId, int groupId, + BasebackupInfoArray *backupsArray) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT * FROM pgautofailover.list_basebackups($1, $2)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, INT4OID }; + IntString groupIdString = intToString(groupId); + const char *paramValues[2] = { formationId, groupIdString.strValue }; + BasebackupInfoArrayParseContext parseContext = { { 0 }, backupsArray, false }; + + backupsArray->count = 0; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &parseContext, &parseBasebackupInfoArray)) + { + log_error("Failed to list base backups from the monitor for " + "\"%s\"/%d", formationId, groupId); + return false; + } + + if (!parseContext.parsedOK) + { + log_error("Failed to parse the list of base backups returned by " + "the monitor for \"%s\"/%d, see previous lines for " + "details", formationId, groupId); + return false; + } + + return true; +} + + +/* + * BasebackupPolicyParseContext/parseBasebackupPolicy parse the 9-column + * row shape both monitor_get_basebackup_policy_for_group() and monitor_ + * get_basebackup_policy() use -- one via get_basebackup_policy_for_group() + * (resolved for a formation/group), the other via get_basebackup_policy() + * (looked up by name for `pg_autoctl show basebackup-policy`) -- both + * wrapped in the same SELECT column list on the C side so a single parser + * serves either. + */ +typedef struct BasebackupPolicyParseContext +{ + char sqlstate[SQLSTATE_LENGTH]; + BasebackupPolicy *policy; + bool found; + bool parsedOk; +} BasebackupPolicyParseContext; + + +static void +parseBasebackupPolicy(void *ctx, PGresult *result) +{ + BasebackupPolicyParseContext *context = + (BasebackupPolicyParseContext *) ctx; + + int ntuples = PQntuples(result); + + if (ntuples != 1) + { + /* zero rows is a valid "no such policy" signal, not a parse error */ + context->parsedOk = (ntuples == 0); + context->found = false; + return; + } + + if (PQgetisnull(result, 0, 0)) + { + /* get_basebackup_policy_for_group() found no policy to resolve -- + * shouldn't happen given the schema's own 'default' row always + * exists, but treat it as "not found" rather than a parse error */ + context->parsedOk = true; + context->found = false; + return; + } + + BasebackupPolicy *policy = context->policy; + + strlcpy(policy->policyName, PQgetvalue(result, 0, 0), NAMEDATALEN); + strlcpy(policy->source, PQgetvalue(result, 0, 1), NAMEDATALEN); + + strlcpy(policy->replayMode, + PQgetisnull(result, 0, 2) ? "" : PQgetvalue(result, 0, 2), + NAMEDATALEN); + + strlcpy(policy->cache, PQgetvalue(result, 0, 3), NAMEDATALEN); + + policy->frequencySeconds = strtol(PQgetvalue(result, 0, 4), NULL, 0); + policy->maxCount = strtol(PQgetvalue(result, 0, 5), NULL, 0); + policy->maxAgeSeconds = strtol(PQgetvalue(result, 0, 6), NULL, 0); + policy->onPromotion = strcmp(PQgetvalue(result, 0, 7), "t") == 0; + policy->concurrency = strtol(PQgetvalue(result, 0, 8), NULL, 0); + + context->found = true; + context->parsedOk = true; +} + + +/* + * monitor_get_basebackup_policy_for_group calls pgautofailover.get_ + * basebackup_policy_for_group(formation, group) -- the policy service_ + * archiver_basebackup.c's own scheduling/retention pass actually applies, + * already resolved through archiver_policy's group-override / formation- + * default / schema-default fallback chain (get_archiver_policy()'s own + * comment). + */ +bool +monitor_get_basebackup_policy_for_group(Monitor *monitor, + const char *formationId, int groupId, + BasebackupPolicy *policy, bool *found) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT policyname, source::text, replaymode::text, cache::text, " + " frequency_seconds, maxcount, maxage_seconds, " + " onpromotion, concurrency " + " FROM pgautofailover.get_basebackup_policy_for_group($1, $2)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, INT4OID }; + IntString groupIdString = intToString(groupId); + const char *paramValues[2] = { formationId, groupIdString.strValue }; + BasebackupPolicyParseContext context = { { 0 }, policy, false, false }; + + *found = false; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseBasebackupPolicy)) + { + log_error("Failed to get the base-backup policy from the monitor " + "for \"%s\"/%d", formationId, groupId); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to parse the base-backup policy returned by the " + "monitor for \"%s\"/%d, see above for details", + formationId, groupId); + return false; + } + + *found = context.found; + + return true; +} + + +/* + * monitor_get_basebackup_policy calls pgautofailover.get_basebackup_policy + * (policyname) -- a named policy looked up directly, for `pg_autoctl show + * basebackup-policy`. Wrapped in the same 9-column SELECT list as monitor_ + * get_basebackup_policy_for_group() above so parseBasebackupPolicy() can + * serve both. *found is false (not an error) when no policy has that name. + */ +bool +monitor_get_basebackup_policy(Monitor *monitor, const char *policyName, + BasebackupPolicy *policy, bool *found) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT policyname, source::text, replaymode::text, cache::text, " + " extract(epoch FROM frequency)::int, maxcount, " + " extract(epoch FROM maxage)::int, onpromotion, concurrency " + " FROM pgautofailover.get_basebackup_policy($1)"; + int paramCount = 1; + Oid paramTypes[1] = { TEXTOID }; + const char *paramValues[1] = { policyName }; + BasebackupPolicyParseContext context = { { 0 }, policy, false, false }; + + *found = false; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseBasebackupPolicy)) + { + log_error("Failed to get base-backup policy \"%s\" from the " + "monitor", policyName); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to parse base-backup policy \"%s\" returned by " + "the monitor, see above for details", policyName); + return false; + } + + *found = context.found; + + return true; +} + + +/* + * monitor_create_basebackup_policy calls pgautofailover.create_ + * basebackup_policy(policyname, policyspec) -- policyspec is a raw JSON + * document text, passed straight through to the monitor's own jsonb + * parsing and per-field coalesce-to-default logic (create_basebackup_ + * policy()'s own body, pgautofailover.sql) rather than parsed twice. + */ +bool +monitor_create_basebackup_policy(Monitor *monitor, + const char *policyName, + const char *jsonSpec, + int64_t *basebackupPolicyId) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.create_basebackup_policy($1, $2::jsonb)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, TEXTOID }; + const char *paramValues[2] = { policyName, jsonSpec }; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BIGINT, false }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseSingleValueResult)) + { + log_error("Failed to create base-backup policy \"%s\" on the " + "monitor", policyName); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to create base-backup policy \"%s\" on the " + "monitor because it returned an unexpected result, see " + "previous lines for details", policyName); + return false; + } + + *basebackupPolicyId = context.bigint; + + return true; +} + + +/* + * monitor_set_basebackup_policy calls pgautofailover.set_basebackup_policy + * (policyname, policyspec) to update an existing named policy -- only the + * fields present in the JSON document change (set_basebackup_policy()'s + * own per-field coalesce, pgautofailover.sql), everything else keeps its + * current value. + */ +bool +monitor_set_basebackup_policy(Monitor *monitor, const char *policyName, + const char *jsonSpec) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.set_basebackup_policy($1, $2::jsonb)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, TEXTOID }; + const char *paramValues[2] = { policyName, jsonSpec }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to set base-backup policy \"%s\" on the monitor", + policyName); + return false; + } + + return true; +} + + +/* + * monitor_set_archiver_policy calls pgautofailover.set_archiver_policy() + * to attach basebackupPolicyId to (formationId, groupId) -- groupId < 0 + * sets the formation-wide default (archiver_policy's own groupid IS NULL + * row, matching set_archiver_policy()'s own in_groupid DEFAULT NULL). + * archiverQuorum <= 0 and replicationQuorumEligible are passed through + * as-is; callers that only want to change the backup policy pass whatever + * this formation/group's own current values already are, since set_ + * archiver_policy() UPSERTs and coalesces NULL to "keep existing" only + * when the row already exists -- a brand new row still needs real values, + * hence no NULL-means-unchanged shortcut is exposed at this C layer. + */ +bool +monitor_set_archiver_policy(Monitor *monitor, + const char *formationId, int groupId, + int archiverQuorum, + int64_t basebackupPolicyId, + bool replicationQuorumEligible) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.set_archiver_policy($1, $2, $3, $4, $5)"; + int paramCount = 5; + Oid paramTypes[5] = { TEXTOID, INT4OID, INT4OID, INT8OID, BOOLOID }; + IntString groupIdString = intToString(groupId); + IntString archiverQuorumString = intToString(archiverQuorum); + IntString basebackupPolicyIdString = intToString(basebackupPolicyId); + const char *paramValues[5] = { + formationId, + groupId < 0 ? NULL : groupIdString.strValue, + archiverQuorumString.strValue, + basebackupPolicyIdString.strValue, + replicationQuorumEligible ? "true" : "false" + }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to set archiver policy for \"%s\"/%d on the " + "monitor", formationId, groupId); + return false; + } + + return true; +} + + bool monitor_register_node(Monitor *monitor, char *formation, char *name, char *host, int port, @@ -2215,7 +3460,15 @@ parseNode(PGresult *result, int rowNumber, NodeAddress *node) value = PQgetvalue(result, rowNumber, 3); - if (!stringToInt(value, &node->port) || node->port == 0) + /* + * nodeport = 0 is a real, intentional value for an ARCHIVING row (see + * pgautofailover.sql's own comment on archiver_add_formation()'s + * INSERT): it has no postmaster of its own to be reachable on. This + * function parses whole-formation node listings (get_nodes/ + * get_other_nodes) that legitimately include those rows now, so a + * parsed zero is not an error -- only a genuine parse failure is. + */ + if (!stringToInt(value, &node->port)) { log_error("Invalid port number \"%s\" returned by monitor", value); return false; @@ -2605,8 +3858,9 @@ parseCurrentNodeState(PGresult *result, int rowNumber, value = PQgetvalue(result, rowNumber, 3); - if (!stringToInt(value, &(nodeState->node.port)) || - nodeState->node.port == 0) + /* nodeport = 0 is a real, intentional value for an ARCHIVING row -- see + * the sibling comment on this same check in parseNode() above */ + if (!stringToInt(value, &(nodeState->node.port))) { log_error("Invalid port number \"%s\" returned by monitor", value); ++errors; diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index 3b0fbe770..733413068 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -36,6 +36,112 @@ typedef struct MonitorAssignedState bool replicationQuorum; } MonitorAssignedState; +#define ARCHIVER_ARRAY_MAX_COUNT 128 + +/* + * One row per archiver attached to a formation, from pgautofailover. + * get_archivers() -- storage stats and FSM state are both nullable on the + * SQL side (usedbytes/freebytes: NULL until the first report; the node_id/ + * reportedState/goalState side of the LEFT JOIN: NULL if this milestone's + * one-'wal-receiver'-row-per-group assumption isn't met yet), hence the + * separate hasStorageStats/hasNode flags rather than a sentinel value. + */ +typedef struct ArchiverInfo +{ + int64_t archiverId; + char archiverName[_POSIX_HOST_NAME_MAX]; + char hostname[_POSIX_HOST_NAME_MAX]; + char region[NAMEDATALEN]; + + bool hasStorageStats; + uint64_t usedBytes; + uint64_t freeBytes; + + bool hasNode; + int64_t nodeId; + NodeState reportedState; + NodeState goalState; +} ArchiverInfo; + +typedef struct ArchiverInfoArray +{ + int count; + ArchiverInfo archivers[ARCHIVER_ARRAY_MAX_COUNT]; +} ArchiverInfoArray; + +#define ARCHIVER_MEMBERSHIP_ARRAY_MAX_COUNT 256 + +/* + * One row per (formation, group) membership an archiver currently holds a + * 'wal-receiver' row in, from pgautofailover.list_archiver_memberships() + * -- what an archiver process itself calls, at startup and periodically + * thereafter, to discover the full set of WAL streams and base-backup + * schedules it is responsible for running (service_archiver_reconciler.c). + */ +typedef struct ArchiverMembership +{ + char formation[NAMEDATALEN]; + int groupId; + int64_t nodeId; + NodeState reportedState; + NodeState goalState; +} ArchiverMembership; + +typedef struct ArchiverMembershipArray +{ + int count; + ArchiverMembership memberships[ARCHIVER_MEMBERSHIP_ARRAY_MAX_COUNT]; +} ArchiverMembershipArray; + +/* + * A base-backup production/retention policy (pgautofailover.basebackup_ + * policy), resolved either by name (monitor_get_basebackup_policy(), `pg_ + * autoctl show basebackup-policy`) or for a (formation, group) pair + * (monitor_get_basebackup_policy_for_group(), what service_archiver_ + * basebackup.c's own scheduling/retention pass actually consumes) -- + * both go through the same SQL-side flattening of the policy's interval + * columns to plain integer seconds (get_basebackup_policy_for_group()'s + * own comment, pgautofailover.sql), so both share this one struct. + * replayMode is empty when source is "live" (basebackup_policy's own + * CHECK constraint: replaymode is NULL unless source = 'replay'). + */ +typedef struct BasebackupPolicy +{ + int64_t basebackupPolicyId; + char policyName[NAMEDATALEN]; + char source[NAMEDATALEN]; + char replayMode[NAMEDATALEN]; + char cache[NAMEDATALEN]; + int frequencySeconds; + int maxCount; + int maxAgeSeconds; + bool onPromotion; + int concurrency; +} BasebackupPolicy; + +#define BASEBACKUP_ARRAY_MAX_COUNT 256 + +/* + * One row per complete base backup for a (formation, group), from + * pgautofailover.list_basebackups() -- just enough for a retention + * decision (age via startedAtEpoch, which of maxcount survives) and to + * act on one once pruned (storageLocation to remove the directory, + * basebackupId to report the deletion). + */ +typedef struct BasebackupInfo +{ + int64_t basebackupId; + char label[NAMEDATALEN]; + char storageLocation[MAXPGPATH]; + int64_t startedAtEpoch; +} BasebackupInfo; + +typedef struct BasebackupInfoArray +{ + int count; + BasebackupInfo backups[BASEBACKUP_ARRAY_MAX_COUNT]; +} BasebackupInfoArray; + typedef struct StateNotification { char message[BUFSIZE]; @@ -154,12 +260,72 @@ bool monitor_print_other_nodes_as_json(Monitor *monitor, bool monitor_get_primary(Monitor *monitor, char *formation, int groupId, NodeAddress *node); +bool monitor_register_archiver(Monitor *monitor, char *name, char *hostname, + char *region, int64_t *archiverId); +bool monitor_archiver_add_formation(Monitor *monitor, int64_t archiverId, + char *formation, int64_t *archiverNodeId); +bool monitor_report_archiver_storage(Monitor *monitor, int64_t archiverId, + uint64_t usedBytes, uint64_t freeBytes); +bool monitor_get_archivers(Monitor *monitor, const char *formation, + ArchiverInfoArray *archiversArray); +bool monitor_list_archiver_memberships(Monitor *monitor, int64_t archiverId, + ArchiverMembershipArray *membershipsArray); +bool monitor_get_latest_basebackup_info(Monitor *monitor, + const char *formationId, int groupId, + const char *preferredSource, + char *storageLocation, size_t storageLocationSize, + char *source, size_t sourceSize, + int *timeline, + bool *found); +bool monitor_get_group_system_identifier(Monitor *monitor, + const char *formationId, int groupId, + uint64_t *systemIdentifier, + bool *found); +bool monitor_report_wal_received(Monitor *monitor, int64_t nodeId, + const char *walFileName, const char *lsn); +bool monitor_report_basebackup_started(Monitor *monitor, int64_t archiverId, + const char *formationId, int groupId, + const char *label, int timeline, + const char *startLsn, + const char *source, + const char *replaymode, + int64_t *basebackupId); +bool monitor_report_basebackup_completed(Monitor *monitor, + int64_t basebackupId, + const char *endLsn, + int64_t sizeBytes, + const char *storageLocation); +bool monitor_report_basebackup_deleted(Monitor *monitor, int64_t basebackupId); +bool monitor_list_basebackups(Monitor *monitor, + const char *formationId, int groupId, + BasebackupInfoArray *backupsArray); +bool monitor_get_basebackup_policy_for_group(Monitor *monitor, + const char *formationId, + int groupId, + BasebackupPolicy *policy, + bool *found); +bool monitor_get_basebackup_policy(Monitor *monitor, const char *policyName, + BasebackupPolicy *policy, bool *found); +bool monitor_create_basebackup_policy(Monitor *monitor, + const char *policyName, + const char *jsonSpec, + int64_t *basebackupPolicyId); +bool monitor_set_basebackup_policy(Monitor *monitor, const char *policyName, + const char *jsonSpec); +bool monitor_set_archiver_policy(Monitor *monitor, + const char *formationId, int groupId, + int archiverQuorum, + int64_t basebackupPolicyId, + bool replicationQuorumEligible); bool monitor_get_coordinator(Monitor *monitor, char *formation, CoordinatorNodeAddress *coordinatorNodeAddress); bool monitor_get_most_advanced_standby(Monitor *monitor, char *formation, int groupId, int64_t callerNodeId, NodeAddress *node, bool *found); +bool monitor_get_archiver_node(Monitor *monitor, + char *formation, int groupId, + NodeAddress *node, bool *found); bool monitor_register_node(Monitor *monitor, char *formation, char *name, diff --git a/src/bin/pg_autoctl/nodespec.c b/src/bin/pg_autoctl/nodespec.c index d57655d2d..fb9ddc74a 100644 --- a/src/bin/pg_autoctl/nodespec.c +++ b/src/bin/pg_autoctl/nodespec.c @@ -203,10 +203,14 @@ nodespec_read(const char *path, NodeSpec *spec) { spec->kind = NODE_KIND_CITUS_WORKER; } + else if (strcmp(kindStr, "archiver") == 0) + { + spec->kind = NODE_KIND_ARCHIVER; + } else { log_error("Unknown node kind \"%s\" in \"%s\"; " - "expected: monitor, postgres, coordinator, worker", + "expected: monitor, postgres, coordinator, worker, archiver", kindStr, path); return false; } @@ -387,6 +391,12 @@ nodespec_write(const NodeSpec *spec, FILE *out) break; } + case NODE_KIND_ARCHIVER: + { + kindStr = "archiver"; + break; + } + default: { kindStr = "postgres"; @@ -445,14 +455,21 @@ nodespec_write(const NodeSpec *spec, FILE *out) fformat(out, "[settings]\n" "candidate_priority = %d\n" - "replication_quorum = %s\n" + "replication_quorum = %s\n", + spec->candidate_priority, + spec->replication_quorum ? "true" : "false"); + + if (!IS_EMPTY_STRING_BUFFER(spec->region)) + { + fformat(out, "region = %s\n", spec->region); + } + + fformat(out, "\n" "[options]\n" "ssl = %s\n" "auth = %s\n" "pg_hba_lan = %s\n", - spec->candidate_priority, - spec->replication_quorum ? "true" : "false", spec->ssl, spec->auth, spec->pg_hba_lan ? "true" : "false"); @@ -556,6 +573,12 @@ nodespec_create_argv(const NodeSpec *spec, break; } + case NODE_KIND_ARCHIVER: + { + PUSH("archiver"); + break; + } + default: { PUSH("postgres"); @@ -563,6 +586,57 @@ nodespec_create_argv(const NodeSpec *spec, } } + /* + * An archiver's own getopts (cli_create_archiver_getopts, + * cli_create_node.c) is deliberately minimal -- no --pgport, --ssl-*, + * --auth, --pg-hba-lan, --candidate-priority, ... -- none of which + * apply to a node with no real PostgresSetup (see haspgdata's own + * design comment, pgautofailover.sql). Building its own argv here + * rather than falling through into the rest of this function (which + * assumes every kind accepts the full postgres flag set) avoids + * "unrecognized option" failures on every one of those. + */ + if (spec->kind == NODE_KIND_ARCHIVER) + { + PUSH("--pgdata"); + PUSH(spec->pgdata); + + if (!IS_EMPTY_STRING_BUFFER(spec->name)) + { + PUSH("--name"); + PUSH(spec->name); + } + + if (!IS_EMPTY_STRING_BUFFER(spec->hostname)) + { + PUSH("--hostname"); + PUSH(spec->hostname); + } + + PUSH("--monitor"); + PUSH(spec->monitor_pguri); + + if (!IS_EMPTY_STRING_BUFFER(spec->formation) && + strcmp(spec->formation, "default") != 0) + { + PUSH("--formation"); + PUSH(spec->formation); + } + + if (!IS_EMPTY_STRING_BUFFER(spec->region) && + strcmp(spec->region, "default") != 0) + { + PUSH("--region"); + PUSH(spec->region); + } + + PUSH("--run"); + + args[i] = NULL; + + return i; + } + PUSH("--pgdata"); PUSH(spec->pgdata); diff --git a/src/bin/pg_autoctl/nodestate_utils.c b/src/bin/pg_autoctl/nodestate_utils.c index 114cc65d5..b0854fc42 100644 --- a/src/bin/pg_autoctl/nodestate_utils.c +++ b/src/bin/pg_autoctl/nodestate_utils.c @@ -458,6 +458,9 @@ nodestateConnectionType(CurrentNodeState *nodeState) case DEMOTE_TIMEOUT_STATE: case DRAINING_STATE: case MAINTENANCE_STATE: + + /* an ARCHIVING row has no PGDATA/postmaster of its own, ever */ + case ARCHIVING_STATE: { return "none"; } diff --git a/src/bin/pg_autoctl/service_archiver.c b/src/bin/pg_autoctl/service_archiver.c new file mode 100644 index 000000000..a301a280e --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver.c @@ -0,0 +1,932 @@ +/* + * src/bin/pg_autoctl/service_archiver.c + * Archiving & Disaster Recovery: supervision of the pg_receivewal child + * process an ARCHIVING node keeps running against its group's current + * primary. + * + * Milestone 2's own scope, per the Build order in + * ~/dev/temp/archiving-disaster-recovery.md: the colocated fast path only. + * pg_receivewal is a real, unmodified Postgres client talking straight to + * the real primary's own walsender -- no new wire protocol needed here at + * all. This file only launches and tracks that one child process; it does + * not yet integrate with supervisor.c's Service/RestartPolicy machinery + * (a liveness check happens on each FSM tick instead, via + * service_archiver_pgreceivewal_is_running(), the same "is it alive" + * check the design doc's own ARCHIVING FSM section describes for + * keeper_ensure_current_state) -- and does not yet use a replication slot + * (WAL retention across a pg_receivewal restart is a follow-up). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "service_archiver.h" + +#include "defaults.h" +#include "file_utils.h" +#include "fsm.h" +#include "log.h" +#include "monitor.h" +#include "service_archiver_basebackup.h" +#include "signals.h" + +/* + * WAL segment filename layout, duplicated from pg_walsender/wal_dir_scan.c: + * pg_autoctl doesn't link that standalone binary's code (see this project's + * Makefile split), so the ~15-line segno/LSN arithmetic is small enough to + * repeat here rather than share. + */ +#define ARCHIVER_WAL_FNAME_LEN 24 +#define ARCHIVER_WAL_SEGMENT_SIZE ((uint64_t) 0x1000000) +#define ARCHIVER_XLOG_SEGMENTS_PER_XLOGID \ + (((uint64_t) 0x100000000) / ARCHIVER_WAL_SEGMENT_SIZE) + +/* + * How often service_archiver_loop() reports storage usage, in ticks + * (PG_AUTOCTL_KEEPER_SLEEP_TIME apart, currently 1s each) -- directory_size() + * walks the archiver's whole pgdata (walcache + basebackups, potentially + * many GB across several retained backups), real I/O work unlike the other + * per-tick checks in this loop, so it isn't worth doing every single tick. + */ +#define ARCHIVER_STORAGE_REPORT_TICKS 30 + +/* + * Last WAL filename already reported to the monitor, so each tick only + * reports newly-appeared segments instead of re-scanning and re-reporting + * the whole cache directory every time (the monitor-side insert is + * idempotent, ON CONFLICT DO NOTHING, but that's a fallback for restarts, + * not meant to be relied on every tick). + */ +static char lastReportedWalFileName[ARCHIVER_WAL_FNAME_LEN + 1] = { 0 }; + +/* + * One pg_receivewal child per archiver process, matching milestone 2's own + * single-membership scope (see this file's own comment) -- a future + * milestone generalizing to several (formation, group) memberships per + * archiver will need one pid per membership instead of this one global. + */ +static pid_t pgReceivewalPid = -1; + + +/* + * service_archiver_pgreceivewal_is_running returns true iff the tracked + * pg_receivewal child is still alive. waitpid(WNOHANG) both checks and + * reaps: called on every FSM tick, so a child that exited between ticks is + * reaped promptly rather than lingering as a zombie. + */ +bool +service_archiver_pgreceivewal_is_running(void) +{ + if (pgReceivewalPid <= 0) + { + return false; + } + + int status = 0; + pid_t ret = waitpid(pgReceivewalPid, &status, WNOHANG); + + if (ret == 0) + { + /* still running */ + return true; + } + + if (ret == pgReceivewalPid) + { + log_warn("pg_receivewal (pid %d) exited with status %d", + pgReceivewalPid, status); + } + else + { + log_warn("Failed to check on pg_receivewal (pid %d): %m", + pgReceivewalPid); + } + + pgReceivewalPid = -1; + return false; +} + + +/* + * service_archiver_stop_pgreceivewal stops the tracked pg_receivewal child, + * if any. Idempotent: a no-op when nothing is tracked or the child has + * already exited on its own. + */ +bool +service_archiver_stop_pgreceivewal(void) +{ + if (!service_archiver_pgreceivewal_is_running()) + { + return true; + } + + log_info("Stopping pg_receivewal (pid %d)", pgReceivewalPid); + + if (kill(pgReceivewalPid, SIGTERM) != 0 && errno != ESRCH) + { + log_error("Failed to send SIGTERM to pg_receivewal (pid %d): %m", + pgReceivewalPid); + return false; + } + + int status = 0; + + if (waitpid(pgReceivewalPid, &status, 0) == -1 && errno != ECHILD) + { + log_error("Failed to wait for pg_receivewal (pid %d) to stop: %m", + pgReceivewalPid); + pgReceivewalPid = -1; + return false; + } + + pgReceivewalPid = -1; + return true; +} + + +/* + * service_archiver_start_pgreceivewal starts pg_receivewal against the + * given primary node, writing captured WAL into the archiver's own local + * storage directory (config->pgSetup.pgdata -- an ARCHIVING node's config + * reuses the same field an ordinary node uses for its real PGDATA, see + * this project's own cli_create_archiver, since it plays the same "this + * node's local root directory" role here without ever holding a real + * Postgres cluster). Idempotent: stops any previously-tracked child first, + * exactly like fsm_init_standby's own upstream reuse pattern. + * + * Passes -S/--slot, naming the slot exactly the way keeper_create_and_drop_ + * replication_slots()/pgsql_replication_slot_create_and_drop() (keeper.c, + * primary_standby.c, pgsql.c) already name it for an ordinary standby -- + * REPLICATION_SLOT_NAME_DEFAULT + "_" + this archiver's own node id. That + * mechanism runs on every primary-role node regardless of the other node's + * kind (AutoFailoverOtherNodesList() has no hasPgData filter, node_active_ + * protocol.c's get_other_nodes()), eagerly creating and maintaining this + * exact slot on whichever node is currently primary the same way it does + * for every real standby -- nothing on the primary side needs to change for + * this to work. Without a slot, a pg_receivewal whose first connection + * attempt loses the startup HBA-propagation race (a real, observed + * scenario) restarts streaming from the server's then-current position + * instead of resuming, permanently and silently skipping every WAL segment + * in between: report_wal_received() never reports them (they were simply + * never captured), and any consumer later asked to stream from inside that + * gap (e.g. pg_walsender's own START_REPLICATION, cmd_start_replication.c) + * would wait forever for a segment that will never exist. A replication + * slot fixes this the same way it does for a standby: the slot pins a + * restart_lsn at creation time and the server retains WAL back to it + * regardless of how many times the consumer disconnects and reconnects. + */ +bool +service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNode) +{ + KeeperConfig *config = &(keeper->config); + + if (!service_archiver_stop_pgreceivewal()) + { + /* errors have already been logged */ + return false; + } + + char pgReceivewalPath[MAXPGPATH] = { 0 }; + + path_in_same_directory(config->pgSetup.pg_ctl, + "pg_receivewal", + pgReceivewalPath); + + if (!file_exists(pgReceivewalPath)) + { + log_error("Failed to find pg_receivewal at \"%s\"", pgReceivewalPath); + return false; + } + + /* + * Create-if-missing only -- never ensure_empty_dir(), which rmtree()s + * first: this directory holds already-captured WAL across restarts, + * the whole point of running an archiver. + */ + if (!directory_exists(config->pgSetup.pgdata) && + mkdir(config->pgSetup.pgdata, 0700) != 0) + { + log_error("Failed to create archiver WAL directory \"%s\": %m", + config->pgSetup.pgdata); + return false; + } + + /* + * A plain key/value conninfo string: trust/no-password authentication, + * matching every other pgaftest docker environment this milestone is + * validated against. A real deployment's --ssl/password handling is a + * follow-up, mirroring pg_basebackup()'s own PGPASSWORD-env dance + * (pgctl.c) once an archiver config carries a replication password. + */ + char primaryConnInfo[MAXCONNINFO] = { 0 }; + + sformat(primaryConnInfo, sizeof(primaryConnInfo), + "host=%s port=%d user=%s application_name=%s", + primaryNode->host, primaryNode->port, + PG_AUTOCTL_REPLICA_USERNAME, config->name); + + char slotName[MAXCONNINFO] = { 0 }; + + sformat(slotName, sizeof(slotName), "%s_%d", + REPLICATION_SLOT_NAME_DEFAULT, keeper->state.current_node_id); + + log_info("Starting pg_receivewal against %s:%d, writing to \"%s\", " + "using replication slot \"%s\"", + primaryNode->host, primaryNode->port, config->pgSetup.pgdata, + slotName); + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("Failed to fork pg_receivewal: %m"); + return false; + } + + if (pid == 0) + { + /* child process: replace ourselves with pg_receivewal */ + char *args[10]; + int argsIndex = 0; + + args[argsIndex++] = pgReceivewalPath; + args[argsIndex++] = "-w"; + args[argsIndex++] = "-d"; + args[argsIndex++] = primaryConnInfo; + args[argsIndex++] = "-D"; + args[argsIndex++] = config->pgSetup.pgdata; + args[argsIndex++] = "--no-sync"; + args[argsIndex++] = "-S"; + args[argsIndex++] = slotName; + args[argsIndex] = NULL; + + execv(pgReceivewalPath, args); + + /* execv only returns on failure */ + log_fatal("execv(\"%s\"): %m", pgReceivewalPath); + _exit(127); + } + + /* parent process: track the child, keep running our own loop */ + pgReceivewalPid = pid; + + return true; +} + + +/* + * is_wal_segment_filename returns true iff name has the shape of a real WAL + * segment file (24 hex digits) -- this also naturally excludes pg_receivewal's + * own ".partial" in-progress file, since it's longer than 24 chars. + */ +static bool +is_wal_segment_filename(const char *name) +{ + size_t len = strlen(name); + + if (len != ARCHIVER_WAL_FNAME_LEN) + { + return false; + } + + for (size_t i = 0; i < len; i++) + { + if (!isxdigit((unsigned char) name[i])) + { + return false; + } + } + + return true; +} + + +/* + * wal_filename_compare is a pg_qsort() comparator over an array of char*, + * ordering WAL segment filenames the same way their fixed-width hex names + * already sort lexicographically (== numerically, oldest to newest). + */ +static int +wal_filename_compare(const void *a, const void *b) +{ + const char *nameA = *(const char *const *) a; + const char *nameB = *(const char *const *) b; + + return strcmp(nameA, nameB); +} + + +/* + * wal_segment_end_lsn computes the LSN just past the end of the WAL segment + * named walFileName -- what report_wal_received() records as "captured up + * to", matching pg_walsender/wal_dir_scan.c's own wal_dir_find_latest() + * arithmetic for the same filename layout. + */ +static void +wal_segment_position_lsn(const char *walFileName, uint64_t offsetInSegment, + char *lsn, size_t lsnSize) +{ + char logIdHex[9] = { 0 }; + char segHex[9] = { 0 }; + + memcpy(logIdHex, walFileName + 8, 8); /* IGNORE-BANNED */ + memcpy(segHex, walFileName + 16, 8); /* IGNORE-BANNED */ + + uint32_t logId = (uint32_t) strtoul(logIdHex, NULL, 16); + uint32_t seg = (uint32_t) strtoul(segHex, NULL, 16); + + uint64_t segno = (uint64_t) logId * ARCHIVER_XLOG_SEGMENTS_PER_XLOGID + seg; + uint64_t position = segno * ARCHIVER_WAL_SEGMENT_SIZE + offsetInSegment; + + sformat(lsn, lsnSize, "%X/%08X", + (uint32_t) (position >> 32), + (uint32_t) (position & 0xFFFFFFFF)); +} + + +static void +wal_segment_end_lsn(const char *walFileName, char *lsn, size_t lsnSize) +{ + wal_segment_position_lsn(walFileName, ARCHIVER_WAL_SEGMENT_SIZE, lsn, lsnSize); +} + + +/* + * partial_segment_real_length reads a ".partial" WAL segment file (pre- + * allocated to its full ARCHIVER_WAL_SEGMENT_SIZE by pg_receivewal the + * moment it's created, matching real Postgres's own WAL file pre- + * allocation, XLogFileInitInternal) and returns the length of its real + * content, trimming the zero-padded unwritten tail -- same technique + * pg_walsender/cmd_start_replication.c's own trim_trailing_zeros() already + * applies when actually serving one of these files. + * + * Trusting a trailing zero run to mean "unwritten" isn't safe in the + * general case -- a real primary's own WAL segments get recycled (renamed + * and reused rather than freshly zero-filled, so old content can linger + * past the real write position) -- but pg_receivewal itself never + * recycles; every ".partial" file it ever creates is fresh, so this holds + * here specifically. + */ +static bool +partial_segment_real_length(const char *path, uint64_t *length) +{ + FILE *file = fopen(path, "rb"); /* IGNORE-BANNED */ + + if (file == NULL) + { + return false; + } + + char *buffer = malloc(ARCHIVER_WAL_SEGMENT_SIZE); + + if (buffer == NULL) + { + fclose(file); + return false; + } + + size_t got = fread(buffer, 1, ARCHIVER_WAL_SEGMENT_SIZE, file); + + fclose(file); + + while (got > 0 && buffer[got - 1] == 0) + { + got--; + } + + free(buffer); + + *length = (uint64_t) got; + + return true; +} + + +/* + * service_archiver_report_captured_wal scans the archiver's local WAL cache + * directory for segments pg_receivewal has completed (i.e. no longer + * ".partial") since the last-reported filename, and reports each one to the + * monitor via monitor_report_wal_received() -- the mechanism backing + * archiver_wal/wal_archived(), so archive_command callers elsewhere in the + * cluster can learn when a segment has landed durably on quorum archivers. + * + * Reports oldest-to-newest and only advances lastReportedWalFileName past a + * segment once its report has actually succeeded, so a monitor hiccup + * retries that segment (and anything after it) on the next tick instead of + * silently skipping it. + */ +bool +service_archiver_report_captured_wal(Keeper *keeper) +{ + const char *walcacheDir = keeper->config.pgSetup.pgdata; + + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + /* nothing captured yet -- not an error */ + return true; + } + + char **names = NULL; + int count = 0; + int capacity = 0; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + if (!is_wal_segment_filename(entry->d_name)) + { + continue; + } + + if (strcmp(entry->d_name, lastReportedWalFileName) <= 0) + { + continue; + } + + if (count == capacity) + { + capacity = capacity == 0 ? 16 : capacity * 2; + names = realloc(names, capacity * sizeof(char *)); + } + + names[count++] = strdup(entry->d_name); + } + + closedir(dir); + + if (count == 0) + { + return true; + } + + pg_qsort(names, count, sizeof(char *), wal_filename_compare); + + bool success = true; + + for (int i = 0; i < count; i++) + { + if (success) + { + char lsn[PG_LSN_MAXLENGTH] = { 0 }; + + wal_segment_end_lsn(names[i], lsn, sizeof(lsn)); + + if (monitor_report_wal_received(&(keeper->monitor), + keeper->state.current_node_id, + names[i], lsn)) + { + strlcpy(lastReportedWalFileName, names[i], + sizeof(lastReportedWalFileName)); + } + else + { + log_error("Failed to report WAL file \"%s\" to the monitor", + names[i]); + success = false; + } + } + + free(names[i]); + } + + free(names); + + return success; +} + + +/* + * service_archiver_position_path computes the local, host-only file both + * the archiver-capture and archiver-serve processes use to exchange the + * current captured LSN. The two are separate fork()ed processes (see + * service_archiver_run.c's own comment on why each gets an independent + * connection) -- each has its own private copy of the Keeper struct after + * the fork, so keeper->postgres.currentLSN as updated by this file's own + * service_archiver_update_current_lsn() is invisible to the archiver-serve + * process no matter how it's written; only a real, external, re-read-each- + * time channel like this file makes the value cross that boundary. Built + * from config->pathnames.config exactly like service_archiver_serve.c's own + * service_archiver_serve_routes_path(), so both independently-started + * processes compute the identical path from their own (identically loaded) + * config, without needing shared memory or IPC. + */ +static void +service_archiver_position_path(KeeperConfig *config, char *dest) +{ + path_in_same_directory(config->pathnames.config, + "archiver-position", dest); +} + + +/* + * service_archiver_persist_current_lsn writes keeper->postgres.currentLSN to + * the local position file (see service_archiver_position_path's own + * comment), atomically (write-to-tmp then rename, matching service_archiver_ + * serve_refresh_routes()'s own pattern) so a concurrent reader never + * observes a partial write. + */ +static bool +service_archiver_persist_current_lsn(Keeper *keeper) +{ + char path[MAXPGPATH] = { 0 }; + + service_archiver_position_path(&(keeper->config), path); + + char tmpPath[MAXPGPATH] = { 0 }; + + sformat(tmpPath, sizeof(tmpPath), "%s.tmp", path); + + FILE *fileStream = fopen_with_umask(tmpPath, "w", FOPEN_FLAGS_W, 0644); + + if (fileStream == NULL) + { + /* errors have already been logged */ + return false; + } + + fformat(fileStream, "%s\n", keeper->postgres.currentLSN); + + if (fclose(fileStream) == EOF) + { + log_warn("Failed to write file \"%s\": %m", tmpPath); + return false; + } + + if (rename(tmpPath, path) != 0) + { + log_warn("Failed to rename \"%s\" to \"%s\": %m", tmpPath, path); + return false; + } + + return true; +} + + +/* + * service_archiver_read_current_lsn reads back the position file written by + * service_archiver_persist_current_lsn(), for use by the (separate process) + * archiver-serve side. Returns false (lsnOut left untouched) when the file + * doesn't exist yet -- the archiver-capture process hasn't completed its + * first tick -- callers should fall back to "0/0" themselves. + */ +bool +service_archiver_read_current_lsn(KeeperConfig *config, + char *lsnOut, size_t lsnOutSize) +{ + char path[MAXPGPATH] = { 0 }; + + service_archiver_position_path(config, path); + + char *contents = NULL; + long fileSize = 0; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + return false; + } + + char *nl = strchr(contents, '\n'); + + if (nl != NULL) + { + *nl = '\0'; + } + + strlcpy(lsnOut, contents, lsnOutSize); + free(contents); + + return lsnOut[0] != '\0'; +} + + +/* + * service_archiver_update_current_lsn scans walcacheDir for the newest WAL + * segment -- complete, or still ".partial" -- and updates keeper->postgres. + * currentLSN to the real, currently-captured position: the full segment + * boundary for a complete one, or the real (zero-tail-trimmed) content + * length within the current ".partial" one when that's the frontier. This + * is the single, out-of-band-maintained source of truth for "how far has + * this archiver actually captured" -- computed here, once, per tick, and + * from here alone: both keeper_node_active()'s own per-tick report to the + * monitor (the same way every other node kind reports its own currentLSN) + * and service_archiver_serve_refresh_routes()'s own routes-file "position" + * key (service_archiver_serve.c) read via service_archiver_read_current_lsn() + * above, rather than each independently re-deriving it by scanning WAL file + * content on their own -- one canonical value, not several that could + * disagree. + * + * This is also what makes an archiving node a real, rankable candidate for + * pgautofailover.get_most_advanced_standby() during a failover election: + * that query already has no kind-based exclusion and already considers any + * node reporting REPORT_LSN_STATE (an archiving node passes through it + * during elections, see ARCHIVING_STATE -> REPORT_LSN_STATE in fsm.c) -- + * the only thing that ever kept an archiver from being selected was this + * value staying "0/0" forever. Falls back to "0/0" itself when nothing has + * been captured yet, matching keeper_update_pg_state()'s own default + * before it has a real reading. + */ +static void +service_archiver_update_current_lsn(Keeper *keeper) +{ + const char *walcacheDir = keeper->config.pgSetup.pgdata; + + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + strlcpy(keeper->postgres.currentLSN, "0/0", + sizeof(keeper->postgres.currentLSN)); + return; + } + + char bestComplete[ARCHIVER_WAL_FNAME_LEN + 1] = { 0 }; + char bestPartial[ARCHIVER_WAL_FNAME_LEN + 1] = { 0 }; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + if (is_wal_segment_filename(entry->d_name)) + { + if (bestComplete[0] == '\0' || strcmp(entry->d_name, bestComplete) > 0) + { + strlcpy(bestComplete, entry->d_name, sizeof(bestComplete)); + } + + continue; + } + + const char *partialSuffix = ".partial"; + size_t nameLen = strlen(entry->d_name); + size_t suffixLen = strlen(partialSuffix); + + if (nameLen == ARCHIVER_WAL_FNAME_LEN + suffixLen && + strcmp(entry->d_name + ARCHIVER_WAL_FNAME_LEN, partialSuffix) == 0) + { + char segPart[ARCHIVER_WAL_FNAME_LEN + 1] = { 0 }; + + memcpy(segPart, entry->d_name, ARCHIVER_WAL_FNAME_LEN); /* IGNORE-BANNED */ + + if (is_wal_segment_filename(segPart) && + (bestPartial[0] == '\0' || strcmp(segPart, bestPartial) > 0)) + { + strlcpy(bestPartial, segPart, sizeof(bestPartial)); + } + } + } + + closedir(dir); + + /* + * A ".partial" file only ever exists for the segment actively being + * written, always the same as or newer than the newest complete one -- + * whenever it exists at all, it's the real frontier. + */ + if (bestPartial[0] != '\0' && + (bestComplete[0] == '\0' || strcmp(bestPartial, bestComplete) >= 0)) + { + char path[MAXPGPATH]; + uint64_t realLength = 0; + + sformat(path, sizeof(path), "%s/%s.partial", walcacheDir, bestPartial); + + if (partial_segment_real_length(path, &realLength)) + { + wal_segment_position_lsn(bestPartial, realLength, + keeper->postgres.currentLSN, + sizeof(keeper->postgres.currentLSN)); + return; + } + + /* fall through to the complete segment below on read failure */ + } + + if (bestComplete[0] == '\0') + { + strlcpy(keeper->postgres.currentLSN, "0/0", + sizeof(keeper->postgres.currentLSN)); + return; + } + + wal_segment_end_lsn(bestComplete, keeper->postgres.currentLSN, + sizeof(keeper->postgres.currentLSN)); +} + + +/* + * service_archiver_report_storage reports this archiver's own disk usage + * (directory_size() over its whole pgdata -- walcache and basebackups + * share the same root, see service_archiver_serve.c's own header comment) + * and free space (statvfs's f_bavail, "available to a non-privileged + * process" -- what actually predicts whether the next base backup or WAL + * segment fits, not f_bfree's superuser-reserved total) to the monitor. + * + * Skips the report outright on a statvfs failure rather than reporting a + * free space of zero: unlike directory_size()'s own "best effort, this is + * informational" stance, a wrong zero here would misleadingly read as + * "completely full" to anything watching (pg_autoctl watch's own archivers + * section). + */ +static bool +service_archiver_report_storage(Keeper *keeper) +{ + KeeperConfig *config = &(keeper->config); + const char *pgdata = config->pgSetup.pgdata; + + uint64_t usedBytes = directory_size(pgdata); + + struct statvfs fsStats = { 0 }; + + if (statvfs(pgdata, &fsStats) != 0) + { + log_warn("Failed to statvfs \"%s\": %m, skipping this storage report", + pgdata); + return false; + } + + uint64_t freeBytes = (uint64_t) fsStats.f_bavail * (uint64_t) fsStats.f_frsize; + + if (!monitor_report_archiver_storage(&(keeper->monitor), config->archiverId, + usedBytes, freeBytes)) + { + log_warn("Failed to report storage usage to the monitor, will retry"); + return false; + } + + return true; +} + + +/* + * service_archiver_loop is the archiver's own node_active() reporting loop + * -- deliberately not keeper_node_active_loop (service_keeper.c): that + * function's own per-tick keeper_update_pg_state()/keeper_ensure_current_ + * state() calls assume a real Postgres instance with a real PGDATA to + * inspect, which an ARCHIVING node never has (see haspgdata's own design + * comment). This loop reuses everything that IS kind-agnostic -- + * keeper_load_state()/keeper_store_state(), keeper_node_active() (the + * monitor RPC wrapper itself only ever reads Keeper's in-memory fields, + * never touches real Postgres), and keeper_fsm_reach_assigned_state() + * dispatching through the very same KeeperFSM[] table -- while replacing + * the two Postgres-specific calls with nothing at all: an ARCHIVING row's + * only "is it running" check is service_archiver_pgreceivewal_is_running(), + * consulted by the FSM transition functions themselves + * (fsm_init_archiver et al., fsm_transition.c), not by this loop. + * + * Milestone 2's own single-membership scope (see this file's own header + * comment): one archiver, one (formation, group) row, reported here + * directly rather than iterating a list the monitor refreshes. + */ +bool +service_archiver_loop(Keeper *keeper) +{ + KeeperStateData *keeperState = &(keeper->state); + + log_info("pg_autoctl archiver service is starting"); + + /* + * An archiver never calls keeper_update_pg_state() -- there's no real + * Postgres instance to query (see haspgdata's own design comment) -- + * so keeper->postgres.currentLSN needs its own source of truth here. + * keeper_node_active() always sends it as one of node_active()'s own + * parameters, and the monitor-side pg_lsn column rejects an empty + * string outright ("invalid input syntax for type pg_lsn"), so it must + * hold a valid value even before the first tick's own scan runs. + */ + strlcpy(keeper->postgres.currentLSN, "0/0", sizeof(keeper->postgres.currentLSN)); + + int tickCount = 0; + + while (!asked_to_stop && !asked_to_stop_fast && !asked_to_quit) + { + MonitorAssignedState assignedState = { 0 }; + + (void) service_archiver_update_current_lsn(keeper); + (void) service_archiver_persist_current_lsn(keeper); + + /* + * An archiver never sets postgres.pgIsRunning through the usual + * keeper_update_pg_state() path (there's no real Postgres to + * query, see haspgdata's own design comment) -- it stays at its + * zero-initialized false forever otherwise. That's not just + * cosmetic: the monitor's own NodeIsHealthy() (node_metadata.c) + * unconditionally requires pgIsRunning to be true before ever + * considering a node healthy, in every one of its branches -- + * including group_state_machine.c's own FAST_FORWARD candidate + * selection, which refuses to assign fast_forward against an + * unhealthy WAL source. Without this, an archiver could never + * legitimately serve as a FAST_FORWARD WAL source no matter how + * caught up it was: the monitor would always see it as unhealthy + * and never select it. + * + * Deliberately NOT tied to service_archiver_pgreceivewal_is_ + * running(): that reflects a narrower "is WAL actively being + * captured from a live primary right now" fact, which is + * legitimately false exactly during the window a FAST_FORWARD + * candidate needs the archiver most -- pg_receivewal has nothing + * to stream from once the primary it was following is dead, but + * the WAL this archiver already captured is still there and still + * servable via pg_walsender regardless. pgIsRunning here means + * "this archiver's own keeper service is alive and reporting", + * the same thing a real node's pgIsRunning=true ultimately proves + * about itself -- a crashed or partitioned archiver is still + * caught by the monitor's own separate report-staleness check + * (NodeIsUnhealthy's reportTime/unhealthyTimeoutMs), which + * doesn't depend on this flag at all. + */ + keeper->postgres.pgIsRunning = true; + + if (!keeper_load_state(keeper)) + { + log_error("Failed to read archiver state file, retrying..."); + } + else if (keeper_node_active(keeper, false, &assignedState)) + { + keeperState->assigned_role = assignedState.state; + + if (keeperState->current_role != keeperState->assigned_role) + { + if (keeper_fsm_reach_assigned_state(keeper)) + { + (void) keeper_store_state(keeper); + } + else + { + log_error("Failed to reach assigned state \"%s\", " + "retrying...", + NodeStateToString(keeperState->assigned_role)); + } + } + + /* + * Liveness check: a state transition only (re)starts + * pg_receivewal at the moment current_role becomes + * ARCHIVING_STATE (fsm_init_archiver/fsm_archiver_follow_new_ + * primary, fsm_transition.c) -- it does not run again on later + * ticks where current_role and assigned_role already agree. + * Without this check, a pg_receivewal that dies (or an archiver + * process that gets restarted while already ARCHIVING) would + * stay down forever instead of being noticed and restarted here, + * exactly the "is it running" check this loop's own header + * comment describes. + */ + if (keeperState->current_role == ARCHIVING_STATE && + !service_archiver_pgreceivewal_is_running()) + { + NodeAddress primaryNode = { 0 }; + + if (!keeper_get_primary(keeper, &primaryNode) || + !service_archiver_start_pgreceivewal(keeper, &primaryNode)) + { + log_error("Failed to restart pg_receivewal, retrying..."); + } + } + + if (!service_archiver_report_captured_wal(keeper)) + { + log_warn("Failed to report newly captured WAL segments to " + "the monitor, will retry"); + } + + if (!service_archiver_maybe_generate_basebackup(keeper)) + { + log_warn("Failed to generate a base backup, will retry"); + } + + if (tickCount % ARCHIVER_STORAGE_REPORT_TICKS == 0) + { + (void) service_archiver_report_storage(keeper); + } + } + else + { + log_warn("Failed to contact the monitor, retrying..."); + } + + if (asked_to_stop || asked_to_stop_fast || asked_to_quit) + { + break; + } + + sleep(PG_AUTOCTL_KEEPER_SLEEP_TIME); + ++tickCount; + } + + (void) service_archiver_stop_pgreceivewal(); + + log_info("pg_autoctl archiver service is stopping"); + + return true; +} diff --git a/src/bin/pg_autoctl/service_archiver.h b/src/bin/pg_autoctl/service_archiver.h new file mode 100644 index 000000000..522b661ff --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver.h @@ -0,0 +1,30 @@ +/* + * src/bin/pg_autoctl/service_archiver.h + * Archiving & Disaster Recovery: supervision of the pg_receivewal child + * process an ARCHIVING node keeps running against its group's current + * primary. See ~/dev/temp/archiving-disaster-recovery.md for the design + * this implements milestone 2 of. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef SERVICE_ARCHIVER_H +#define SERVICE_ARCHIVER_H + +#include "keeper.h" +#include "pgsql.h" + +bool service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNode); +bool service_archiver_stop_pgreceivewal(void); +bool service_archiver_pgreceivewal_is_running(void); + +bool service_archiver_report_captured_wal(Keeper *keeper); + +bool service_archiver_read_current_lsn(KeeperConfig *config, + char *lsnOut, size_t lsnOutSize); + +bool service_archiver_loop(Keeper *keeper); + +#endif /* SERVICE_ARCHIVER_H */ diff --git a/src/bin/pg_autoctl/service_archiver_basebackup.c b/src/bin/pg_autoctl/service_archiver_basebackup.c new file mode 100644 index 000000000..9235e9276 --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_basebackup.c @@ -0,0 +1,1260 @@ +/* + * src/bin/pg_autoctl/service_archiver_basebackup.c + * Archiving & Disaster Recovery: base backup generation, both `live` and + * `replay`/`volatile` sources (Milestone 5, per the Build order in + * ~/dev/temp/archiving-disaster-recovery.md: "live first, then + * replay/volatile"), plus policy-driven scheduling and retention -- + * appended to M5 rather than left as a follow-up, so the archiver's own + * base-backup production is a real, bounded resource (frequency-gated, + * count/age-pruned) before Milestones 6/7/8 (warm standby, PITR, cloud + * push) start building on top of it. `replay`/`persistent` is still a + * later milestone -- that mode keeps its staging instance resident as a + * `warm-standby` `archiver_node` row, which doesn't exist until + * Milestone 6. + * + * Trigger scope: bootstrap is always `live` (nothing to replay from yet on + * the first run, matching the design doc's own bootstrap rule), every + * backup after that follows basebackup_policy's own `source`/`replaymode` + * (resolved via monitor_get_basebackup_policy_for_group(), which chains + * get_archiver_policy()'s group-override / formation-default / schema- + * default fallback the same way wal_archived()'s own archiver_quorum + * lookup does), gated on `frequency` seconds having elapsed since the + * newest existing backup -- or fired immediately regardless of frequency + * when `onpromotion` is set and the group's primary has changed since the + * last tick that checked (see get_current_primary_node_id()'s own + * comment). After each successful completion, retention prunes anything + * beyond `maxcount` or older than `maxage` (apply_basebackup_retention()): + * removes the directory, then report_basebackup_deleted() on the monitor, + * which cascades to prune_archiver_wal() on its own. `concurrency` is + * read but not enforced: this milestone's own single-membership scope + * (one archiver, one group) already limits this file to one base backup + * production job in flight at a time (basebackup_child_is_running()) -- + * running several concurrently only has meaning once an archiver can serve + * more than one (formation, group) at once, a later milestone's concern. + * + * Target selection ('live') follows the design doc's own precedence, + * minus its warm-standby tier (a later milestone, nothing to select from + * yet): the first healthy secondary in the group, falling back to the + * primary when none exists. "Healthy" here just means "reachable via + * pgautofailover.get_nodes()", not "least-loaded" -- picking between + * several healthy secondaries by load is a refinement, not required for + * base-backup generation to work at all. + * + * Target selection ('replay') is entirely local: a throwaway staging + * Postgres instance, extracted from the last retained base backup and + * replayed forward using this archiver's own already-captured WAL (no + * network round trip to any live node at all) until it promotes on its + * own (see write_replay_recovery_config()'s own comment on why this + * targets "everything locally available" rather than a specific LSN), + * then sourced via pg_basebackup over loopback and discarded ('volatile': + * no persistent archiver_node row, nothing left running or on disk between + * cycles). + * + * Base backup generation itself is a one-shot forked child (tracked via + * basebackupPid, the same pattern service_archiver.c uses for + * pgReceivewalPid), not a persistent supervised service: it runs to + * completion and exits, so it must not block service_archiver_loop()'s own + * per-tick node_active()/WAL-report cycle for however long it takes. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "service_archiver_basebackup.h" + +#include "defaults.h" +#include "file_utils.h" +#include "log.h" +#include "monitor.h" +#include "pgsql.h" +#include "runprogram.h" +#include "signals.h" +#include "string_utils.h" +#include "supervisor.h" + +/* + * One base backup generation child at a time, mirroring + * service_archiver.c's own pgReceivewalPid tracking pattern. + */ +static pid_t basebackupPid = -1; + +/* accumulator for directory_size()'s nftw() callback -- nftw() has no + * user-data parameter, so this has to be file-scope */ +static uint64_t directorySizeAccumulator = 0; + +/* how long to wait for the replay staging instance to finish replaying + * available WAL and promote before giving up on this cycle */ +#define ARCHIVER_REPLAY_PROMOTE_TIMEOUT_SECONDS 60 + + +static bool +basebackup_child_is_running(void) +{ + if (basebackupPid <= 0) + { + return false; + } + + int status = 0; + pid_t ret = waitpid(basebackupPid, &status, WNOHANG); + + if (ret == 0) + { + /* still running */ + return true; + } + + if (ret == basebackupPid) + { + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) + { + log_warn("Base backup generation process (pid %d) exited " + "with status %d", basebackupPid, status); + } + } + else + { + log_warn("Failed to check on base backup generation process " + "(pid %d): %m", basebackupPid); + } + + basebackupPid = -1; + return false; +} + + +/* + * select_basebackup_source picks the `live` target: the first healthy + * secondary in the group, falling back to the primary when none exists. + * Rows with port == 0 are ARCHIVING memberships (this node's own row among + * them, per the port == 0 sentinel documented in pgautofailover.sql) -- + * never a valid pg_basebackup source, so they are skipped outright. + */ +static bool +select_basebackup_source(Keeper *keeper, NodeAddress *source) +{ + NodeAddressArray nodeArray = { 0 }; + + if (!monitor_get_nodes(&(keeper->monitor), + keeper->config.formation, + keeper->config.groupId, + &nodeArray)) + { + /* errors already logged */ + return false; + } + + NodeAddress *primary = NULL; + + for (int i = 0; i < nodeArray.count; i++) + { + NodeAddress *node = &(nodeArray.nodes[i]); + + if (node->port == 0) + { + continue; + } + + if (node->isPrimary) + { + primary = node; + continue; + } + + *source = *node; + return true; + } + + if (primary != NULL) + { + *source = *primary; + return true; + } + + return false; +} + + +/* + * read_basebackup_label extracts "START WAL LOCATION" and "START TIMELINE" + * from a just-completed pg_basebackup's own backup_label file -- the + * authoritative start position, matching pg_walsender/cmd_base_backup.c's + * own read_backup_label() (duplicated rather than shared: pg_autoctl + * doesn't link that standalone binary's code, see this project's Makefile + * split). + */ +static bool +read_basebackup_label(const char *backupDir, char *lsnOut, size_t lsnOutSize, + int *timelineOut) +{ + char path[MAXPGPATH]; + + sformat(path, sizeof(path), "%s/backup_label", backupDir); + + char *contents = NULL; + long fileSize = 0; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + return false; + } + + bool foundLsn = false; + bool foundTimeline = false; + char *line = contents; + + while (line != NULL && *line != '\0') + { + char *nl = strchr(line, '\n'); + + if (nl != NULL) + { + *nl = '\0'; + } + + const char *lsnPrefix = "START WAL LOCATION: "; + const char *tliPrefix = "START TIMELINE: "; + + if (strncmp(line, lsnPrefix, strlen(lsnPrefix)) == 0) + { + const char *value = line + strlen(lsnPrefix); + const char *end = value; + + while (*end && !isspace((unsigned char) *end)) + { + end++; + } + + size_t len = Min((size_t) (end - value), lsnOutSize - 1); + + memcpy(lsnOut, value, len); /* IGNORE-BANNED */ + lsnOut[len] = '\0'; + foundLsn = true; + } + else if (strncmp(line, tliPrefix, strlen(tliPrefix)) == 0) + { + foundTimeline = stringToInt(line + strlen(tliPrefix), timelineOut); + } + + line = (nl != NULL) ? nl + 1 : NULL; + } + + free(contents); + + return foundLsn && foundTimeline; +} + + +/* + * query_wal_position runs a single ad hoc query against connInfo, used + * right after a base backup finishes to capture the source's current WAL + * write position (primary) or replay position (standby/staging instance) + * -- recorded as the backup's endlsn. Not the exact internal stop-backup + * LSN real pg_basebackup computes server-side (not observable from a plain + * CLI wrapper around it), but a reasonable upper bound: "WAL up to at + * least this point must be replayed to reach consistency." + */ +static bool +query_wal_position(const char *connInfo, bool isPrimary, + char *lsn, size_t lsnSize) +{ + PGSQL client = { 0 }; + + if (!pgsql_init(&client, (char *) connInfo, PGSQL_CONN_UPSTREAM)) + { + return false; + } + + const char *sql = isPrimary + ? "SELECT pg_current_wal_lsn()::text" + : "SELECT pg_last_wal_replay_lsn()::text"; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_STRING, false }; + + bool result = pgsql_execute_with_params(&client, sql, 0, NULL, NULL, + &context, &parseSingleValueResult); + + PQfinish(client.connection); + + if (!result || !context.parsedOk || context.strVal == NULL) + { + return false; + } + + strlcpy(lsn, context.strVal, lsnSize); + free(context.strVal); + + return true; +} + + +static int +accumulate_file_size(const char *path, const struct stat *sb, + int typeflag, struct FTW *ftwbuf) +{ + if (typeflag == FTW_F) + { + directorySizeAccumulator += (uint64_t) sb->st_size; + } + + return 0; +} + + +/* + * directory_size adds up the apparent size of every regular file under + * dirPath. Best effort: sizebytes is informational only (nothing in the + * monitor schema's own logic -- prune_archiver_wal() included -- reads it + * back), so a failure here is not worth failing an otherwise-successful + * base backup over. Exposed (service_archiver_basebackup.h) for service_ + * archiver.c's own periodic storage-usage report, over the archiver's + * whole pgdata rather than just one backup directory. + */ +uint64_t +directory_size(const char *dirPath) +{ + directorySizeAccumulator = 0; + + (void) nftw(dirPath, accumulate_file_size, 16, FTW_PHYS); + + return directorySizeAccumulator; +} + + +/* + * run_pg_basebackup execs the real, unmodified pg_basebackup client + * against source, writing into backupDir. --wal-method=none: this backup + * is deliberately not self-consistent on its own -- for a `live` backup, + * the archiver's already-running WAL capture (service_archiver.c) is what + * supplies the WAL needed to reach consistency on replay; for a `replay` + * backup, the source is itself already paused at a known-consistent LSN, + * so there is nothing further to bundle either way. + */ +static bool +run_pg_basebackup(KeeperConfig *config, NodeAddress *source, + const char *backupDir, const char *label) +{ + char pgBasebackupPath[MAXPGPATH] = { 0 }; + + path_in_same_directory(config->pgSetup.pg_ctl, "pg_basebackup", + pgBasebackupPath); + + if (!file_exists(pgBasebackupPath)) + { + log_error("Failed to find pg_basebackup at \"%s\"", pgBasebackupPath); + return false; + } + + log_info("Generating base backup \"%s\" from %s:%d into \"%s\"", + label, source->host, source->port, backupDir); + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("Failed to fork pg_basebackup: %m"); + return false; + } + + if (pid == 0) + { + char portStr[NAMEDATALEN]; + + sformat(portStr, sizeof(portStr), "%d", source->port); + + char *args[16]; + int argsIndex = 0; + + args[argsIndex++] = pgBasebackupPath; + args[argsIndex++] = "-h"; + args[argsIndex++] = source->host; + args[argsIndex++] = "-p"; + args[argsIndex++] = portStr; + args[argsIndex++] = "-U"; + args[argsIndex++] = PG_AUTOCTL_REPLICA_USERNAME; + args[argsIndex++] = "-D"; + args[argsIndex++] = (char *) backupDir; + args[argsIndex++] = "--format=plain"; + args[argsIndex++] = "--wal-method=none"; + args[argsIndex++] = "--checkpoint=fast"; + args[argsIndex++] = "--label"; + args[argsIndex++] = (char *) label; + args[argsIndex++] = "--no-password"; + args[argsIndex] = NULL; + + execv(pgBasebackupPath, args); + + /* execv only returns on failure */ + log_fatal("execv(\"%s\"): %m", pgBasebackupPath); + _exit(127); + } + + int status = 0; + + if (waitpid(pid, &status, 0) == -1) + { + log_error("Failed to wait for pg_basebackup (pid %d): %m", pid); + return false; + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) + { + log_error("pg_basebackup failed while generating base backup \"%s\"", + label); + return false; + } + + return true; +} + + +/* + * report_basebackup reads backupDir's own backup_label for the + * authoritative start position, then reports both the start and + * completion of this base backup to the monitor. Shared by the live and + * replay paths; source/replaymode is the one thing that differs. + */ +static bool +report_basebackup(Keeper *keeper, NodeAddress *endLsnSource, + const char *backupDir, const char *label, + const char *source, const char *replaymode) +{ + KeeperConfig *config = &(keeper->config); + + char startLsn[PG_LSN_MAXLENGTH] = { 0 }; + int timeline = 1; + + if (!read_basebackup_label(backupDir, startLsn, sizeof(startLsn), + &timeline)) + { + log_error("Failed to read backup_label from \"%s\" after " + "pg_basebackup completed", backupDir); + return false; + } + + if (!monitor_init(&(keeper->monitor), config->monitor_pguri)) + { + log_error("Failed to contact the monitor to report base backup " + "\"%s\"", label); + return false; + } + + int64_t basebackupId = 0; + + if (!monitor_report_basebackup_started(&(keeper->monitor), + config->archiverId, + config->formation, + config->groupId, + label, timeline, startLsn, + source, replaymode, + &basebackupId)) + { + /* errors already logged */ + return false; + } + + /* + * dbname is otherwise unknown here -- an ARCHIVING node has no real + * PostgresSetup of its own to read one from (haspgdata's own design + * comment). DEFAULT_DATABASE_NAME ("postgres") is what every ordinary + * node defaults its own --dbname to (cli_create_node.c), and is always + * present regardless of that default, so it is a safe target for a + * plain read-only SQL query -- true of the replay staging instance too, + * copied verbatim from a `live` backup of an ordinary node. + */ + char connInfo[MAXCONNINFO] = { 0 }; + + sformat(connInfo, sizeof(connInfo), + "host=%s port=%d user=%s dbname=%s application_name=%s", + endLsnSource->host, endLsnSource->port, + PG_AUTOCTL_REPLICA_USERNAME, DEFAULT_DATABASE_NAME, config->name); + + char endLsn[PG_LSN_MAXLENGTH] = { 0 }; + + if (!query_wal_position(connInfo, endLsnSource->isPrimary, + endLsn, sizeof(endLsn))) + { + /* not fatal: the backup itself succeeded, only this one piece of + * informational metadata is missing -- fall back to the start + * position rather than failing an otherwise-successful backup */ + strlcpy(endLsn, startLsn, sizeof(endLsn)); + } + + uint64_t sizeBytes = directory_size(backupDir); + + return monitor_report_basebackup_completed(&(keeper->monitor), + basebackupId, endLsn, + (int64_t) sizeBytes, + backupDir); +} + + +/* + * apply_basebackup_retention lists every complete base backup for this + * group (newest first, list_basebackups()'s own ordering) and prunes + * whatever policy says shouldn't survive: anything beyond the newest + * maxcount, or older than maxage, whichever fires first for a given + * backup -- a backup can be pruned for either reason independently, not + * only once maxcount is already exceeded. maxcount <= 0 or maxage_seconds + * <= 0 disables that particular rule (there is no real-world policy where + * "keep zero backups" or "expire instantly" is the intended behavior; the + * schema's own CHECK constraints don't allow either as a stored value, + * but this stays defensive against a hand-edited row or a future relaxed + * constraint). + * + * Best effort past the first failure: one backup's directory failing to + * remove (e.g. a permissions issue) does not stop the rest of the list + * from being evaluated -- each one is independent, and the failed one + * simply gets retried on the next cycle since it's still 'complete' and + * still over its own retention rule. + */ +static bool +apply_basebackup_retention(Keeper *keeper, BasebackupPolicy *policy) +{ + KeeperConfig *config = &(keeper->config); + BasebackupInfoArray backups = { 0 }; + + if (!monitor_list_basebackups(&(keeper->monitor), + config->formation, config->groupId, + &backups)) + { + log_warn("Failed to list base backups for retention, will retry " + "on the next cycle"); + return false; + } + + time_t now = time(NULL); + bool success = true; + + for (int i = 0; i < backups.count; i++) + { + BasebackupInfo *backup = &(backups.backups[i]); + + bool beyondMaxCount = policy->maxCount > 0 && i >= policy->maxCount; + bool beyondMaxAge = policy->maxAgeSeconds > 0 && + (now - (time_t) backup->startedAtEpoch) > + policy->maxAgeSeconds; + + if (!beyondMaxCount && !beyondMaxAge) + { + continue; + } + + log_info("Pruning base backup \"%s\" (%s)", + backup->label, + beyondMaxCount ? "beyond maxcount" : "past maxage"); + + if (directory_exists(backup->storageLocation) && + !rmtree(backup->storageLocation, true)) + { + log_warn("Failed to remove base backup directory \"%s\", will " + "retry on the next cycle", backup->storageLocation); + success = false; + continue; + } + + if (!monitor_report_basebackup_deleted(&(keeper->monitor), + backup->basebackupId)) + { + log_warn("Failed to report base backup %" PRId64 " as deleted " + "to the monitor, will retry on the next cycle", + backup->basebackupId); + success = false; + } + } + + return success; +} + + +/* + * generate_live_basebackup is the forked child's own body for a `live` + * backup: run pg_basebackup against source to completion, report it, then + * apply retention. Runs in its own process, with its own monitor + * connection (the parent's keeper->monitor is not fork-safe to share, + * exactly as service_archiver_run.c's own supervised children already + * document). + */ +static bool +generate_live_basebackup(Keeper *keeper, NodeAddress *source, + const char *backupDir, const char *label, + BasebackupPolicy *policy) +{ + if (!run_pg_basebackup(&(keeper->config), source, backupDir, label)) + { + return false; + } + + if (!report_basebackup(keeper, source, backupDir, label, "live", NULL)) + { + return false; + } + + (void) apply_basebackup_retention(keeper, policy); + + return true; +} + + +/* + * copy_directory_tree shells out to `cp -R -p` (POSIX-portable across this + * project's actual dev/CI targets, unlike GNU cp's `-a`) to seed the + * replay staging directory from the last retained base backup. No + * existing recursive-copy helper exists in this codebase to reuse, and + * reimplementing one (special files, symlinks, permissions) is a much + * larger and riskier undertaking than reusing a battle-tested system + * utility -- the same reasoning this project already applies to + * pg_basebackup/pg_receivewal/pg_ctl themselves. Uses run_program() + * (runprogram.h), this project's own subprocess helper, rather than a + * hand-rolled fork()/exec(): matches every other external-program call in + * this codebase, and captures stderr for the error message below. + */ +static bool +copy_directory_tree(const char *sourceDir, const char *destDir) +{ + char cpPath[MAXPGPATH] = { 0 }; + + if (!search_path_first("cp", cpPath, LOG_ERROR)) + { + log_error("Failed to find \"cp\" in PATH"); + return false; + } + + Program program = run_program(cpPath, "-R", "-p", sourceDir, destDir, NULL); + bool success = program.returnCode == 0; + + if (!success) + { + log_error("cp -R -p \"%s\" \"%s\" failed: %s", + sourceDir, destDir, + program.stdErr != NULL ? program.stdErr : ""); + } + + free_program(&program); + + return success; +} + + +/* + * write_replay_recovery_config points the staging instance's recovery at + * this archiver's own local WAL cache (the colocated fast path -- no + * network round trip needed, matching service_archiver.c's own philosophy). + * No recovery_target_lsn: an idle-ish source produces mostly-zero-padded + * segments (a "complete", renamed segment file is always its full fixed + * size regardless of how much of it is real WAL), so "the end of the + * latest complete segment" is not actually a reachable record boundary -- + * recovery correctly refuses to pause at a target that doesn't correspond + * to any real record, and errors out instead ("recovery ended before + * configured recovery target was reached"). Instead, this replays every + * available locally-captured record and lets Postgres promote once + * restore_command runs out of segments to fetch -- for a snapshot that + * gets pg_basebackup'd and discarded immediately after (this is + * `volatile`: nothing persists between cycles), a promoted instance is + * exactly as usable a source as a paused one; only a `persistent` replica + * kept resident between cycles (a later milestone) would need the more + * precise pause-at-target-LSN behavior the design doc describes for + * `pg_autoctl warm-standby advance`. + * + * recovery.signal, not standby.signal: this is a one-shot archive recovery + * of already-captured WAL, not open-ended standby streaming. + */ +static bool +write_replay_recovery_config(const char *stagingDir, const char *walcacheDir) +{ + /* + * recovery.signal is what actually puts Postgres into archive recovery + * at startup (PG12+): without it, a data directory that still has + * backup_label is treated as an ordinary crash-recovery restart, which + * fails outright since the copied backup's pg_wal has no local WAL to + * replay from ("could not locate required checkpoint record") -- + * restore_command is only ever consulted once recovery.signal (or + * standby.signal) says this is a recovery in the first place. + */ + char signalPath[MAXPGPATH] = { 0 }; + + sformat(signalPath, sizeof(signalPath), "%s/recovery.signal", stagingDir); + + if (!write_file("", 0, signalPath)) + { + return false; + } + + char confPath[MAXPGPATH] = { 0 }; + + sformat(confPath, sizeof(confPath), "%s/postgresql.auto.conf", stagingDir); + + char conf[BUFSIZE] = { 0 }; + + /* + * ssl = off: the copied postgresql.conf/postgresql.auto.conf still + * carries the source node's own ssl_cert_file/ssl_key_file settings + * (typically absolute paths into *that node's* PGDATA, e.g. from + * --ssl-self-signed) -- meaningless here, since this archiver has no + * Postgres SSL certs of its own to begin with. Left enabled, the + * staging instance fails outright at startup ("could not load server + * certificate file ...: No such file or directory"). Safe to disable + * unconditionally: this instance only ever accepts the loopback + * pg_basebackup connection below, for the lifetime of one throwaway + * cycle. + */ + sformat(conf, sizeof(conf), + "\n" + "# added by pg_autoctl's archiver replay/volatile base backup generation\n" + "restore_command = 'cp \"%s/%%f\" \"%%p\"'\n" + "ssl = off\n", + walcacheDir); + + return append_to_file(conf, strlen(conf), confPath); +} + + +/* + * pid of the currently-running replay staging instance, if any -- tracked + * the same way service_archiver.c tracks pgReceivewalPid, so + * stop_staging_postgres() knows what to signal. + */ +static pid_t stagingPostgresPid = -1; + + +/* + * start_staging_postgres execs the real "postgres" binary directly against + * stagingDir, loopback-only, on PG_AUTOCTL_ARCHIVER_REPLAY_PORT -- the same + * fork()/execv() pattern already used for pg_receivewal + * (service_archiver.c) and pg_basebackup (run_pg_basebackup(), this file), + * rather than going through pg_ctl: readiness is confirmed by + * wait_for_replay_pause()'s own connection-retry loop below, so pg_ctl's + * own "-w" startup wait buys nothing here, and this sidesteps it -- and the + * SQL-connection-based readiness check this needs anyway. + */ +static bool +start_staging_postgres(KeeperConfig *config, const char *stagingDir) +{ + char postgresPath[MAXPGPATH] = { 0 }; + + path_in_same_directory(config->pgSetup.pg_ctl, "postgres", postgresPath); + + if (!file_exists(postgresPath)) + { + log_error("Failed to find postgres at \"%s\"", postgresPath); + return false; + } + + char portStr[NAMEDATALEN] = { 0 }; + + sformat(portStr, sizeof(portStr), "%d", PG_AUTOCTL_ARCHIVER_REPLAY_PORT); + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("Failed to fork postgres: %m"); + return false; + } + + if (pid == 0) + { + char *args[8]; + int argsIndex = 0; + + args[argsIndex++] = postgresPath; + args[argsIndex++] = "-D"; + args[argsIndex++] = (char *) stagingDir; + args[argsIndex++] = "-p"; + args[argsIndex++] = portStr; + args[argsIndex++] = "-h"; + args[argsIndex++] = "127.0.0.1"; + args[argsIndex] = NULL; + + execv(postgresPath, args); + + /* execv only returns on failure */ + log_fatal("execv(\"%s\"): %m", postgresPath); + _exit(127); + } + + stagingPostgresPid = pid; + + return true; +} + + +/* + * stop_staging_postgres stops the replay staging instance. Best effort: + * called during cleanup, including on failure paths where the instance may + * or may not have actually started. + */ +static void +stop_staging_postgres(void) +{ + if (stagingPostgresPid <= 0) + { + return; + } + + if (kill(stagingPostgresPid, SIGTERM) != 0 && errno != ESRCH) + { + log_warn("Failed to send SIGTERM to the replay staging instance " + "(pid %d): %m", stagingPostgresPid); + } + + int status = 0; + + if (waitpid(stagingPostgresPid, &status, 0) == -1 && errno != ECHILD) + { + log_warn("Failed to wait for the replay staging instance " + "(pid %d) to stop: %m", stagingPostgresPid); + } + + stagingPostgresPid = -1; +} + + +/* + * wait_for_replay_promotion connects to the staging instance (retrying: it + * takes a moment after fork()/execv() to start accepting connections) and + * polls pg_is_in_recovery() until it reports false -- Postgres promotes on + * its own once restore_command runs out of segments to fetch (see + * write_replay_recovery_config()'s own comment on why this replays to "no + * more locally-captured WAL" rather than a specific target LSN) -- or + * timeoutSeconds elapses. + */ +static bool +wait_for_replay_promotion(const char *connInfo, int timeoutSeconds) +{ + time_t deadline = time(NULL) + timeoutSeconds; + bool promoted = false; + + while (!promoted && time(NULL) < deadline) + { + PGSQL client = { 0 }; + + if (pgsql_init(&client, (char *) connInfo, PGSQL_CONN_UPSTREAM)) + { + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BOOL, false }; + const char *sql = "SELECT pg_is_in_recovery()"; + + if (pgsql_execute_with_params(&client, sql, 0, NULL, NULL, + &context, &parseSingleValueResult) && + context.parsedOk) + { + promoted = !context.boolVal; + } + + PQfinish(client.connection); + } + + if (!promoted) + { + sleep(1); + } + } + + return promoted; +} + + +/* + * generate_replay_basebackup is the forked child's own body for a + * `replay`/`volatile` backup: extract the last retained base backup into a + * fresh staging directory, replay this archiver's own locally-captured WAL + * forward until it promotes (see write_replay_recovery_config()'s own + * comment for why this targets "everything locally available" rather than + * a specific LSN), snapshot the promoted instance via pg_basebackup over + * loopback, report it, then stop and discard the staging instance -- + * 'volatile' means nothing survives between cycles, each one replays the + * whole gap since the last retained backup again. + */ +static bool +generate_replay_basebackup(Keeper *keeper, const char *sourceBackupDir, + const char *backupDir, const char *label, + BasebackupPolicy *policy) +{ + KeeperConfig *config = &(keeper->config); + + char stagingDir[MAXPGPATH] = { 0 }; + + sformat(stagingDir, sizeof(stagingDir), "%s/replay-staging", + config->pgSetup.pgdata); + + if (directory_exists(stagingDir) && !rmtree(stagingDir, true)) + { + log_error("Failed to remove leftover replay staging directory " + "\"%s\" from a previous cycle", stagingDir); + return false; + } + + log_info("Generating a replay base backup, extracting \"%s\" into \"%s\"", + sourceBackupDir, stagingDir); + + if (!copy_directory_tree(sourceBackupDir, stagingDir)) + { + return false; + } + + if (!write_replay_recovery_config(stagingDir, config->pgSetup.pgdata)) + { + log_error("Failed to write replay recovery configuration in \"%s\"", + stagingDir); + return false; + } + + if (!start_staging_postgres(config, stagingDir)) + { + return false; + } + + char stagingConnInfo[MAXCONNINFO] = { 0 }; + + sformat(stagingConnInfo, sizeof(stagingConnInfo), + "host=127.0.0.1 port=%d user=%s dbname=%s application_name=%s", + PG_AUTOCTL_ARCHIVER_REPLAY_PORT, + PG_AUTOCTL_REPLICA_USERNAME, DEFAULT_DATABASE_NAME, config->name); + + bool ok = wait_for_replay_promotion(stagingConnInfo, + ARCHIVER_REPLAY_PROMOTE_TIMEOUT_SECONDS); + + if (!ok) + { + log_error("Replay staging instance at \"%s\" failed to replay " + "available WAL and promote within %d seconds", + stagingDir, ARCHIVER_REPLAY_PROMOTE_TIMEOUT_SECONDS); + } + else + { + NodeAddress stagingNode = { 0 }; + + strlcpy(stagingNode.host, "127.0.0.1", sizeof(stagingNode.host)); + stagingNode.port = PG_AUTOCTL_ARCHIVER_REPLAY_PORT; + + /* promoted by the time wait_for_replay_promotion() returns true -- + * report_basebackup()'s own endlsn query needs to know to use + * pg_current_wal_lsn(), not pg_last_wal_replay_lsn() (NULL outside + * recovery) */ + stagingNode.isPrimary = true; + + ok = run_pg_basebackup(config, &stagingNode, backupDir, label) && + report_basebackup(keeper, &stagingNode, backupDir, label, + "replay", policy->replayMode); + + if (ok) + { + (void) apply_basebackup_retention(keeper, policy); + } + } + + stop_staging_postgres(); + + /* volatile: discard the staging instance unconditionally, success or not */ + if (!rmtree(stagingDir, true)) + { + log_warn("Failed to remove replay staging directory \"%s\" after " + "use, will be overwritten on the next cycle", stagingDir); + } + + return ok; +} + + +/* + * lastKnownPrimaryNodeId tracks the group's primary across ticks, purely + * in-memory (reset on archiver restart, same lifetime as basebackupPid/ + * stagingPostgresPid above) -- -1 means "not observed yet", which the + * onpromotion check below treats as "nothing to compare against", not "a + * promotion just happened" (that would misfire a forced backup on this + * process's very first tick). + */ +static int64_t lastKnownPrimaryNodeId = -1; + + +/* + * get_current_primary_node_id finds the group's current primary via the + * same monitor_get_nodes() call select_basebackup_source() already makes + * for its own, different purpose (picking a live source) -- kept as a + * separate round trip rather than sharing state across the two call + * sites, since either can run without the other on a given tick + * (onpromotion is checked unconditionally; select_basebackup_source() only + * runs once a backup already turns out to be due). Returns false (not an + * error) when the group currently has no primary at all (mid-election) -- + * callers should skip the comparison for this tick rather than treat that + * as "no promotion". + */ +static bool +get_current_primary_node_id(Keeper *keeper, int64_t *primaryNodeId) +{ + NodeAddressArray nodeArray = { 0 }; + + if (!monitor_get_nodes(&(keeper->monitor), + keeper->config.formation, + keeper->config.groupId, + &nodeArray)) + { + return false; + } + + for (int i = 0; i < nodeArray.count; i++) + { + if (nodeArray.nodes[i].isPrimary) + { + *primaryNodeId = nodeArray.nodes[i].nodeId; + return true; + } + } + + return false; +} + + +/* + * notify_archiver_serve_of_new_basebackup signals the archiver-serve + * process (SIGUSR1) to refresh its routes file immediately, rather than + * leaving pg_walsender to serve a stale route for up to ARCHIVER_SERVE_ + * ROUTES_REFRESH_TICKS more ticks after the monitor already knows this + * backup is complete. Best-effort: archiver-serve's own periodic refresh + * is still there as a fallback, so any failure here (pidfile missing or + * stale, process already gone) is logged and otherwise ignored -- it must + * never turn an already-successful base backup into a failure. + * + * config->archiverPidFilePath is the archiver-level *supervisor's* own + * shared pidfile, with one " " line per supervised + * service (archiver-serve, archiver-reconciler, each archiver-capture-*) + * -- not a dedicated pidfile of archiver-serve's own. Reading its first + * line (as a plain read_pidfile() would) gives the supervisor's own pid, + * not archiver-serve's; supervisor_find_service_pid() is what actually + * looks a specific service up by name. + */ +static void +notify_archiver_serve_of_new_basebackup(KeeperConfig *config) +{ + if (IS_EMPTY_STRING_BUFFER(config->archiverPidFilePath)) + { + return; + } + + pid_t archiverServePid = 0; + + if (!supervisor_find_service_pid(config->archiverPidFilePath, + SERVICE_NAME_ARCHIVER_SERVE, + &archiverServePid) || + archiverServePid <= 0) + { + log_debug("Could not find archiver-serve's pid in \"%s\" to " + "prompt an immediate routes refresh; it will pick up " + "this base backup on its own next periodic tick", + config->archiverPidFilePath); + return; + } + + if (kill(archiverServePid, SIGUSR1) != 0) + { + log_debug("Could not signal archiver-serve (pid %d) to prompt an " + "immediate routes refresh: %m; it will pick up this " + "base backup on its own next periodic tick", + archiverServePid); + } +} + + +/* + * service_archiver_maybe_generate_basebackup checks, once per + * service_archiver_loop() tick, whether a base backup generation is due + * for this group and -- if so, and no generation is already in flight -- + * forks a child to produce one. See this file's own header comment for + * the full policy-driven trigger scope this implements. + */ +bool +service_archiver_maybe_generate_basebackup(Keeper *keeper) +{ + if (basebackup_child_is_running()) + { + return true; + } + + KeeperConfig *config = &(keeper->config); + + BasebackupPolicy policy = { 0 }; + bool foundPolicy = false; + + if (!monitor_get_basebackup_policy_for_group(&(keeper->monitor), + config->formation, + config->groupId, + &policy, &foundPolicy)) + { + /* errors already logged */ + return false; + } + + if (!foundPolicy) + { + /* shouldn't happen: the schema's own 'default' policy always + * exists, and get_archiver_policy()'s own three-way fallback + * always resolves to at least that row */ + log_warn("Failed to resolve a base-backup policy for \"%s\"/%d, " + "skipping this cycle", config->formation, config->groupId); + return true; + } + + BasebackupInfoArray backups = { 0 }; + + if (!monitor_list_basebackups(&(keeper->monitor), + config->formation, config->groupId, + &backups)) + { + /* errors already logged */ + return false; + } + + bool bootstrap = (backups.count == 0); + + /* + * Runs every tick regardless of whether a backup is otherwise due, so + * lastKnownPrimaryNodeId always reflects the most recently observed + * primary -- skipping this update on a due-anyway tick would compare + * a future promotion against a stale value from several ticks back + * and misfire. + */ + bool forcedByPromotion = false; + + if (policy.onPromotion) + { + int64_t currentPrimaryNodeId = 0; + + if (get_current_primary_node_id(keeper, ¤tPrimaryNodeId)) + { + if (lastKnownPrimaryNodeId >= 0 && + lastKnownPrimaryNodeId != currentPrimaryNodeId) + { + forcedByPromotion = true; + + log_info("Forcing a new base backup: the group's primary " + "changed (node %" PRId64 " -> node %" PRId64 ")", + lastKnownPrimaryNodeId, currentPrimaryNodeId); + } + + lastKnownPrimaryNodeId = currentPrimaryNodeId; + } + } + + bool due = bootstrap || forcedByPromotion; + + if (!due) + { + time_t now = time(NULL); + time_t elapsed = now - (time_t) backups.backups[0].startedAtEpoch; + + due = elapsed >= (time_t) policy.frequencySeconds; + } + + if (!due) + { + return true; + } + + char backupsDir[MAXPGPATH] = { 0 }; + + sformat(backupsDir, sizeof(backupsDir), "%s/basebackups", + config->pgSetup.pgdata); + + if (!directory_exists(backupsDir) && mkdir(backupsDir, 0700) != 0) + { + log_error("Failed to create \"%s\": %m", backupsDir); + return false; + } + + /* bootstrap is always 'live' -- nothing to replay from yet, matching + * the design doc's own bootstrap rule -- every backup after that + * follows the resolved policy's own source */ + bool useReplay = !bootstrap && strcmp(policy.source, "replay") == 0; + + time_t now = time(NULL); + struct tm nowUTC = { 0 }; + + gmtime_r(&now, &nowUTC); + + char label[NAMEDATALEN] = { 0 }; + + strftime(label, sizeof(label), + useReplay ? "basebackup-replay-%Y%m%dT%H%M%SZ" + : "basebackup-%Y%m%dT%H%M%SZ", + &nowUTC); + + char backupDir[MAXPGPATH] = { 0 }; + + sformat(backupDir, sizeof(backupDir), "%s/%s", backupsDir, label); + + /* + * sourceBackupDir/policy must be captured now, in the parent, into + * buffers the forked child can safely read after fork(): both are + * local, stack-allocated, still valid across fork() (the child gets + * its own copy of the whole address space). + */ + char sourceBackupDir[MAXPGPATH] = { 0 }; + + if (useReplay) + { + strlcpy(sourceBackupDir, backups.backups[0].storageLocation, + sizeof(sourceBackupDir)); + } + + NodeAddress liveSource = { 0 }; + bool haveLiveSource = false; + + if (!useReplay) + { + if (!select_basebackup_source(keeper, &liveSource)) + { + log_warn("No eligible node to source a live base backup from " + "yet, will retry"); + return true; + } + + haveLiveSource = true; + } + + fflush(stdout); + fflush(stderr); + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("Failed to fork the base backup generation process: %m"); + return false; + } + + if (pid == 0) + { + (void) set_signal_handlers(false); + (void) set_ps_title("pg_autoctl: archiver basebackup"); + + bool ok = haveLiveSource + ? generate_live_basebackup(keeper, &liveSource, backupDir, + label, &policy) + : generate_replay_basebackup(keeper, sourceBackupDir, + backupDir, label, &policy); + + if (ok) + { + notify_archiver_serve_of_new_basebackup(config); + } + + exit(ok ? EXIT_CODE_QUIT : EXIT_CODE_INTERNAL_ERROR); + } + + log_debug("pg_autoctl archiver basebackup process started in " + "subprocess %d", pid); + basebackupPid = pid; + + return true; +} diff --git a/src/bin/pg_autoctl/service_archiver_basebackup.h b/src/bin/pg_autoctl/service_archiver_basebackup.h new file mode 100644 index 000000000..c005016f6 --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_basebackup.h @@ -0,0 +1,20 @@ +/* + * src/bin/pg_autoctl/service_archiver_basebackup.h + * Archiving & Disaster Recovery: base backup generation, `live` source + * only (Milestone 5's own first half). See service_archiver_basebackup.c + * for the full scope note. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef SERVICE_ARCHIVER_BASEBACKUP_H +#define SERVICE_ARCHIVER_BASEBACKUP_H + +#include "keeper.h" + +bool service_archiver_maybe_generate_basebackup(Keeper *keeper); +uint64_t directory_size(const char *dirPath); + +#endif /* SERVICE_ARCHIVER_BASEBACKUP_H */ diff --git a/src/bin/pg_autoctl/service_archiver_reconciler.c b/src/bin/pg_autoctl/service_archiver_reconciler.c new file mode 100644 index 000000000..7132e167f --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_reconciler.c @@ -0,0 +1,713 @@ +/* + * src/bin/pg_autoctl/service_archiver_reconciler.c + * Archiving & Disaster Recovery: an archiver's own membership + * reconciler. + * + * One archiver identity can hold more than one (formation, group) + * membership at once -- every group of a Citus formation, or several + * unrelated formations altogether. Each membership needs its own WAL + * capture (service_archiver.c's service_archiver_loop(), one + * pg_receivewal per membership) and, through it, its own base-backup + * scheduling -- but only ever one shared pg_walsender/routes file + * (service_archiver_serve.c already multiplexes every membership's own + * data from a single process). + * + * This file is the intermediate supervised process (started by + * start_archiver(), service_archiver_run.c, alongside "serve") whose one + * job is to keep the set of running per-membership capture processes in + * sync with what the monitor currently reports this archiver is attached + * to -- discovered via pgautofailover.list_archiver_memberships(), + * diffed against supervisor.c's own dynamic Service array + * (supervisor_add_service()/supervisor_remove_service()) on a periodic + * tick. + * + * Living as its own intermediate process, rather than folding this + * directly into start_archiver()'s own top-level supervisor, is a + * deliberate blast-radius choice: a bug in this genuinely new dynamic- + * reconciliation logic can only crash and restart this one process (via + * the top-level supervisor's own plain, unmodified RP_PERMANENT restart + * policy) -- "serve" keeps running throughout, unaffected, still able to + * serve whatever every membership already has captured on disk. + * + * Crash recovery: if this process itself is restarted, its own in-memory + * record of which pid belongs to which membership is gone -- but the + * capture processes it had started are not (a crashed parent doesn't + * kill its children). Blindly re-discovering memberships and starting a + * fresh capture for each would risk two pg_receivewal processes fighting + * over the same replication slot. Rather than trying to adopt those + * still-running orphans (subtle, and a running pg_receivewal restarting + * against its own slot is already a safe, ordinary occurrence elsewhere + * in this project), this file takes the simpler and equally safe path: + * on startup, read back a small persisted "pid formation group" tracking + * file, SIGTERM anything in it that is still alive, and then start every + * currently-discovered membership fresh. A replication slot retains WAL + * back to its own restart_lsn regardless of how many times its consumer + * reconnects, so this brief, deliberate restart costs nothing. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "service_archiver_reconciler.h" + +#include "defaults.h" +#include "file_utils.h" +#include "log.h" +#include "monitor.h" +#include "service_archiver_run.h" +#include "signals.h" +#include "state.h" +#include "string_utils.h" +#include "supervisor.h" + +/* + * How often the reconciler actually re-queries the monitor for this + * archiver's current membership list. The periodic callback itself may + * be invoked much more often than this by supervisor_loop() (as often as + * every 100ms when otherwise idle) -- this is a wall-clock gate on top + * of that, not a tick count, so it stays correct regardless of how fast + * the underlying loop happens to be running. + */ +#define ARCHIVER_RECONCILER_INTERVAL_SECONDS 30 + +/* + * The name every reconciler-managed capture service is given, so this + * file's own diffing logic can tell a capture service apart from + * anything else that might end up on the same supervisor (there is + * nothing else on this one today, but matching the prefix rather than + * assuming makes that an explicit invariant instead of an accident). + */ +#define ARCHIVER_CAPTURE_SERVICE_NAME_PREFIX "archiver-capture-" + + +static char * archiver_reconciler_tracking_path(Keeper *templateKeeper, char *dest); +static char * archiver_reconciler_pidfile_path(Keeper *templateKeeper, char *dest); +static void archiver_reconciler_cleanup_stale_children(Keeper *templateKeeper); +static bool archiver_reconciler_write_tracking_file(Keeper *templateKeeper, + Supervisor *supervisor); +static bool build_membership_keeper(Keeper *templateKeeper, + ArchiverMembership *membership, + Keeper **outKeeper); +static bool membership_service_name(ArchiverMembership *membership, + char *dest, size_t destSize); +static bool find_membership_service(Supervisor *supervisor, + const char *formation, int groupId, + Service **result); +static void archiver_reconciler_tick(Supervisor *supervisor, void *context); + + +/* + * archiver_reconciler_tracking_path computes the path of the small + * "pid formation group" tracking file this reconciler persists across + * its own restarts, one line per currently-managed capture child -- + * sibling of the archiver's own pg_autoctl.cfg, matching every other + * archiver-root-level bookkeeping file (archiver-routes.ini) this + * project already writes there. + */ +static char * +archiver_reconciler_tracking_path(Keeper *templateKeeper, char *dest) +{ + path_in_same_directory(templateKeeper->config.pathnames.config, + "archiver-reconciler-children", dest); + return dest; +} + + +/* + * archiver_reconciler_pidfile_path computes the pidfile this + * reconciler's own inner supervisor_start_with_callback() call tracks + * its capture children with -- distinct from templateKeeper->config. + * pathnames.pid, which belongs to start_archiver()'s own top-level + * supervisor (tracking "serve" and this reconciler process itself): + * both are real, independently-owned pidfiles, and must never collide + * on the same path. + */ +static char * +archiver_reconciler_pidfile_path(Keeper *templateKeeper, char *dest) +{ + path_in_same_directory(templateKeeper->config.pathnames.pid, + "archiver-reconciler.pid", dest); + return dest; +} + + +/* + * archiver_reconciler_cleanup_stale_children reads back the tracking + * file left behind by a previous instance of this same process (if any) + * and SIGTERMs any pid still alive -- see this file's own header comment + * for why a clean restart, rather than adoption, is the deliberate + * choice here. Best effort throughout: a missing file, an unparsable + * line, or a signal failure (the process was already gone) are all + * simply skipped, never fatal -- whatever is genuinely still running + * gets caught by the fresh discovery pass that follows regardless. + */ +static void +archiver_reconciler_cleanup_stale_children(Keeper *templateKeeper) +{ + char path[MAXPGPATH] = { 0 }; + + (void) archiver_reconciler_tracking_path(templateKeeper, path); + + if (!file_exists(path)) + { + return; + } + + char *contents = NULL; + long fileSize = 0; + + if (!read_file(path, &contents, &fileSize) || contents == NULL) + { + return; + } + + char *lines[BUFSIZE] = { 0 }; + int lineCount = splitLines(contents, lines, BUFSIZE); + + for (int i = 0; i < lineCount; i++) + { + int pid = 0; + char formation[NAMEDATALEN] = { 0 }; + int groupId = 0; + + if (sscanf(lines[i], "%d %63s %d", /* IGNORE-BANNED */ + &pid, formation, &groupId) != 3) + { + continue; + } + + if (pid <= 0) + { + continue; + } + + if (kill((pid_t) pid, 0) == 0) + { + log_info("Stopping leftover archiver capture process %d for " + "\"%s\"/%d from a previous reconciler instance", + pid, formation, groupId); + + if (kill((pid_t) pid, SIGTERM) != 0) + { + log_warn("Failed to signal leftover process %d: %m", pid); + } + } + } + + free(contents); +} + + +/* + * archiver_reconciler_write_tracking_file persists the current set of + * reconciler-managed capture services, atomically (write to a .tmp path, + * then rename) so a concurrent reader (this same process, on its own + * next restart) never observes a partial write. + */ +static bool +archiver_reconciler_write_tracking_file(Keeper *templateKeeper, + Supervisor *supervisor) +{ + char path[MAXPGPATH] = { 0 }; + + (void) archiver_reconciler_tracking_path(templateKeeper, path); + + char tmpPath[MAXPGPATH] = { 0 }; + + sformat(tmpPath, sizeof(tmpPath), "%s.tmp", path); + + FILE *fileStream = fopen_with_umask(tmpPath, "w", FOPEN_FLAGS_W, 0644); + + if (fileStream == NULL) + { + /* errors have already been logged */ + return false; + } + + for (int i = 0; i < supervisor->serviceCount; i++) + { + Service *service = &(supervisor->services[i]); + + if (strncmp(service->name, ARCHIVER_CAPTURE_SERVICE_NAME_PREFIX, + strlen(ARCHIVER_CAPTURE_SERVICE_NAME_PREFIX)) != 0) + { + continue; + } + + Keeper *membershipKeeper = (Keeper *) service->context; + + fformat(fileStream, "%d %s %d\n", + service->pid, + membershipKeeper->config.formation, + membershipKeeper->config.groupId); + } + + if (fclose(fileStream) == EOF) + { + log_warn("Failed to write file \"%s\": %m", tmpPath); + return false; + } + + if (rename(tmpPath, path) != 0) + { + log_warn("Failed to rename \"%s\" to \"%s\": %m", tmpPath, path); + return false; + } + + return true; +} + + +/* + * build_membership_keeper builds a full, independent Keeper for one + * membership out of the shared archiver-level template Keeper: the + * archiver identity (archiverId, monitor_pguri, pg_ctl, hostname, name, + * ...) is copied as-is, while formation/groupId and pgSetup.pgdata are + * overridden to this membership's own values -- a dedicated + * /// subdirectory, created here if missing, + * that becomes this membership's own WAL cache, basebackups/, and (via + * keeper_config_set_pathnames_from_pgdata below, which derives every + * pathname from pgdata) its own state/config/pid file paths, entirely + * distinct from every other membership's. + * + * Every existing service_archiver_ or service_archiver_basebackup_ + * function already takes this same Keeper / KeeperConfig shape and + * derives everything it does from config->formation/groupId/pgSetup. + * pgdata -- so building N of these and forking one capture child per + * instance (service_archiver_capture_start(), unchanged) is what makes + * multi-membership support possible without touching that code at all. + * + * The returned Keeper is heap-allocated and becomes the long-lived + * Service.context for its own capture child -- owned by the caller from + * here on (freed on removal, see archiver_reconciler_tick()). + */ +static bool +build_membership_keeper(Keeper *templateKeeper, ArchiverMembership *membership, + Keeper **outKeeper) +{ + Keeper *membershipKeeper = (Keeper *) calloc(1, sizeof(Keeper)); + + if (membershipKeeper == NULL) + { + log_error("Failed to allocate memory for archiver membership " + "\"%s\"/%d", membership->formation, membership->groupId); + return false; + } + + /* start from a shallow copy of the shared archiver identity -- every + * field is a plain value (char arrays, ints), never a pointer this + * process doesn't already own, so a shallow copy is a real copy */ + *membershipKeeper = *templateKeeper; + + /* + * Stash the archiver-level supervisor's own shared pidfile path (still + * correct at this exact point, inherited from templateKeeper) into its + * own dedicated field before the pathnames recompute below overwrites + * config.pathnames.pid with this membership's own value -- see + * KeeperConfig's own comment on archiverPidFilePath for why a capture + * child needs this. + */ + strlcpy(membershipKeeper->config.archiverPidFilePath, + templateKeeper->config.pathnames.pid, + sizeof(membershipKeeper->config.archiverPidFilePath)); + + strlcpy(membershipKeeper->config.formation, membership->formation, + sizeof(membershipKeeper->config.formation)); + membershipKeeper->config.groupId = membership->groupId; + + sformat(membershipKeeper->config.pgSetup.pgdata, + sizeof(membershipKeeper->config.pgSetup.pgdata), + "%s/%s/%d", + templateKeeper->config.pgSetup.pgdata, + membership->formation, membership->groupId); + + if (!directory_exists(membershipKeeper->config.pgSetup.pgdata) && + pg_mkdir_p(membershipKeeper->config.pgSetup.pgdata, 0700) != 0) + { + log_error("Failed to create archiver membership directory \"%s\": %m", + membershipKeeper->config.pgSetup.pgdata); + free(membershipKeeper); + return false; + } + + /* + * The shallow copy above inherited the template keeper's own + * already-computed pathnames (config/state/nodes/pid, derived from + * the archiver-level pgdata). keeper_config_set_pathnames_from_pgdata()'s + * setters each skip an already-nonempty field, so without this reset + * every membership beyond the first would silently keep pointing at + * the template's (or an earlier membership's) files instead of its + * own -- clear them so they're recomputed from this membership's own + * pgdata below. + */ + memset(&(membershipKeeper->config.pathnames), 0, + sizeof(membershipKeeper->config.pathnames)); + + if (!keeper_config_set_pathnames_from_pgdata( + &(membershipKeeper->config.pathnames), + membershipKeeper->config.pgSetup.pgdata)) + { + log_error("Failed to compute pathnames for archiver membership " + "\"%s\"/%d", membership->formation, membership->groupId); + free(membershipKeeper); + return false; + } + + /* + * Only lay down initial state the first time this membership is ever + * captured (no state file yet) -- service_archiver_loop()'s own + * first action every tick is keeper_load_state(), so an already- + * existing file (this membership was captured before, e.g. across an + * archiver restart) must be left alone rather than clobbered back to + * this tick's snapshot of reported/goal state. + */ + if (!file_exists(membershipKeeper->config.pathnames.state)) + { + keeper_state_init(&(membershipKeeper->state)); + membershipKeeper->state.current_node_id = membership->nodeId; + membershipKeeper->state.current_group = membership->groupId; + membershipKeeper->state.current_role = membership->reportedState; + membershipKeeper->state.assigned_role = membership->goalState; + + if (!keeper_store_state(membershipKeeper)) + { + log_error("Failed to write initial state for archiver " + "membership \"%s\"/%d", membership->formation, + membership->groupId); + free(membershipKeeper); + return false; + } + } + + *outKeeper = membershipKeeper; + return true; +} + + +/* + * membership_service_name computes the supervised-service name for one + * membership's capture process -- ARCHIVER_CAPTURE_SERVICE_NAME_PREFIX + * plus "-", what this file's own diffing and tracking- + * file logic key on. + */ +static bool +membership_service_name(ArchiverMembership *membership, char *dest, size_t destSize) +{ + sformat(dest, destSize, "%s%s-%d", ARCHIVER_CAPTURE_SERVICE_NAME_PREFIX, + membership->formation, membership->groupId); + return true; +} + + +/* + * find_membership_service looks for an already-supervised capture + * service for (formation, groupId) among supervisor->services, matching + * on each service's own Keeper context rather than re-parsing its name. + */ +static bool +find_membership_service(Supervisor *supervisor, const char *formation, + int groupId, Service **result) +{ + for (int i = 0; i < supervisor->serviceCount; i++) + { + Service *service = &(supervisor->services[i]); + + if (strncmp(service->name, ARCHIVER_CAPTURE_SERVICE_NAME_PREFIX, + strlen(ARCHIVER_CAPTURE_SERVICE_NAME_PREFIX)) != 0) + { + continue; + } + + Keeper *membershipKeeper = (Keeper *) service->context; + + if (streq(membershipKeeper->config.formation, formation) && + membershipKeeper->config.groupId == groupId) + { + *result = service; + return true; + } + } + + return false; +} + + +/* + * archiver_reconciler_tick is this file's own Supervisor.periodicCallback + * (see supervisor.h): rate-limited to ARCHIVER_RECONCILER_INTERVAL_SECONDS + * by wall-clock time regardless of how often supervisor_loop() actually + * invokes it, it re-lists this archiver's current memberships and adds + * or removes supervised capture services to match -- the only place + * outside archiver_reconciler_loop() itself that calls + * supervisor_add_service()/supervisor_remove_service(). + */ +static void +archiver_reconciler_tick(Supervisor *supervisor, void *context) +{ + Keeper *templateKeeper = (Keeper *) context; + static time_t lastCheckedAt = 0; + time_t now = time(NULL); + + if (lastCheckedAt != 0 && (now - lastCheckedAt) < + ARCHIVER_RECONCILER_INTERVAL_SECONDS) + { + return; + } + + lastCheckedAt = now; + + ArchiverMembershipArray memberships = { 0 }; + + if (!monitor_list_archiver_memberships(&(templateKeeper->monitor), + templateKeeper->config.archiverId, + &memberships)) + { + log_warn("Failed to list archiver memberships from the monitor, " + "will retry"); + return; + } + + /* additions: a discovered membership with no matching supervised + * service yet */ + for (int i = 0; i < memberships.count; i++) + { + ArchiverMembership *membership = &(memberships.memberships[i]); + Service *existing = NULL; + + if (find_membership_service(supervisor, membership->formation, + membership->groupId, &existing)) + { + continue; + } + + Keeper *membershipKeeper = NULL; + + if (!build_membership_keeper(templateKeeper, membership, &membershipKeeper)) + { + /* errors have already been logged; try again on the next tick */ + continue; + } + + Service newService = { + { 0 }, RP_PERMANENT, -1, + &service_archiver_capture_start, + (void *) membershipKeeper, { 0 } + }; + + (void) membership_service_name(membership, newService.name, + sizeof(newService.name)); + + log_info("Archiver reconciler: adding membership \"%s\"/%d", + membership->formation, membership->groupId); + + if (!supervisor_add_service(supervisor, newService)) + { + log_warn("Failed to start capture for membership \"%s\"/%d, " + "will retry", membership->formation, membership->groupId); + free(membershipKeeper); + } + } + + /* + * removals: a supervised capture service whose membership is no + * longer in the fresh discovery list -- collected first, then + * removed in a second pass, since supervisor_remove_service() packs + * the array and would otherwise invalidate this loop's own indices. + */ + Service *toRemove[ARCHIVER_MEMBERSHIP_ARRAY_MAX_COUNT] = { 0 }; + int toRemoveCount = 0; + + for (int i = 0; i < supervisor->serviceCount; i++) + { + Service *service = &(supervisor->services[i]); + + if (strncmp(service->name, ARCHIVER_CAPTURE_SERVICE_NAME_PREFIX, + strlen(ARCHIVER_CAPTURE_SERVICE_NAME_PREFIX)) != 0) + { + continue; + } + + Keeper *membershipKeeper = (Keeper *) service->context; + bool stillMember = false; + + for (int j = 0; j < memberships.count; j++) + { + ArchiverMembership *membership = &(memberships.memberships[j]); + + if (streq(membershipKeeper->config.formation, membership->formation) && + membershipKeeper->config.groupId == membership->groupId) + { + stillMember = true; + break; + } + } + + if (!stillMember && toRemoveCount < ARCHIVER_MEMBERSHIP_ARRAY_MAX_COUNT) + { + toRemove[toRemoveCount++] = service; + } + } + + for (int i = 0; i < toRemoveCount; i++) + { + Keeper *membershipKeeper = (Keeper *) toRemove[i]->context; + pid_t pid = toRemove[i]->pid; + char formation[NAMEDATALEN] = { 0 }; + int groupId = membershipKeeper->config.groupId; + + strlcpy(formation, membershipKeeper->config.formation, sizeof(formation)); + + log_info("Archiver reconciler: removing membership \"%s\"/%d", + formation, groupId); + + if (supervisor_remove_service(supervisor, pid, SIGTERM)) + { + free(membershipKeeper); + } + else + { + log_warn("Failed to remove capture for membership \"%s\"/%d, " + "will retry", formation, groupId); + } + } + + if (toRemoveCount > 0 || memberships.count != supervisor->serviceCount) + { + (void) archiver_reconciler_write_tracking_file(templateKeeper, supervisor); + } +} + + +/* + * service_archiver_reconciler_loop is the reconciler process's own body: + * clean up after any previous instance of itself (see this file's own + * header comment), discover this archiver's current memberships, start + * one capture child per membership, persist the tracking file, then hand + * off to supervisor_start_with_callback() with archiver_reconciler_tick() + * as the periodic callback for everything from here on. + */ +static bool +service_archiver_reconciler_loop(Keeper *templateKeeper) +{ + (void) archiver_reconciler_cleanup_stale_children(templateKeeper); + + ArchiverMembershipArray memberships = { 0 }; + + if (!monitor_list_archiver_memberships(&(templateKeeper->monitor), + templateKeeper->config.archiverId, + &memberships)) + { + log_fatal("Failed to list archiver memberships from the monitor, " + "see above for details"); + return false; + } + + int serviceCount = memberships.count; + Service *services = (Service *) calloc(serviceCount > 0 ? serviceCount : 1, + sizeof(Service)); + + if (services == NULL) + { + log_fatal("Failed to allocate memory for %d archiver memberships", + serviceCount); + return false; + } + + for (int i = 0; i < memberships.count; i++) + { + ArchiverMembership *membership = &(memberships.memberships[i]); + Keeper *membershipKeeper = NULL; + + if (!build_membership_keeper(templateKeeper, membership, &membershipKeeper)) + { + log_fatal("Failed to prepare archiver membership \"%s\"/%d, " + "see above for details", + membership->formation, membership->groupId); + free(services); + return false; + } + + services[i].policy = RP_PERMANENT; + services[i].pid = -1; + services[i].startFunction = &service_archiver_capture_start; + services[i].context = (void *) membershipKeeper; + + (void) membership_service_name(membership, services[i].name, + sizeof(services[i].name)); + } + + log_info("Archiver reconciler: starting capture for %d membership(s)", + memberships.count); + + char pidfile[MAXPGPATH] = { 0 }; + + (void) archiver_reconciler_pidfile_path(templateKeeper, pidfile); + + return supervisor_start_with_callback(services, serviceCount, pidfile, + &archiver_reconciler_tick, + (void *) templateKeeper); +} + + +/* + * service_archiver_reconciler_start forks the reconciler process itself + * -- matching service_archiver_capture_start()/service_archiver_serve_ + * start_service()'s own fork-without-exec shape (service_archiver_run.c), + * one level up: this is what start_archiver() supervises directly. + */ +bool +service_archiver_reconciler_start(void *context, pid_t *pid) +{ + Keeper *keeper = (Keeper *) context; + + fflush(stdout); + fflush(stderr); + + pid_t fpid = fork(); + + switch (fpid) + { + case -1: + { + log_error("Failed to fork the archiver reconciler process"); + return false; + } + + case 0: + { + (void) set_signal_handlers(false); + (void) set_ps_title("pg_autoctl: archiver reconciler"); + + /* see service_archiver_capture_start()'s own comment on why + * each supervised child re-connects independently */ + if (!monitor_init(&(keeper->monitor), keeper->config.monitor_pguri)) + { + log_fatal("Failed to contact the monitor, see above for details"); + exit(EXIT_CODE_MONITOR); + } + + if (!service_archiver_reconciler_loop(keeper)) + { + exit(EXIT_CODE_INTERNAL_ERROR); + } + + exit(EXIT_CODE_QUIT); + } + + default: + { + log_debug("pg_autoctl archiver reconciler process started in " + "subprocess %d", fpid); + *pid = fpid; + return true; + } + } +} diff --git a/src/bin/pg_autoctl/service_archiver_reconciler.h b/src/bin/pg_autoctl/service_archiver_reconciler.h new file mode 100644 index 000000000..4381cc122 --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_reconciler.h @@ -0,0 +1,21 @@ +/* + * src/bin/pg_autoctl/service_archiver_reconciler.h + * Archiving & Disaster Recovery: an archiver's own membership + * reconciler -- the intermediate supervised process that lets one + * archiver manage WAL capture (and, through it, base-backup + * scheduling) for every (formation, group) membership it holds, not + * just one. See service_archiver_reconciler.c for the full design. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef SERVICE_ARCHIVER_RECONCILER_H +#define SERVICE_ARCHIVER_RECONCILER_H + +#include "keeper.h" + +bool service_archiver_reconciler_start(void *context, pid_t *pid); + +#endif /* SERVICE_ARCHIVER_RECONCILER_H */ diff --git a/src/bin/pg_autoctl/service_archiver_run.c b/src/bin/pg_autoctl/service_archiver_run.c new file mode 100644 index 000000000..ca9419f06 --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_run.c @@ -0,0 +1,191 @@ +/* + * src/bin/pg_autoctl/service_archiver_run.c + * See service_archiver_run.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "service_archiver_run.h" + +#include "cli_root.h" +#include "file_utils.h" +#include "log.h" +#include "monitor.h" +#include "service_archiver.h" +#include "service_archiver_reconciler.h" +#include "service_archiver_serve.h" +#include "signals.h" +#include "supervisor.h" + + +/* + * service_archiver_capture_start forks a child that runs + * service_archiver_loop() (service_archiver.c) -- the outbound WAL-capture + * half, supervising pg_receivewal against one (formation, group) + * membership's own primary. No exec(): this project's own binary already + * implements the loop, matching service_keeper_start()'s sibling shape for + * an ordinary node minus the execv() re-exec (that one replaces the + * process image to get a fresh "node-active"-titled process; forking + * straight into the loop function is simpler and just as correct here). + * + * Exported (not static): start_archiver() below no longer calls this + * directly -- it's service_archiver_reconciler.c that does, once per + * membership this archiver holds, since an archiver can hold more than + * one at once. See that file's own header comment for the full design. + */ +bool +service_archiver_capture_start(void *context, pid_t *pid) +{ + Keeper *keeper = (Keeper *) context; + + fflush(stdout); + fflush(stderr); + + pid_t fpid = fork(); + + switch (fpid) + { + case -1: + { + log_error("Failed to fork the archiver capture process"); + return false; + } + + case 0: + { + (void) set_signal_handlers(false); + (void) set_ps_title("pg_autoctl: archiver capture"); + + /* + * Re-connect: the parent's own keeper->monitor connection is + * not fork-safe to share, and may already have been closed by + * the caller (cli_service.c's cli_keeper_run finishes its own + * connection before starting services) -- each supervised + * child establishes its own, exactly like a freshly exec'd + * process would. + */ + if (!monitor_init(&(keeper->monitor), keeper->config.monitor_pguri)) + { + log_fatal("Failed to contact the monitor, see above for details"); + exit(EXIT_CODE_MONITOR); + } + + if (!service_archiver_loop(keeper)) + { + exit(EXIT_CODE_INTERNAL_ERROR); + } + + exit(EXIT_CODE_QUIT); + } + + default: + { + log_debug("pg_autoctl archiver capture process started in " + "subprocess %d", fpid); + *pid = fpid; + return true; + } + } +} + + +/* + * service_archiver_serve_start_service forks a child that runs + * service_archiver_serve_loop() (service_archiver_serve.c) -- the inbound + * serving half, exec'ing and supervising pg_walsender. Named with a + * "_service" suffix to avoid colliding with service_archiver_serve.c's own + * service_archiver_serve_start_walsender(), a different function one level + * down (that one starts pg_walsender itself; this one starts the loop that + * in turn starts and monitors pg_walsender). + */ +static bool +service_archiver_serve_start_service(void *context, pid_t *pid) +{ + Keeper *keeper = (Keeper *) context; + + fflush(stdout); + fflush(stderr); + + pid_t fpid = fork(); + + switch (fpid) + { + case -1: + { + log_error("Failed to fork the archiver serve process"); + return false; + } + + case 0: + { + (void) set_signal_handlers(false); + (void) set_ps_title("pg_autoctl: archiver serve"); + + /* see service_archiver_capture_start()'s own comment on why + * each supervised child re-connects independently */ + if (!monitor_init(&(keeper->monitor), keeper->config.monitor_pguri)) + { + log_fatal("Failed to contact the monitor, see above for details"); + exit(EXIT_CODE_MONITOR); + } + + if (!service_archiver_serve_loop(keeper)) + { + exit(EXIT_CODE_INTERNAL_ERROR); + } + + exit(EXIT_CODE_QUIT); + } + + default: + { + log_debug("pg_autoctl archiver serve process started in " + "subprocess %d", fpid); + *pid = fpid; + return true; + } + } +} + + +/* + * start_archiver supervises exactly two top-level children: "serve" (one + * pg_walsender for every membership this archiver holds, unchanged) and + * "reconciler" (service_archiver_reconciler.c), which in turn keeps one + * WAL-capture child per membership running, added and removed as this + * archiver's own attachments change. Both use the plain, unmodified + * supervisor_start() -- a fixed two-element array like every other node + * kind's own top-level supervisor -- so a bug in the reconciler's own, + * genuinely new dynamic-membership logic can only crash and restart the + * reconciler itself; "serve" is never affected. + */ +bool +start_archiver(Keeper *keeper) +{ + const char *pidfile = keeper->config.pathnames.pid; + + Service subprocesses[] = { + { + SERVICE_NAME_ARCHIVER_SERVE, + RP_PERMANENT, + -1, + &service_archiver_serve_start_service, + (void *) keeper + }, + { + SERVICE_NAME_ARCHIVER_RECONCILER, + RP_PERMANENT, + -1, + &service_archiver_reconciler_start, + (void *) keeper + } + }; + + int subprocessesCount = sizeof(subprocesses) / sizeof(subprocesses[0]); + + return supervisor_start(subprocesses, subprocessesCount, pidfile); +} diff --git a/src/bin/pg_autoctl/service_archiver_run.h b/src/bin/pg_autoctl/service_archiver_run.h new file mode 100644 index 000000000..d4bad3c0a --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_run.h @@ -0,0 +1,35 @@ +/* + * src/bin/pg_autoctl/service_archiver_run.h + * Archiving & Disaster Recovery: `pg_autoctl run` support for + * kind = archiver (milestone 3's own build-order line). Supervises the + * archiver's two halves -- WAL capture (service_archiver.c's + * service_archiver_loop, outbound pg_receivewal against the primary) + * and serving (service_archiver_serve.c's service_archiver_serve_loop, + * inbound pg_walsender) -- as two real supervisor.c Service[] entries + * under one supervised process tree, restart-on-crash, the same way + * start_keeper() already supervises postgres + node-active together for + * an ordinary node. Replaces needing two separately-managed processes + * (`create archiver --run` for capture, `archiver serve` for serving) + * with the one unified entry point operators already expect from + * `pg_autoctl run`. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef SERVICE_ARCHIVER_RUN_H +#define SERVICE_ARCHIVER_RUN_H + +#include "keeper.h" + +bool start_archiver(Keeper *keeper); + +/* + * Exported for service_archiver_reconciler.c, which starts one of these + * per (formation, group) membership this archiver holds -- see that + * file's own header comment. + */ +bool service_archiver_capture_start(void *context, pid_t *pid); + +#endif /* SERVICE_ARCHIVER_RUN_H */ diff --git a/src/bin/pg_autoctl/service_archiver_serve.c b/src/bin/pg_autoctl/service_archiver_serve.c new file mode 100644 index 000000000..556db7881 --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_serve.c @@ -0,0 +1,607 @@ +/* + * src/bin/pg_autoctl/service_archiver_serve.c + * See service_archiver_serve.h. + * + * pg_walsender is exec'd exactly once per archiver process, but serves + * every (formation, group) membership that archiver holds through the one + * shared routes file -- one "[formation/group]" section per membership, + * refreshed from the monitor's own membership list + * (monitor_list_archiver_memberships) each time, mirroring the fan-out + * service_archiver_reconciler.c does on the capture side (one process per + * membership there, since pg_receivewal can only ever follow one primary + * at a time; a single pg_walsender can multiplex any number of client + * connections instead, so no such fan-out is needed here). + * + * Each route's walcache/position paths are deliberately *derived* rather + * than looked up on the monitor: archiver_add_formation()'s own SQL + * (pgautofailover.sql) inserts the new archiver_node row's pgdata as an + * empty string -- the monitor has no way to know an archiver's local WAL + * cache path, that's inherently archiver-host-local information never sent + * to it. Instead each route's paths are computed the same way service_ + * archiver_reconciler.c's own build_membership_keeper() computes them for + * the capture side, from the same /// convention, + * so both independently arrive at identical paths. The one thing genuinely + * worth asking the monitor is the latest base backup's storage location + * (monitor_get_latest_basebackup_info), which is real, monitor-tracked + * state. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "service_archiver_serve.h" + +#include "cli_root.h" /* pg_autoctl_program */ +#include "defaults.h" +#include "file_utils.h" +#include "log.h" +#include "monitor.h" +#include "service_archiver.h" +#include "signals.h" + +/* how often service_archiver_serve_loop() re-checks pg_walsender's + * liveness and refreshes the routes file, in seconds */ +#define ARCHIVER_SERVE_TICK_SECONDS 1 + +/* matches service_archiver.c's own ARCHIVER_WAL_FNAME_LEN: a real WAL + * segment filename is 24 hex digits (8 TLI + 8 logId + 8 seg) */ +#define ARCHIVER_SERVE_WAL_FNAME_LEN 24 +#define ARCHIVER_SERVE_ROUTES_REFRESH_TICKS 30 + +/* + * One pg_walsender child per archiver process, matching service_archiver. + * c's own single-membership scope (see this file's own header comment). + */ +static pid_t pgWalsenderPid = -1; +static int archiverServePort = 0; + + +void +service_archiver_serve_set_port(int port) +{ + archiverServePort = port; +} + + +static void +service_archiver_serve_routes_path(KeeperConfig *config, char *dest) +{ + path_in_same_directory(config->pathnames.config, + "archiver-routes.ini", dest); +} + + +bool +service_archiver_serve_walsender_is_running(void) +{ + if (pgWalsenderPid <= 0) + { + return false; + } + + int status = 0; + pid_t ret = waitpid(pgWalsenderPid, &status, WNOHANG); + + if (ret == 0) + { + /* still running */ + return true; + } + + if (ret == pgWalsenderPid) + { + log_warn("pg_walsender (pid %d) exited", pgWalsenderPid); + } + else if (ret == -1 && errno != ECHILD) + { + log_warn("Failed to waitpid() on pg_walsender (pid %d): %m", pgWalsenderPid); + } + + pgWalsenderPid = -1; + return false; +} + + +bool +service_archiver_serve_stop_walsender(void) +{ + if (pgWalsenderPid <= 0) + { + return true; + } + + log_info("Stopping pg_walsender (pid %d)", pgWalsenderPid); + + if (kill(pgWalsenderPid, SIGTERM) != 0 && errno != ESRCH) + { + log_error("Failed to send SIGTERM to pg_walsender (pid %d): %m", + pgWalsenderPid); + return false; + } + + int status = 0; + + if (waitpid(pgWalsenderPid, &status, 0) == -1 && errno != ECHILD) + { + log_error("Failed to waitpid() on pg_walsender (pid %d): %m", + pgWalsenderPid); + pgWalsenderPid = -1; + return false; + } + + pgWalsenderPid = -1; + return true; +} + + +bool +service_archiver_serve_start_walsender(Keeper *keeper) +{ + KeeperConfig *config = &(keeper->config); + + if (!service_archiver_serve_stop_walsender()) + { + /* errors have already been logged */ + return false; + } + + char pgWalsenderPath[MAXPGPATH] = { 0 }; + + path_in_same_directory(pg_autoctl_program, "pg_walsender", pgWalsenderPath); + + if (!file_exists(pgWalsenderPath)) + { + log_error("Failed to find pg_walsender at \"%s\"", pgWalsenderPath); + return false; + } + + char routesPath[MAXPGPATH] = { 0 }; + + service_archiver_serve_routes_path(config, routesPath); + + int port = archiverServePort > 0 ? archiverServePort : PG_AUTOCTL_ARCHIVER_SERVE_PORT; + char portStr[16] = { 0 }; + + sformat(portStr, sizeof(portStr), "%d", port); + + log_info("Starting pg_walsender on port %d, routes \"%s\"", port, routesPath); + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("Failed to fork pg_walsender: %m"); + return false; + } + + if (pid == 0) + { + /* child process: replace ourselves with pg_walsender */ + char *args[6]; + int argsIndex = 0; + + args[argsIndex++] = pgWalsenderPath; + args[argsIndex++] = "--port"; + args[argsIndex++] = portStr; + args[argsIndex++] = "--routes"; + args[argsIndex++] = routesPath; + args[argsIndex] = NULL; + + execv(pgWalsenderPath, args); + + /* execv only returns on failure */ + log_fatal("execv(\"%s\"): %m", pgWalsenderPath); + _exit(127); + } + + /* parent process: track the child, keep running our own loop */ + pgWalsenderPid = pid; + + return true; +} + + +/* + * walcache_current_timeline scans walcacheDir for the newest captured WAL + * segment (same 24-hex-digit filename shape and sort order as service_ + * archiver.c's own is_wal_segment_filename/wal_filename_compare, matching + * pg_walsender/wal_dir_scan.c's own wal_dir_find_latest arithmetic for the + * same layout) and returns its embedded timeline (the filename's first 8 + * hex digits). Returns false (not an error) when the walcache has no + * complete segment yet -- too early to know, not "timeline 0". + */ +static bool +walcache_current_timeline(const char *walcacheDir, int *timeline) +{ + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + return false; + } + + char best[ARCHIVER_SERVE_WAL_FNAME_LEN + 1] = { 0 }; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + size_t len = strlen(entry->d_name); + bool isWalSegment = (len == ARCHIVER_SERVE_WAL_FNAME_LEN); + + for (size_t i = 0; isWalSegment && i < len; i++) + { + isWalSegment = isxdigit((unsigned char) entry->d_name[i]); + } + + if (!isWalSegment) + { + continue; + } + + if (best[0] == '\0' || strcmp(entry->d_name, best) > 0) + { + strlcpy(best, entry->d_name, sizeof(best)); + } + } + + closedir(dir); + + if (best[0] == '\0') + { + return false; + } + + char tliHex[9] = { 0 }; + + memcpy(tliHex, best, 8); /* IGNORE-BANNED */ + *timeline = (int) strtol(tliHex, NULL, 16); + + return true; +} + + +/* + * service_archiver_serve_membership_config derives a membership's own + * KeeperConfig from the archiver-level template config, exactly the way + * service_archiver_reconciler.c's own build_membership_keeper() derives a + * membership's Keeper: same /// walcache + * subdirectory (templateConfig->pgSetup.pgdata is the archiver's root, not + * any one membership's own cache), and pathnames re-derived from it, so + * this process independently computes the identical paths the capture + * side is writing into without any IPC between the two. + */ +static bool +service_archiver_serve_membership_config(KeeperConfig *templateConfig, + const char *formation, + int groupId, + KeeperConfig *outConfig) +{ + *outConfig = *templateConfig; + + strlcpy(outConfig->formation, formation, sizeof(outConfig->formation)); + outConfig->groupId = groupId; + + sformat(outConfig->pgSetup.pgdata, sizeof(outConfig->pgSetup.pgdata), + "%s/%s/%d", templateConfig->pgSetup.pgdata, formation, groupId); + + return keeper_config_set_pathnames_from_pgdata(&(outConfig->pathnames), + outConfig->pgSetup.pgdata); +} + + +/* + * service_archiver_serve_write_route writes one "[formation/group]" section + * to an already-open routes file, for a single membership. Split out of + * service_archiver_serve_refresh_routes() so that function can call this + * once per membership this archiver holds -- an archiver serves every + * membership it is attached to through the one shared pg_walsender/routes + * file, unlike the capture side, which runs one process per membership + * (service_archiver_reconciler.c). + */ +static bool +service_archiver_serve_write_route(Monitor *monitor, KeeperConfig *config, + FILE *fileStream) +{ + char basebackupLocation[MAXPGPATH] = { 0 }; + char basebackupSource[NAMEDATALEN] = { 0 }; + int basebackupTimeline = 0; + bool found = false; + + /* + * preferredSource = "live": a "replay" base backup (basebackup_replay_ + * mode) promotes a throwaway extracted copy to make it self- + * consistent, which genuinely puts it on a *later* timeline than + * whatever the walcache itself has captured (which only ever advances + * on the real primary's own timeline). A real pg_basebackup rejects + * that combination outright once it reaches its own background WAL + * streaming step ("starting timeline N is not present in the server", + * receivelog.c comparing the backup's own timeline against IDENTIFY_ + * SYSTEM's -- and IDENTIFY_SYSTEM itself correctly reports the + * walcache's real captured timeline, see cmd_identify_system.c). A + * "live" backup is taken directly from the actively-followed primary, + * so it always shares the walcache's timeline by construction -- ask + * for one specifically rather than "whatever is newest regardless of + * type", which would otherwise serve an unusable pairing as soon as a + * newer 'replay' backup exists (get_latest_basebackup's own comment, + * pgautofailover.sql). + */ + if (!monitor_get_latest_basebackup_info(monitor, + config->formation, + config->groupId, + "live", + basebackupLocation, + sizeof(basebackupLocation), + basebackupSource, + sizeof(basebackupSource), + &basebackupTimeline, + &found)) + { + log_warn("Failed to fetch the latest base backup location from the " + "monitor for \"%s/%d\"; the routes file will omit it for now", + config->formation, config->groupId); + found = false; + } + + /* + * Defense in depth against the same mismatch, in case a future + * 'live'-sourced backup mode is ever added that doesn't actually + * guarantee walcache-timeline compatibility: never advertise a pairing + * we can independently tell apart, even though preferredSource = + * "live" above should already make this unreachable today. + */ + if (found) + { + int walcacheTimeline = 0; + + if (walcache_current_timeline(config->pgSetup.pgdata, &walcacheTimeline) && + walcacheTimeline != basebackupTimeline) + { + log_warn("The latest base backup for \"%s/%d\" is on timeline " + "%d, but the walcache is capturing timeline %d; " + "omitting the base backup from the routes file until " + "they match", + config->formation, config->groupId, + basebackupTimeline, walcacheTimeline); + found = false; + } + } + + uint64_t systemIdentifier = 0; + bool foundSystemIdentifier = false; + + if (!monitor_get_group_system_identifier(monitor, + config->formation, + config->groupId, + &systemIdentifier, + &foundSystemIdentifier)) + { + log_warn("Failed to fetch the system identifier for \"%s/%d\" from " + "the monitor; the routes file will omit it for now", + config->formation, config->groupId); + foundSystemIdentifier = false; + } + + fformat(fileStream, "[%s/%d]\n", config->formation, config->groupId); + fformat(fileStream, "walcache = %s\n", config->pgSetup.pgdata); + + /* + * The single, out-of-band-maintained "how far have I actually + * captured" value -- see service_archiver_update_current_lsn()'s own + * comment (service_archiver.c) for why pg_walsender should read this + * rather than re-derive it by scanning WAL file content itself: + * cmd_base_backup.c's own end-of-backup position, and cmd_identify_ + * system.c's own xlogpos, both prefer this route-file value when + * present, falling back to their own (WAL-cache-scanning) logic only + * when it's missing -- an older archiver-serve binary talking to a + * newer routes file, or vice versa, during a rolling upgrade. + * + * Read via service_archiver_read_current_lsn() rather than a Keeper's + * own postgres.currentLSN directly: this process (archiver-serve) and + * the one that actually maintains that value (archiver-capture, + * service_archiver.c, one per membership since service_archiver_ + * reconciler.c) are separate fork()ed processes (service_archiver_ + * run.c) with independent copies of their own Keeper struct after the + * fork -- no in-memory value in *this* process is ever updated by a + * sibling process's own writes. The position file is the real, + * re-read-every-refresh channel that actually crosses that boundary, + * and config here already carries this membership's own pathnames + * (service_archiver_serve_membership_config above), so it resolves to + * the right file regardless of how many memberships this archiver has. + */ + char currentLSN[PG_LSN_MAXLENGTH] = "0/0"; + + (void) service_archiver_read_current_lsn(config, currentLSN, sizeof(currentLSN)); + + fformat(fileStream, "position = %s\n", currentLSN); + + if (found) + { + fformat(fileStream, "basebackup = %s\n", basebackupLocation); + fformat(fileStream, "timeline = %d\n", basebackupTimeline); + } + + if (foundSystemIdentifier) + { + fformat(fileStream, "systemid = %" PRIu64 "\n", systemIdentifier); + } + + return true; +} + + +/* + * service_archiver_serve_refresh_routes writes one "[formation/group]" + * section per (formation, group) membership this archiver currently holds + * -- discovered fresh from the monitor every refresh, the same source of + * truth service_archiver_reconciler.c's own capture-side fan-out uses, so + * pg_walsender always has a route for every membership regardless of how + * many there are. Falls back to this archiver's own local config (the + * formation/group it was first created against) only when the monitor + * can't be reached or genuinely reports no memberships yet -- e.g. between + * `pg_autoctl create archiver` and its own first archiver_add_formation() + * completing. + */ +bool +service_archiver_serve_refresh_routes(Keeper *keeper) +{ + KeeperConfig *config = &(keeper->config); + + ArchiverMembershipArray membershipsArray = { 0 }; + + if (!monitor_list_archiver_memberships(&(keeper->monitor), + config->archiverId, + &membershipsArray)) + { + log_warn("Failed to list this archiver's memberships from the " + "monitor; the routes file will only cover \"%s/%d\" for now", + config->formation, config->groupId); + membershipsArray.count = 0; + } + + if (membershipsArray.count == 0) + { + strlcpy(membershipsArray.memberships[0].formation, config->formation, + sizeof(membershipsArray.memberships[0].formation)); + membershipsArray.memberships[0].groupId = config->groupId; + membershipsArray.count = 1; + } + + char routesPath[MAXPGPATH] = { 0 }; + + service_archiver_serve_routes_path(config, routesPath); + + char tmpPath[MAXPGPATH] = { 0 }; + + sformat(tmpPath, sizeof(tmpPath), "%s.tmp", routesPath); + + FILE *fileStream = fopen_with_umask(tmpPath, "w", FOPEN_FLAGS_W, 0644); + + if (fileStream == NULL) + { + /* errors have already been logged */ + return false; + } + + bool success = true; + + for (int i = 0; i < membershipsArray.count; i++) + { + ArchiverMembership *membership = &(membershipsArray.memberships[i]); + + KeeperConfig membershipConfig = { 0 }; + + if (!service_archiver_serve_membership_config(config, + membership->formation, + membership->groupId, + &membershipConfig) || + !service_archiver_serve_write_route(&(keeper->monitor), + &membershipConfig, + fileStream)) + { + log_warn("Failed to write the route for \"%s/%d\"", + membership->formation, membership->groupId); + success = false; + } + } + + if (fclose(fileStream) == EOF) + { + log_error("Failed to write file \"%s\": %m", tmpPath); + return false; + } + + if (rename(tmpPath, routesPath) != 0) + { + log_error("Failed to rename \"%s\" to \"%s\": %m", tmpPath, routesPath); + return false; + } + + log_debug("Refreshed archiver routes file \"%s\" (%d membership(s))", + routesPath, membershipsArray.count); + + return success; +} + + +bool +service_archiver_serve_loop(Keeper *keeper) +{ + log_info("pg_autoctl archiver serve: archiver %" PRId64 ", formation " + "\"%s\", group %d", + keeper->config.archiverId, keeper->config.formation, + keeper->config.groupId); + + if (!service_archiver_serve_refresh_routes(keeper)) + { + log_warn("Failed to write the initial routes file; pg_walsender " + "will start without one route resolved yet"); + } + + if (!service_archiver_serve_start_walsender(keeper)) + { + log_fatal("Failed to start pg_walsender, see above for details"); + return false; + } + + int tickCount = 0; + + for (;;) + { + if (asked_to_stop || asked_to_stop_fast || asked_to_quit) + { + break; + } + + if (asked_to_reload) + { + asked_to_reload = 0; + (void) service_archiver_serve_refresh_routes(keeper); + } + + /* + * SIGUSR1: a capture child just finished generating and reporting + * a base backup (service_archiver_maybe_generate_basebackup(), + * service_archiver_basebackup.c) and is prompting an immediate + * refresh rather than leaving pg_walsender to serve a stale route + * for up to ARCHIVER_SERVE_ROUTES_REFRESH_TICKS more ticks. + */ + if (asked_to_refresh_routes) + { + asked_to_refresh_routes = 0; + (void) service_archiver_serve_refresh_routes(keeper); + } + + if (!service_archiver_serve_walsender_is_running()) + { + log_warn("pg_walsender is not running anymore, restarting it"); + + if (!service_archiver_serve_start_walsender(keeper)) + { + log_error("Failed to restart pg_walsender, will retry on " + "the next tick"); + } + } + + if (tickCount > 0 && + tickCount % ARCHIVER_SERVE_ROUTES_REFRESH_TICKS == 0) + { + (void) service_archiver_serve_refresh_routes(keeper); + } + + sleep(ARCHIVER_SERVE_TICK_SECONDS); + ++tickCount; + } + + (void) service_archiver_serve_stop_walsender(); + + return true; +} diff --git a/src/bin/pg_autoctl/service_archiver_serve.h b/src/bin/pg_autoctl/service_archiver_serve.h new file mode 100644 index 000000000..7f801745a --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_serve.h @@ -0,0 +1,31 @@ +/* + * src/bin/pg_autoctl/service_archiver_serve.h + * Archiving & Disaster Recovery: supervision of the pg_walsender child + * process an archiver runs to serve its captured WAL and base backups to + * downstream consumers (warm standbies, PITR nodes, `create postgres + * --from-archiver` rebuilds) -- the inbound counterpart to + * service_archiver.c's outbound pg_receivewal supervision. See + * ~/dev/temp/archiving-disaster-recovery.md and + * src/bin/pg_walsender/walsender.h for the protocol this serves. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef SERVICE_ARCHIVER_SERVE_H +#define SERVICE_ARCHIVER_SERVE_H + +#include "keeper.h" + +void service_archiver_serve_set_port(int port); + +bool service_archiver_serve_start_walsender(Keeper *keeper); +bool service_archiver_serve_stop_walsender(void); +bool service_archiver_serve_walsender_is_running(void); + +bool service_archiver_serve_refresh_routes(Keeper *keeper); + +bool service_archiver_serve_loop(Keeper *keeper); + +#endif /* SERVICE_ARCHIVER_SERVE_H */ diff --git a/src/bin/pg_autoctl/state.c b/src/bin/pg_autoctl/state.c index c4df76266..4da841519 100644 --- a/src/bin/pg_autoctl/state.c +++ b/src/bin/pg_autoctl/state.c @@ -484,6 +484,11 @@ NodeStateToString(NodeState s) return "dropped"; } + case ARCHIVING_STATE: + { + return "archiving"; + } + case ANY_STATE: { return "#any state#"; @@ -592,6 +597,10 @@ NodeStateFromString(const char *str) { return DROPPED_STATE; } + else if (strcmp(str, "archiving") == 0) + { + return ARCHIVING_STATE; + } else { log_fatal("Failed to parse state string \"%s\"", str); diff --git a/src/bin/pg_autoctl/state.h b/src/bin/pg_autoctl/state.h index 92243347d..04dc02ffd 100644 --- a/src/bin/pg_autoctl/state.h +++ b/src/bin/pg_autoctl/state.h @@ -52,6 +52,7 @@ typedef enum FAST_FORWARD_STATE, JOIN_SECONDARY_STATE, DROPPED_STATE, + ARCHIVING_STATE, /* Allow some wildcard-matching transitions (from ANY state to) */ ANY_STATE = 128 diff --git a/src/bin/pg_autoctl/supervisor.c b/src/bin/pg_autoctl/supervisor.c index a6f02bf23..e9b6f9c0b 100644 --- a/src/bin/pg_autoctl/supervisor.c +++ b/src/bin/pg_autoctl/supervisor.c @@ -65,6 +65,8 @@ static bool supervisor_may_restart(Service *service); static bool supervisor_update_pidfile(Supervisor *supervisor); +static bool supervisor_wait_for_exit(pid_t pid, int maxWaitMs); + /* * supervisor_start starts given services as sub-processes and then supervise @@ -72,12 +74,36 @@ static bool supervisor_update_pidfile(Supervisor *supervisor); */ bool supervisor_start(Service services[], int serviceCount, const char *pidfile) +{ + return supervisor_start_with_callback(services, serviceCount, pidfile, + NULL, NULL); +} + + +/* + * supervisor_start_with_callback is supervisor_start()'s full + * implementation, with an optional periodic callback -- see + * Supervisor.periodicCallback's own comment (supervisor.h) for what it's + * for and the constraints it comes with. supervisor_start() itself is a + * thin wrapper passing NULL/NULL, so every existing caller is unaffected + * by this function's existence. + */ +bool +supervisor_start_with_callback(Service services[], int serviceCount, + const char *pidfile, + void (*periodicCallback)(Supervisor *supervisor, + void *context), + void *periodicCallbackContext) { int serviceIndex = 0; bool success = true; Supervisor supervisor = { services, serviceCount, { 0 }, -1 }; + supervisor.periodicCallback = periodicCallback; + supervisor.periodicCallbackContext = periodicCallbackContext; + supervisor.pendingSubprocessCount = serviceCount; + /* copy the pidfile over to our supervisor structure */ strlcpy(supervisor.pidfile, pidfile, MAXPGPATH); @@ -222,11 +248,10 @@ supervisor_start(Service services[], int serviceCount, const char *pidfile) static SupervisorExitMode supervisor_loop(Supervisor *supervisor) { - int subprocessCount = supervisor->serviceCount; bool firstLoop = true; /* wait until all subprocesses are done */ - while (subprocessCount > 0) + while (supervisor->pendingSubprocessCount > 0) { pid_t pid; int status; @@ -258,6 +283,18 @@ supervisor_loop(Supervisor *supervisor) */ (void) nodespec_watcher_check(&supervisor->watcher, &supervisor->watchedSpec); + + /* + * Optional caller-supplied periodic callback -- see + * Supervisor.periodicCallback's own comment (supervisor.h). + * A no-op for every caller except supervisor_start_with_ + * callback()'s own explicit users. + */ + if (supervisor->periodicCallback != NULL) + { + (void) supervisor->periodicCallback( + supervisor, supervisor->periodicCallbackContext); + } } /* ignore errors */ @@ -336,12 +373,12 @@ supervisor_loop(Supervisor *supervisor) } /* one child process is no more */ - --subprocessCount; + --supervisor->pendingSubprocessCount; /* apply the service restart policy */ if (supervisor_restart_service(supervisor, dead, status)) { - ++subprocessCount; + ++supervisor->pendingSubprocessCount; } break; @@ -1124,6 +1161,202 @@ supervisor_update_pidfile(Supervisor *supervisor) } +/* + * supervisor_wait_for_exit waits, up to maxWaitMs, for pid to actually be + * reaped (waitpid(WNOHANG) returning that exact pid, or ECHILD meaning it + * was already reaped elsewhere). Polls every 10ms; returns true as soon + * as the child is gone, false if it's still around once the deadline is + * reached. + */ +static bool +supervisor_wait_for_exit(pid_t pid, int maxWaitMs) +{ + int elapsedMs = 0; + + while (elapsedMs < maxWaitMs) + { + int status = 0; + pid_t reaped = waitpid(pid, &status, WNOHANG); + + if (reaped == pid) + { + return true; + } + + if (reaped == -1 && errno == ECHILD) + { + /* already reaped elsewhere -- fine, treat as done */ + return true; + } + + pg_usleep(10 * 1000); + elapsedMs += 10; + } + + return false; +} + + +/* + * supervisor_add_service adds a new service to an already-running + * supervisor, starts it, and updates the pidfile to include it. + * + * Requires supervisor->services to be a heap-allocated array -- see + * Supervisor.periodicCallback's own comment (supervisor.h) for why: this + * function reallocs it to grow by one slot. Only ever safe to call from + * a supervisor started via supervisor_start_with_callback() with its own + * heap-allocated initial array, never from a plain supervisor_start() + * caller's stack/static one. + */ +bool +supervisor_add_service(Supervisor *supervisor, Service service) +{ + int newCount = supervisor->serviceCount + 1; + Service *grown = realloc(supervisor->services, newCount * sizeof(Service)); + + if (grown == NULL) + { + log_error("Failed to allocate memory to add service \"%s\"", + service.name); + return false; + } + + supervisor->services = grown; + supervisor->services[supervisor->serviceCount] = service; + + Service *added = &(supervisor->services[supervisor->serviceCount]); + + log_debug("Starting pg_autoctl %s service", added->name); + + if (!(*added->startFunction)(added->context, &(added->pid))) + { + log_error("Failed to start service \"%s\"", added->name); + + /* undo the growth -- this slot never became real */ + Service *shrunk = realloc(supervisor->services, + supervisor->serviceCount * sizeof(Service)); + + if (shrunk != NULL) + { + supervisor->services = shrunk; + } + + return false; + } + + uint64_t now = time(NULL); + RestartCounters *counters = &(added->restartCounters); + + counters->count = 1; + counters->position = 0; + counters->startTime[counters->position] = now; + + log_info("Started pg_autoctl %s service with pid %d", + added->name, added->pid); + + supervisor->serviceCount = newCount; + supervisor->pendingSubprocessCount++; + + if (!supervisor_update_pidfile(supervisor)) + { + log_error("Failed to update pidfile \"%s\" after adding service \"%s\"", + supervisor->pidfile, added->name); + return false; + } + + return true; +} + + +/* + * supervisor_remove_service stops a currently-supervised service (found + * by pid) and removes it from the supervisor's own array, so it is no + * longer restarted on exit and no longer written to the pidfile. + * + * Sends `signal` (typically SIGTERM) and waits, briefly and boundedly, + * for the child to actually exit -- reaping it synchronously here rather + * than via supervisor_loop()'s own waitpid(WNOHANG) path, so the caller + * knows the removal is complete (and the slot genuinely reusable) by the + * time this returns, instead of racing the main loop's next iteration. + * A child still stuck after the wait is removed from supervision anyway + * (logged as a warning): whatever asked for this removal -- typically a + * membership that no longer exists -- has already decided this process + * shouldn't be tracked, stuck or not. + * + * Requires supervisor->services to be heap-allocated, same as + * supervisor_add_service() above. + */ +bool +supervisor_remove_service(Supervisor *supervisor, pid_t pid, int signal) +{ + Service *found = NULL; + + if (!supervisor_find_service(supervisor, pid, &found)) + { + log_error("Failed to remove service with pid %d: not found", pid); + return false; + } + + char name[NAMEDATALEN] = { 0 }; + + strlcpy(name, found->name, NAMEDATALEN); + int foundIndex = found - supervisor->services; + + if (kill(pid, signal) != 0 && errno != ESRCH) + { + log_error("Failed to send signal %s to service \"%s\" with pid %d: %m", + strsignal(signal), name, pid); + return false; + } + + if (!supervisor_wait_for_exit(pid, SUPERVISOR_REMOVE_SERVICE_MAX_WAIT_MS)) + { + log_warn("Service \"%s\" (pid %d) did not exit within %d ms of " + "signal %s; removing it from supervision anyway", + name, pid, SUPERVISOR_REMOVE_SERVICE_MAX_WAIT_MS, + strsignal(signal)); + } + + /* close the gap in the array, keeping it packed */ + for (int i = foundIndex; i < supervisor->serviceCount - 1; i++) + { + supervisor->services[i] = supervisor->services[i + 1]; + } + + supervisor->serviceCount--; + supervisor->pendingSubprocessCount--; + + if (supervisor->serviceCount > 0) + { + Service *shrunk = realloc(supervisor->services, + supervisor->serviceCount * sizeof(Service)); + + if (shrunk != NULL) + { + supervisor->services = shrunk; + } + + /* + * A failed shrink-realloc is harmless: the buffer is still valid + * and still holds every remaining service correctly, just larger + * than strictly needed -- keep using it as-is rather than fail + * the whole removal over it. + */ + } + + log_info("Removed service \"%s\" (was pid %d) from supervision", name, pid); + + if (!supervisor_update_pidfile(supervisor)) + { + log_error("Failed to update pidfile \"%s\" after removing service \"%s\"", + supervisor->pidfile, name); + return false; + } + + return true; +} + + /* * supervisor_find_service_pid reads the pidfile contents and process it line * by line to find the pid of the given service name. diff --git a/src/bin/pg_autoctl/supervisor.h b/src/bin/pg_autoctl/supervisor.h index 0cddf99d7..d80242d97 100644 --- a/src/bin/pg_autoctl/supervisor.h +++ b/src/bin/pg_autoctl/supervisor.h @@ -26,6 +26,15 @@ #define SERVICE_NAME_KEEPER "node-active" #define SERVICE_NAME_MONITOR "listener" +/* an archiver's two top-level halves, supervised together by + * start_archiver() (service_archiver_run.c) -- see that file's own header + * comment. "reconciler" in turn supervises one WAL-capture child per + * (formation, group) membership this archiver holds -- named + * "archiver-capture--" each, not a single fixed name, + * since there can be any number of them (service_archiver_reconciler.c). */ +#define SERVICE_NAME_ARCHIVER_SERVE "archiver-serve" +#define SERVICE_NAME_ARCHIVER_RECONCILER "archiver-reconciler" + /* * At pg_autoctl create time we use a transient service to initialize our local * node. When using the --run option, the transient service is terminated and @@ -74,6 +83,12 @@ typedef enum #define SUPERVISOR_SERVICE_MAX_RETRY 5 #define SUPERVISOR_SERVICE_MAX_TIME 300 /* in seconds */ +/* + * How long supervisor_remove_service() waits for a signalled service to + * actually exit before giving up and removing it from supervision anyway. + */ +#define SUPERVISOR_REMOVE_SERVICE_MAX_WAIT_MS 5000 + /* * We use a "ring buffer" of the MaxR most recent retries. * @@ -144,16 +159,63 @@ typedef struct Supervisor */ NodeSpecWatcher watcher; NodeSpec watchedSpec; /* last-applied spec — baseline for diff */ + + /* + * Optional periodic callback, invoked once per supervisor_loop() + * iteration -- the same cadence as the node spec watcher above (as + * often as every 100ms when otherwise idle, more often during child + * churn). NULL (the default, set via supervisor_start()) is a no-op. + * A caller that wants periodic work done at a coarser cadence than + * that -- the archiver reconciler's own membership-list polling, for + * instance, see service_archiver_reconciler.c -- is expected to + * track elapsed wall-clock time itself and mostly return + * immediately, the same way service_archiver_serve.c's own + * tick-counted routes refresh does at a different layer. Set via + * supervisor_start_with_callback() rather than supervisor_start(), + * so every existing caller is unaffected. + * + * This periodic callback is also the intended, and only supported, + * way to call supervisor_add_service()/supervisor_remove_service() + * below: both realloc `services`, which requires it to already be a + * heap-allocated array -- true only for a caller that built its own + * initial array that way before calling + * supervisor_start_with_callback(), never for the plain stack/static + * arrays every ordinary supervisor_start() caller passes in. + */ + void (*periodicCallback)(struct Supervisor *supervisor, void *context); + void *periodicCallbackContext; + + /* + * How many currently-tracked services are still expected to + * eventually report exiting before supervisor_loop() may return -- + * decremented as each one permanently exits, incremented when one is + * restarted instead. Used to be a plain local variable inside + * supervisor_loop() itself; promoted onto the struct so that + * supervisor_add_service()/supervisor_remove_service(), called from + * outside that function (via the periodic callback above), can keep + * it consistent too. + */ + int pendingSubprocessCount; } Supervisor; bool supervisor_start(Service services[], int serviceCount, const char *pidfile); +bool supervisor_start_with_callback(Service services[], int serviceCount, + const char *pidfile, + void (*periodicCallback)(Supervisor *supervisor, + void *context), + void *periodicCallbackContext); + bool supervisor_stop(Supervisor *supervisor); bool supervisor_find_service_pid(const char *pidfile, const char *serviceName, pid_t *pid); +bool supervisor_add_service(Supervisor *supervisor, Service service); + +bool supervisor_remove_service(Supervisor *supervisor, pid_t pid, int signal); + #endif /* SUPERVISOR_H */ diff --git a/src/bin/pg_autoctl/watch.c b/src/bin/pg_autoctl/watch.c index 9426d107d..c979f2e09 100644 --- a/src/bin/pg_autoctl/watch.c +++ b/src/bin/pg_autoctl/watch.c @@ -40,6 +40,7 @@ #include "pidfile.h" #include "state.h" #include "string_utils.h" +#include "system_utils.h" #include "watch.h" #include "watch_colspecs.h" @@ -51,6 +52,7 @@ static bool cli_watch_process_keys(WatchContext *context); static int print_watch_header(WatchContext *context, int r); static int print_watch_footer(WatchContext *context); static int print_nodes_array(WatchContext *context, int r, int c); +static int print_archivers_array(WatchContext *context, int r, int c); static int print_events_array(WatchContext *context, int r, int c); static void print_current_time(WatchContext *context, int r); @@ -249,6 +251,7 @@ cli_watch_update_from_monitor(WatchContext *context) { Monitor *monitor = &(context->monitor); CurrentNodeStateArray *nodesArray = &(context->nodesArray); + ArchiverInfoArray *archiversArray = &(context->archiversArray); MonitorEventsArray *eventsArray = &(context->eventsArray); /* @@ -268,6 +271,12 @@ cli_watch_update_from_monitor(WatchContext *context) return false; } + if (!monitor_get_archivers(monitor, context->formation, archiversArray)) + { + /* errors have already been logged */ + return false; + } + if (!monitor_get_formation_number_sync_standbys( monitor, context->formation, @@ -493,7 +502,20 @@ cli_watch_render(WatchContext *context, WatchContext *previous) int firstNodeRow = nodeHeaderRow + 1; int lastNodeRow = firstNodeRow + context->nodesArray.count - 1; - int eventHeaderRow = lastNodeRow + 2; /* blank line, evenzt headers */ + /* + * The archivers area only takes up screen space when there's at least + * one archiver attached to the formation -- an ordinary cluster with no + * archivers should look exactly like it did before this section existed + * (firstArchiverRow > lastArchiverRow makes it an empty range, which the + * area-selection cascade below naturally skips over). + */ + int archiverHeaderRow = lastNodeRow + 2; /* blank line, archiver headers */ + int firstArchiverRow = archiverHeaderRow + 1; + int lastArchiverRow = firstArchiverRow + context->archiversArray.count - 1; + + int eventHeaderRow = (context->archiversArray.count > 0) + ? lastArchiverRow + 2 + : lastNodeRow + 2; /* blank line, event headers */ int firstEventRow = eventHeaderRow + 1; int lastEventRow = firstEventRow + context->eventsArray.count - 1; @@ -513,9 +535,11 @@ cli_watch_render(WatchContext *context, WatchContext *previous) * that's part of the data: avoid empty separation lines, avoid header * lines. * - * We conceptually divide the screen in two areas: first, the nodes array - * area, and then the events area. When scrolling away from an area we may - * jump to the other area directly. + * We conceptually divide the screen in three areas: the nodes array + * area, the archivers area, and the events area. When scrolling away + * from an area we may jump to the other area directly -- area 2 + * (archivers) is skipped over entirely when there are no archivers to + * show (firstArchiverRow > lastArchiverRow). */ if (context->selectedArea == 1) { @@ -525,17 +549,46 @@ cli_watch_render(WatchContext *context, WatchContext *previous) } else if (context->selectedRow > lastNodeRow) { - context->selectedArea = 2; - context->selectedRow = firstEventRow; + if (context->archiversArray.count > 0) + { + context->selectedArea = 2; + context->selectedRow = firstArchiverRow; + } + else + { + context->selectedArea = 3; + context->selectedRow = firstEventRow; + } } } else if (context->selectedArea == 2) { - if (context->selectedRow < firstEventRow) + if (context->selectedRow < firstArchiverRow) { context->selectedArea = 1; context->selectedRow = lastNodeRow; } + else if (context->selectedRow > lastArchiverRow) + { + context->selectedArea = 3; + context->selectedRow = firstEventRow; + } + } + else if (context->selectedArea == 3) + { + if (context->selectedRow < firstEventRow) + { + if (context->archiversArray.count > 0) + { + context->selectedArea = 2; + context->selectedRow = lastArchiverRow; + } + else + { + context->selectedArea = 1; + context->selectedRow = lastNodeRow; + } + } else if (context->selectedRow > lastEventRow) { context->selectedRow = lastEventRow; @@ -556,6 +609,16 @@ cli_watch_render(WatchContext *context, WatchContext *previous) (void) clear_line_at(printedRows); + if (context->archiversArray.count > 0) + { + (void) clear_line_at(++printedRows); + + int archiverRows = print_archivers_array(context, archiverHeaderRow, 0); + printedRows += archiverRows; + + (void) clear_line_at(printedRows); + } + /* * Now print the events array. Because that operation is more expensive, * and because most of the times there is no event happening, we compare @@ -751,6 +814,96 @@ print_nodes_array(WatchContext *context, int r, int c) } +/* + * print_archivers_array prints one row per archiver attached to the current + * formation: name, host, region, its ARCHIVING membership's FSM state, and + * its most recently reported storage usage/free space. Unlike print_nodes_ + * array, this doesn't go through the ColPolicy width-matching machinery + * (watch_colspecs.h) -- six fixed-width columns is simple enough not to + * need it, and this section only ever appears at all when there's at least + * one archiver to show. + */ +#define ARCHIVER_NAME_COL_LEN 20 +#define ARCHIVER_HOST_COL_LEN 20 +#define ARCHIVER_REGION_COL_LEN 12 +#define ARCHIVER_STATE_COL_LEN 12 +#define ARCHIVER_SIZE_COL_LEN 10 + +static int +print_archivers_array(WatchContext *context, int r, int c) +{ + ArchiverInfoArray *archiversArray = &(context->archiversArray); + + int lines = 0; + int currentRow = r; + + clear_line_at(currentRow); + + attron(A_STANDOUT); + mvprintw(currentRow, c, "%-*s %-*s %-*s %-*s %*s %*s ", + ARCHIVER_NAME_COL_LEN, "Archiver Name", + ARCHIVER_HOST_COL_LEN, "Host", + ARCHIVER_REGION_COL_LEN, "Region", + ARCHIVER_STATE_COL_LEN, "State", + ARCHIVER_SIZE_COL_LEN, "Used", + ARCHIVER_SIZE_COL_LEN, "Free"); + attroff(A_STANDOUT); + + ++currentRow; + ++lines; + + for (int index = 0; index < archiversArray->count; index++) + { + ArchiverInfo *archiver = &(archiversArray->archivers[index]); + bool selected = currentRow == context->selectedRow; + + char usedStr[NAMEDATALEN] = "?"; + char freeStr[NAMEDATALEN] = "?"; + + if (archiver->hasStorageStats) + { + pretty_print_bytes(usedStr, sizeof(usedStr), archiver->usedBytes); + pretty_print_bytes(freeStr, sizeof(freeStr), archiver->freeBytes); + } + + const char *stateStr = + archiver->hasNode + ? NodeStateToString(archiver->reportedState) + : "?"; + + clear_line_at(currentRow); + + if (selected) + { + attron(A_REVERSE); + } + + mvprintw(currentRow, c, "%-*s %-*s %-*s %-*s %*s %*s ", + ARCHIVER_NAME_COL_LEN, archiver->archiverName, + ARCHIVER_HOST_COL_LEN, archiver->hostname, + ARCHIVER_REGION_COL_LEN, archiver->region, + ARCHIVER_STATE_COL_LEN, stateStr, + ARCHIVER_SIZE_COL_LEN, usedStr, + ARCHIVER_SIZE_COL_LEN, freeStr); + + if (selected) + { + attroff(A_REVERSE); + } + + ++currentRow; + ++lines; + + if (context->rows <= currentRow) + { + break; + } + } + + return lines; +} + + /* * pick_column_spec chooses which column spec should be used depending on the * current size (rows, cols) of the display, and given update column specs with diff --git a/src/bin/pg_autoctl/watch.h b/src/bin/pg_autoctl/watch.h index 0a2e1e748..f41bc1472 100644 --- a/src/bin/pg_autoctl/watch.h +++ b/src/bin/pg_autoctl/watch.h @@ -51,7 +51,8 @@ typedef struct WatchContext int rows; int cols; int selectedRow; - int selectedArea; /* area 1: node states, area 2: node events */ + int selectedArea; /* area 1: node states, area 2: archivers, + * area 3: node events */ int startCol; WatchMoveFocus move; @@ -69,6 +70,7 @@ typedef struct WatchContext /* data to display */ CurrentNodeStateArray nodesArray; + ArchiverInfoArray archiversArray; MonitorEventsArray eventsArray; MonitorEventsHeaders eventsHeaders; } WatchContext; diff --git a/src/bin/pg_walsender/Makefile b/src/bin/pg_walsender/Makefile new file mode 100644 index 000000000..79598500d --- /dev/null +++ b/src/bin/pg_walsender/Makefile @@ -0,0 +1,64 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the PostgreSQL License. +# +# pg_walsender -- the archiver's own replication-protocol server. Standalone +# binary: does NOT link any pg_autoctl/*.c, only src/bin/common/ and +# src/bin/lib/log/, so it can be exec'd and tested independently of +# pg_autoctl (see ~/dev/temp/archiving-disaster-recovery.md and +# src/bin/pg_autoctl/service_archiver.c's own "colocated fast path" note). + +PG_WALSENDER = ./pg_walsender + +SRC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +# Must be set before include so that targets in Makefile.common don't +# become the default goal when included before the all: rule below. +.DEFAULT_GOAL := all + +include $(SRC_DIR)../common/Makefile.common + +override CFLAGS += -I$(SRC_DIR) -I$(SRC_DIR)vendor + +# ----------------------------------------------------------------------- +# Sources that live in this directory +# ----------------------------------------------------------------------- +LOCAL_SRC = main.c accept_loop.c startup.c auth.c framing.c repl_command.c \ + routes.c cmd_identify_system.c cmd_show.c cmd_base_backup.c \ + tar_stream.c cmd_fetch_file.c fetch_client.c \ + cmd_timeline_history.c cmd_replication_slot.c \ + cmd_start_replication.c wal_dir_scan.c + +LOCAL_OBJS = $(patsubst %.c,%.o,$(LOCAL_SRC)) + +# ----------------------------------------------------------------------- +# Vendored PostgreSQL source (see vendor/tar.c's own header comment) -- +# compiled as vendor-%.o to keep it visually distinct from this project's +# own code. +# ----------------------------------------------------------------------- +VENDOR_SRC = tar.c +VENDOR_OBJS = $(patsubst %.c,vendor-%.o,$(VENDOR_SRC)) + +vendor-%.o: $(SRC_DIR)vendor/%.c + @if test ! -d $(DEPDIR); then mkdir -p $(DEPDIR); fi + $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/vendor-$(*F).Po -o $@ $< + +OBJS = $(LOCAL_OBJS) $(VENDOR_OBJS) +OBJS += lib-log.o lib-snprintf.o lib-strerror.o +OBJS += $(COMMON_LIB) + +INCLUDES = $(wildcard $(SRC_DIR)*.h) + +all: $(COMMON_LIB) $(PG_WALSENDER) ; + +$(PG_WALSENDER): $(OBJS) $(INCLUDES) + $(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) $(LIBS) -o $@ + +clean: + rm -f $(OBJS) $(PG_WALSENDER) + rm -rf $(DEPDIR) + +install: $(PG_WALSENDER) + install -d $(DESTDIR)$(BINDIR) + install -m 0755 $(PG_WALSENDER) $(DESTDIR)$(BINDIR) + +.PHONY: all clean install diff --git a/src/bin/pg_walsender/accept_loop.c b/src/bin/pg_walsender/accept_loop.c new file mode 100644 index 000000000..c2d357473 --- /dev/null +++ b/src/bin/pg_walsender/accept_loop.c @@ -0,0 +1,345 @@ +/* + * src/bin/pg_walsender/accept_loop.c + * See accept_loop.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "accept_loop.h" +#include "auth.h" +#include "cmd_fetch_file.h" +#include "defaults.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" +#include "repl_command.h" +#include "routes.h" +#include "signals.h" +#include "startup.h" + +/* dbname prefix that routes a connection to the FETCH_FILE side-channel + * instead of the normal replication command loop -- see cmd_fetch_file.h */ +#define WS_FETCH_DBNAME_PREFIX "fetch/" + +/* + * A real, unmodified Postgres standby's own internal walreceiver process + * (primary_conninfo-driven physical replication) always sends this literal + * string as its startup packet's dbname -- confirmed against a real + * standby: it does not forward whatever dbname the operator wrote into + * primary_conninfo the way a generic libpq client (psql, pg_receivewal, + * this project's own FETCH_FILE client) does. See the routeKey fallback + * below. + */ +#define WS_REAL_WALRECEIVER_DBNAME "replication" + + +static int +create_listen_socket(int port) +{ + int sock = socket(AF_INET, SOCK_STREAM, 0); + + if (sock < 0) + { + log_error("Failed to create the listening socket: %m"); + return -1; + } + + int reuse = 1; + + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + + struct sockaddr_in addr; + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = INADDR_ANY; + addr.sin_port = htons(port); + + if (bind(sock, (struct sockaddr *) &addr, sizeof(addr)) != 0) + { + log_error("Failed to bind port %d: %m", port); + close(sock); + return -1; + } + + if (listen(sock, 64) != 0) + { + log_error("Failed to listen on port %d: %m", port); + close(sock); + return -1; + } + + return sock; +} + + +/* + * handle_connection runs the full lifecycle of one accepted connection: + * startup negotiation, routes-based auth, the initial handshake messages a + * real client expects (AuthenticationOk/ParameterStatus/BackendKeyData/ + * ReadyForQuery), and then the simple-query command loop replication + * connections use (see pgsql.c's own comment elsewhere in this project: + * "extended query protocol not supported in a replication connection"). + * Runs entirely inside the forked child; the caller _exit()s right after. + */ +static void +handle_connection(int clientSock, const WsServerConfig *config) +{ + WsStartupParams params; + + if (!ws_startup_negotiate(clientSock, ¶ms)) + { + close(clientSock); + return; + } + + WsRoute *routes = NULL; + int routeCount = 0; + + if (config->routesPath[0] != '\0') + { + if (!routes_load(config->routesPath, &routes, &routeCount)) + { + close(clientSock); + return; + } + } + + bool isFetchMode = (strncmp(params.database, WS_FETCH_DBNAME_PREFIX, + strlen(WS_FETCH_DBNAME_PREFIX)) == 0); + const char *routeKey = isFetchMode + ? params.database + strlen(WS_FETCH_DBNAME_PREFIX) + : params.database; + + /* + * dbname-based routing cannot work for a real walreceiver connection + * (see WS_REAL_WALRECEIVER_DBNAME's own comment) -- fall back to the + * single configured route unambiguously, matching this milestone's own + * one-membership-per-archiver scope. Multiple routes with a real + * walreceiver connecting is left as a clean auth rejection (routeKey + * stays "replication", which never matches a real route.key) rather + * than guessing; a multi-route archiver needs a different mechanism + * for a real standby to identify its route (e.g. application_name, + * which real walreceiver does forward from primary_conninfo, unlike + * dbname) -- a later milestone's problem, not this one's. + */ + if (!isFetchMode && + strcmp(routeKey, WS_REAL_WALRECEIVER_DBNAME) == 0 && + routeCount == 1) + { + routeKey = routes[0].key; + } + + const WsRoute *route = NULL; + + if (!ws_authenticate(clientSock, ¶ms, routeKey, routes, routeCount, &route)) + { + routes_free(routes); + close(clientSock); + return; + } + + char title[256]; + + sformat(title, sizeof(title), "pg_autoctl: walsender %s%s", + isFetchMode ? "fetch " : "", route != NULL ? route->key : routeKey); + set_ps_title(title); + + if (isFetchMode) + { + cmd_fetch_file(clientSock, route); + routes_free(routes); + close(clientSock); + return; + } + + if (!ws_send_authentication_ok(clientSock) || + !ws_send_parameter_status(clientSock, "server_version", WS_SERVER_VERSION) || + !ws_send_parameter_status(clientSock, "client_encoding", "UTF8") || + !ws_send_parameter_status(clientSock, "server_encoding", "UTF8") || + !ws_send_parameter_status(clientSock, "integer_datetimes", "on") || + !ws_send_parameter_status(clientSock, "default_transaction_read_only", "off") || + !ws_send_backend_key_data(clientSock, getpid(), 0) || + !ws_send_ready_for_query(clientSock)) + { + routes_free(routes); + close(clientSock); + return; + } + + for (;;) + { + char type; + char *payload = NULL; + int32_t payloadLen = 0; + + if (!ws_read_message(clientSock, &type, &payload, &payloadLen)) + { + free(payload); + break; + } + + if (type == 'X') /* Terminate */ + { + free(payload); + break; + } + + if (type != 'Q') /* Query -- the only message replication + * connections send commands through */ + { + ws_send_error_response(clientSock, "08P01", + "pg_walsender only accepts simple query " + "protocol messages"); + free(payload); + break; + } + + WsCommand cmd; + + if (!repl_command_parse(payload, &cmd)) + { + ws_send_error_response(clientSock, "42601", + "unrecognized replication command"); + } + else + { + ws_dispatch_command(clientSock, &cmd, route, + params.replicationDatabase ? params.database : NULL); + } + + free(payload); + + if (!ws_send_ready_for_query(clientSock)) + { + break; + } + } + + routes_free(routes); + close(clientSock); +} + + +bool +ws_accept_loop(const WsServerConfig *config) +{ + int listenSock = create_listen_socket(config->port); + + if (listenSock < 0) + { + return false; + } + + /* + * Auto-reap forked children: SIGCHLD/SIG_IGN is enough here since we + * never need a child's exit status, only that it not linger as a + * zombie -- simpler than an explicit waitpid(WNOHANG) loop. + */ + signal(SIGCHLD, SIG_IGN); + + set_signal_handlers(false); + + log_info("pg_walsender listening on port %d%s%s", + config->port, + config->routesPath[0] != '\0' ? ", routes " : " (no routes file)", + config->routesPath[0] != '\0' ? config->routesPath : ""); + + while (!asked_to_stop && !asked_to_stop_fast) + { + /* + * pqsignal() (signals.c, via postgres_fe.h) installs our handlers + * with SA_RESTART, so a blocking accept() is never interrupted by + * SIGTERM -- it would just keep sleeping through shutdown forever. + * Poll with a short timeout instead, so the loop condition above + * gets re-checked promptly after asked_to_stop is set. + */ + fd_set readSet; + + FD_ZERO(&readSet); + FD_SET(listenSock, &readSet); + + struct timeval timeout = { 1, 0 }; /* 1 second */ + + int selectRet = select(listenSock + 1, &readSet, NULL, NULL, &timeout); + + if (selectRet < 0) + { + if (errno == EINTR) + { + continue; + } + + log_error("select() failed: %m"); + continue; + } + + if (selectRet == 0) + { + /* timed out, no pending connection -- loop back to the + * asked_to_stop check above */ + continue; + } + + struct sockaddr_storage clientAddr; + socklen_t clientAddrLen = sizeof(clientAddr); + + int clientSock = accept(listenSock, + (struct sockaddr *) &clientAddr, + &clientAddrLen); + + if (clientSock < 0) + { + if (errno == EINTR) + { + continue; + } + + log_error("accept() failed: %m"); + continue; + } + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("fork() failed: %m"); + close(clientSock); + continue; + } + + if (pid == 0) + { + /* + * Child: no exec(), just call straight into the connection + * handler -- matches real Postgres's BackendMain() model (see + * the design doc's "Process model" section). + */ + close(listenSock); + handle_connection(clientSock, config); + _exit(0); + } + + /* parent: keep accepting; SIGCHLD/SIG_IGN reaps the child for us */ + close(clientSock); + } + + close(listenSock); + log_info("pg_walsender shutting down"); + + return true; +} diff --git a/src/bin/pg_walsender/accept_loop.h b/src/bin/pg_walsender/accept_loop.h new file mode 100644 index 000000000..c3e5b1f1f --- /dev/null +++ b/src/bin/pg_walsender/accept_loop.h @@ -0,0 +1,29 @@ +/* + * src/bin/pg_walsender/accept_loop.h + * The bare accept loop: socket()/bind()/listen()/accept(), fork() + * per connection with no exec() (matching real Postgres's + * BackendStartup()/BackendMain() model for cheap concurrency -- see + * the design doc's "Process model" section), each forked child running + * the full startup/auth/command-loop for exactly one connection. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_ACCEPT_LOOP_H +#define WS_ACCEPT_LOOP_H + +#include + +#include "postgres_fe.h" + +typedef struct WsServerConfig +{ + int port; + char routesPath[MAXPGPATH]; /* empty: no routing, manual-testing mode */ +} WsServerConfig; + +bool ws_accept_loop(const WsServerConfig *config); + +#endif /* WS_ACCEPT_LOOP_H */ diff --git a/src/bin/pg_walsender/auth.c b/src/bin/pg_walsender/auth.c new file mode 100644 index 000000000..a8cd74198 --- /dev/null +++ b/src/bin/pg_walsender/auth.c @@ -0,0 +1,101 @@ +/* + * src/bin/pg_walsender/auth.c + * See auth.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include + +#include "postgres_fe.h" + +#include "auth.h" +#include "defaults.h" +#include "framing.h" +#include "log.h" + + +static bool +ws_get_peer_ip(int sock, char *ipBuf, size_t ipBufSize) +{ + struct sockaddr_storage addr; + socklen_t addrLen = sizeof(addr); + + if (getpeername(sock, (struct sockaddr *) &addr, &addrLen) != 0) + { + log_error("Failed to getpeername() on the accepted connection: %m"); + return false; + } + + if (getnameinfo((struct sockaddr *) &addr, addrLen, + ipBuf, ipBufSize, NULL, 0, NI_NUMERICHOST) != 0) + { + log_error("Failed to resolve the peer's numeric address: %m"); + return false; + } + + return true; +} + + +bool +ws_authenticate(int sock, const WsStartupParams *params, const char *routeKey, + const WsRoute *routes, int routeCount, + const WsRoute **foundRoute) +{ + *foundRoute = NULL; + + if (strcmp(params->user, PG_AUTOCTL_REPLICA_USERNAME) != 0) + { + log_warn("Rejecting connection for unknown user \"%s\"", params->user); + ws_send_error_response(sock, "28000", + "role is not permitted to connect to pg_walsender"); + return false; + } + + if (routeCount == 0) + { + /* + * No routes file was supplied at all: manual/standalone testing + * mode, accept unconditionally now that the role matched. A real + * deployment always passes --routes (see main.c), so this branch + * never applies to a pg_autoctl-supervised pg_walsender. + */ + return true; + } + + const WsRoute *route = routes_find(routes, routeCount, routeKey); + + if (route == NULL) + { + log_warn("Rejecting connection for unknown route \"%s\"", routeKey); + ws_send_error_response(sock, "3D000", + "unknown formation/group requested as dbname"); + return false; + } + + char peerIP[NI_MAXHOST]; + + if (!ws_get_peer_ip(sock, peerIP, sizeof(peerIP))) + { + ws_send_error_response(sock, "08000", "failed to identify peer address"); + return false; + } + + if (!routes_host_allowed(route, peerIP)) + { + log_warn("Rejecting connection from %s: not in the allowed_hosts list " + "for route \"%s\"", peerIP, route->key); + ws_send_error_response(sock, "28000", + "no pg_hba.conf-equivalent entry for this host"); + return false; + } + + *foundRoute = route; + + return true; +} diff --git a/src/bin/pg_walsender/auth.h b/src/bin/pg_walsender/auth.h new file mode 100644 index 000000000..197ed0a4e --- /dev/null +++ b/src/bin/pg_walsender/auth.h @@ -0,0 +1,44 @@ +/* + * src/bin/pg_walsender/auth.h + * Trust-equivalent authentication, matching this project's existing + * convention: no password/SCRAM infrastructure exists anywhere in + * pg_auto_failover today (pghba.c installs plain "trust" entries for the + * replicator role, defaults.h's REPLICATION_PASSWORD_DEFAULT is NULL). + * pg_walsender mirrors that: accept iff the startup packet's user is the + * replicator role and, when the resolved route carries an allowed_hosts + * list, the peer address matches -- routes.c's allowed_hosts is + * effectively pg_walsender's own pg_hba.conf, since it has no PGDATA of + * its own to carry a real one. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_AUTH_H +#define WS_AUTH_H + +#include + +#include "walsender.h" +#include "routes.h" + +/* + * ws_authenticate checks params against the replicator username and, if + * routes/routeCount is non-empty, against the route matching routeKey and + * its allowed_hosts. routeKey is passed explicitly rather than read from + * params->database because the FETCH_FILE side-channel (see + * cmd_fetch_file.h) reuses this same auth path with a "fetch/" prefix + * stripped off the connection's actual dbname -- the caller (accept_loop.c) + * decides what routeKey means, this function only ever looks it up. On + * success returns true and sets *foundRoute (NULL when routes were not + * supplied at all -- a manual-testing convenience, see main.c's --routes + * option). On failure, an ErrorResponse has already been sent to sock; the + * caller only needs to close the connection. + */ +bool ws_authenticate(int sock, const WsStartupParams *params, + const char *routeKey, + const WsRoute *routes, int routeCount, + const WsRoute **foundRoute); + +#endif /* WS_AUTH_H */ diff --git a/src/bin/pg_walsender/cmd_base_backup.c b/src/bin/pg_walsender/cmd_base_backup.c new file mode 100644 index 000000000..d916e996f --- /dev/null +++ b/src/bin/pg_walsender/cmd_base_backup.c @@ -0,0 +1,666 @@ +/* + * src/bin/pg_walsender/cmd_base_backup.c + * See cmd_base_backup.h. + * + * Wire sequence for a successful, synchronous BASE_BACKUP (traced from + * basebackup_copy.c's bbsink_copystream_* callbacks and cross-checked + * against the exact PQgetResult() loop in pg_basebackup.c around its own + * "BASE_BACKUP" psprintf call -- both in + * /Users/dim/dev/PostgreSQL/postgresql): + * + * 1. RowDescription(recptr text, tli int8) + DataRow + CommandComplete + * "SELECT" -- the start position + * 2. RowDescription(spcoid oid, spclocation text, size int8) + + * DataRow(NULL, NULL, NULL) + CommandComplete "SELECT" -- one row, + * the base directory itself (path NULL means "not a tablespace") + * 3. CopyOutResponse(format 0, natts 0) + * 4. CopyData['n', "base.tar\0", "\0"] -- PqBackupMsg_NewArchive + * 5. CopyData['d', ] x N -- PqMsg_CopyData + * 6. CopyDone + * 7. RowDescription(recptr text, tli int8) + DataRow + CommandComplete + * "SELECT" -- the end position + * 8. CommandComplete "BASE_BACKUP" -- EndReplicationCommand + * + * pg_basebackup.c calls PQgetResult() exactly four times for this (steps + * 1, 2, [3-6 consumed internally by ReceiveArchiveStream], 7, 8), and + * explicitly checks step 8's PQresultStatus() == PGRES_COMMAND_OK. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include + +#include "postgres_fe.h" + +#include "pqexpbuffer.h" + +#include "cmd_base_backup.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" +#include "string_utils.h" +#include "tar_stream.h" + +typedef struct BaseBackupOptions +{ + char label[256]; + bool sendWal; + bool manifestRequested; + bool compressionRequested; + char target[64]; +} BaseBackupOptions; + + +/* + * scan_options tolerantly parses the BASE_BACKUP option list real + * pg_basebackup sends, e.g.: + * LABEL 'pg_basebackup base backup', CHECKPOINT 'fast', TARGET 'client' + * Options this MVP doesn't act on (PROGRESS, CHECKPOINT, WAIT, MAX_RATE, + * TABLESPACE_MAP, VERIFY_CHECKSUMS, MANIFEST_CHECKSUMS) are recognized and + * ignored rather than rejected -- only WAL/MANIFEST/COMPRESSION/a non- + * "client" TARGET actually change behavior (see cmd_base_backup()'s own + * validation right after calling this). + */ +static void +scan_options(const char *raw, BaseBackupOptions *opts) +{ + memset(opts, 0, sizeof(BaseBackupOptions)); + + const char *p = raw; + + while (*p) + { + while (isspace((unsigned char) *p) || *p == ',' || *p == '(' || *p == ')') + { + p++; + } + + if (*p == '\0') + { + break; + } + + const char *keyStart = p; + + while (*p && !isspace((unsigned char) *p) && *p != ',' && *p != ')') + { + p++; + } + + char key[64]; + size_t keyLen = Min((size_t) (p - keyStart), sizeof(key) - 1); + + memcpy(key, keyStart, keyLen); /* IGNORE-BANNED */ + key[keyLen] = '\0'; + + while (isspace((unsigned char) *p)) + { + p++; + } + + char value[512] = { 0 }; + + if (*p == '\'') + { + p++; + + char *out = value; + char *outEnd = value + sizeof(value) - 1; + + while (*p && !(*p == '\'' && p[1] != '\'')) + { + if (*p == '\'' && p[1] == '\'') + { + if (out < outEnd) + { + *out++ = '\''; + } + p += 2; + continue; + } + + if (out < outEnd) + { + *out++ = *p; + } + + p++; + } + + *out = '\0'; + + if (*p == '\'') + { + p++; + } + } + else if (*p && *p != ',' && *p != ')') + { + const char *valStart = p; + + while (*p && *p != ',' && *p != ')' && !isspace((unsigned char) *p)) + { + p++; + } + + size_t valLen = Min((size_t) (p - valStart), sizeof(value) - 1); + + memcpy(value, valStart, valLen); /* IGNORE-BANNED */ + value[valLen] = '\0'; + } + + if (strcasecmp(key, "LABEL") == 0) + { + strlcpy(opts->label, value, sizeof(opts->label)); + } + else if (strcasecmp(key, "WAL") == 0) + { + opts->sendWal = true; + } + else if (strcasecmp(key, "MANIFEST") == 0) + { + /* pg_basebackup only ever sends this key when it wants one + * ("yes"/"force-encode"); --no-manifest omits it entirely */ + opts->manifestRequested = true; + } + else if (strcasecmp(key, "TARGET") == 0) + { + strlcpy(opts->target, value, sizeof(opts->target)); + } + else if (strcasecmp(key, "COMPRESSION") == 0) + { + opts->compressionRequested = true; + } + + while (isspace((unsigned char) *p) || *p == ',') + { + p++; + } + } +} + + +/* + * read_backup_label extracts the "START WAL LOCATION" and "START TIMELINE" + * fields real pg_basebackup already wrote into basebackupDir/backup_label + * when the archiver originally took this backup (see cmd_base_backup.h's + * own header comment: do_pg_backup_start() is never called here, this file + * already exists on disk). Returns false (caller falls back to the + * route's own systemid/timeline, "0/0" for the LSN) if the file is + * missing or doesn't parse -- a base backup taken by a later milestone's + * own machinery is expected to always have one. + */ +static bool +read_backup_label(const char *basebackupDir, char *lsnOut, size_t lsnOutSize, + int *timelineOut) +{ + char path[MAXPGPATH]; + + sformat(path, sizeof(path), "%s/backup_label", basebackupDir); + + char *contents = NULL; + long fileSize = 0; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + return false; + } + + bool foundLsn = false; + bool foundTimeline = false; + char *line = contents; + + while (line != NULL && *line != '\0') + { + char *nl = strchr(line, '\n'); + + if (nl != NULL) + { + *nl = '\0'; + } + + const char *lsnPrefix = "START WAL LOCATION: "; + const char *tliPrefix = "START TIMELINE: "; + + if (strncmp(line, lsnPrefix, strlen(lsnPrefix)) == 0) + { + const char *value = line + strlen(lsnPrefix); + const char *end = value; + + while (*end && !isspace((unsigned char) *end)) + { + end++; + } + + size_t len = Min((size_t) (end - value), lsnOutSize - 1); + + memcpy(lsnOut, value, len); /* IGNORE-BANNED */ + lsnOut[len] = '\0'; + foundLsn = true; + } + else if (strncmp(line, tliPrefix, strlen(tliPrefix)) == 0) + { + foundTimeline = stringToInt(line + strlen(tliPrefix), timelineOut); + } + + line = (nl != NULL) ? nl + 1 : NULL; + } + + free(contents); + + return foundLsn && foundTimeline; +} + + +typedef struct TarStreamCbContext +{ + int sock; + bool ok; +} TarStreamCbContext; + + +static bool +tar_chunk_cb(void *context, const char *data, size_t len) +{ + TarStreamCbContext *ctx = (TarStreamCbContext *) context; + PQExpBuffer buf = createPQExpBuffer(); + + appendPQExpBufferChar(buf, 'd'); /* PqMsg_CopyData content tag */ + appendBinaryPQExpBuffer(buf, data, len); + + bool ok = !PQExpBufferBroken(buf) && + ws_send_copy_data(ctx->sock, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + if (!ok) + { + ctx->ok = false; + } + + return ok; +} + + +static bool +send_position_row(int sock, const char *lsn, const char *tli) +{ + WsColumn columns[] = { + { "recptr", WS_TEXTOID, -1 }, + { "tli", WS_INT8OID, 8 }, + }; + + const char *values[] = { lsn, tli }; + + return ws_send_row_description(sock, columns, 2) && + ws_send_data_row(sock, values, 2) && + ws_send_command_complete(sock, "SELECT"); +} + + +/* + * find_reachable_end_position and its helpers below compute a base + * backup's "end of backup" position -- see this file's own header comment + * for where that fits in the wire sequence, and cmd_base_backup()'s own + * call site for why it must be a real, currently-reachable target rather + * than a stale re-send of the start position. + * + * Deliberately not pg_walsender/wal_dir_scan.c's own wal_dir_find_latest() + * (this project doesn't share code across its own binaries, see this + * file's own precedent of small, self-contained helpers): that function + * only ever considers a *complete* (non-".partial") segment, which is the + * right, conservative choice for IDENTIFY_SYSTEM/CREATE_REPLICATION_SLOT's + * own "confirmed durable" needs, but wrong here -- an archiver whose only + * WAL activity so far is still sitting in the current ".partial" segment + * (a real, common case: nothing has forced a segment switch yet) would + * make wal_dir_find_latest() report "nothing captured", sending BASE_ + * BACKUP straight back to the same stale start-of-backup fallback this + * whole mechanism exists to avoid. The archiver's walcache always has + * *something* real captured by the time a base backup exists at all + * (pg_receivewal streams from the moment archiving starts); the position + * within the current in-progress segment is exactly as reachable via + * START_REPLICATION as a completed one, once its zero-padded unwritten + * tail (pg_receivewal's own pre-allocation, matching real Postgres's + * XLogFileInitInternal) is trimmed off -- the same trim_trailing_zeros() + * logic cmd_start_replication.c already applies when actually serving it, + * applied here once, up front, to find where its real content ends. + */ +#define CBB_WAL_SEGMENT_SIZE UINT64CONST(0x1000000) +#define CBB_XLOG_SEGMENTS_PER_XLOGID (UINT64CONST(0x100000000) / CBB_WAL_SEGMENT_SIZE) +#define CBB_WAL_FNAME_LEN 24 + + +static bool +is_wal_segment_filename(const char *name) +{ + size_t len = strlen(name); + + if (len != CBB_WAL_FNAME_LEN) + { + return false; + } + + for (size_t i = 0; i < len; i++) + { + if (!isxdigit((unsigned char) name[i])) + { + return false; + } + } + + return true; +} + + +static bool +partial_segment_real_length(const char *path, uint64_t *length) +{ + FILE *file = fopen(path, "rb"); /* IGNORE-BANNED */ + + if (file == NULL) + { + return false; + } + + char *buffer = malloc(CBB_WAL_SEGMENT_SIZE); + + if (buffer == NULL) + { + fclose(file); + return false; + } + + size_t got = fread(buffer, 1, CBB_WAL_SEGMENT_SIZE, file); + + fclose(file); + + while (got > 0 && buffer[got - 1] == 0) + { + got--; + } + + free(buffer); + + *length = (uint64_t) got; + + return true; +} + + +static bool +find_reachable_end_position(const char *walcacheDir, uint32_t *timeline, + char *endLsn, size_t endLsnSize) +{ + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + return false; + } + + char bestComplete[CBB_WAL_FNAME_LEN + 1] = { 0 }; + char bestPartial[CBB_WAL_FNAME_LEN + 1] = { 0 }; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + if (is_wal_segment_filename(entry->d_name)) + { + if (bestComplete[0] == '\0' || strcmp(entry->d_name, bestComplete) > 0) + { + strlcpy(bestComplete, entry->d_name, sizeof(bestComplete)); + } + + continue; + } + + const char *partialSuffix = ".partial"; + size_t nameLen = strlen(entry->d_name); + size_t suffixLen = strlen(partialSuffix); + + if (nameLen == CBB_WAL_FNAME_LEN + suffixLen && + strcmp(entry->d_name + CBB_WAL_FNAME_LEN, partialSuffix) == 0) + { + char segPart[CBB_WAL_FNAME_LEN + 1] = { 0 }; + + memcpy(segPart, entry->d_name, CBB_WAL_FNAME_LEN); /* IGNORE-BANNED */ + + if (is_wal_segment_filename(segPart) && + (bestPartial[0] == '\0' || strcmp(segPart, bestPartial) > 0)) + { + strlcpy(bestPartial, segPart, sizeof(bestPartial)); + } + } + } + + closedir(dir); + + /* + * The current frontier is whichever of the two is numerically later -- + * a ".partial" file only ever exists for the segment actively being + * written, always the same as or newer than the newest complete one. + */ + bool usePartial = bestPartial[0] != '\0' && + (bestComplete[0] == '\0' || + strcmp(bestPartial, bestComplete) >= 0); + + const char *chosen = usePartial ? bestPartial : bestComplete; + + if (chosen[0] == '\0') + { + return false; + } + + char tliHex[9] = { 0 }; + char logIdHex[9] = { 0 }; + char segHex[9] = { 0 }; + + memcpy(tliHex, chosen, 8); /* IGNORE-BANNED */ + memcpy(logIdHex, chosen + 8, 8); /* IGNORE-BANNED */ + memcpy(segHex, chosen + 16, 8); /* IGNORE-BANNED */ + + uint32_t tli = (uint32_t) strtoul(tliHex, NULL, 16); + uint32_t logId = (uint32_t) strtoul(logIdHex, NULL, 16); + uint32_t seg = (uint32_t) strtoul(segHex, NULL, 16); + + uint64_t segno = (uint64_t) logId * CBB_XLOG_SEGMENTS_PER_XLOGID + seg; + uint64_t segStart = segno * CBB_WAL_SEGMENT_SIZE; + uint64_t position; + + if (usePartial) + { + char path[MAXPGPATH]; + uint64_t realLength = 0; + + sformat(path, sizeof(path), "%s/%s.partial", walcacheDir, bestPartial); + + if (!partial_segment_real_length(path, &realLength)) + { + return false; + } + + position = segStart + realLength; + } + else + { + position = segStart + CBB_WAL_SEGMENT_SIZE; + } + + *timeline = tli; + sformat(endLsn, endLsnSize, "%X/%08X", + (uint32_t) (position >> 32), (uint32_t) (position & 0xFFFFFFFF)); + + return true; +} + + +void +cmd_base_backup(int sock, const WsRoute *route, const char *rawOptions) +{ + if (route == NULL || route->basebackupDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no base backup configured for this route " + "(the archiver hasn't taken one yet, or this " + "route wasn't given a basebackup directory)"); + return; + } + + BaseBackupOptions opts; + + scan_options(rawOptions, &opts); + + if (opts.sendWal) + { + ws_send_error_response(sock, "0A000", + "WAL-inclusive BASE_BACKUP is not supported " + "yet -- retry with pg_basebackup's -X none"); + return; + } + + if (opts.manifestRequested) + { + ws_send_error_response(sock, "0A000", + "backup manifests are not supported yet -- " + "retry with pg_basebackup's --no-manifest"); + return; + } + + if (opts.compressionRequested) + { + ws_send_error_response(sock, "0A000", + "server-side compression is not supported yet"); + return; + } + + if (opts.target[0] != '\0' && strcasecmp(opts.target, "client") != 0) + { + ws_send_error_response(sock, "0A000", + "only the default client-streaming BASE_BACKUP " + "target is supported"); + return; + } + + char lsn[32] = "0/0"; + int timeline = (route->timeline > 0) ? route->timeline : 1; + + if (!read_backup_label(route->basebackupDir, lsn, sizeof(lsn), &timeline)) + { + log_warn("No parseable backup_label under \"%s\"; reporting a " + "placeholder start position", route->basebackupDir); + } + + char tliStr[16]; + + sformat(tliStr, sizeof(tliStr), "%d", timeline); + + if (!send_position_row(sock, lsn, tliStr)) + { + return; + } + + WsColumn tsColumns[] = { + { "spcoid", WS_INT4OID, 4 }, + { "spclocation", WS_TEXTOID, -1 }, + { "size", WS_INT8OID, 8 }, + }; + + const char *tsValues[] = { NULL, NULL, NULL }; + + if (!ws_send_row_description(sock, tsColumns, 3) || + !ws_send_data_row(sock, tsValues, 3) || + !ws_send_command_complete(sock, "SELECT")) + { + return; + } + + if (!ws_send_copy_out_response(sock, 0)) + { + return; + } + + { + PQExpBuffer buf = createPQExpBuffer(); + + appendPQExpBufferChar(buf, 'n'); /* PqBackupMsg_NewArchive */ + appendBinaryPQExpBuffer(buf, "base.tar", strlen("base.tar") + 1); + appendBinaryPQExpBuffer(buf, "", 1); /* empty path: not a tablespace */ + + bool ok = !PQExpBufferBroken(buf) && + ws_send_copy_data(sock, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + if (!ok) + { + return; + } + } + + TarStreamCbContext ctx = { sock, true }; + + if (!tar_stream_directory(route->basebackupDir, tar_chunk_cb, &ctx) || !ctx.ok) + { + log_error("Failed to stream base backup tar contents from \"%s\"", + route->basebackupDir); + return; + } + + if (!ws_send_copy_done(sock)) + { + return; + } + + /* + * The end-of-backup position must be a real, currently-reachable target + * -- re-sending the same (potentially long-stale) start position here + * would tell a real pg_basebackup's own background WAL streamer + * (--wal-method=stream) to wait for a target it may have already + * passed hours ago, or, worse, one from a since-pruned segment it can + * never reach; either way its background thread hangs the whole + * command forever waiting on a position that will never legitimately + * arrive as "new" data. + * + * route->position is the canonical, out-of-band-maintained value -- + * see service_archiver_update_current_lsn()'s own comment (pg_autoctl's + * service_archiver.c) for why the archiver-serve supervisor computes + * this once, itself, and writes it into the routes file, rather than + * every reader (this one included) independently re-deriving it by + * scanning WAL file content on its own. find_reachable_end_position() + * (this file's own comment) is the fallback for a route that doesn't + * carry one yet (an older archiver-serve binary against a newer pg_ + * walsender, during a rolling upgrade) -- still a real, reachable + * position, just independently re-derived. Falls back further still to + * the start position only if the walcache is completely empty (no base + * backup should exist at all in that case). + */ + char endLsn[32]; + uint32_t endTimeline; + const char *endLsnPtr = lsn; + const char *endTliStr = tliStr; + char endTliBuf[16]; + + if (route->position[0] != '\0') + { + endLsnPtr = route->position; + endTliStr = tliStr; + } + else if (find_reachable_end_position(route->walcacheDir, &endTimeline, endLsn, + sizeof(endLsn))) + { + sformat(endTliBuf, sizeof(endTliBuf), "%u", endTimeline); + endLsnPtr = endLsn; + endTliStr = endTliBuf; + } + + if (!send_position_row(sock, endLsnPtr, endTliStr)) + { + return; + } + + ws_send_command_complete(sock, "BASE_BACKUP"); +} diff --git a/src/bin/pg_walsender/cmd_base_backup.h b/src/bin/pg_walsender/cmd_base_backup.h new file mode 100644 index 000000000..71dcce70b --- /dev/null +++ b/src/bin/pg_walsender/cmd_base_backup.h @@ -0,0 +1,33 @@ +/* + * src/bin/pg_walsender/cmd_base_backup.h + * BASE_BACKUP: streams route->basebackupDir as a ustar archive over the + * real multiplexed-COPY-stream wire format modern (>= 15) pg_basebackup + * clients expect (traced from + * /Users/dim/dev/PostgreSQL/postgresql's src/backend/backup/ + * basebackup_copy.c and src/bin/pg_basebackup/pg_basebackup.c -- see + * this file's own .c for the exact message sequence, with citations). + * + * MVP scope: a single archive (the base directory itself, no separate + * tablespaces), no server-side compression, no backup manifest, no + * WAL-inclusive backup (`-X none` on the client side) -- each rejected + * up front with a clean ErrorResponse rather than silently ignored. + * do_pg_backup_start()/do_pg_backup_stop() (live-instance, backend-only) + * are never called: the archiver's basebackupDir is already a complete, + * at-rest backup (produced by a real pg_basebackup run against a live + * server -- the "Base backup generation" milestone, not yet + * implemented), so the start/end LSN this command reports comes from + * that backup's own backup_label file, not a live checkpoint. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_BASE_BACKUP_H +#define WS_CMD_BASE_BACKUP_H + +#include "routes.h" + +void cmd_base_backup(int sock, const WsRoute *route, const char *rawOptions); + +#endif /* WS_CMD_BASE_BACKUP_H */ diff --git a/src/bin/pg_walsender/cmd_fetch_file.c b/src/bin/pg_walsender/cmd_fetch_file.c new file mode 100644 index 000000000..4854d2e87 --- /dev/null +++ b/src/bin/pg_walsender/cmd_fetch_file.c @@ -0,0 +1,107 @@ +/* + * src/bin/pg_walsender/cmd_fetch_file.c + * See cmd_fetch_file.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "postgres_fe.h" + +#include "cmd_fetch_file.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" + +#define WS_FETCH_FILENAME_MAX 256 + + +/* + * filename_is_safe rejects anything that isn't a bare filename: no path + * separators, no leading dot (rules out "." / ".." / hidden files), not + * empty. WAL segment names and ".history" files are both plain + * [0-9A-F.history]-shaped basenames, never nested paths, so this is not a + * meaningful restriction for real callers -- only for a hostile one trying + * to walk out of walcacheDir. + */ +static bool +filename_is_safe(const char *filename) +{ + if (filename[0] == '\0' || filename[0] == '.') + { + return false; + } + + if (strchr(filename, '/') != NULL || strchr(filename, '\\') != NULL) + { + return false; + } + + return true; +} + + +void +cmd_fetch_file(int sock, const WsRoute *route) +{ + if (!ws_send_authentication_ok(sock)) + { + return; + } + + char filename[WS_FETCH_FILENAME_MAX]; + + if (!ws_read_line(sock, filename, sizeof(filename))) + { + ws_send_error_response(sock, "08P01", + "expected a single filename line after " + "authentication"); + return; + } + + if (!filename_is_safe(filename)) + { + log_warn("Rejecting FETCH_FILE request for unsafe filename \"%s\"", + filename); + ws_send_error_response(sock, "22023", "invalid filename"); + return; + } + + if (route == NULL || route->walcacheDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no WAL cache directory configured for this route"); + return; + } + + char path[MAXPGPATH]; + + sformat(path, sizeof(path), "%s/%s", route->walcacheDir, filename); + + char *contents = NULL; + long fileSize = 0; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + log_info("FETCH_FILE: \"%s\" not found under \"%s\"", + filename, route->walcacheDir); + ws_send_error_response(sock, "58P01", "requested file not found"); + return; + } + + if (!ws_send_copy_data(sock, contents, (int32_t) fileSize)) + { + log_error("Failed to send \"%s\" (%ld bytes) to a FETCH_FILE client", + filename, fileSize); + } + else + { + log_info("FETCH_FILE: served \"%s\" (%ld bytes) from \"%s\"", + filename, fileSize, route->walcacheDir); + } + + free(contents); +} diff --git a/src/bin/pg_walsender/cmd_fetch_file.h b/src/bin/pg_walsender/cmd_fetch_file.h new file mode 100644 index 000000000..88b2d2d99 --- /dev/null +++ b/src/bin/pg_walsender/cmd_fetch_file.h @@ -0,0 +1,36 @@ +/* + * src/bin/pg_walsender/cmd_fetch_file.h + * FETCH_FILE: a non-standard side-channel, not a replication-protocol + * command, for restore_command-style single-WAL-file fetch (see the + * design doc's own reasoning: restore_command spawns a fresh subprocess + * once per segment, with no persistent session to reuse -- riding the + * replication grammar would add protocol surface no real client ever + * exercises). Reuses the same connection's startup-packet + auth + * machinery (accept_loop.c routes a dbname of the form + * "fetch//" here instead of into the normal + * replication command loop), so it's gated by the same trust/ + * allowed_hosts check, no new auth surface. + * + * Wire shape, deliberately minimal since the only caller is + * fetch_client.c (this project's own code, not a real Postgres tool): + * after AuthenticationOk, the client sends the bare filename as a single + * '\n'-terminated line (ws_read_line, not a real protocol message), and + * the server replies with exactly one message: CopyData carrying the + * raw file bytes on success, or ErrorResponse on failure. Then the + * connection closes -- no CopyOutResponse/CopyDone, this isn't a real + * COPY sub-protocol, just reusing CopyData as a convenient length- + * prefixed binary envelope. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_FETCH_FILE_H +#define WS_CMD_FETCH_FILE_H + +#include "routes.h" + +void cmd_fetch_file(int sock, const WsRoute *route); + +#endif /* WS_CMD_FETCH_FILE_H */ diff --git a/src/bin/pg_walsender/cmd_identify_system.c b/src/bin/pg_walsender/cmd_identify_system.c new file mode 100644 index 000000000..b60210ee9 --- /dev/null +++ b/src/bin/pg_walsender/cmd_identify_system.c @@ -0,0 +1,73 @@ +/* + * src/bin/pg_walsender/cmd_identify_system.c + * See cmd_identify_system.h. + * + * systemid comes straight from the route (written by pg_autoctl's + * archiver-serve supervisor from the monitor's own tracked values -- see + * routes.h). timeline/xlogpos prefer the newest fully-captured WAL + * segment's own boundary (wal_dir_scan.h, filename-derived, not a + * parsed WAL record position) when the WAL cache has one, falling back + * to the route's static timeline and "0/0" when it doesn't (a brand + * new archiver with nothing captured yet). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "postgres_fe.h" + +#include "cmd_identify_system.h" +#include "file_utils.h" +#include "framing.h" +#include "wal_dir_scan.h" + + +void +cmd_identify_system(int sock, const WsRoute *route, const char *dbname) +{ + WsColumn columns[] = { + { "systemid", WS_TEXTOID, -1 }, + { "timeline", WS_INT4OID, 4 }, + { "xlogpos", WS_TEXTOID, -1 }, + { "dbname", WS_TEXTOID, -1 }, + }; + + char timelineStr[16]; + char xlogpos[32] = "0/0"; + const char *systemId = (route != NULL && route->systemId[0] != '\0') + ? route->systemId + : "0"; + int timeline = (route != NULL && route->timeline > 0) ? route->timeline : 1; + + if (route != NULL && route->walcacheDir[0] != '\0') + { + uint32_t foundTimeline; + + if (wal_dir_find_latest(route->walcacheDir, &foundTimeline, + xlogpos, sizeof(xlogpos))) + { + timeline = (int) foundTimeline; + } + } + + sformat(timelineStr, sizeof(timelineStr), "%d", timeline); + + const char *values[] = { + systemId, + timelineStr, + xlogpos, + dbname, + }; + + if (!ws_send_row_description(sock, columns, 4) || + !ws_send_data_row(sock, values, 4) || + !ws_send_command_complete(sock, "IDENTIFY_SYSTEM")) + { + /* the connection is likely dead at this point; the command loop's + * next ws_read_message() will notice and close it */ + return; + } +} diff --git a/src/bin/pg_walsender/cmd_identify_system.h b/src/bin/pg_walsender/cmd_identify_system.h new file mode 100644 index 000000000..62cf8060d --- /dev/null +++ b/src/bin/pg_walsender/cmd_identify_system.h @@ -0,0 +1,19 @@ +/* + * src/bin/pg_walsender/cmd_identify_system.h + * IDENTIFY_SYSTEM: reports systemid/timeline/xlogpos/dbname for the + * resolved route. See cmd_identify_system.c for what's a placeholder in + * this milestone vs. wired to real data. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_IDENTIFY_SYSTEM_H +#define WS_CMD_IDENTIFY_SYSTEM_H + +#include "routes.h" + +void cmd_identify_system(int sock, const WsRoute *route, const char *dbname); + +#endif /* WS_CMD_IDENTIFY_SYSTEM_H */ diff --git a/src/bin/pg_walsender/cmd_replication_slot.c b/src/bin/pg_walsender/cmd_replication_slot.c new file mode 100644 index 000000000..7c2fc6ea0 --- /dev/null +++ b/src/bin/pg_walsender/cmd_replication_slot.c @@ -0,0 +1,297 @@ +/* + * src/bin/pg_walsender/cmd_replication_slot.c + * See cmd_replication_slot.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include + +#include "postgres_fe.h" + +#include "cmd_replication_slot.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" +#include "wal_dir_scan.h" + +#define WS_SLOT_NAME_MAX 64 + + +/* + * parse_slot_name reads a possibly-quoted identifier (matching real + * Postgres's AppendQuotedIdentifier on the client side -- unquoted for a + * simple lowercase name, double-quoted otherwise) from the front of *p, + * advancing *p past it. + */ +static bool +parse_slot_name(const char **p, char *nameOut, size_t nameOutSize) +{ + const char *s = *p; + + while (isspace((unsigned char) *s)) + { + s++; + } + + if (*s == '"') + { + s++; + + char *out = nameOut; + char *outEnd = nameOut + nameOutSize - 1; + + while (*s && *s != '"') + { + if (out < outEnd) + { + *out++ = *s; + } + s++; + } + + if (*s != '"') + { + return false; + } + + *out = '\0'; + s++; + } + else + { + const char *start = s; + + while (*s && !isspace((unsigned char) *s)) + { + s++; + } + + size_t len = Min((size_t) (s - start), nameOutSize - 1); + + memcpy(nameOut, start, len); /* IGNORE-BANNED */ + nameOut[len] = '\0'; + } + + *p = s; + + return nameOut[0] != '\0'; +} + + +static bool +slot_name_is_safe(const char *name) +{ + if (name[0] == '\0') + { + return false; + } + + for (const char *p = name; *p; p++) + { + if (!(isalnum((unsigned char) *p) || *p == '_' || *p == '-')) + { + return false; + } + } + + return true; +} + + +static void +slot_marker_path(const WsRoute *route, const char *slotName, char *dest, size_t destSize) +{ + sformat(dest, destSize, "%s/.slot_%s", route->walcacheDir, slotName); +} + + +void +cmd_create_replication_slot(int sock, const WsRoute *route, const char *rawArgs) +{ + if (route == NULL || route->walcacheDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no WAL cache directory configured for this route"); + return; + } + + const char *p = rawArgs; + char slotName[WS_SLOT_NAME_MAX]; + + if (!parse_slot_name(&p, slotName, sizeof(slotName)) || !slot_name_is_safe(slotName)) + { + ws_send_error_response(sock, "22023", "invalid or missing slot name"); + return; + } + + bool sawPhysical = false; + bool sawLogical = false; + char word[64]; + + while (*p) + { + while (*p && (isspace((unsigned char) *p) || *p == ',' || *p == '(' || *p == ')')) + { + p++; + } + + if (!*p) + { + break; + } + + const char *start = p; + + while (*p && !isspace((unsigned char) *p) && *p != ',' && + *p != '(' && *p != ')') + { + p++; + } + + size_t len = Min((size_t) (p - start), sizeof(word) - 1); + + memcpy(word, start, len); /* IGNORE-BANNED */ + word[len] = '\0'; + + if (strcasecmp(word, "PHYSICAL") == 0) + { + sawPhysical = true; + } + else if (strcasecmp(word, "LOGICAL") == 0) + { + sawLogical = true; + } + + /* TEMPORARY and RESERVE_WAL are accepted but not enforced yet -- + * see this file's own header comment on retention */ + } + + if (sawLogical || !sawPhysical) + { + ws_send_error_response(sock, "0A000", + "only physical replication slots are supported"); + return; + } + + char consistentPoint[32] = "0/0"; + uint32_t timeline; + + (void) wal_dir_find_latest(route->walcacheDir, &timeline, consistentPoint, + sizeof(consistentPoint)); + + char path[MAXPGPATH]; + + slot_marker_path(route, slotName, path, sizeof(path)); + + char contents[128]; + + sformat(contents, sizeof(contents), "restart_lsn=%s\n", consistentPoint); + + if (!write_file(contents, strlen(contents), path)) + { + log_error("Failed to write replication slot marker \"%s\"", path); + ws_send_error_response(sock, "58030", "failed to persist the replication slot"); + return; + } + + WsColumn columns[] = { + { "slot_name", WS_TEXTOID, -1 }, + { "consistent_point", WS_TEXTOID, -1 }, + { "snapshot_name", WS_TEXTOID, -1 }, + { "output_plugin", WS_TEXTOID, -1 }, + }; + + const char *values[] = { slotName, consistentPoint, NULL, NULL }; + + if (ws_send_row_description(sock, columns, 4) && + ws_send_data_row(sock, values, 4)) + { + ws_send_command_complete(sock, "CREATE_REPLICATION_SLOT"); + } +} + + +void +cmd_read_replication_slot(int sock, const WsRoute *route, const char *rawArgs) +{ + if (route == NULL || route->walcacheDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no WAL cache directory configured for this route"); + return; + } + + const char *p = rawArgs; + char slotName[WS_SLOT_NAME_MAX]; + + if (!parse_slot_name(&p, slotName, sizeof(slotName)) || !slot_name_is_safe(slotName)) + { + ws_send_error_response(sock, "22023", "invalid or missing slot name"); + return; + } + + char path[MAXPGPATH]; + + slot_marker_path(route, slotName, path, sizeof(path)); + + char *contents = NULL; + long fileSize = 0; + + WsColumn columns[] = { + { "slot_type", WS_TEXTOID, -1 }, + { "restart_lsn", WS_TEXTOID, -1 }, + { "restart_tli", WS_INT8OID, 8 }, + }; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + /* matches real Postgres: slot doesn't exist -> one all-NULL row, + * not an ErrorResponse -- the client checks PQgetisnull() itself */ + const char *nullValues[] = { NULL, NULL, NULL }; + + if (ws_send_row_description(sock, columns, 3) && + ws_send_data_row(sock, nullValues, 3)) + { + ws_send_command_complete(sock, "READ_REPLICATION_SLOT"); + } + + return; + } + + char restartLsn[32] = "0/0"; + const char *prefix = "restart_lsn="; + char *line = strstr(contents, prefix); + + if (line != NULL) + { + line += strlen(prefix); + + char *nl = strchr(line, '\n'); + + if (nl != NULL) + { + *nl = '\0'; + } + + strlcpy(restartLsn, line, sizeof(restartLsn)); + } + + free(contents); + + uint32_t timeline = (route->timeline > 0) ? (uint32_t) route->timeline : 1; + char timelineStr[16]; + + sformat(timelineStr, sizeof(timelineStr), "%u", timeline); + + const char *values[] = { "physical", restartLsn, timelineStr }; + + if (ws_send_row_description(sock, columns, 3) && + ws_send_data_row(sock, values, 3)) + { + ws_send_command_complete(sock, "READ_REPLICATION_SLOT"); + } +} diff --git a/src/bin/pg_walsender/cmd_replication_slot.h b/src/bin/pg_walsender/cmd_replication_slot.h new file mode 100644 index 000000000..f48ddb0a1 --- /dev/null +++ b/src/bin/pg_walsender/cmd_replication_slot.h @@ -0,0 +1,23 @@ +/* + * src/bin/pg_walsender/cmd_replication_slot.h + * CREATE_REPLICATION_SLOT / READ_REPLICATION_SLOT, physical slots only + * (matching the design doc's own scope). A slot here is a bookkeeping + * marker file under the route's WAL cache directory -- not a real + * Postgres slot on a live server (there's no live server), and not yet + * wired into any WAL-retention enforcement (that's the prune/retention + * milestone's job, see prune_archiver_wal() in the SQL schema). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_REPLICATION_SLOT_H +#define WS_CMD_REPLICATION_SLOT_H + +#include "routes.h" + +void cmd_create_replication_slot(int sock, const WsRoute *route, const char *rawArgs); +void cmd_read_replication_slot(int sock, const WsRoute *route, const char *rawArgs); + +#endif /* WS_CMD_REPLICATION_SLOT_H */ diff --git a/src/bin/pg_walsender/cmd_show.c b/src/bin/pg_walsender/cmd_show.c new file mode 100644 index 000000000..50ed409ca --- /dev/null +++ b/src/bin/pg_walsender/cmd_show.c @@ -0,0 +1,53 @@ +/* + * src/bin/pg_walsender/cmd_show.c + * See cmd_show.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "postgres_fe.h" + +#include "cmd_show.h" +#include "framing.h" + + +void +cmd_show(int sock, const char *name) +{ + const char *value = NULL; + + if (strcasecmp(name, "wal_segment_size") == 0) + { + /* matches the real default; a non-default segment size would need + * to come from the archived group's own tracked configuration -- + * not wired in yet, see the identify_system placeholder note */ + value = "16MB"; + } + else if (strcasecmp(name, "data_directory_mode") == 0) + { + value = "0700"; + } + + if (value == NULL) + { + ws_send_error_response(sock, "42704", "unrecognized configuration parameter"); + return; + } + + WsColumn columns[] = { + { name, WS_TEXTOID, -1 }, + }; + + const char *values[] = { value }; + + if (!ws_send_row_description(sock, columns, 1) || + !ws_send_data_row(sock, values, 1) || + !ws_send_command_complete(sock, "SHOW")) + { + return; + } +} diff --git a/src/bin/pg_walsender/cmd_show.h b/src/bin/pg_walsender/cmd_show.h new file mode 100644 index 000000000..8e23a404d --- /dev/null +++ b/src/bin/pg_walsender/cmd_show.h @@ -0,0 +1,17 @@ +/* + * src/bin/pg_walsender/cmd_show.h + * SHOW : real pg_basebackup/pg_receivewal only ever query + * wal_segment_size and data_directory_mode (see streamutil.c in the + * Postgres source), so those are the only two GUCs this needs to answer. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_SHOW_H +#define WS_CMD_SHOW_H + +void cmd_show(int sock, const char *name); + +#endif /* WS_CMD_SHOW_H */ diff --git a/src/bin/pg_walsender/cmd_start_replication.c b/src/bin/pg_walsender/cmd_start_replication.c new file mode 100644 index 000000000..66b369f6a --- /dev/null +++ b/src/bin/pg_walsender/cmd_start_replication.c @@ -0,0 +1,555 @@ +/* + * src/bin/pg_walsender/cmd_start_replication.c + * See cmd_start_replication.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "port/pg_bswap.h" +#include "pqexpbuffer.h" + +#include "cmd_start_replication.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" +#include "signals.h" +#include "wal_dir_scan.h" + +#define WS_WAL_SEGMENT_SIZE UINT64CONST(0x1000000) +#define WS_STREAM_CHUNK_SIZE (32 * 1024) +#define WS_KEEPALIVE_INTERVAL_SEC 5 +#define WS_POLL_INTERVAL_USEC (200 * 1000) + + +static void +append_int64(PQExpBuffer buf, int64_t v) +{ + uint64_t n = pg_hton64((uint64_t) v); + + appendBinaryPQExpBuffer(buf, (const char *) &n, 8); +} + + +static bool +send_xlogdata(int sock, uint64_t dataStart, uint64_t walEnd, + const char *data, size_t len) +{ + PQExpBuffer buf = createPQExpBuffer(); + + appendPQExpBufferChar(buf, 'w'); /* PqReplMsg_WALData */ + append_int64(buf, (int64_t) dataStart); + append_int64(buf, (int64_t) walEnd); + append_int64(buf, (int64_t) 0); /* sendTime, not load-bearing here */ + appendBinaryPQExpBuffer(buf, data, len); + + bool ok = !PQExpBufferBroken(buf) && ws_send_copy_data(sock, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +static bool +send_keepalive(int sock, uint64_t walEnd) +{ + PQExpBuffer buf = createPQExpBuffer(); + + appendPQExpBufferChar(buf, 'k'); /* PqReplMsg_Keepalive */ + append_int64(buf, (int64_t) walEnd); + append_int64(buf, (int64_t) 0); /* sendTime */ + appendPQExpBufferChar(buf, 0); /* replyRequested = false */ + + bool ok = !PQExpBufferBroken(buf) && ws_send_copy_data(sock, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +/* + * wait_for_more_data_or_client waits up to WS_POLL_INTERVAL_USEC for + * either more WAL bytes to become available or a message from the client, + * draining (and ignoring the content of) any standby status update the + * client sends meanwhile -- this project has no cascading/retention logic + * that needs to react to it yet. Returns false when the client has + * disconnected/terminated or we've been asked to stop, in which case the + * caller should end the stream. + */ +static bool +wait_for_more_data_or_client(int sock, uint64_t currentLsn, time_t *lastKeepalive) +{ + if (asked_to_stop || asked_to_stop_fast) + { + return false; + } + + fd_set readSet; + + FD_ZERO(&readSet); + FD_SET(sock, &readSet); + + struct timeval timeout = { 0, WS_POLL_INTERVAL_USEC }; + + int selectRet = select(sock + 1, &readSet, NULL, NULL, &timeout); + + if (selectRet < 0 && errno != EINTR) + { + return false; + } + + if (selectRet > 0 && FD_ISSET(sock, &readSet)) + { + char type; + char *payload = NULL; + int32_t payloadLen = 0; + + if (!ws_read_message(sock, &type, &payload, &payloadLen)) + { + free(payload); + return false; /* client disconnected */ + } + + free(payload); + + if (type == 'X' || type == 'c') /* Terminate or CopyDone */ + { + return false; + } + + /* 'd' CopyData: a standby status update / hot-standby feedback we + * don't act on yet -- already consumed above, nothing more to do */ + } + + time_t now = time(NULL); + + if (now - *lastKeepalive >= WS_KEEPALIVE_INTERVAL_SEC) + { + if (!send_keepalive(sock, currentLsn)) + { + return false; + } + + *lastKeepalive = now; + } + + return true; +} + + +/* + * trim_trailing_zeros returns the length of buffer with any trailing run of + * zero bytes removed. A ".partial" segment is pre-allocated to its full + * WS_WAL_SEGMENT_SIZE by pg_receivewal the moment it's created (matching + * real Postgres's own WAL file pre-allocation, XLogFileInitInternal) -- + * unlike a real primary's own walsender, which only ever knows about bytes + * it has actually flushed, a plain fread() from a ".partial" file cannot + * tell real WAL content apart from the not-yet-written tail, which reads + * back as zeros. Sending that tail as if it were real WAL data is exactly + * what a real standby's own recovery logic detects as "invalid record + * length ... got 0" -- and, on that response, terminates its walreceiver + * outright rather than treating it as "no more data yet, retry" (which is + * pg_receivewal's own polling behavior, so it never noticed). + * + * Trimming any trailing zero run before ever sending it means an in- + * progress chunk boundary is re-read (and re-trimmed) on the next + * iteration rather than shipped as real data -- self-correcting, at worst + * a few bytes of redundant re-reads per tick, never sent out early. + */ +static size_t +trim_trailing_zeros(const char *buffer, size_t len) +{ + while (len > 0 && buffer[len - 1] == 0) + { + len--; + } + + return len; +} + + +static bool +parse_lsn(const char *s, uint64_t *lsn, const char **endptr) +{ + char *afterHi; + unsigned long hi = strtoul(s, &afterHi, 16); + + if (afterHi == s || *afterHi != '/') + { + return false; + } + + char *afterLo; + unsigned long lo = strtoul(afterHi + 1, &afterLo, 16); + + if (afterLo == afterHi + 1) + { + return false; + } + + *lsn = ((uint64_t) hi << 32) | (uint32_t) lo; + *endptr = afterLo; + + return true; +} + + +static const char * +skip_ws(const char *p) +{ + while (isspace((unsigned char) *p)) + { + p++; + } + + return p; +} + + +/* + * find_oldest_segno scans walcacheDir for the lowest-numbered WAL segment + * present on the given timeline (complete or still ".partial" -- either + * counts as "this archiver has it"). Returns false (*oldestSegno untouched) + * if nothing has been captured on that timeline at all yet. + * + * This is what lets the main streaming loop below tell "the requested + * segment hasn't been captured *yet*" (segno >= oldest present -- normal, + * just wait) apart from "the requested segment predates everything this + * archiver has ever captured" (segno < oldest present -- a real, permanent + * gap, not a timing issue): pg_receivewal has no replication slot before + * this project's own recent fix (service_archiver_start_pgreceivewal(), + * pg_autoctl's service_archiver.c), so a pg_receivewal whose very first + * connection attempt loses the startup HBA-propagation race restarts + * streaming from the server's then-current position instead of resuming, + * silently skipping every segment in between -- observed in practice + * during this milestone's own end-to-end testing. Without this check, a + * client asking to stream from inside that permanent gap (e.g. a real pg_ + * basebackup's own --wal-method=stream background receiver, replaying from + * the position a BASE_BACKUP response advertised) would sit in this file's + * own wait_for_more_data_or_client() loop forever, waiting for a segment + * that can never arrive. + */ +static bool +find_oldest_segno(const char *walcacheDir, uint32_t timeline, uint64_t *oldestSegno) +{ + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + return false; + } + + bool found = false; + uint64_t best = 0; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + size_t len = strlen(entry->d_name); + char segPart[25] = { 0 }; + + if (len == 24) + { + memcpy(segPart, entry->d_name, 24); /* IGNORE-BANNED */ + } + else if (len == 24 + 8 && strcmp(entry->d_name + 24, ".partial") == 0) + { + memcpy(segPart, entry->d_name, 24); /* IGNORE-BANNED */ + } + else + { + continue; + } + + bool isHex = true; + + for (size_t i = 0; i < 24 && isHex; i++) + { + isHex = isxdigit((unsigned char) segPart[i]); + } + + if (!isHex) + { + continue; + } + + char tliHex[9] = { 0 }; + + memcpy(tliHex, segPart, 8); /* IGNORE-BANNED */ + + if ((uint32_t) strtoul(tliHex, NULL, 16) != timeline) + { + continue; + } + + char logIdHex[9] = { 0 }; + char segHex[9] = { 0 }; + + memcpy(logIdHex, segPart + 8, 8); /* IGNORE-BANNED */ + memcpy(segHex, segPart + 16, 8); /* IGNORE-BANNED */ + + uint32_t logId = (uint32_t) strtoul(logIdHex, NULL, 16); + uint32_t seg = (uint32_t) strtoul(segHex, NULL, 16); + uint64_t segno = (uint64_t) logId * + (UINT64CONST(0x100000000) / WS_WAL_SEGMENT_SIZE) + seg; + + if (!found || segno < best) + { + best = segno; + found = true; + } + } + + closedir(dir); + + if (found) + { + *oldestSegno = best; + } + + return found; +} + + +void +cmd_start_replication(int sock, const WsRoute *route, const char *rawArgs) +{ + if (route == NULL || route->walcacheDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no WAL cache directory configured for this route"); + return; + } + + const char *p = skip_ws(rawArgs); + + if (strncasecmp(p, "SLOT", 4) == 0 && isspace((unsigned char) p[4])) + { + p = skip_ws(p + 4); + + /* consume a possibly-quoted slot name, positioning is unaffected + * by which slot (if any) was named -- see this file's own header + * comment on why no real slot-based retention exists yet */ + if (*p == '"') + { + p++; + while (*p && *p != '"') + { + p++; + } + if (*p == '"') + { + p++; + } + } + else + { + while (*p && !isspace((unsigned char) *p)) + { + p++; + } + } + + p = skip_ws(p); + } + + if (strncasecmp(p, "PHYSICAL", 8) == 0 && + (isspace((unsigned char) p[8]) || p[8] == '\0')) + { + p = skip_ws(p + 8); + } + + uint64_t startLsn; + const char *after; + + if (!parse_lsn(p, &startLsn, &after)) + { + ws_send_error_response(sock, "22023", "invalid or missing start LSN"); + return; + } + + p = skip_ws(after); + + uint32_t timeline = (route->timeline > 0) ? (uint32_t) route->timeline : 1; + + if (strncasecmp(p, "TIMELINE", 8) == 0) + { + p = skip_ws(p + 8); + timeline = (uint32_t) strtoul(p, NULL, 10); + } + + if (!ws_send_copy_both_response(sock, 0)) + { + return; + } + + log_info("START_REPLICATION: streaming from %X/%08X on timeline %u " + "from \"%s\"", + (uint32_t) (startLsn >> 32), (uint32_t) startLsn, timeline, + route->walcacheDir); + + uint64_t segno = startLsn / WS_WAL_SEGMENT_SIZE; + uint64_t offset = startLsn % WS_WAL_SEGMENT_SIZE; + uint64_t currentLsn = startLsn; + time_t lastKeepalive = time(NULL); + + for (;;) + { + if (asked_to_stop || asked_to_stop_fast) + { + break; + } + + char filename[32]; + + wal_segment_filename(timeline, segno, filename, sizeof(filename)); + + char completePath[MAXPGPATH]; + + sformat(completePath, sizeof(completePath), "%s/%s", + route->walcacheDir, filename); + + bool isComplete = file_exists(completePath); + + char partialPath[MAXPGPATH]; + + sformat(partialPath, sizeof(partialPath), "%s.partial", completePath); + + const char *readPath = isComplete ? completePath : partialPath; + + if (!isComplete && !file_exists(partialPath)) + { + uint64_t oldestSegno; + + if (find_oldest_segno(route->walcacheDir, timeline, &oldestSegno) && + segno < oldestSegno) + { + char oldestName[32]; + + wal_segment_filename(timeline, oldestSegno, + oldestName, sizeof(oldestName)); + + log_error("START_REPLICATION: requested segment \"%s\" " + "predates the oldest segment this archiver has " + "captured (\"%s\") -- it was never captured and " + "can never become available, refusing to wait " + "forever for it", + filename, oldestName); + + ws_send_error_response(sock, "58P01", + "requested WAL segment predates this " + "archiver's captured history and will " + "never become available"); + return; + } + + /* nothing captured for this segment yet -- wait for it */ + if (!wait_for_more_data_or_client(sock, currentLsn, &lastKeepalive)) + { + break; + } + + continue; + } + + FILE *file = fopen(readPath, "rb"); /* IGNORE-BANNED */ + + if (file == NULL) + { + log_warn("Failed to open \"%s\": %m (will retry)", readPath); + + if (!wait_for_more_data_or_client(sock, currentLsn, &lastKeepalive)) + { + break; + } + + continue; + } + + if (fseeko(file, (off_t) offset, SEEK_SET) != 0) + { + log_error("Failed to seek to offset %" PRIu64 " in \"%s\": %m", + offset, readPath); + fclose(file); + break; + } + + char buffer[WS_STREAM_CHUNK_SIZE]; + size_t got = fread(buffer, 1, sizeof(buffer), file); + + fclose(file); + + if (!isComplete) + { + got = trim_trailing_zeros(buffer, got); + } + + if (got == 0) + { + if (isComplete) + { + /* fully drained this now-complete segment: move on */ + segno++; + offset = 0; + continue; + } + + if (!wait_for_more_data_or_client(sock, currentLsn, &lastKeepalive)) + { + break; + } + + continue; + } + + if (!send_xlogdata(sock, currentLsn, currentLsn + got, buffer, got)) + { + break; /* client gone */ + } + + currentLsn += got; + offset += got; + + if (offset >= WS_WAL_SEGMENT_SIZE) + { + segno++; + offset = 0; + } + } + + (void) ws_send_copy_done(sock); + + /* + * Real walsender.c's own controlled-shutdown path (WalSndDone) follows + * CopyDone with a CommandComplete tagged "COPY" before returning to + * the command loop -- required protocol, not optional decoration: a + * real client's receivelog.c (ReceiveXlogStream) only accepts an + * ended stream as a *successful* stop when it can read a matching + * PGRES_COMMAND_OK result afterward; without it, a client that decided + * on its own to stop here (e.g. pg_basebackup's --wal-method=stream + * background receiver, once it reaches its target LSN) falls through + * to "unexpected termination of replication stream" and exits + * non-zero, even though nothing on the wire was actually wrong. A + * genuinely long-lived streaming client (real walreceiver, primary_ + * conninfo) never triggers this path at all -- it never decides to + * stop on its own -- which is why this went unnoticed until a real + * pg_basebackup was tested end to end. + */ + (void) ws_send_command_complete(sock, "COPY"); + + log_info("START_REPLICATION: stream ended at %X/%08X", + (uint32_t) (currentLsn >> 32), (uint32_t) currentLsn); +} diff --git a/src/bin/pg_walsender/cmd_start_replication.h b/src/bin/pg_walsender/cmd_start_replication.h new file mode 100644 index 000000000..de151d950 --- /dev/null +++ b/src/bin/pg_walsender/cmd_start_replication.h @@ -0,0 +1,26 @@ +/* + * src/bin/pg_walsender/cmd_start_replication.h + * START_REPLICATION [SLOT ] TIMELINE : streams WAL + * bytes straight out of the route's WAL cache directory, physical-only. + * + * Deliberately does NOT vendor xlogreader.c for this: real walsender's + * own WalSndSegmentOpen (walsender.c) just computes a path from TLI+segno + * and opens it -- streaming raw bytes needs no WAL *record* decoding at + * all, only byte-range bookkeeping this file does directly. xlogreader.c + * would only earn its keep here for validating record boundaries, not + * required for a client (a real pg_receivewal) that already does its own + * validation on the bytes it receives. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_START_REPLICATION_H +#define WS_CMD_START_REPLICATION_H + +#include "routes.h" + +void cmd_start_replication(int sock, const WsRoute *route, const char *rawArgs); + +#endif /* WS_CMD_START_REPLICATION_H */ diff --git a/src/bin/pg_walsender/cmd_timeline_history.c b/src/bin/pg_walsender/cmd_timeline_history.c new file mode 100644 index 000000000..b2bf35c5a --- /dev/null +++ b/src/bin/pg_walsender/cmd_timeline_history.c @@ -0,0 +1,71 @@ +/* + * src/bin/pg_walsender/cmd_timeline_history.c + * See cmd_timeline_history.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "postgres_fe.h" + +#include "cmd_timeline_history.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" + +/* matches xlog_internal.h's own MAXFNAMELEN (backend-only header, not + * pulled in here) -- "%08X.history" is always exactly 17 bytes + NUL */ +#define WS_MAXFNAMELEN 64 + + +void +cmd_timeline_history(int sock, const WsRoute *route, int timeline) +{ + if (route == NULL || route->walcacheDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no WAL cache directory configured for this route"); + return; + } + + /* matches real Postgres's TLHistoryFileName() macro exactly */ + char filename[WS_MAXFNAMELEN]; + + sformat(filename, sizeof(filename), "%08X.history", timeline); + + char path[MAXPGPATH]; + + sformat(path, sizeof(path), "%s/%s", route->walcacheDir, filename); + + char *contents = NULL; + long fileSize = 0; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + /* matches real walsender.c: no history file for this timeline is + * an ERROR there too, not a soft "empty" fallback */ + log_info("TIMELINE_HISTORY: \"%s\" not found under \"%s\"", + filename, route->walcacheDir); + ws_send_error_response(sock, "58P01", + "requested timeline history file not found"); + return; + } + + WsColumn columns[] = { + { "filename", WS_TEXTOID, -1 }, + { "content", WS_TEXTOID, -1 }, + }; + + const char *values[] = { filename, contents }; + + if (ws_send_row_description(sock, columns, 2) && + ws_send_data_row(sock, values, 2)) + { + ws_send_command_complete(sock, "TIMELINE_HISTORY"); + } + + free(contents); +} diff --git a/src/bin/pg_walsender/cmd_timeline_history.h b/src/bin/pg_walsender/cmd_timeline_history.h new file mode 100644 index 000000000..4e93ae251 --- /dev/null +++ b/src/bin/pg_walsender/cmd_timeline_history.h @@ -0,0 +1,23 @@ +/* + * src/bin/pg_walsender/cmd_timeline_history.h + * TIMELINE_HISTORY : serves a ".history" file straight out of + * the route's WAL cache directory. Traced from walsender.c's own + * SendTimeLineHistory() (backend, not linked -- see walsender.h's own + * header comment): a single RowDescription(filename text, content text) + * + one DataRow + CommandComplete, no COPY involved. Genuinely just a + * flat-file read; the only real-instance-shaped input is which timeline + * was asked for. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_TIMELINE_HISTORY_H +#define WS_CMD_TIMELINE_HISTORY_H + +#include "routes.h" + +void cmd_timeline_history(int sock, const WsRoute *route, int timeline); + +#endif /* WS_CMD_TIMELINE_HISTORY_H */ diff --git a/src/bin/pg_walsender/defaults.h b/src/bin/pg_walsender/defaults.h new file mode 100644 index 000000000..95a90392d --- /dev/null +++ b/src/bin/pg_walsender/defaults.h @@ -0,0 +1,38 @@ +/* + * src/bin/pg_walsender/defaults.h + * A handful of constants pg_walsender needs that would otherwise come + * from pg_autoctl/defaults.h -- duplicated rather than included, since + * pg_walsender is deliberately a standalone binary that does not link + * any of pg_autoctl's own sources (see + * ~/dev/temp/archiving-disaster-recovery.md). Keep + * PG_AUTOCTL_REPLICA_USERNAME in sync with pg_autoctl/defaults.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_DEFAULTS_H +#define WS_DEFAULTS_H + +#include "postgres_fe.h" + +#define PG_AUTOCTL_REPLICA_USERNAME "pgautofailover_replicator" + +#define WS_DEFAULT_PORT 6543 + +/* + * Reported as the "server_version" startup parameter so that real libpq + * clients (pg_basebackup, pg_receivewal) compute a sane PQserverVersion(). + * PG_VERSION/PG_VERSION_NUM (from pg_config.h, pulled in via postgres_fe.h) + * are this build's own real target version -- pg_walsender is built once + * per PGVERSION, against that version's own server headers (Makefile.common's + * pg_config --includedir-server), so this is already the archived group's + * actual pg_version, not a stand-in for it. A previous fixed "16.4" value + * here made every non-PG16 build report a version mismatch to real + * pg_basebackup/pg_receivewal clients ("incompatible server version"). + */ +#define WS_SERVER_VERSION PG_VERSION +#define WS_SERVER_VERSION_NUM PG_VERSION_NUM + +#endif /* WS_DEFAULTS_H */ diff --git a/src/bin/pg_walsender/fetch_client.c b/src/bin/pg_walsender/fetch_client.c new file mode 100644 index 000000000..bde3857d6 --- /dev/null +++ b/src/bin/pg_walsender/fetch_client.c @@ -0,0 +1,262 @@ +/* + * src/bin/pg_walsender/fetch_client.c + * See fetch_client.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "pqexpbuffer.h" + +#include "fetch_client.h" +#include "defaults.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" + + +static int +connect_to(const char *host, int port) +{ + char portStr[16]; + + sformat(portStr, sizeof(portStr), "%d", port); + + struct addrinfo hints; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + + struct addrinfo *res = NULL; + int rc = getaddrinfo(host, portStr, &hints, &res); + + if (rc != 0) + { + log_error("Failed to resolve \"%s\": %s", host, gai_strerror(rc)); + return -1; + } + + int sock = -1; + + for (struct addrinfo *rp = res; rp != NULL; rp = rp->ai_next) + { + sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + + if (sock < 0) + { + continue; + } + + if (connect(sock, rp->ai_addr, rp->ai_addrlen) == 0) + { + break; + } + + close(sock); + sock = -1; + } + + freeaddrinfo(res); + + if (sock < 0) + { + log_error("Failed to connect to %s:%d: %m", host, port); + } + + return sock; +} + + +static bool +send_startup_message(int sock, const char *database) +{ + PQExpBuffer buf = createPQExpBuffer(); + int32_t version = htonl(196608); /* protocol 3.0 */ + + appendBinaryPQExpBuffer(buf, (const char *) &version, 4); + + appendBinaryPQExpBuffer(buf, "user", strlen("user") + 1); + appendBinaryPQExpBuffer(buf, PG_AUTOCTL_REPLICA_USERNAME, + strlen(PG_AUTOCTL_REPLICA_USERNAME) + 1); + + appendBinaryPQExpBuffer(buf, "database", strlen("database") + 1); + appendBinaryPQExpBuffer(buf, database, strlen(database) + 1); + + appendPQExpBufferChar(buf, '\0'); /* terminates the parameter list */ + + int32_t totalLen = htonl(buf->len + 4); + bool ok = !PQExpBufferBroken(buf) && + ws_write_bytes(sock, &totalLen, 4) && + ws_write_bytes(sock, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +static void +extract_error_message(const char *payload, int32_t payloadLen, + char *out, size_t outSize) +{ + out[0] = '\0'; + + const char *p = payload; + const char *end = payload + payloadLen; + + while (p < end && *p != '\0') + { + char code = *p++; + const char *value = p; + + while (p < end && *p != '\0') + { + p++; + } + + if (code == 'M') + { + size_t len = Min((size_t) (p - value), outSize - 1); + + memcpy(out, value, len); /* IGNORE-BANNED */ + out[len] = '\0'; + } + + if (p < end) + { + p++; /* skip this field's NUL terminator */ + } + } +} + + +int +ws_fetch_file_client(const char *host, int port, const char *routeKey, + const char *filename, const char *outputPath) +{ + int sock = connect_to(host, port); + + if (sock < 0) + { + return 1; + } + + char database[512]; + + sformat(database, sizeof(database), "fetch/%s", routeKey); + + if (!send_startup_message(sock, database)) + { + log_error("Failed to send the startup packet to %s:%d: %m", host, port); + close(sock); + return 1; + } + + char type; + char *payload = NULL; + int32_t payloadLen = 0; + + if (!ws_read_message(sock, &type, &payload, &payloadLen)) + { + log_error("Failed to read the authentication response from %s:%d", + host, port); + free(payload); + close(sock); + return 1; + } + + if (type == 'E') + { + char message[512]; + + extract_error_message(payload, payloadLen, message, sizeof(message)); + log_error("Authentication failed: %s", message); + free(payload); + close(sock); + return 1; + } + + free(payload); + + if (type != 'R') + { + log_error("Unexpected message type '%c' from %s:%d (expected " + "AuthenticationOk)", type, host, port); + close(sock); + return 1; + } + + char line[300]; + + sformat(line, sizeof(line), "%s\n", filename); + + if (!ws_write_bytes(sock, line, strlen(line))) + { + log_error("Failed to send the filename request to %s:%d: %m", host, port); + close(sock); + return 1; + } + + if (!ws_read_message(sock, &type, &payload, &payloadLen)) + { + log_error("Failed to read the file response from %s:%d", host, port); + free(payload); + close(sock); + return 1; + } + + if (type == 'E') + { + char message[512]; + + extract_error_message(payload, payloadLen, message, sizeof(message)); + log_error("Failed to fetch \"%s\": %s", filename, message); + free(payload); + close(sock); + return 1; + } + + if (type != 'd') + { + log_error("Unexpected message type '%c' from %s:%d (expected CopyData)", + type, host, port); + free(payload); + close(sock); + return 1; + } + + close(sock); + + char tmpPath[MAXPGPATH]; + + sformat(tmpPath, sizeof(tmpPath), "%s.pg_walsender_fetch_tmp", outputPath); + + if (!write_file(payload, payloadLen, tmpPath)) + { + log_error("Failed to write \"%s\": %m", tmpPath); + free(payload); + return 1; + } + + free(payload); + + if (rename(tmpPath, outputPath) != 0) + { + log_error("Failed to rename \"%s\" to \"%s\": %m", tmpPath, outputPath); + return 1; + } + + log_info("Fetched \"%s\" (%d bytes) to \"%s\"", filename, payloadLen, outputPath); + + return 0; +} diff --git a/src/bin/pg_walsender/fetch_client.h b/src/bin/pg_walsender/fetch_client.h new file mode 100644 index 000000000..995cc6c68 --- /dev/null +++ b/src/bin/pg_walsender/fetch_client.h @@ -0,0 +1,31 @@ +/* + * src/bin/pg_walsender/fetch_client.h + * The client side of the FETCH_FILE side-channel (cmd_fetch_file.h) -- + * the only caller of that protocol, matching its header comment ("the + * only caller here is pg_autoctl's own restore_command wrapper, which + * this project fully controls end to end"). Exposed as `pg_walsender + * fetch-file ...` (see main.c) so pg_autoctl's restore_command can shell + * out to it directly, the same way it already shells out to real + * pg_receivewal/pg_basebackup elsewhere in this project. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_FETCH_CLIENT_H +#define WS_FETCH_CLIENT_H + +/* + * Connects to host:port, requests filename for routeKey ("/ + * "), and writes the result to outputPath (via a same-directory + * temp file + rename, so a killed/interrupted fetch never leaves a + * partial file at outputPath). Returns 0 on success, 1 on any failure + * (connection, auth, missing file, short write) -- always with a + * human-readable message already logged, matching restore_command's own + * "non-zero means retry me" contract. + */ +int ws_fetch_file_client(const char *host, int port, const char *routeKey, + const char *filename, const char *outputPath); + +#endif /* WS_FETCH_CLIENT_H */ diff --git a/src/bin/pg_walsender/framing.c b/src/bin/pg_walsender/framing.c new file mode 100644 index 000000000..4965dc839 --- /dev/null +++ b/src/bin/pg_walsender/framing.c @@ -0,0 +1,501 @@ +/* + * src/bin/pg_walsender/framing.c + * See framing.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "pqexpbuffer.h" + +#include "framing.h" +#include "log.h" + +/* startup-packet body larger than this is rejected outright as malformed */ +#define WS_MAX_STARTUP_PACKET_SIZE 10000 + +/* an ordinary post-startup message body larger than this is rejected */ +#define WS_MAX_MESSAGE_SIZE (64 * 1024 * 1024) + +#define SSL_REQUEST_CODE 80877103 +#define GSS_REQUEST_CODE 80877104 +#define CANCEL_REQUEST_CODE 80877102 + + +bool +ws_read_bytes(int sock, void *buf, size_t len) +{ + char *ptr = (char *) buf; + size_t remaining = len; + + while (remaining > 0) + { + ssize_t n = read(sock, ptr, remaining); + + if (n < 0) + { + if (errno == EINTR) + { + continue; + } + return false; + } + + if (n == 0) + { + /* peer closed the connection */ + return false; + } + + ptr += n; + remaining -= n; + } + + return true; +} + + +bool +ws_write_bytes(int sock, const void *buf, size_t len) +{ + const char *ptr = (const char *) buf; + size_t remaining = len; + + while (remaining > 0) + { + ssize_t n = write(sock, ptr, remaining); + + if (n < 0) + { + if (errno == EINTR) + { + continue; + } + return false; + } + + ptr += n; + remaining -= n; + } + + return true; +} + + +bool +ws_write_raw_byte(int sock, char c) +{ + return ws_write_bytes(sock, &c, 1); +} + + +bool +ws_read_line(int sock, char *line, size_t maxLen) +{ + size_t n = 0; + + while (n < maxLen - 1) + { + char c; + + if (!ws_read_bytes(sock, &c, 1)) + { + return false; + } + + if (c == '\n') + { + line[n] = '\0'; + return true; + } + + line[n++] = c; + } + + return false; /* line too long */ +} + + +bool +ws_read_startup_payload(int sock, char **payload, int32_t *payloadLen) +{ + unsigned char lenBuf[4]; + + *payload = NULL; + *payloadLen = 0; + + if (!ws_read_bytes(sock, lenBuf, 4)) + { + return false; + } + + int32_t len = ((int32_t) lenBuf[0] << 24) | ((int32_t) lenBuf[1] << 16) | + ((int32_t) lenBuf[2] << 8) | (int32_t) lenBuf[3]; + + if (len < 4 || len > WS_MAX_STARTUP_PACKET_SIZE) + { + log_error("Received an invalid startup packet length: %d", len); + return false; + } + + int32_t bodyLen = len - 4; + char *buf = (char *) malloc(bodyLen + 1); + + if (buf == NULL) + { + log_error("Failed to allocate %d bytes for a startup packet: %m", bodyLen); + return false; + } + + if (bodyLen > 0 && !ws_read_bytes(sock, buf, bodyLen)) + { + free(buf); + return false; + } + + buf[bodyLen] = '\0'; + + *payload = buf; + *payloadLen = bodyLen; + + return true; +} + + +bool +ws_read_message(int sock, char *type, char **payload, int32_t *payloadLen) +{ + *payload = NULL; + *payloadLen = 0; + + if (!ws_read_bytes(sock, type, 1)) + { + return false; + } + + unsigned char lenBuf[4]; + + if (!ws_read_bytes(sock, lenBuf, 4)) + { + return false; + } + + int32_t len = ((int32_t) lenBuf[0] << 24) | ((int32_t) lenBuf[1] << 16) | + ((int32_t) lenBuf[2] << 8) | (int32_t) lenBuf[3]; + + if (len < 4 || len > WS_MAX_MESSAGE_SIZE) + { + log_error("Received an invalid message length %d for message type '%c'", + len, *type); + return false; + } + + int32_t bodyLen = len - 4; + char *buf = (char *) malloc(bodyLen + 1); + + if (buf == NULL) + { + log_error("Failed to allocate %d bytes for a protocol message: %m", bodyLen); + return false; + } + + if (bodyLen > 0 && !ws_read_bytes(sock, buf, bodyLen)) + { + free(buf); + return false; + } + + buf[bodyLen] = '\0'; + + *payload = buf; + *payloadLen = bodyLen; + + return true; +} + + +bool +ws_send_message(int sock, char type, const char *data, int32_t dataLen) +{ + char header[5]; + int32_t netLen = htonl(dataLen + 4); + + header[0] = type; + memcpy(header + 1, &netLen, 4); /* IGNORE-BANNED */ + + if (!ws_write_bytes(sock, header, 5)) + { + return false; + } + + if (dataLen > 0 && !ws_write_bytes(sock, data, dataLen)) + { + return false; + } + + return true; +} + + +bool +ws_send_authentication_ok(int sock) +{ + int32_t zero = 0; + + return ws_send_message(sock, 'R', (const char *) &zero, 4); +} + + +bool +ws_send_parameter_status(int sock, const char *name, const char *value) +{ + PQExpBuffer buf = createPQExpBuffer(); + + appendBinaryPQExpBuffer(buf, name, strlen(name) + 1); + appendBinaryPQExpBuffer(buf, value, strlen(value) + 1); + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, 'S', buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +bool +ws_send_backend_key_data(int sock, int32_t pid, int32_t secret) +{ + char data[8]; + int32_t netPid = htonl(pid); + int32_t netSecret = htonl(secret); + + memcpy(data, &netPid, 4); /* IGNORE-BANNED */ + memcpy(data + 4, &netSecret, 4); /* IGNORE-BANNED */ + + return ws_send_message(sock, 'K', data, 8); +} + + +/* + * ws_send_negotiate_protocol_version sends the 'v' NegotiateProtocolVersion + * message. Per the wire protocol, the first Int32 is *not* a bare minor + * version -- it's the full negotiated protocol version (major<<16|minor), + * exactly like the version code in a StartupMessage; real libpq's + * pqGetNegotiateProtocolVersion3() compares it against PG_PROTOCOL(3, 0) + * and rejects anything smaller as "downgrade to pre-3.0 protocol version". + * newestMinor is the highest minor protocol version we actually support + * (always 0 -- only protocol 3.0 is implemented), combined here with major + * version 3. unsupportedOptions/nUnsupportedOptions lists any "_pq_.*" + * startup options the client asked for that we don't recognize (we don't + * parse any, so this is every "_pq_.*" key seen) -- real libpq's own + * protocol-GREASE self-test requires the server to echo back + * "_pq_.test_protocol_negotiation" here, or it fails the connection with + * "server did not report the unsupported ... parameter". See startup.c's + * own caller for why this exists. + */ +bool +ws_send_negotiate_protocol_version(int sock, int32_t newestMinor, + const char **unsupportedOptions, + int nUnsupportedOptions) +{ + PQExpBuffer buf = createPQExpBuffer(); + + int32_t netVersion = htonl((3 << 16) | (newestMinor & 0xFFFF)); + int32_t netOptionCount = htonl(nUnsupportedOptions); + + appendBinaryPQExpBuffer(buf, (const char *) &netVersion, 4); + appendBinaryPQExpBuffer(buf, (const char *) &netOptionCount, 4); + + for (int i = 0; i < nUnsupportedOptions; i++) + { + appendBinaryPQExpBuffer(buf, unsupportedOptions[i], + strlen(unsupportedOptions[i]) + 1); + } + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, 'v', buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +bool +ws_send_ready_for_query(int sock) +{ + char status = 'I'; + + return ws_send_message(sock, 'Z', &status, 1); +} + + +bool +ws_send_error_response(int sock, const char *sqlstate, const char *message) +{ + PQExpBuffer buf = createPQExpBuffer(); + + appendPQExpBufferChar(buf, 'S'); + appendBinaryPQExpBuffer(buf, "ERROR", strlen("ERROR") + 1); + + appendPQExpBufferChar(buf, 'C'); + appendBinaryPQExpBuffer(buf, sqlstate, strlen(sqlstate) + 1); + + appendPQExpBufferChar(buf, 'M'); + appendBinaryPQExpBuffer(buf, message, strlen(message) + 1); + + appendPQExpBufferChar(buf, '\0'); /* terminates the field list */ + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, 'E', buf->data, buf->len); + + destroyPQExpBuffer(buf); + + log_debug("walsender: sent ErrorResponse %s: %s", sqlstate, message); + + return ok; +} + + +bool +ws_send_command_complete(int sock, const char *tag) +{ + return ws_send_message(sock, 'C', tag, strlen(tag) + 1); +} + + +bool +ws_send_row_description(int sock, const WsColumn *columns, int ncols) +{ + PQExpBuffer buf = createPQExpBuffer(); + int16_t n = htons((int16_t) ncols); + + appendBinaryPQExpBuffer(buf, (const char *) &n, 2); + + for (int i = 0; i < ncols; i++) + { + int32_t zero32 = 0; + int16_t zero16 = 0; + int32_t typeOid = htonl(columns[i].typeOid); + int16_t typeLen = htons(columns[i].typeLen); + int32_t typeMod = htonl(-1); + int16_t format = 0; /* text */ + + appendBinaryPQExpBuffer(buf, columns[i].name, strlen(columns[i].name) + 1); + appendBinaryPQExpBuffer(buf, (const char *) &zero32, 4); /* table Oid */ + appendBinaryPQExpBuffer(buf, (const char *) &zero16, 2); /* column attnum */ + appendBinaryPQExpBuffer(buf, (const char *) &typeOid, 4); + appendBinaryPQExpBuffer(buf, (const char *) &typeLen, 2); + appendBinaryPQExpBuffer(buf, (const char *) &typeMod, 4); + appendBinaryPQExpBuffer(buf, (const char *) &format, 2); + } + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, 'T', buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +bool +ws_send_data_row(int sock, const char **values, int ncols) +{ + PQExpBuffer buf = createPQExpBuffer(); + int16_t n = htons((int16_t) ncols); + + appendBinaryPQExpBuffer(buf, (const char *) &n, 2); + + for (int i = 0; i < ncols; i++) + { + if (values[i] == NULL) + { + int32_t neg1 = htonl(-1); + + appendBinaryPQExpBuffer(buf, (const char *) &neg1, 4); + } + else + { + int32_t len = htonl((int32_t) strlen(values[i])); + + appendBinaryPQExpBuffer(buf, (const char *) &len, 4); + appendBinaryPQExpBuffer(buf, values[i], strlen(values[i])); + } + } + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, 'D', buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +static bool +ws_send_copy_response(int sock, char type, int ncols) +{ + PQExpBuffer buf = createPQExpBuffer(); + + /* overall format code: 0 (textual) -- we only ever send raw bytes, not + * a real column, so this is a formality real clients don't inspect for + * a CopyBoth/CopyOut stream driven by BASE_BACKUP/START_REPLICATION */ + appendPQExpBufferChar(buf, 0); + + int16_t n = htons((int16_t) ncols); + + appendBinaryPQExpBuffer(buf, (const char *) &n, 2); + + for (int i = 0; i < ncols; i++) + { + int16_t fmt = 0; + + appendBinaryPQExpBuffer(buf, (const char *) &fmt, 2); + } + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, type, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +bool +ws_send_copy_out_response(int sock, int ncols) +{ + return ws_send_copy_response(sock, 'H', ncols); +} + + +bool +ws_send_copy_both_response(int sock, int ncols) +{ + return ws_send_copy_response(sock, 'W', ncols); +} + + +bool +ws_send_copy_data(int sock, const char *data, int32_t dataLen) +{ + return ws_send_message(sock, 'd', data, dataLen); +} + + +bool +ws_send_copy_done(int sock) +{ + return ws_send_message(sock, 'c', NULL, 0); +} diff --git a/src/bin/pg_walsender/framing.h b/src/bin/pg_walsender/framing.h new file mode 100644 index 000000000..c7184de0b --- /dev/null +++ b/src/bin/pg_walsender/framing.h @@ -0,0 +1,82 @@ +/* + * src/bin/pg_walsender/framing.h + * Hand-written wire-level primitives for the Postgres frontend/backend + * protocol's server side: message read/write, CopyData framing, + * RowDescription/DataRow, ReadyForQuery, ErrorResponse. This is the + * pqcomm.c + pqformat.c equivalent -- no reusable library exists for + * this anywhere in Postgres (see walsender.h's own header comment), so + * it's hand-rolled directly from the documented wire format. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_FRAMING_H +#define WS_FRAMING_H + +#include +#include +#include + +/* well-known type Oids used in the RowDescriptions we hand back */ +#define WS_TEXTOID 25 +#define WS_INT4OID 23 +#define WS_INT8OID 20 + +typedef struct WsColumn +{ + const char *name; + int32_t typeOid; + int16_t typeLen; /* -1 for varlena types such as text */ +} WsColumn; + +/* raw byte I/O, EINTR-safe, short-read/short-write safe */ +bool ws_read_bytes(int sock, void *buf, size_t len); +bool ws_write_bytes(int sock, const void *buf, size_t len); + +/* + * ws_read_line reads a single '\n'-terminated line (the '\n' consumed but + * not included in *line), up to maxLen-1 bytes, NUL-terminated. Used only + * by the FETCH_FILE side-channel (cmd_fetch_file.c) for its one-shot + * "filename\n" request -- not part of the real Postgres wire protocol, + * deliberately as simple as the exchange it serves. + */ +bool ws_read_line(int sock, char *line, size_t maxLen); + +/* + * Startup-phase framing: before authentication, messages have no leading + * type byte (StartupMessage, SSLRequest, GSSENCRequest, CancelRequest are + * all just a length-prefixed body). + */ +bool ws_read_startup_payload(int sock, char **payload, int32_t *payloadLen); +bool ws_write_raw_byte(int sock, char c); + +/* + * Post-startup framing: 1-byte type + int32 length (length includes + * itself, matching the real protocol) + payload. ws_read_message + * NUL-terminates the returned payload for convenience (Query message + * bodies are C strings); callers that need the raw length still get it. + */ +bool ws_read_message(int sock, char *type, char **payload, int32_t *payloadLen); +bool ws_send_message(int sock, char type, const char *data, int32_t dataLen); + +bool ws_send_authentication_ok(int sock); +bool ws_send_parameter_status(int sock, const char *name, const char *value); +bool ws_send_backend_key_data(int sock, int32_t pid, int32_t secret); +bool ws_send_negotiate_protocol_version(int sock, int32_t newestMinor, + const char **unsupportedOptions, + int nUnsupportedOptions); +bool ws_send_ready_for_query(int sock); +bool ws_send_error_response(int sock, const char *sqlstate, const char *message); +bool ws_send_command_complete(int sock, const char *tag); + +bool ws_send_row_description(int sock, const WsColumn *columns, int ncols); +bool ws_send_data_row(int sock, const char **values, int ncols); + +bool ws_send_copy_out_response(int sock, int ncols); +bool ws_send_copy_both_response(int sock, int ncols); +bool ws_send_copy_data(int sock, const char *data, int32_t dataLen); +bool ws_send_copy_done(int sock); + +#endif /* WS_FRAMING_H */ diff --git a/src/bin/pg_walsender/main.c b/src/bin/pg_walsender/main.c new file mode 100644 index 000000000..d028ee149 --- /dev/null +++ b/src/bin/pg_walsender/main.c @@ -0,0 +1,230 @@ +/* + * src/bin/pg_walsender/main.c + * Entry point for pg_walsender. Two modes, dispatched on argv[1]: + * + * pg_walsender --port [--routes ] + * Runs the accept loop (see accept_loop.h). Exec'd by pg_autoctl's + * `archiver serve` supervisor (service_archiver_serve.c), but fully + * runnable and testable on its own against real psql/pg_basebackup/ + * pg_receivewal. + * + * pg_walsender fetch-file --host --port

--route / + * --filename --output + * Runs the FETCH_FILE client (fetch_client.h) once and exits -- + * pg_autoctl's restore_command shells out to this, the same way it + * already shells out to real pg_receivewal/pg_basebackup elsewhere + * in this project. + * + * Standalone binary (see the Makefile's own header comment) -- links + * neither of these modes against pg_autoctl's own sources. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "lock_utils.h" + +#include "accept_loop.h" +#include "defaults.h" +#include "fetch_client.h" +#include "file_utils.h" +#include "log.h" +#include "string_utils.h" + +/* + * Globals required by shared common/ sources (file_utils.c's + * init_ps_buffer/set_ps_title in particular) -- pg_walsender owns these + * stub definitions itself, exactly like pgaftest's main.c does, since it + * doesn't link pg_autoctl's own main.c. + */ +char pg_autoctl_argv0[MAXPGPATH] = "pg_walsender"; +char pg_autoctl_program[MAXPGPATH] = "pg_walsender"; +int pgconnect_timeout = 2; +char *ps_buffer; +size_t ps_buffer_size; +size_t last_status_len; +Semaphore log_semaphore = { 0 }; + + +static void +usage(const char *argv0) +{ + fprintf(stderr, /* IGNORE-BANNED */ + "Usage: %s --port [--routes ]\n" + " %s fetch-file --host --port

--route / " + "--filename --output \n\n" + " --port port to listen on (server mode default: %d)\n" + " --routes path to the routes INI file mapping " + "\"/\" to\n" + " { walcache, basebackup, allowed_hosts } -- " + "omit only for manual\n" + " standalone testing (accepts any dbname, no " + "host restriction)\n" + " fetch-file one-shot FETCH_FILE client, for use as a " + "restore_command\n", + argv0, argv0, WS_DEFAULT_PORT); +} + + +static int +main_fetch_file(int argc, char **argv) +{ + char host[256] = { 0 }; + int port = WS_DEFAULT_PORT; + char route[256] = { 0 }; + char filename[256] = { 0 }; + char output[MAXPGPATH] = { 0 }; + + static struct option longOptions[] = { + { "host", required_argument, NULL, 'H' }, + { "port", required_argument, NULL, 'p' }, + { "route", required_argument, NULL, 'r' }, + { "filename", required_argument, NULL, 'f' }, + { "output", required_argument, NULL, 'o' }, + { NULL, 0, NULL, 0 } + }; + + int c; + + while ((c = getopt_long(argc, argv, "H:p:r:f:o:", longOptions, NULL)) != -1) + { + switch (c) + { + case 'H': + { + strlcpy(host, optarg, sizeof(host)); + break; + } + + case 'p': + { + if (!stringToInt(optarg, &port)) + { + log_fatal("Invalid --port value \"%s\"", optarg); + return 1; + } + break; + } + + case 'r': + { + strlcpy(route, optarg, sizeof(route)); + break; + } + + case 'f': + { + strlcpy(filename, optarg, sizeof(filename)); + break; + } + + case 'o': + { + strlcpy(output, optarg, sizeof(output)); + break; + } + + default: + { + usage(argv[0]); + return 1; + } + } + } + + if (host[0] == '\0' || route[0] == '\0' || filename[0] == '\0' || + output[0] == '\0') + { + fprintf(stderr, /* IGNORE-BANNED */ + "fetch-file: --host, --route, --filename, and " + "--output are all required\n"); + usage(argv[0]); + return 1; + } + + return ws_fetch_file_client(host, port, route, filename, output); +} + + +int +main(int argc, char **argv) +{ + strlcpy(pg_autoctl_program, argv[0], sizeof(pg_autoctl_program)); + init_ps_buffer(argc, argv); + + log_set_level(LOG_INFO); + + if (argc >= 2 && strcmp(argv[1], "fetch-file") == 0) + { + /* shift argv so getopt_long in main_fetch_file() skips "fetch-file" */ + return main_fetch_file(argc - 1, argv + 1); + } + + WsServerConfig config = { 0 }; + + config.port = WS_DEFAULT_PORT; + + static struct option longOptions[] = { + { "port", required_argument, NULL, 'p' }, + { "routes", required_argument, NULL, 'r' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + int c; + + while ((c = getopt_long(argc, argv, "p:r:h", longOptions, NULL)) != -1) + { + switch (c) + { + case 'p': + { + if (!stringToInt(optarg, &(config.port))) + { + log_fatal("Invalid --port value \"%s\"", optarg); + return 1; + } + break; + } + + case 'r': + { + strlcpy(config.routesPath, optarg, sizeof(config.routesPath)); + break; + } + + case 'h': + { + usage(argv[0]); + return 0; + } + + default: + { + usage(argv[0]); + return 1; + } + } + } + + if (config.port <= 0 || config.port > 65535) + { + log_fatal("Invalid --port value"); + return 1; + } + + if (!ws_accept_loop(&config)) + { + return 1; + } + + return 0; +} diff --git a/src/bin/pg_walsender/repl_command.c b/src/bin/pg_walsender/repl_command.c new file mode 100644 index 000000000..fa31a4bef --- /dev/null +++ b/src/bin/pg_walsender/repl_command.c @@ -0,0 +1,186 @@ +/* + * src/bin/pg_walsender/repl_command.c + * See repl_command.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include + +#include "postgres_fe.h" + +#include "string_utils.h" + +#include "repl_command.h" +#include "cmd_base_backup.h" +#include "cmd_identify_system.h" +#include "cmd_replication_slot.h" +#include "cmd_show.h" +#include "cmd_start_replication.h" +#include "cmd_timeline_history.h" +#include "framing.h" + + +static const char * +skip_whitespace(const char *p) +{ + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') + { + p++; + } + + return p; +} + + +static void +rtrim(char *s) +{ + size_t n = strlen(s); + + while (n > 0 && + (s[n - 1] == ' ' || s[n - 1] == '\t' || s[n - 1] == '\n' || + s[n - 1] == '\r' || s[n - 1] == ';')) + { + s[--n] = '\0'; + } +} + + +bool +repl_command_parse(const char *query, WsCommand *cmd) +{ + memset(cmd, 0, sizeof(WsCommand)); + + const char *p = skip_whitespace(query); + + if (strncasecmp(p, "IDENTIFY_SYSTEM", strlen("IDENTIFY_SYSTEM")) == 0) + { + cmd->type = WS_CMD_IDENTIFY_SYSTEM; + return true; + } + + if (strncasecmp(p, "SHOW", strlen("SHOW")) == 0 && isspace((unsigned char) p[4])) + { + p = skip_whitespace(p + 4); + strlcpy(cmd->showName, p, sizeof(cmd->showName)); + rtrim(cmd->showName); + cmd->type = WS_CMD_SHOW; + return true; + } + + if (strncasecmp(p, "BASE_BACKUP", strlen("BASE_BACKUP")) == 0) + { + p = skip_whitespace(p + strlen("BASE_BACKUP")); + strlcpy(cmd->rawOptions, p, sizeof(cmd->rawOptions)); + rtrim(cmd->rawOptions); + cmd->type = WS_CMD_BASE_BACKUP; + return true; + } + + if (strncasecmp(p, "TIMELINE_HISTORY", strlen("TIMELINE_HISTORY")) == 0) + { + p = skip_whitespace(p + strlen("TIMELINE_HISTORY")); + + if (!stringToInt(p, &(cmd->timeline))) + { + return false; + } + + cmd->type = WS_CMD_TIMELINE_HISTORY; + return true; + } + + if (strncasecmp(p, "CREATE_REPLICATION_SLOT", + strlen("CREATE_REPLICATION_SLOT")) == 0) + { + p = skip_whitespace(p + strlen("CREATE_REPLICATION_SLOT")); + strlcpy(cmd->rawArgs, p, sizeof(cmd->rawArgs)); + rtrim(cmd->rawArgs); + cmd->type = WS_CMD_CREATE_REPLICATION_SLOT; + return true; + } + + if (strncasecmp(p, "READ_REPLICATION_SLOT", + strlen("READ_REPLICATION_SLOT")) == 0) + { + p = skip_whitespace(p + strlen("READ_REPLICATION_SLOT")); + strlcpy(cmd->rawArgs, p, sizeof(cmd->rawArgs)); + rtrim(cmd->rawArgs); + cmd->type = WS_CMD_READ_REPLICATION_SLOT; + return true; + } + + if (strncasecmp(p, "START_REPLICATION", strlen("START_REPLICATION")) == 0) + { + p = skip_whitespace(p + strlen("START_REPLICATION")); + strlcpy(cmd->rawArgs, p, sizeof(cmd->rawArgs)); + rtrim(cmd->rawArgs); + cmd->type = WS_CMD_START_REPLICATION; + return true; + } + + cmd->type = WS_CMD_UNKNOWN; + return false; +} + + +void +ws_dispatch_command(int sock, const WsCommand *cmd, + const WsRoute *route, const char *dbname) +{ + switch (cmd->type) + { + case WS_CMD_IDENTIFY_SYSTEM: + { + cmd_identify_system(sock, route, dbname); + break; + } + + case WS_CMD_SHOW: + { + cmd_show(sock, cmd->showName); + break; + } + + case WS_CMD_BASE_BACKUP: + { + cmd_base_backup(sock, route, cmd->rawOptions); + break; + } + + case WS_CMD_TIMELINE_HISTORY: + { + cmd_timeline_history(sock, route, cmd->timeline); + break; + } + + case WS_CMD_CREATE_REPLICATION_SLOT: + { + cmd_create_replication_slot(sock, route, cmd->rawArgs); + break; + } + + case WS_CMD_READ_REPLICATION_SLOT: + { + cmd_read_replication_slot(sock, route, cmd->rawArgs); + break; + } + + case WS_CMD_START_REPLICATION: + { + cmd_start_replication(sock, route, cmd->rawArgs); + break; + } + + default: + { + ws_send_error_response(sock, "42601", "unsupported replication command"); + break; + } + } +} diff --git a/src/bin/pg_walsender/repl_command.h b/src/bin/pg_walsender/repl_command.h new file mode 100644 index 000000000..fe53e5964 --- /dev/null +++ b/src/bin/pg_walsender/repl_command.h @@ -0,0 +1,69 @@ +/* + * src/bin/pg_walsender/repl_command.h + * Parses the Query-message command strings real replication clients send + * (e.g. "IDENTIFY_SYSTEM", "SHOW wal_segment_size") and dispatches to the + * matching cmd_*.c handler. This is repl_gram.y/repl_scanner.l's + * equivalent, hand-rolled: the real grammar is backend-locked (bison + * output building backend Node types via palloc, see the design + * research), and the fixed ~7-command surface this project needs doesn't + * justify vendoring bison/flex infrastructure for it -- plain C + * tokenizing is enough. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_REPL_COMMAND_H +#define WS_REPL_COMMAND_H + +#include + +#include "routes.h" + +typedef enum WsCommandType +{ + WS_CMD_IDENTIFY_SYSTEM, + WS_CMD_SHOW, + WS_CMD_BASE_BACKUP, + WS_CMD_TIMELINE_HISTORY, + WS_CMD_CREATE_REPLICATION_SLOT, + WS_CMD_READ_REPLICATION_SLOT, + WS_CMD_START_REPLICATION, + WS_CMD_UNKNOWN +} WsCommandType; + +typedef struct WsCommand +{ + WsCommandType type; + char showName[NAMEDATALEN]; /* WS_CMD_SHOW only */ + char rawOptions[1024]; /* WS_CMD_BASE_BACKUP only: the "(...)" or + * trailing-token option list verbatim, + * parsed by cmd_base_backup.c itself */ + int timeline; /* WS_CMD_TIMELINE_HISTORY only */ + char rawArgs[512]; /* WS_CMD_{CREATE,READ}_REPLICATION_SLOT / + * WS_CMD_START_REPLICATION: everything + * after the keyword, verbatim, parsed by + * each command's own cmd_*.c */ +} WsCommand; + +/* + * repl_command_parse fills *cmd from the given Query-message string. + * Returns false (cmd->type == WS_CMD_UNKNOWN) for anything not yet + * recognized -- the caller sends the ErrorResponse, this function doesn't + * touch the socket. + */ +bool repl_command_parse(const char *query, WsCommand *cmd); + +/* + * ws_dispatch_command runs cmd against the connection's resolved route + * (NULL in manual-testing mode, see auth.h) and the dbname the client + * originally requested (always set, even without a route -- see + * startup.c), sending whatever RowDescription/DataRow/CommandComplete or + * ErrorResponse the command produces. Never sends ReadyForQuery -- the + * caller's command loop does that once per Query message, uniformly. + */ +void ws_dispatch_command(int sock, const WsCommand *cmd, + const WsRoute *route, const char *dbname); + +#endif /* WS_REPL_COMMAND_H */ diff --git a/src/bin/pg_walsender/routes.c b/src/bin/pg_walsender/routes.c new file mode 100644 index 000000000..107cef1b4 --- /dev/null +++ b/src/bin/pg_walsender/routes.c @@ -0,0 +1,238 @@ +/* + * src/bin/pg_walsender/routes.c + * See routes.h. Deliberately built on the low-level, dynamic-section + * ini.h API (ini_load/ini_section_count/...) rather than this project's + * own ini_file.c wrapper: ini_file.c's IniOption model assumes a fixed, + * compile-time-known set of section/key names, which doesn't fit a file + * whose sections are one per archived (formation, group) -- unknown in + * advance. ini.h's lower-level, enumerable API is exactly the right + * shape and is already vendored into this project (src/bin/lib/libs/ + * ini.h, compiled into libpgaf_common.a via common/ini_implementation.c). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "ini.h" + +#include "routes.h" +#include "file_utils.h" +#include "log.h" +#include "string_utils.h" + + +bool +routes_load(const char *path, WsRoute **routesOut, int *countOut) +{ + *routesOut = NULL; + *countOut = 0; + + char *contents = NULL; + long fileSize = 0; + + if (!read_file(path, &contents, &fileSize)) + { + log_error("Failed to read routes file \"%s\"", path); + return false; + } + + ini_t *ini = ini_load(contents, NULL); + + free(contents); + + if (ini == NULL) + { + log_error("Failed to parse routes file \"%s\"", path); + return false; + } + + int sectionCount = ini_section_count(ini); + + /* section 0 is ini.h's implicit global section: never a real route */ + WsRoute *routes = (WsRoute *) calloc(sectionCount, sizeof(WsRoute)); + + if (routes == NULL && sectionCount > 0) + { + log_error("Failed to allocate memory for %d routes", sectionCount); + ini_destroy(ini); + return false; + } + + int n = 0; + + for (int s = 0; s < sectionCount; s++) + { + const char *name = ini_section_name(ini, s); + + if (name == NULL || name[0] == '\0') + { + continue; /* the global section */ + } + + WsRoute *route = &routes[n]; + + memset(route, 0, sizeof(WsRoute)); + strlcpy(route->key, name, sizeof(route->key)); + + int propCount = ini_property_count(ini, s); + + for (int p = 0; p < propCount; p++) + { + const char *rawPropName = ini_property_name(ini, s, p); + const char *propValue = ini_property_value(ini, s, p); + + if (rawPropName == NULL || propValue == NULL) + { + continue; + } + + /* + * ini.h's own parser (src/bin/lib/libs/ini.h's ini_load) trims + * whitespace around the value but NOT trailing whitespace + * between a key and '=' -- "walcache = /path" parses the key + * as "walcache " with a trailing space. Trim defensively here + * rather than relying on every routes file being written with + * no space before '='. + */ + char propName[128]; + + strlcpy(propName, rawPropName, sizeof(propName)); + + size_t nameLen = strlen(propName); + + while (nameLen > 0 && isspace((unsigned char) propName[nameLen - 1])) + { + propName[--nameLen] = '\0'; + } + + if (strcmp(propName, "walcache") == 0) + { + strlcpy(route->walcacheDir, propValue, sizeof(route->walcacheDir)); + } + else if (strcmp(propName, "basebackup") == 0) + { + strlcpy(route->basebackupDir, propValue, sizeof(route->basebackupDir)); + } + else if (strcmp(propName, "allowed_hosts") == 0) + { + strlcpy(route->allowedHosts, propValue, sizeof(route->allowedHosts)); + } + else if (strcmp(propName, "systemid") == 0) + { + strlcpy(route->systemId, propValue, sizeof(route->systemId)); + } + else if (strcmp(propName, "timeline") == 0) + { + (void) stringToInt(propValue, &(route->timeline)); + } + else if (strcmp(propName, "position") == 0) + { + strlcpy(route->position, propValue, sizeof(route->position)); + } + else + { + log_warn("Ignoring unknown routes file key \"%s\" in section [%s]", + propName, name); + } + } + + n++; + } + + ini_destroy(ini); + + *routesOut = routes; + *countOut = n; + + return true; +} + + +void +routes_free(WsRoute *routes) +{ + free(routes); +} + + +const WsRoute * +routes_find(const WsRoute *routes, int count, const char *key) +{ + for (int i = 0; i < count; i++) + { + if (strcmp(routes[i].key, key) == 0) + { + return &routes[i]; + } + } + + return NULL; +} + + +bool +routes_host_allowed(const WsRoute *route, const char *peerIP) +{ + if (route->allowedHosts[0] == '\0') + { + return true; /* no restriction configured for this route */ + } + + char list[sizeof(route->allowedHosts)]; + + strlcpy(list, route->allowedHosts, sizeof(list)); + + char *saveptr = NULL; + + for (char *tok = strtok_r(list, ",", &saveptr); + tok != NULL; + tok = strtok_r(NULL, ",", &saveptr)) + { + while (*tok == ' ' || *tok == '\t') + { + tok++; + } + + if (strcmp(tok, peerIP) == 0) + { + return true; + } + + /* also resolve hostnames in the allow-list and compare addresses */ + struct addrinfo hints; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + + struct addrinfo *res = NULL; + + if (getaddrinfo(tok, NULL, &hints, &res) == 0) + { + for (struct addrinfo *rp = res; rp != NULL; rp = rp->ai_next) + { + char resolved[NI_MAXHOST]; + + if (getnameinfo(rp->ai_addr, rp->ai_addrlen, + resolved, sizeof(resolved), + NULL, 0, NI_NUMERICHOST) == 0 && + strcmp(resolved, peerIP) == 0) + { + freeaddrinfo(res); + return true; + } + } + + freeaddrinfo(res); + } + } + + return false; +} diff --git a/src/bin/pg_walsender/routes.h b/src/bin/pg_walsender/routes.h new file mode 100644 index 000000000..e54941545 --- /dev/null +++ b/src/bin/pg_walsender/routes.h @@ -0,0 +1,59 @@ +/* + * src/bin/pg_walsender/routes.h + * The archiver's own "pg_hba.conf" equivalent: a small INI file, one + * section per "/" this archiver serves, mapping the + * incoming connection's dbname to a WAL-cache directory, a base-backup + * directory, and an optional allowed-hosts list. Written and periodically + * refreshed by pg_autoctl's archiver-serve supervisor + * (service_archiver_serve.c) from the monitor's archiver_node/basebackup + * rows; pg_walsender itself never talks to the monitor (see the + * "Routing" section of ~/dev/temp/archiving-disaster-recovery.md's + * implementation plan). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_ROUTES_H +#define WS_ROUTES_H + +#include + +#include "postgres_fe.h" + +typedef struct WsRoute +{ + char key[NAMEDATALEN + 16]; /* "/", matches dbname */ + char walcacheDir[MAXPGPATH]; + char basebackupDir[MAXPGPATH]; + char allowedHosts[1024]; /* comma-separated, empty = unrestricted */ + char systemId[32]; /* decimal uint64, as text; "" = unknown */ + int timeline; /* 0 = unknown */ + char position[32]; /* "%X/%08X" pg_lsn text; "" = unknown -- + * see service_archiver_update_current_lsn()'s + * own comment (service_archiver.c) for what + * this is and why it lives here rather than + * being re-derived from WAL file content by + * each reader */ +} WsRoute; + +/* + * routes_load parses the routes file at path into a freshly malloc'ed + * array. Returns true with *routesOut and *countOut set (possibly count + * == 0 for an empty file) on success, false on a missing/malformed file. + */ +bool routes_load(const char *path, WsRoute **routesOut, int *countOut); +void routes_free(WsRoute *routes); + +const WsRoute * routes_find(const WsRoute *routes, int count, const char *key); + +/* + * routes_host_allowed checks peerIP (a numeric address string, as returned + * by getnameinfo(..., NI_NUMERICHOST)) against route->allowedHosts, which + * may contain either numeric addresses or hostnames (resolved via DNS at + * check time). An empty allowedHosts list means "no restriction." + */ +bool routes_host_allowed(const WsRoute *route, const char *peerIP); + +#endif /* WS_ROUTES_H */ diff --git a/src/bin/pg_walsender/startup.c b/src/bin/pg_walsender/startup.c new file mode 100644 index 000000000..e5d61b624 --- /dev/null +++ b/src/bin/pg_walsender/startup.c @@ -0,0 +1,183 @@ +/* + * src/bin/pg_walsender/startup.c + * See startup.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include + +#include "postgres_fe.h" + +#include "startup.h" +#include "framing.h" +#include "log.h" + +#define SSL_REQUEST_CODE 80877103 +#define GSS_REQUEST_CODE 80877104 +#define CANCEL_REQUEST_CODE 80877102 + + +bool +ws_startup_negotiate(int sock, WsStartupParams *params) +{ + memset(params, 0, sizeof(WsStartupParams)); + + for (;;) + { + char *payload = NULL; + int32_t payloadLen = 0; + + if (!ws_read_startup_payload(sock, &payload, &payloadLen)) + { + free(payload); + return false; + } + + if (payloadLen < 4) + { + log_error("Received a malformed startup packet (%d bytes)", payloadLen); + free(payload); + return false; + } + + int32_t code; + + memcpy(&code, payload, 4); /* IGNORE-BANNED */ + code = ntohl(code); + + if (code == SSL_REQUEST_CODE || code == GSS_REQUEST_CODE) + { + free(payload); + + /* + * MVP: no SSL/GSS support yet (see the design doc's Auth + * section) -- decline, real libpq's default sslmode=prefer + * falls back to plaintext automatically on 'N'. + */ + if (!ws_write_raw_byte(sock, 'N')) + { + return false; + } + + continue; + } + + if (code == CANCEL_REQUEST_CODE) + { + log_debug("Ignoring a CancelRequest on a walsender connection"); + free(payload); + return false; + } + + if ((code >> 16) != 3) + { + log_error("Unsupported startup protocol version 0x%08x", code); + free(payload); + return false; + } + + /* + * Parse the NUL-separated key/value pairs following the version + * code first -- we need to know which "_pq_.*" options (if any) the + * client sent *before* we can answer NegotiateProtocolVersion below: + * real libpq's protocol-GREASE self-test sends + * "_pq_.test_protocol_negotiation" and requires the server to echo + * it back as unsupported (we don't parse any "_pq_.*" options, so + * every one seen here is unsupported by definition). + */ + const char *ptr = payload + 4; + const char *end = payload + payloadLen; + + enum + { + WS_MAX_UNSUPPORTED_OPTIONS = 16 + }; + const char *unsupportedOptions[WS_MAX_UNSUPPORTED_OPTIONS]; + int nUnsupportedOptions = 0; + + while (ptr < end && *ptr != '\0') + { + const char *key = ptr; + + ptr += strlen(ptr) + 1; + + if (ptr >= end) + { + break; + } + + const char *value = ptr; + + ptr += strlen(ptr) + 1; + + if (strcmp(key, "user") == 0) + { + strlcpy(params->user, value, sizeof(params->user)); + } + else if (strcmp(key, "database") == 0) + { + strlcpy(params->database, value, sizeof(params->database)); + } + else if (strcmp(key, "application_name") == 0) + { + strlcpy(params->applicationName, value, sizeof(params->applicationName)); + } + else if (strcmp(key, "replication") == 0) + { + params->replicationDatabase = (strcasecmp(value, "database") == 0); + params->replication = (strcmp(value, "1") == 0 || + strcasecmp(value, "true") == 0 || + params->replicationDatabase); + } + else if (strncmp(key, "_pq_.", 5) == 0 && + nUnsupportedOptions < WS_MAX_UNSUPPORTED_OPTIONS) + { + unsupportedOptions[nUnsupportedOptions++] = key; + } + } + + /* + * Only protocol 3.0 is implemented. A client is free to ask for a + * newer minor version than we understand -- real libpq deliberately + * probes with a bogus one (protocol "GREASE", e.g. 3.9999) to + * verify a server properly negotiates rather than silently + * accepting whatever was asked for, and refuses to proceed against + * a server that gets this wrong. Tell it the newest minor version + * we actually speak (0) via NegotiateProtocolVersion, matching real + * Postgres's own backend behaviour, then continue the connection at + * that version rather than closing it. + */ + if ((code & 0xFFFF) != 0) + { + if (!ws_send_negotiate_protocol_version(sock, 0, + unsupportedOptions, + nUnsupportedOptions)) + { + free(payload); + return false; + } + } + + free(payload); + + /* + * A real replication connection always carries "database" too when + * replication=database is used (that's how pg_basebackup connects); + * a bare replication=1/true connection (pg_receivewal's style) may + * not set "database" at all. Default it to the "user" so downstream + * routing always has *something* to look up rather than an empty + * key -- callers that require a real "/" key + * still get a clean "unknown route" ErrorResponse from auth.c. + */ + if (params->database[0] == '\0') + { + strlcpy(params->database, params->user, sizeof(params->database)); + } + + return true; + } +} diff --git a/src/bin/pg_walsender/startup.h b/src/bin/pg_walsender/startup.h new file mode 100644 index 000000000..d1c4477c5 --- /dev/null +++ b/src/bin/pg_walsender/startup.h @@ -0,0 +1,30 @@ +/* + * src/bin/pg_walsender/startup.h + * Startup-packet negotiation: SSL/GSS decline, protocol version check, + * and StartupMessage key/value parsing. Structurally mirrors real + * Postgres's ProcessStartupPacket() (backend_startup.c), reimplemented + * frontend-only -- that function is backend-locked (palloc/List/ereport, + * see the design research in ~/dev/temp/archiving-disaster-recovery.md's + * companion investigation), not something we can call into directly. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_STARTUP_H +#define WS_STARTUP_H + +#include + +#include "walsender.h" + +/* + * ws_startup_negotiate reads (and answers) SSLRequest/GSSENCRequest + * probes until the client sends a real StartupMessage, then parses it into + * *params. Returns false on any protocol error or if the client gives up + * (socket already unusable at that point; caller should just close it). + */ +bool ws_startup_negotiate(int sock, WsStartupParams *params); + +#endif /* WS_STARTUP_H */ diff --git a/src/bin/pg_walsender/tar_stream.c b/src/bin/pg_walsender/tar_stream.c new file mode 100644 index 000000000..fc0a48a67 --- /dev/null +++ b/src/bin/pg_walsender/tar_stream.c @@ -0,0 +1,271 @@ +/* + * src/bin/pg_walsender/tar_stream.c + * See tar_stream.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "pgtar.h" + +#include "tar_stream.h" +#include "file_utils.h" +#include "log.h" + +/* matches basebackup.c's own TAR_NUM_TERMINATION_BLOCKS */ +#define TAR_NUM_TERMINATION_BLOCKS 2 + +#define TAR_READ_CHUNK_SIZE (64 * 1024) + +typedef struct TarWalkState +{ + TarChunkCallback callback; + void *context; + bool ok; +} TarWalkState; + + +static bool +emit(TarWalkState *state, const char *data, size_t len) +{ + if (!state->ok) + { + return false; + } + + if (!state->callback(state->context, data, len)) + { + state->ok = false; + } + + return state->ok; +} + + +static bool +emit_header(TarWalkState *state, const char *memberName, + const char *linkTarget, struct stat *st) +{ + char header[TAR_BLOCK_SIZE]; + + enum tarError rc = tarCreateHeader(header, memberName, linkTarget, + st->st_size, st->st_mode, + st->st_uid, st->st_gid, st->st_mtime); + + if (rc != TAR_OK) + { + log_error("Failed to build a tar header for \"%s\": %s", memberName, + rc == TAR_NAME_TOO_LONG + ? "file name too long for tar format" + : "symbolic link target too long for tar format"); + return false; + } + + return emit(state, header, TAR_BLOCK_SIZE); +} + + +static bool +emit_file_contents(TarWalkState *state, const char *path, off_t size) +{ + FILE *file = fopen(path, "rb"); /* IGNORE-BANNED */ + + if (file == NULL) + { + log_error("Failed to open \"%s\": %m", path); + return false; + } + + char buffer[TAR_READ_CHUNK_SIZE]; + off_t remaining = size; + + while (remaining > 0) + { + size_t want = (size_t) Min(remaining, (off_t) sizeof(buffer)); + size_t got = fread(buffer, 1, want, file); + + if (got == 0) + { + log_error("Short read on \"%s\" while building a base backup tar " + "stream (file changed size mid-read?)", path); + fclose(file); + return false; + } + + if (!emit(state, buffer, got)) + { + fclose(file); + return false; + } + + remaining -= (off_t) got; + } + + fclose(file); + + size_t pad = tarPaddingBytesRequired((size_t) size); + + if (pad > 0) + { + char zeros[TAR_BLOCK_SIZE] = { 0 }; + + if (!emit(state, zeros, pad)) + { + return false; + } + } + + return true; +} + + +static bool +walk_directory(TarWalkState *state, const char *rootDir, const char *relDir) +{ + char fullDir[MAXPGPATH]; + + if (relDir[0] == '\0') + { + strlcpy(fullDir, rootDir, sizeof(fullDir)); + } + else + { + sformat(fullDir, sizeof(fullDir), "%s/%s", rootDir, relDir); + } + + DIR *dir = opendir(fullDir); + + if (dir == NULL) + { + log_error("Failed to open directory \"%s\": %m", fullDir); + return false; + } + + struct dirent *entry; + + while (state->ok && (entry = readdir(dir)) != NULL) + { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) + { + continue; + } + + char fullPath[MAXPGPATH]; + char relPath[MAXPGPATH]; + + sformat(fullPath, sizeof(fullPath), "%s/%s", fullDir, entry->d_name); + + if (relDir[0] == '\0') + { + strlcpy(relPath, entry->d_name, sizeof(relPath)); + } + else + { + sformat(relPath, sizeof(relPath), "%s/%s", relDir, entry->d_name); + } + + struct stat st; + + if (lstat(fullPath, &st) != 0) + { + log_error("Failed to stat \"%s\": %m", fullPath); + state->ok = false; + break; + } + + if (S_ISLNK(st.st_mode)) + { + char linkTarget[MAXPGPATH]; + ssize_t len = readlink(fullPath, linkTarget, sizeof(linkTarget) - 1); + + if (len < 0) + { + log_error("Failed to read symbolic link \"%s\": %m", fullPath); + state->ok = false; + break; + } + + linkTarget[len] = '\0'; + + /* + * A symlink to a directory (Postgres uses this for tablespace + * links under pg_tblspc/) is written as a directory entry with + * a link target, matching tarCreateHeader()'s own convention + * (see its S_ISDIR/linktarget handling) -- but we don't + * recurse through it: multi-tablespace archives are a later + * milestone (see this file's own header comment), a symlink + * here is emitted as a bare tar entry, not expanded. + */ + if (!emit_header(state, relPath, linkTarget, &st)) + { + state->ok = false; + break; + } + + continue; + } + + if (S_ISDIR(st.st_mode)) + { + if (!emit_header(state, relPath, NULL, &st)) + { + state->ok = false; + break; + } + + if (!walk_directory(state, rootDir, relPath)) + { + state->ok = false; + break; + } + + continue; + } + + if (!S_ISREG(st.st_mode)) + { + /* skip anything else (sockets, fifos, device files) */ + continue; + } + + if (!emit_header(state, relPath, NULL, &st)) + { + state->ok = false; + break; + } + + if (!emit_file_contents(state, fullPath, st.st_size)) + { + state->ok = false; + break; + } + } + + closedir(dir); + + return state->ok; +} + + +bool +tar_stream_directory(const char *rootDir, TarChunkCallback callback, void *context) +{ + TarWalkState state = { callback, context, true }; + + if (!walk_directory(&state, rootDir, "")) + { + return false; + } + + char zeros[TAR_BLOCK_SIZE * TAR_NUM_TERMINATION_BLOCKS] = { 0 }; + + return emit(&state, zeros, sizeof(zeros)); +} diff --git a/src/bin/pg_walsender/tar_stream.h b/src/bin/pg_walsender/tar_stream.h new file mode 100644 index 000000000..5c4fcce6f --- /dev/null +++ b/src/bin/pg_walsender/tar_stream.h @@ -0,0 +1,48 @@ +/* + * src/bin/pg_walsender/tar_stream.h + * Walks a directory tree and emits it as a ustar-format byte stream via a + * callback, chunked for CopyData framing. Reproduces the shape of + * basebackup.c's sendDir()/sendFile()/_tarWriteHeader() pattern -- not + * linked (backend-only, tied to the bbsink sink-chain and palloc/ + * ereport), but the tar-header math itself comes straight from the + * vendored vendor/tar.c (tarCreateHeader(), the real Postgres source + * both basebackup.c and pg_basebackup itself build on). + * + * Deliberately simpler than basebackup.c's own sendDir(): this walks an + * already-complete, static backup directory (produced by a real + * pg_basebackup run against a live server -- see the "Base backup + * generation" milestone, not yet implemented), so none of basebackup.c's + * live-PGDATA special-casing (skipping pg_wal/pg_stat_tmp/postmaster + * files, injecting a synthesized backup_label, tracking WAL positions + * mid-walk) applies -- the directory is tarred up exactly as it sits on + * disk. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_TAR_STREAM_H +#define WS_TAR_STREAM_H + +#include +#include + +/* + * Called with successive chunks of the tar byte stream (header blocks, + * file content, padding, and the final end-of-archive zero blocks all flow + * through this same callback) -- return false to abort the walk early + * (e.g. the client disconnected mid-stream). + */ +typedef bool (*TarChunkCallback) (void *context, const char *data, size_t len); + +/* + * tar_stream_directory walks rootDir recursively and invokes callback with + * the resulting ustar byte stream, including the standard two-zero-block + * end-of-archive marker. Tar member names are rootDir-relative, with no + * leading "./" (matching real Postgres's own convention -- see + * basebackup.c's sendDir()). + */ +bool tar_stream_directory(const char *rootDir, TarChunkCallback callback, void *context); + +#endif /* WS_TAR_STREAM_H */ diff --git a/src/bin/pg_walsender/vendor/pgtar.h b/src/bin/pg_walsender/vendor/pgtar.h new file mode 100644 index 000000000..9f80583b5 --- /dev/null +++ b/src/bin/pg_walsender/vendor/pgtar.h @@ -0,0 +1,98 @@ +/*------------------------------------------------------------------------- + * + * pgtar.h + * Functions for manipulating tarfile datastructures (vendor/tar.c) + * + * Vendored from PostgreSQL's src/include/pgtar.h (checked against + * /Users/dim/dev/PostgreSQL/postgresql; logic unchanged, reformatted to + * this project's own brace style via citus_indent) -- pure ustar-format + * constants and one inline helper, no backend dependency, genuinely + * reusable as-is (same PostgreSQL License). pg_walsender's own + * tar_stream.c builds the BASE_BACKUP tar stream on top of + * tarCreateHeader() from vendor/tar.c, the same way basebackup.c's + * _tarWriteHeader() does. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/pgtar.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_TAR_H +#define PG_TAR_H + +#define TAR_BLOCK_SIZE 512 + +enum tarError +{ + TAR_OK = 0, + TAR_NAME_TOO_LONG, + TAR_SYMLINK_TOO_LONG, +}; + +/* + * Offsets of fields within a 512-byte tar header. + * + * "tar number" values should be generated using print_tar_number() and can be + * read using read_tar_number(). Fields that contain strings are generally + * both filled and read using strlcpy(). + * + * The value for the checksum field can be computed using tarChecksum(). + * + * Some fields are not used by PostgreSQL; see tarCreateHeader(). + */ +enum tarHeaderOffset +{ + TAR_OFFSET_NAME = 0, /* 100 byte string */ + TAR_OFFSET_MODE = 100, /* 8 byte tar number, excludes S_IFMT */ + TAR_OFFSET_UID = 108, /* 8 byte tar number */ + TAR_OFFSET_GID = 116, /* 8 byte tar number */ + TAR_OFFSET_SIZE = 124, /* 8 byte tar number */ + TAR_OFFSET_MTIME = 136, /* 12 byte tar number */ + TAR_OFFSET_CHECKSUM = 148, /* 8 byte tar number */ + TAR_OFFSET_TYPEFLAG = 156, /* 1 byte file type, see TAR_FILETYPE_* */ + TAR_OFFSET_LINKNAME = 157, /* 100 byte string */ + TAR_OFFSET_MAGIC = 257, /* "ustar" with terminating zero byte */ + TAR_OFFSET_VERSION = 263, /* "00" */ + TAR_OFFSET_UNAME = 265, /* 32 byte string */ + TAR_OFFSET_GNAME = 297, /* 32 byte string */ + TAR_OFFSET_DEVMAJOR = 329, /* 8 byte tar number */ + TAR_OFFSET_DEVMINOR = 337, /* 8 byte tar number */ + TAR_OFFSET_PREFIX = 345, /* 155 byte string */ + /* last 12 bytes of the 512-byte block are unassigned */ +}; + +/* See POSIX (not all the standard file type codes are listed here) */ +enum tarFileType +{ + TAR_FILETYPE_PLAIN = '0', + TAR_FILETYPE_PLAIN_OLD = '\0', /* backwards compatibility, per POSIX */ + TAR_FILETYPE_SYMLINK = '2', + TAR_FILETYPE_DIRECTORY = '5', + TAR_FILETYPE_PAX_EXTENDED = 'x', + TAR_FILETYPE_PAX_EXTENDED_GLOBAL = 'g', +}; + +extern enum tarError tarCreateHeader(char *h, const char *filename, + const char *linktarget, pgoff_t size, + mode_t mode, uid_t uid, gid_t gid, + time_t mtime); +extern uint64 read_tar_number(const char *s, int len); +extern void print_tar_number(char *s, int len, uint64 val); +extern int tarChecksum(const char *header); +extern bool isValidTarHeader(const char *header); + +/* + * Compute the number of padding bytes required for an entry in a tar + * archive. We must pad out to a multiple of TAR_BLOCK_SIZE. Since that's + * a power of 2, we can use TYPEALIGN(). + */ +static inline size_t +tarPaddingBytesRequired(size_t len) +{ + return TYPEALIGN(TAR_BLOCK_SIZE, len) - len; +} + + +#endif diff --git a/src/bin/pg_walsender/vendor/tar.c b/src/bin/pg_walsender/vendor/tar.c new file mode 100644 index 000000000..626439053 --- /dev/null +++ b/src/bin/pg_walsender/vendor/tar.c @@ -0,0 +1,278 @@ +/* + * vendor/tar.c + * Vendored from PostgreSQL's src/port/tar.c (checked against + * /Users/dim/dev/PostgreSQL/postgresql; logic unchanged, reformatted to + * this project's own brace style via citus_indent) -- ustar header + * construction/checksum logic, pure C with no backend dependency (only + * c.h/pgtar.h), already proven frontend-safe since it's what + * pg_basebackup's own client-side tar handling and the backend's + * basebackup.c both build on. pg_walsender's tar_stream.c uses + * tarCreateHeader()/tarPaddingBytesRequired() directly rather than + * re-deriving the ustar byte layout by hand. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/port/tar.c + * + */ + +#include "c.h" + +#include + +#include "pgtar.h" + +/* + * Print a numeric field in a tar header. The field starts at *s and is of + * length len; val is the value to be written. + * + * Per POSIX, the way to write a number is in octal with leading zeroes and + * one trailing space (or NUL, but we use space) at the end of the specified + * field width. + * + * However, the given value may not fit in the available space in octal form. + * If that's true, we use the GNU extension of writing \200 followed by the + * number in base-256 form (ie, stored in binary MSB-first). (Note: here we + * support only non-negative numbers, so we don't worry about the GNU rules + * for handling negative numbers.) + */ +void +print_tar_number(char *s, int len, uint64 val) +{ + if (val < (((uint64) 1) << ((len - 1) * 3))) + { + /* Use octal with trailing space */ + s[--len] = ' '; + while (len) + { + s[--len] = (val & 7) + '0'; + val >>= 3; + } + } + else + { + /* Use base-256 with leading \200 */ + s[0] = '\200'; + while (len > 1) + { + s[--len] = (val & 255); + val >>= 8; + } + } +} + + +/* + * Read a numeric field in a tar header. The field starts at *s and is of + * length len. + * + * The POSIX-approved format for a number is octal, ending with a space or + * NUL. However, for values that don't fit, we recognize the GNU extension + * of \200 followed by the number in base-256 form (ie, stored in binary + * MSB-first). (Note: here we support only non-negative numbers, so we don't + * worry about the GNU rules for handling negative numbers.) + */ +uint64 +read_tar_number(const char *s, int len) +{ + uint64 result = 0; + + if (*s == '\200') + { + /* base-256 */ + while (--len) + { + result <<= 8; + result |= (unsigned char) (*++s); + } + } + else + { + /* octal */ + while (len-- && *s >= '0' && *s <= '7') + { + result <<= 3; + result |= (*s - '0'); + s++; + } + } + return result; +} + + +/* + * Calculate the tar checksum for a header. The header is assumed to always + * be 512 bytes, per the tar standard. + */ +int +tarChecksum(const char *header) +{ + int i, + sum; + + /* + * Per POSIX, the checksum is the simple sum of all bytes in the header, + * treating the bytes as unsigned, and treating the checksum field (at + * offset TAR_OFFSET_CHECKSUM) as though it contained 8 spaces. + */ + sum = 8 * ' '; /* presumed value for checksum field */ + for (i = 0; i < TAR_BLOCK_SIZE; i++) + { + if (i < TAR_OFFSET_CHECKSUM || i >= TAR_OFFSET_CHECKSUM + 8) + { + sum += 0xFF & header[i]; + } + } + return sum; +} + + +/* + * Check validity of a tar header (assumed to be 512 bytes long). + * We verify the checksum and the magic number / version. + */ +bool +isValidTarHeader(const char *header) +{ + int sum; + int chk = tarChecksum(header); + + sum = read_tar_number(&header[TAR_OFFSET_CHECKSUM], 8); + + if (sum != chk) + { + return false; + } + + /* POSIX tar format */ + if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar\0", 6) == 0 && + memcmp(&header[TAR_OFFSET_VERSION], "00", 2) == 0) + { + return true; + } + + /* GNU tar format */ + if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar \0", 8) == 0) + { + return true; + } + + /* not-quite-POSIX format written by pre-9.3 pg_dump */ + if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar00\0", 8) == 0) + { + return true; + } + + return false; +} + + +/* + * Fill in the buffer pointed to by h with a tar format header. This buffer + * must always have space for 512 characters, which is a requirement of + * the tar format. + */ +enum tarError +tarCreateHeader(char *h, const char *filename, const char *linktarget, + pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime) +{ + if (strlen(filename) > 99) + { + return TAR_NAME_TOO_LONG; + } + + if (linktarget && strlen(linktarget) > 99) + { + return TAR_SYMLINK_TOO_LONG; + } + + memset(h, 0, TAR_BLOCK_SIZE); + + /* Name 100 */ + strlcpy(&h[TAR_OFFSET_NAME], filename, 100); + if (linktarget != NULL || S_ISDIR(mode)) + { + /* + * We only support symbolic links to directories, and this is + * indicated in the tar format by adding a slash at the end of the + * name, the same as for regular directories. + */ + int flen = strlen(filename); + + flen = Min(flen, 99); + h[flen] = '/'; + h[flen + 1] = '\0'; + } + + /* Mode 8 - this doesn't include the file type bits (S_IFMT) */ + print_tar_number(&h[TAR_OFFSET_MODE], 8, (mode & 07777)); + + /* User ID 8 */ + print_tar_number(&h[TAR_OFFSET_UID], 8, uid); + + /* Group 8 */ + print_tar_number(&h[TAR_OFFSET_GID], 8, gid); + + /* File size 12 */ + if (linktarget != NULL || S_ISDIR(mode)) + { + /* Symbolic link or directory has size zero */ + print_tar_number(&h[TAR_OFFSET_SIZE], 12, 0); + } + else + { + print_tar_number(&h[TAR_OFFSET_SIZE], 12, size); + } + + /* Mod Time 12 */ + print_tar_number(&h[TAR_OFFSET_MTIME], 12, mtime); + + /* Checksum 8 cannot be calculated until we've filled all other fields */ + + if (linktarget != NULL) + { + /* Type - Symbolic link */ + h[TAR_OFFSET_TYPEFLAG] = TAR_FILETYPE_SYMLINK; + + /* Link Name 100 */ + strlcpy(&h[TAR_OFFSET_LINKNAME], linktarget, 100); + } + else if (S_ISDIR(mode)) + { + /* Type - directory */ + h[TAR_OFFSET_TYPEFLAG] = TAR_FILETYPE_DIRECTORY; + } + else + { + /* Type - regular file */ + h[TAR_OFFSET_TYPEFLAG] = TAR_FILETYPE_PLAIN; + } + + /* Magic 6 */ + strcpy(&h[TAR_OFFSET_MAGIC], "ustar"); /* IGNORE-BANNED */ + + /* Version 2 */ + memcpy(&h[TAR_OFFSET_VERSION], "00", 2); /* IGNORE-BANNED */ + + /* User 32 */ + /* XXX: Do we need to care about setting correct username? */ + strlcpy(&h[TAR_OFFSET_UNAME], "postgres", 32); + + /* Group 32 */ + /* XXX: Do we need to care about setting correct group name? */ + strlcpy(&h[TAR_OFFSET_GNAME], "postgres", 32); + + /* Major Dev 8 */ + print_tar_number(&h[TAR_OFFSET_DEVMAJOR], 8, 0); + + /* Minor Dev 8 */ + print_tar_number(&h[TAR_OFFSET_DEVMINOR], 8, 0); + + /* Prefix 155 - not used, leave as nulls */ + + /* Finally, compute and insert the checksum */ + print_tar_number(&h[TAR_OFFSET_CHECKSUM], 8, tarChecksum(h)); + + return TAR_OK; +} diff --git a/src/bin/pg_walsender/wal_dir_scan.c b/src/bin/pg_walsender/wal_dir_scan.c new file mode 100644 index 000000000..cbd9d43bf --- /dev/null +++ b/src/bin/pg_walsender/wal_dir_scan.c @@ -0,0 +1,114 @@ +/* + * src/bin/pg_walsender/wal_dir_scan.c + * See wal_dir_scan.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "wal_dir_scan.h" +#include "file_utils.h" + +/* default WAL segment size (16MB), matching cmd_show.c's own + * "SHOW wal_segment_size" -> "16MB" answer */ +#define WS_WAL_SEGMENT_SIZE UINT64CONST(0x1000000) +#define WS_XLOG_SEGMENTS_PER_XLOGID (UINT64CONST(0x100000000) / WS_WAL_SEGMENT_SIZE) + +#define WS_WAL_FNAME_LEN 24 + + +static bool +is_wal_segment_filename(const char *name) +{ + size_t len = strlen(name); + + if (len != WS_WAL_FNAME_LEN) + { + return false; + } + + for (size_t i = 0; i < len; i++) + { + if (!isxdigit((unsigned char) name[i])) + { + return false; + } + } + + return true; +} + + +void +wal_segment_filename(uint32_t timeline, uint64_t segno, char *dest, size_t destSize) +{ + uint32_t logId = (uint32_t) (segno / WS_XLOG_SEGMENTS_PER_XLOGID); + uint32_t seg = (uint32_t) (segno % WS_XLOG_SEGMENTS_PER_XLOGID); + + sformat(dest, destSize, "%08X%08X%08X", timeline, logId, seg); +} + + +bool +wal_dir_find_latest(const char *walcacheDir, uint32_t *timeline, + char *endLsn, size_t endLsnSize) +{ + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + return false; + } + + char best[WS_WAL_FNAME_LEN + 1] = { 0 }; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + if (!is_wal_segment_filename(entry->d_name)) + { + continue; + } + + if (best[0] == '\0' || strcmp(entry->d_name, best) > 0) + { + strlcpy(best, entry->d_name, sizeof(best)); + } + } + + closedir(dir); + + if (best[0] == '\0') + { + return false; + } + + char tliHex[9] = { 0 }; + char logIdHex[9] = { 0 }; + char segHex[9] = { 0 }; + + memcpy(tliHex, best, 8); /* IGNORE-BANNED */ + memcpy(logIdHex, best + 8, 8); /* IGNORE-BANNED */ + memcpy(segHex, best + 16, 8); /* IGNORE-BANNED */ + + uint32_t tli = (uint32_t) strtoul(tliHex, NULL, 16); + uint32_t logId = (uint32_t) strtoul(logIdHex, NULL, 16); + uint32_t seg = (uint32_t) strtoul(segHex, NULL, 16); + + uint64_t segno = (uint64_t) logId * WS_XLOG_SEGMENTS_PER_XLOGID + seg; + uint64_t endOfSegment = (segno + 1) * WS_WAL_SEGMENT_SIZE; + + *timeline = tli; + sformat(endLsn, endLsnSize, "%X/%08X", + (uint32_t) (endOfSegment >> 32), (uint32_t) (endOfSegment & 0xFFFFFFFF)); + + return true; +} diff --git a/src/bin/pg_walsender/wal_dir_scan.h b/src/bin/pg_walsender/wal_dir_scan.h new file mode 100644 index 000000000..1e5382c18 --- /dev/null +++ b/src/bin/pg_walsender/wal_dir_scan.h @@ -0,0 +1,45 @@ +/* + * src/bin/pg_walsender/wal_dir_scan.h + * Finds the newest fully-captured (non-.partial) WAL segment in an + * archiver's WAL cache directory and derives its boundary LSNs from the + * segment filename alone (standard 24-hex-digit XLogFileName format, + * assuming the fixed 16MB default segment size this project's own SHOW + * wal_segment_size already reports -- see cmd_show.c). + * + * This is a segment-boundary approximation, not a real-record-level + * position: it doesn't parse WAL contents, just the filename. Good + * enough for CREATE_REPLICATION_SLOT's consistent_point and + * IDENTIFY_SYSTEM's xlogpos; START_REPLICATION's actual segment + * streaming (wal_segment_source.c) reads the real bytes. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_WAL_DIR_SCAN_H +#define WS_WAL_DIR_SCAN_H + +#include +#include + +/* + * wal_dir_find_latest scans walcacheDir for the highest-numbered complete + * WAL segment (24 hex chars, no ".partial" suffix). On success, returns + * true with *timeline set and endLsn filled with that segment's end-of- + * segment LSN (formatted "%X/%08X", matching pg_lsn's own text form) -- + * the natural "resume from here" position once this segment is fully + * captured. Returns false (not an error, *timeline and *endLsn untouched) + * if the directory has no WAL segments yet. + */ +bool wal_dir_find_latest(const char *walcacheDir, uint32_t *timeline, + char *endLsn, size_t endLsnSize); + +/* + * wal_segment_filename formats a filename the same way real Postgres does + * (XLogFileName), for a given timeline and 0-based segment number. + */ +void wal_segment_filename(uint32_t timeline, uint64_t segno, + char *dest, size_t destSize); + +#endif /* WS_WAL_DIR_SCAN_H */ diff --git a/src/bin/pg_walsender/walsender.h b/src/bin/pg_walsender/walsender.h new file mode 100644 index 000000000..6eecfe10b --- /dev/null +++ b/src/bin/pg_walsender/walsender.h @@ -0,0 +1,54 @@ +/* + * src/bin/pg_walsender/walsender.h + * Shared types for pg_walsender, the archiver's own replication-protocol + * server (see ~/dev/temp/archiving-disaster-recovery.md, "Process model" + * and "Build order" milestone 2). Reimplements the wire-level surface of + * the real Postgres walsender well enough to serve IDENTIFY_SYSTEM, SHOW, + * and (later milestones) BASE_BACKUP/START_REPLICATION/TIMELINE_HISTORY + * to unmodified pg_basebackup/pg_receivewal clients, backed by an + * archiver's local WAL cache and base backups instead of a live + * postmaster. No frontend-linkable server-side protocol library exists + * anywhere in Postgres (confirmed against + * /Users/dim/dev/PostgreSQL/postgresql's pqcomm.c/backend_startup.c/ + * repl_gram.y/walsender.c, all backend-only) -- this is a genuine + * reimplementation guided by that source, not a linking exercise. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_WALSENDER_H +#define WS_WALSENDER_H + +#include + +#include "postgres_fe.h" + +/* one entry per "/" the archiver serves, see routes.h */ +typedef struct WsRoute WsRoute; + +/* + * Parsed StartupMessage contents we care about. "database" doubles as our + * routing key ("/", see the design doc's own worked + * process-title example, "pg_autoctl: walsender default/0"). + */ +typedef struct WsStartupParams +{ + char user[NAMEDATALEN]; + char database[NAMEDATALEN + 16]; /* "/", may exceed a bare NAMEDATALEN */ + char applicationName[NAMEDATALEN]; + bool replication; + + /* + * True only when the client's startup packet set replication=database + * (pg_basebackup's style) rather than a plain replication=1/true + * (pg_receivewal's style). IDENTIFY_SYSTEM's own dbname column must be + * NULL for the latter -- real pg_receivewal fatals out ("unexpectedly + * database specific") if it isn't, since a non-NULL dbname is its + * signal that the connection was accidentally database-qualified. + */ + bool replicationDatabase; +} WsStartupParams; + +#endif /* WS_WALSENDER_H */ diff --git a/src/bin/pgaftest/Makefile b/src/bin/pgaftest/Makefile index 6564f559f..19a2af41a 100644 --- a/src/bin/pgaftest/Makefile +++ b/src/bin/pgaftest/Makefile @@ -32,6 +32,7 @@ SHARED_SRCS = cli_common.c config.c coordinator.c fsm.c fsm_transition.c \ fsm_transition_citus.c keeper.c keeper_config.c keeper_pg_init.c \ monitor.c monitor_config.c monitor_pg_init.c \ nodespec.c nodestate_utils.c pghba.c primary_standby.c \ + service_archiver.c service_archiver_basebackup.c \ service_keeper.c service_keeper_init.c service_monitor.c \ service_monitor_init.c service_postgres.c service_postgres_ctl.c \ state.c step_socket.c supervisor.c systemd_config.c timeline_history.c diff --git a/src/bin/pgaftest/cli_indent.c b/src/bin/pgaftest/cli_indent.c index 3732652d5..c2048fd78 100644 --- a/src/bin/pgaftest/cli_indent.c +++ b/src/bin/pgaftest/cli_indent.c @@ -415,6 +415,10 @@ print_node(FILE *out, const TestNode *n, int baseIndent) strlcpy(kindbuf, "worker", sizeof(kindbuf)); } } + else if (n->kind == NODE_KIND_ARCHIVER) + { + strlcpy(kindbuf, "archiver", sizeof(kindbuf)); + } const char *kind = kindbuf; #define ADD(k, v) do { props[pc].kw = (k); strlcpy(props[pc].val, (v), \ @@ -896,6 +900,26 @@ print_cmd(FILE *out, const TestCmd *cmd, int indent) break; } + case CMD_WAIT_SQL: + { + /* + * Always the canonical generic form: "wal segment ... archived", + * "archiver state is ...", and "basebackup ... is ..." are all + * sugar folded into a plain SQL/expected pair at parse time, so + * there's no surface syntax left to distinguish and round-trip + * -- this always re-renders as the generic form, same as + * CMD_SQL normalises embedded newlines rather than preserving + * original formatting. + */ + char norm[8192]; + normalize_sql(cmd->args, norm, sizeof(norm)); + + fformat(out, "%*swait until sql %s { %s } is { %s } timeout %ds\n", + indent, "", cmd->service, norm, cmd->expected, + cmd->timeoutSeconds); + break; + } + case CMD_PROMOTE: { fformat(out, "%*spromote", indent, ""); diff --git a/src/bin/pgaftest/compose_gen.c b/src/bin/pgaftest/compose_gen.c index 12acb35eb..fa558f5c9 100644 --- a/src/bin/pgaftest/compose_gen.c +++ b/src/bin/pgaftest/compose_gen.c @@ -1631,6 +1631,12 @@ compose_gen_write_node_ini(const TestCluster *cluster, break; } + case NODE_KIND_ARCHIVER: + { + kindStr = "archiver"; + break; + } + default: { kindStr = "postgres"; diff --git a/src/bin/pgaftest/test_runner.c b/src/bin/pgaftest/test_runner.c index 40cec5b2e..8bf4d1288 100644 --- a/src/bin/pgaftest/test_runner.c +++ b/src/bin/pgaftest/test_runner.c @@ -116,6 +116,15 @@ test_cmd_print(FILE *f, const TestCmd *cmd, int indent) break; } + case CMD_WAIT_SQL: + { + fprintf(f, /* IGNORE-BANNED */ + "%swait until sql %s { %s } is { %s } timeout %ds\n", + pad, cmd->service, cmd->args, cmd->expected, + cmd->timeoutSeconds); + break; + } + case CMD_EXPECT: { fprintf(f, "%sexpect { %s }\n", pad, cmd->expected); /* IGNORE-BANNED */ @@ -3329,6 +3338,50 @@ runner_exec_cmd(TestRunner *r, TestCmd *cmd, char *errBuf, int errLen) return true; } + case CMD_WAIT_SQL: + { + /* + * Generic SQL-condition poll: re-run cmd->args on cmd->service + * every second until its output contains cmd->expected (same + * substring semantics as CMD_EXPECT) or the timeout elapses. + * Reuses exec_sql_on_service() rather than the LISTEN/NOTIFY + * machinery wait_for_state()/wait_for_states() use: those key + * off specific goalstate/reportedstate convergence events, + * which an arbitrary scalar SQL expression has none of. + */ + time_t deadline = time(NULL) + cmd->timeoutSeconds; + char output[4096] = ""; + bool matched = false; + + for (;;) + { + if (exec_sql_on_service(r, cmd->service, cmd->args, + output, sizeof(output)) && + strstr(output, cmd->expected) != NULL) + { + matched = true; + break; + } + + if (time(NULL) >= deadline) + { + break; + } + + sleep(1); + } + + if (!matched) + { + sformat(errBuf, errLen, + "timeout: sql on %s never matched \"%s\" " + "(last output: \"%s\")", + cmd->service, cmd->expected, output); + return false; + } + return true; + } + case CMD_EXPECT_ERROR: { if (!r->lastSqlFailed) @@ -4338,6 +4391,14 @@ cmd_label(const TestCmd *cmd, char *buf, int len) break; } + case CMD_WAIT_SQL: + { + inline_text(cmd->args, tmp, sizeof(tmp)); + sformat(buf, len, "wait until sql %s { %s } is { %s } timeout %ds", + cmd->service, tmp, cmd->expected, cmd->timeoutSeconds); + break; + } + case CMD_EXPECT: { if (strchr(cmd->expected, '\n')) diff --git a/src/bin/pgaftest/test_spec.h b/src/bin/pgaftest/test_spec.h index 3acb7bc9b..18939dfe4 100644 --- a/src/bin/pgaftest/test_spec.h +++ b/src/bin/pgaftest/test_spec.h @@ -18,11 +18,16 @@ #define PGAF_MAX_STEPS 256 #define PGAF_MAX_SEQ 256 #define PGAF_TIMEOUT_DEFAULT 90 +#define PGAF_MAX_ARCHIVERS 8 +#define PGAF_MAX_ARCHIVER_FORMATIONS 8 /* ----------------------------------------------------------------------- * Cluster topology (from the cluster { } block) * * Hierarchy: cluster → monitor + formations → nodes + * ↘ archivers (attach to one or more formations by name, + * not nested inside any one of them -- see + * TestArchiverNode's own comment below) * * Syntax: * @@ -45,6 +50,16 @@ * w1 worker group 1 * w2 worker group 1 * } + * + * # Top-level archiver, sibling to monitor/formation -- see + * # TestArchiverNode's own comment for why this isn't nested inside + * # a formation_block the way ordinary nodes are. Braces are + * # mandatory here (unlike monitor's own bare form) -- see + * # archiver_block's own comment in test_spec_parse.y for why. + * archiver archiver1 { + * formation default + * region dc1 + * } * } * * When "formation" has no name it defaults to "default". @@ -98,11 +113,78 @@ typedef struct TestFormation int nodeCount; } TestFormation; +/* ----------------------------------------------------------------------- + * Top-level archiver nodes (from a cluster-level "archiver { ... }" + * declaration, sibling to "monitor" and "formation" -- NOT nested inside a + * formation_block's node_list the way ordinary/coordinator/worker nodes + * are). This matches the real data model: pgautofailover.archiver has no + * formationid column at all, and attaches to one or more formations + * through the separate archiver_formation join table -- an archiver is a + * process identity that formations attach to, not a member of any one of + * them. + * + * Syntax: + * + * archiver archiver1 { + * formation default + * region eu-west # optional; defaults to "default" + * create and launch deferred # optional; see below + * } + * + * Despite being declared at the top level, an archiver ends up represented + * internally as an ordinary TestNode (kind = NODE_KIND_ARCHIVER), appended + * to its own declared formation's own node list -- see parse_test_spec()'s + * own fold_archivers_into_formations() call, run once right after + * yyparse() returns. This means compose_gen.c's existing, fully-featured + * per-node machinery (writing a real pg_autoctl_node.ini, "pg_autoctl node + * run " as the container's own command, healthcheck/depends_on + * ordering, create/launch-deferred handling) already used for an archiver + * nested directly inside a formation_block -- the older, still-supported + * spelling -- just works for these too, completely unmodified. Only the + * *declaration* needs to be top-level, to match the real data model + * (pgautofailover.archiver has no formationid column at all, it attaches + * to formations through the separate archiver_formation join table, so an + * archiver isn't a member of any one formation the way an ordinary node + * genuinely is) -- once parsed, there is no other difference left. + * + * Only ever attaches to the FIRST formation listed: pg_autoctl create + * archiver's own ini-driven bootstrap (nodespec.c) has no notion of more + * than one --formation at create time. An archiver that needs to cover + * more than one formation from the very start should still declare just + * that first one here, then attach the rest once it's running (see + * archiver_multi_formation.pgaf for the pattern: a direct `sql monitor { + * SELECT pgautofailover.archiver_add_formation(...) }` step) -- + * fold_archivers_into_formations() exits with a clear error rather than + * silently dropping any formation past the first. + * + * "create and launch deferred" (or either half alone) behaves exactly as + * it does for an ordinary node: the container still runs "pg_autoctl node + * run ", but the ini's own [launch] section makes that command poll + * and wait rather than actually registering -- `exec pg_autoctl + * node start` un-defers it explicitly, same as any other deferred node + * (see citus_basic_operation.pgaf's own test_011 for why a Citus + * formation's archiver needs this: it must not attempt to register before + * every worker group already exists, and nothing here waits for that on + * its own). + * ----------------------------------------------------------------------- */ +typedef struct TestArchiverNode +{ + char name[128]; + char region[64]; /* --region NAME; "" = omit (defaults to "default") */ + char formations[PGAF_MAX_ARCHIVER_FORMATIONS][128]; + int formationCount; + bool createDeferred; /* node waits before pg_autoctl create */ + bool launchDeferred; /* node waits for pg_autoctl node start */ +} TestArchiverNode; + typedef struct TestCluster { TestFormation formations[PGAF_MAX_FORMATIONS]; int formationCount; + TestArchiverNode archivers[PGAF_MAX_ARCHIVERS]; + int archiverCount; + bool withMonitor; /* true when "monitor" keyword appears in cluster{} */ bool withCitus; bool bindSource; /* bind-source: mount repo root → /usr/src/pg_auto_failover */ @@ -196,6 +278,17 @@ typedef enum TestCmdKind * last-replayed LSN has caught up to that captured * value. service = node to poll, state = source * node to capture the LSN from. */ + CMD_WAIT_SQL, /* wait until sql { SQL } is { value } [timeout Ns] + * — polls an arbitrary scalar SQL expression until + * its (substring-matched, same semantics as + * CMD_EXPECT) result contains , or times + * out. The building block "wait until wal segment + * ... archived", "wait until archiver state is + * ...", and "wait until basebackup ... is ..." are + * all sugar for at parse time -- reach for this + * directly only when none of those fit. + * service = target service (e.g. "monitor"), + * args = SQL text, expected = value to match. */ } TestCmdKind; typedef struct TestCmd diff --git a/src/bin/pgaftest/test_spec_parse.c b/src/bin/pgaftest/test_spec_parse.c index 153815756..cf7e61f06 100644 --- a/src/bin/pgaftest/test_spec_parse.c +++ b/src/bin/pgaftest/test_spec_parse.c @@ -85,105 +85,111 @@ T_NUM_SYNC = 274, T_COORDINATOR = 275, T_WORKER = 276, - T_ASYNC = 277, - T_NO_MONITOR = 278, - T_SUSPENDED = 279, - T_LAUNCH = 280, - T_CREATE = 281, - T_DEFERRED = 282, - T_IMMEDIATE = 283, - T_FALSE = 284, - T_TRUE = 285, - T_INITIALLY = 286, - T_VOLUME = 287, - T_LISTEN = 288, - T_CITUS_SECONDARY = 289, - T_CANDIDATE_PRIORITY = 290, - T_PORT = 291, - T_PASSWORD = 292, - T_MONITOR_PASSWORD = 293, - T_CITUS_CLUSTER_NAME = 294, - T_DEBIAN_CLUSTER = 295, - T_REPLICATION_QUORUM = 296, - T_REPLICATION_PASSWORD = 297, - T_EXTENSION_VERSION = 298, - T_BIND_SOURCE = 299, - T_LEGACY_STARTUP = 300, - T_REGION = 301, - T_NODEINI = 302, - T_FS_INIT = 303, - T_FS_SINGLE = 304, - T_FS_PRIMARY = 305, - T_FS_WAIT_PRIMARY = 306, - T_FS_WAIT_STANDBY = 307, - T_FS_DEMOTED = 308, - T_FS_DEMOTE_TIMEOUT = 309, - T_FS_DRAINING = 310, - T_FS_SECONDARY = 311, - T_FS_CATCHINGUP = 312, - T_FS_PREP_PROMOTION = 313, - T_FS_STOP_REPLICATION = 314, - T_FS_MAINTENANCE = 315, - T_FS_JOIN_PRIMARY = 316, - T_FS_APPLY_SETTINGS = 317, - T_FS_PREPARE_MAINTENANCE = 318, - T_FS_WAIT_MAINTENANCE = 319, - T_FS_REPORT_LSN = 320, - T_FS_FAST_FORWARD = 321, - T_FS_JOIN_SECONDARY = 322, - T_FS_DROPPED = 323, - T_EXEC = 324, - T_EXEC_FAILS = 325, - T_RUN = 326, - T_PG_AUTOCTL = 327, - T_WAIT = 328, - T_UNTIL = 329, - T_TIMEOUT = 330, - T_AND = 331, - T_IS = 332, - T_WITH = 333, - T_REPLAYS = 334, - T_ASSERT = 335, - T_SQL = 336, - T_EXPECT = 337, - T_ERROR = 338, - T_PROMOTE = 339, - T_PERFORM = 340, - T_FAILOVER = 341, - T_NETWORK = 342, - T_DISCONNECT = 343, - T_CONNECT = 344, - T_SLEEP = 345, - T_COMPOSE = 346, - T_DOWN = 347, - T_START = 348, - T_STOP = 349, - T_STOPPED = 350, - T_KILL = 351, - T_INJECT = 352, - T_STATE = 353, - T_ASSIGNED_STATE = 354, - T_IN = 355, - T_GROUP = 356, - T_LBRACE = 357, - T_RBRACE = 358, - T_COMMA = 359, - T_POSTGRES = 360, - T_STAYS = 361, - T_WHILE = 362, - T_THROUGH = 363, - T_SET = 364, - T_GET = 365, - T_FSM = 366, - T_LOGS = 367, - T_NOT = 368, - T_CONTAINS = 369, - T_MATCHES = 370, - T_INTEGER = 371, - T_IDENT = 372, - T_STRING = 373, - T_BLOCK = 374, - T_SHELL_ARGS = 375 + T_ARCHIVER = 277, + T_ASYNC = 278, + T_NO_MONITOR = 279, + T_SUSPENDED = 280, + T_LAUNCH = 281, + T_CREATE = 282, + T_DEFERRED = 283, + T_IMMEDIATE = 284, + T_FALSE = 285, + T_TRUE = 286, + T_INITIALLY = 287, + T_VOLUME = 288, + T_LISTEN = 289, + T_CITUS_SECONDARY = 290, + T_CANDIDATE_PRIORITY = 291, + T_PORT = 292, + T_PASSWORD = 293, + T_MONITOR_PASSWORD = 294, + T_CITUS_CLUSTER_NAME = 295, + T_DEBIAN_CLUSTER = 296, + T_REPLICATION_QUORUM = 297, + T_REPLICATION_PASSWORD = 298, + T_EXTENSION_VERSION = 299, + T_BIND_SOURCE = 300, + T_LEGACY_STARTUP = 301, + T_REGION = 302, + T_NODEINI = 303, + T_FS_INIT = 304, + T_FS_SINGLE = 305, + T_FS_PRIMARY = 306, + T_FS_WAIT_PRIMARY = 307, + T_FS_WAIT_STANDBY = 308, + T_FS_DEMOTED = 309, + T_FS_DEMOTE_TIMEOUT = 310, + T_FS_DRAINING = 311, + T_FS_SECONDARY = 312, + T_FS_CATCHINGUP = 313, + T_FS_PREP_PROMOTION = 314, + T_FS_STOP_REPLICATION = 315, + T_FS_MAINTENANCE = 316, + T_FS_JOIN_PRIMARY = 317, + T_FS_APPLY_SETTINGS = 318, + T_FS_PREPARE_MAINTENANCE = 319, + T_FS_WAIT_MAINTENANCE = 320, + T_FS_REPORT_LSN = 321, + T_FS_FAST_FORWARD = 322, + T_FS_JOIN_SECONDARY = 323, + T_FS_DROPPED = 324, + T_EXEC = 325, + T_EXEC_FAILS = 326, + T_RUN = 327, + T_PG_AUTOCTL = 328, + T_WAIT = 329, + T_UNTIL = 330, + T_TIMEOUT = 331, + T_AND = 332, + T_IS = 333, + T_WITH = 334, + T_REPLAYS = 335, + T_ASSERT = 336, + T_SQL = 337, + T_EXPECT = 338, + T_ERROR = 339, + T_PROMOTE = 340, + T_PERFORM = 341, + T_FAILOVER = 342, + T_NETWORK = 343, + T_DISCONNECT = 344, + T_CONNECT = 345, + T_SLEEP = 346, + T_COMPOSE = 347, + T_DOWN = 348, + T_START = 349, + T_STOP = 350, + T_STOPPED = 351, + T_KILL = 352, + T_INJECT = 353, + T_STATE = 354, + T_ASSIGNED_STATE = 355, + T_IN = 356, + T_GROUP = 357, + T_LBRACE = 358, + T_RBRACE = 359, + T_COMMA = 360, + T_POSTGRES = 361, + T_STAYS = 362, + T_WHILE = 363, + T_THROUGH = 364, + T_SET = 365, + T_GET = 366, + T_FSM = 367, + T_LOGS = 368, + T_NOT = 369, + T_CONTAINS = 370, + T_MATCHES = 371, + T_WAL = 372, + T_SEGMENT = 373, + T_ARCHIVED = 374, + T_BASEBACKUP = 375, + T_SLASH = 376, + T_INTEGER = 377, + T_IDENT = 378, + T_STRING = 379, + T_BLOCK = 380, + T_SHELL_ARGS = 381 }; #endif /* Tokens. */ @@ -206,105 +212,111 @@ #define T_NUM_SYNC 274 #define T_COORDINATOR 275 #define T_WORKER 276 -#define T_ASYNC 277 -#define T_NO_MONITOR 278 -#define T_SUSPENDED 279 -#define T_LAUNCH 280 -#define T_CREATE 281 -#define T_DEFERRED 282 -#define T_IMMEDIATE 283 -#define T_FALSE 284 -#define T_TRUE 285 -#define T_INITIALLY 286 -#define T_VOLUME 287 -#define T_LISTEN 288 -#define T_CITUS_SECONDARY 289 -#define T_CANDIDATE_PRIORITY 290 -#define T_PORT 291 -#define T_PASSWORD 292 -#define T_MONITOR_PASSWORD 293 -#define T_CITUS_CLUSTER_NAME 294 -#define T_DEBIAN_CLUSTER 295 -#define T_REPLICATION_QUORUM 296 -#define T_REPLICATION_PASSWORD 297 -#define T_EXTENSION_VERSION 298 -#define T_BIND_SOURCE 299 -#define T_LEGACY_STARTUP 300 -#define T_REGION 301 -#define T_NODEINI 302 -#define T_FS_INIT 303 -#define T_FS_SINGLE 304 -#define T_FS_PRIMARY 305 -#define T_FS_WAIT_PRIMARY 306 -#define T_FS_WAIT_STANDBY 307 -#define T_FS_DEMOTED 308 -#define T_FS_DEMOTE_TIMEOUT 309 -#define T_FS_DRAINING 310 -#define T_FS_SECONDARY 311 -#define T_FS_CATCHINGUP 312 -#define T_FS_PREP_PROMOTION 313 -#define T_FS_STOP_REPLICATION 314 -#define T_FS_MAINTENANCE 315 -#define T_FS_JOIN_PRIMARY 316 -#define T_FS_APPLY_SETTINGS 317 -#define T_FS_PREPARE_MAINTENANCE 318 -#define T_FS_WAIT_MAINTENANCE 319 -#define T_FS_REPORT_LSN 320 -#define T_FS_FAST_FORWARD 321 -#define T_FS_JOIN_SECONDARY 322 -#define T_FS_DROPPED 323 -#define T_EXEC 324 -#define T_EXEC_FAILS 325 -#define T_RUN 326 -#define T_PG_AUTOCTL 327 -#define T_WAIT 328 -#define T_UNTIL 329 -#define T_TIMEOUT 330 -#define T_AND 331 -#define T_IS 332 -#define T_WITH 333 -#define T_REPLAYS 334 -#define T_ASSERT 335 -#define T_SQL 336 -#define T_EXPECT 337 -#define T_ERROR 338 -#define T_PROMOTE 339 -#define T_PERFORM 340 -#define T_FAILOVER 341 -#define T_NETWORK 342 -#define T_DISCONNECT 343 -#define T_CONNECT 344 -#define T_SLEEP 345 -#define T_COMPOSE 346 -#define T_DOWN 347 -#define T_START 348 -#define T_STOP 349 -#define T_STOPPED 350 -#define T_KILL 351 -#define T_INJECT 352 -#define T_STATE 353 -#define T_ASSIGNED_STATE 354 -#define T_IN 355 -#define T_GROUP 356 -#define T_LBRACE 357 -#define T_RBRACE 358 -#define T_COMMA 359 -#define T_POSTGRES 360 -#define T_STAYS 361 -#define T_WHILE 362 -#define T_THROUGH 363 -#define T_SET 364 -#define T_GET 365 -#define T_FSM 366 -#define T_LOGS 367 -#define T_NOT 368 -#define T_CONTAINS 369 -#define T_MATCHES 370 -#define T_INTEGER 371 -#define T_IDENT 372 -#define T_STRING 373 -#define T_BLOCK 374 -#define T_SHELL_ARGS 375 +#define T_ARCHIVER 277 +#define T_ASYNC 278 +#define T_NO_MONITOR 279 +#define T_SUSPENDED 280 +#define T_LAUNCH 281 +#define T_CREATE 282 +#define T_DEFERRED 283 +#define T_IMMEDIATE 284 +#define T_FALSE 285 +#define T_TRUE 286 +#define T_INITIALLY 287 +#define T_VOLUME 288 +#define T_LISTEN 289 +#define T_CITUS_SECONDARY 290 +#define T_CANDIDATE_PRIORITY 291 +#define T_PORT 292 +#define T_PASSWORD 293 +#define T_MONITOR_PASSWORD 294 +#define T_CITUS_CLUSTER_NAME 295 +#define T_DEBIAN_CLUSTER 296 +#define T_REPLICATION_QUORUM 297 +#define T_REPLICATION_PASSWORD 298 +#define T_EXTENSION_VERSION 299 +#define T_BIND_SOURCE 300 +#define T_LEGACY_STARTUP 301 +#define T_REGION 302 +#define T_NODEINI 303 +#define T_FS_INIT 304 +#define T_FS_SINGLE 305 +#define T_FS_PRIMARY 306 +#define T_FS_WAIT_PRIMARY 307 +#define T_FS_WAIT_STANDBY 308 +#define T_FS_DEMOTED 309 +#define T_FS_DEMOTE_TIMEOUT 310 +#define T_FS_DRAINING 311 +#define T_FS_SECONDARY 312 +#define T_FS_CATCHINGUP 313 +#define T_FS_PREP_PROMOTION 314 +#define T_FS_STOP_REPLICATION 315 +#define T_FS_MAINTENANCE 316 +#define T_FS_JOIN_PRIMARY 317 +#define T_FS_APPLY_SETTINGS 318 +#define T_FS_PREPARE_MAINTENANCE 319 +#define T_FS_WAIT_MAINTENANCE 320 +#define T_FS_REPORT_LSN 321 +#define T_FS_FAST_FORWARD 322 +#define T_FS_JOIN_SECONDARY 323 +#define T_FS_DROPPED 324 +#define T_EXEC 325 +#define T_EXEC_FAILS 326 +#define T_RUN 327 +#define T_PG_AUTOCTL 328 +#define T_WAIT 329 +#define T_UNTIL 330 +#define T_TIMEOUT 331 +#define T_AND 332 +#define T_IS 333 +#define T_WITH 334 +#define T_REPLAYS 335 +#define T_ASSERT 336 +#define T_SQL 337 +#define T_EXPECT 338 +#define T_ERROR 339 +#define T_PROMOTE 340 +#define T_PERFORM 341 +#define T_FAILOVER 342 +#define T_NETWORK 343 +#define T_DISCONNECT 344 +#define T_CONNECT 345 +#define T_SLEEP 346 +#define T_COMPOSE 347 +#define T_DOWN 348 +#define T_START 349 +#define T_STOP 350 +#define T_STOPPED 351 +#define T_KILL 352 +#define T_INJECT 353 +#define T_STATE 354 +#define T_ASSIGNED_STATE 355 +#define T_IN 356 +#define T_GROUP 357 +#define T_LBRACE 358 +#define T_RBRACE 359 +#define T_COMMA 360 +#define T_POSTGRES 361 +#define T_STAYS 362 +#define T_WHILE 363 +#define T_THROUGH 364 +#define T_SET 365 +#define T_GET 366 +#define T_FSM 367 +#define T_LOGS 368 +#define T_NOT 369 +#define T_CONTAINS 370 +#define T_MATCHES 371 +#define T_WAL 372 +#define T_SEGMENT 373 +#define T_ARCHIVED 374 +#define T_BASEBACKUP 375 +#define T_SLASH 376 +#define T_INTEGER 377 +#define T_IDENT 378 +#define T_STRING 379 +#define T_BLOCK 380 +#define T_SHELL_ARGS 381 @@ -452,6 +464,7 @@ static TestCmd *current_promote_cmd = NULL; static TestCmd *current_pass_cmd = NULL; /* for opt_passing_through */ static TestFormation *current_formation = NULL; static TestNode *current_node = NULL; +static TestArchiverNode *current_archiver = NULL; @@ -475,7 +488,7 @@ static TestNode *current_node = NULL; #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED typedef union YYSTYPE -#line 145 "test_spec_parse.y" +#line 146 "test_spec_parse.y" { int ival; char *str; @@ -483,7 +496,7 @@ typedef union YYSTYPE TestCmd *cmd; } /* Line 193 of yacc.c. */ -#line 487 "test_spec_parse.c" +#line 500 "test_spec_parse.c" YYSTYPE; # define yystype YYSTYPE /* obsolescent; will be withdrawn */ # define YYSTYPE_IS_DECLARED 1 @@ -496,7 +509,7 @@ typedef union YYSTYPE /* Line 216 of yacc.c. */ -#line 500 "test_spec_parse.c" +#line 513 "test_spec_parse.c" #ifdef short # undef short @@ -711,20 +724,20 @@ union yyalloc /* YYFINAL -- State number of the termination state. */ #define YYFINAL 21 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 620 +#define YYLAST 683 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 121 +#define YYNTOKENS 127 /* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 65 +#define YYNNTS 71 /* YYNRULES -- Number of rules. */ -#define YYNRULES 214 +#define YYNRULES 234 /* YYNRULES -- Number of states. */ -#define YYNSTATES 355 +#define YYNSTATES 412 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ #define YYUNDEFTOK 2 -#define YYMAXUTOK 375 +#define YYMAXUTOK 381 #define YYTRANSLATE(YYX) \ ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) @@ -769,7 +782,8 @@ static const yytype_uint8 yytranslate[] = 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, 118, 119, 120 + 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, + 125, 126 }; #if YYDEBUG @@ -779,126 +793,139 @@ static const yytype_uint16 yyprhs[] = { 0, 0, 3, 5, 8, 10, 12, 14, 16, 18, 19, 25, 26, 29, 31, 33, 35, 37, 39, 41, - 43, 45, 47, 51, 55, 59, 63, 68, 73, 80, - 83, 86, 89, 92, 95, 98, 101, 102, 109, 110, - 113, 115, 117, 119, 121, 123, 125, 128, 131, 132, - 135, 137, 139, 140, 141, 146, 147, 155, 156, 159, - 161, 163, 165, 167, 169, 171, 174, 177, 182, 185, - 187, 189, 191, 194, 197, 200, 203, 206, 209, 212, - 215, 218, 221, 224, 227, 230, 233, 237, 241, 244, - 247, 251, 255, 256, 259, 261, 263, 265, 267, 269, - 271, 273, 275, 277, 279, 281, 283, 285, 287, 289, - 291, 295, 298, 302, 305, 309, 312, 316, 319, 321, - 323, 325, 330, 335, 337, 341, 342, 345, 347, 349, - 353, 357, 358, 368, 369, 379, 387, 395, 401, 408, - 414, 421, 423, 425, 429, 433, 434, 437, 440, 445, - 446, 449, 453, 460, 467, 474, 481, 485, 488, 491, - 495, 499, 502, 504, 508, 511, 516, 522, 530, 534, - 538, 544, 550, 553, 556, 560, 564, 568, 573, 577, - 581, 585, 586, 592, 598, 602, 607, 613, 618, 624, - 627, 628, 631, 633, 635, 637, 639, 641, 643, 645, - 647, 649, 651, 653, 655, 657, 659, 661, 663, 665, - 667, 669, 671, 673, 675 + 43, 45, 47, 48, 55, 56, 59, 62, 65, 68, + 73, 76, 79, 81, 85, 89, 93, 97, 102, 107, + 114, 117, 120, 123, 126, 129, 132, 135, 136, 143, + 144, 147, 149, 151, 153, 155, 157, 159, 162, 165, + 166, 169, 171, 173, 174, 175, 180, 181, 189, 190, + 193, 195, 197, 199, 201, 203, 205, 207, 210, 213, + 218, 221, 223, 225, 227, 230, 233, 236, 239, 242, + 245, 248, 251, 254, 257, 260, 263, 266, 269, 273, + 277, 280, 283, 287, 291, 292, 295, 297, 299, 301, + 303, 305, 307, 309, 311, 313, 315, 317, 319, 321, + 323, 325, 327, 331, 334, 338, 341, 345, 348, 352, + 355, 357, 359, 361, 366, 371, 373, 377, 378, 381, + 383, 385, 389, 393, 394, 404, 405, 415, 423, 431, + 437, 444, 450, 457, 466, 478, 489, 501, 503, 505, + 509, 513, 514, 517, 520, 525, 526, 529, 533, 540, + 547, 554, 561, 565, 568, 571, 575, 579, 582, 584, + 588, 591, 596, 602, 610, 614, 618, 624, 630, 633, + 636, 640, 644, 648, 653, 657, 661, 665, 666, 672, + 678, 682, 687, 693, 698, 704, 707, 708, 711, 713, + 715, 717, 719, 721, 723, 725, 727, 729, 731, 733, + 735, 737, 739, 741, 743, 745, 747, 749, 751, 753, + 755, 757, 759, 761, 762 }; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ static const yytype_int16 yyrhs[] = { - 122, 0, -1, 123, -1, 122, 123, -1, 124, -1, - 146, -1, 147, -1, 148, -1, 182, -1, -1, 3, - 102, 125, 126, 103, -1, -1, 126, 127, -1, 128, - -1, 129, -1, 131, -1, 132, -1, 130, -1, 133, - -1, 44, -1, 45, -1, 4, -1, 4, 40, 117, - -1, 4, 14, 117, -1, 4, 36, 116, -1, 4, - 37, 118, -1, 4, 117, 25, 27, -1, 4, 117, - 31, 95, -1, 4, 117, 25, 27, 37, 118, -1, - 13, 118, -1, 13, 117, -1, 43, 117, -1, 43, - 118, -1, 15, 117, -1, 16, 117, -1, 17, 117, - -1, -1, 18, 134, 135, 102, 138, 103, -1, -1, - 135, 137, -1, 117, -1, 118, -1, 16, -1, 4, - -1, 5, -1, 136, -1, 19, 116, -1, 56, 29, - -1, -1, 138, 141, -1, 117, -1, 4, -1, -1, - -1, 139, 140, 142, 144, -1, -1, 5, 117, 140, - 143, 102, 144, 103, -1, -1, 144, 145, -1, 20, - -1, 21, -1, 22, -1, 23, -1, 24, -1, 27, - -1, 25, 27, -1, 26, 27, -1, 26, 76, 25, - 27, -1, 25, 28, -1, 28, -1, 33, -1, 34, - -1, 35, 116, -1, 46, 117, -1, 46, 118, -1, - 101, 116, -1, 36, 116, -1, 39, 117, -1, 40, - 117, -1, 15, 117, -1, 16, 117, -1, 17, 117, - -1, 41, 30, -1, 41, 29, -1, 42, 118, -1, - 38, 118, -1, 32, 117, 117, -1, 32, 117, 118, - -1, 8, 149, -1, 9, 149, -1, 10, 185, 149, - -1, 102, 150, 103, -1, -1, 150, 151, -1, 152, - -1, 158, -1, 165, -1, 166, -1, 167, -1, 168, - -1, 170, -1, 171, -1, 173, -1, 174, -1, 175, - -1, 176, -1, 179, -1, 180, -1, 181, -1, 172, - -1, 69, 117, 120, -1, 69, 117, -1, 70, 117, - 120, -1, 70, 117, -1, 71, 117, 120, -1, 71, - 117, -1, 72, 117, 120, -1, 72, 117, -1, 72, - -1, 12, -1, 77, -1, 117, 98, 153, 184, -1, - 117, 98, 153, 117, -1, 154, -1, 155, 76, 154, - -1, -1, 108, 157, -1, 184, -1, 117, -1, 157, - 104, 184, -1, 157, 104, 117, -1, -1, 73, 74, - 117, 98, 153, 184, 159, 156, 164, -1, -1, 73, - 74, 117, 98, 153, 117, 160, 156, 164, -1, 73, - 74, 117, 99, 153, 184, 164, -1, 73, 74, 117, - 99, 153, 117, 164, -1, 73, 74, 117, 95, 164, - -1, 73, 74, 117, 79, 117, 164, -1, 73, 74, - 161, 162, 164, -1, 73, 74, 154, 76, 155, 164, - -1, 184, -1, 117, -1, 161, 104, 184, -1, 161, - 104, 117, -1, -1, 100, 163, -1, 101, 116, -1, - 163, 104, 101, 116, -1, -1, 75, 116, -1, 78, - 75, 116, -1, 80, 117, 98, 153, 184, 164, -1, - 80, 117, 98, 153, 117, 164, -1, 80, 117, 99, - 153, 184, 164, -1, 80, 117, 99, 153, 117, 164, - -1, 81, 117, 119, -1, 82, 119, -1, 82, 83, - -1, 82, 83, 117, -1, 82, 83, 116, -1, 84, - 169, -1, 117, -1, 169, 104, 117, -1, 85, 86, - -1, 85, 86, 101, 116, -1, 85, 86, 100, 18, - 117, -1, 85, 86, 100, 18, 117, 101, 116, -1, - 87, 88, 117, -1, 87, 89, 117, -1, 47, 109, - 117, 117, 117, -1, 47, 110, 117, 117, 117, -1, - 90, 116, -1, 91, 92, -1, 91, 93, 117, -1, - 91, 94, 117, -1, 91, 96, 117, -1, 91, 97, - 117, 120, -1, 94, 105, 139, -1, 93, 105, 139, - -1, 111, 10, 139, -1, -1, 107, 178, 102, 150, - 103, -1, 80, 139, 106, 184, 177, -1, 109, 117, - 117, -1, 112, 117, 114, 118, -1, 112, 117, 113, - 114, 118, -1, 112, 117, 115, 118, -1, 112, 117, - 113, 115, 118, -1, 11, 183, -1, -1, 183, 185, - -1, 48, -1, 49, -1, 50, -1, 51, -1, 52, - -1, 53, -1, 54, -1, 55, -1, 56, -1, 57, - -1, 58, -1, 59, -1, 60, -1, 61, -1, 62, - -1, 63, -1, 64, -1, 65, -1, 66, -1, 67, - -1, 68, -1, 117, -1, 118, -1 + 128, 0, -1, 129, -1, 128, 129, -1, 130, -1, + 156, -1, 157, -1, 158, -1, 192, -1, -1, 3, + 103, 131, 132, 104, -1, -1, 132, 133, -1, 138, + -1, 139, -1, 141, -1, 142, -1, 140, -1, 143, + -1, 134, -1, 45, -1, 46, -1, -1, 22, 123, + 135, 103, 136, 104, -1, -1, 136, 137, -1, 18, + 123, -1, 47, 123, -1, 47, 124, -1, 27, 77, + 26, 28, -1, 26, 28, -1, 27, 28, -1, 4, + -1, 4, 41, 123, -1, 4, 14, 123, -1, 4, + 37, 122, -1, 4, 38, 124, -1, 4, 123, 26, + 28, -1, 4, 123, 32, 96, -1, 4, 123, 26, + 28, 38, 124, -1, 13, 124, -1, 13, 123, -1, + 44, 123, -1, 44, 124, -1, 15, 123, -1, 16, + 123, -1, 17, 123, -1, -1, 18, 144, 145, 103, + 148, 104, -1, -1, 145, 147, -1, 123, -1, 124, + -1, 16, -1, 4, -1, 5, -1, 146, -1, 19, + 122, -1, 57, 30, -1, -1, 148, 151, -1, 123, + -1, 4, -1, -1, -1, 149, 150, 152, 154, -1, + -1, 5, 123, 150, 153, 103, 154, 104, -1, -1, + 154, 155, -1, 20, -1, 21, -1, 22, -1, 23, + -1, 24, -1, 25, -1, 28, -1, 26, 28, -1, + 27, 28, -1, 27, 77, 26, 28, -1, 26, 29, + -1, 29, -1, 34, -1, 35, -1, 36, 122, -1, + 47, 123, -1, 47, 124, -1, 102, 122, -1, 37, + 122, -1, 40, 123, -1, 41, 123, -1, 15, 123, + -1, 16, 123, -1, 17, 123, -1, 42, 31, -1, + 42, 30, -1, 43, 124, -1, 39, 124, -1, 33, + 123, 123, -1, 33, 123, 124, -1, 8, 159, -1, + 9, 159, -1, 10, 195, 159, -1, 103, 160, 104, + -1, -1, 160, 161, -1, 162, -1, 168, -1, 175, + -1, 176, -1, 177, -1, 178, -1, 180, -1, 181, + -1, 183, -1, 184, -1, 185, -1, 186, -1, 189, + -1, 190, -1, 191, -1, 182, -1, 70, 123, 126, + -1, 70, 123, -1, 71, 123, 126, -1, 71, 123, + -1, 72, 123, 126, -1, 72, 123, -1, 73, 123, + 126, -1, 73, 123, -1, 73, -1, 12, -1, 78, + -1, 123, 99, 163, 194, -1, 123, 99, 163, 123, + -1, 164, -1, 165, 77, 164, -1, -1, 109, 167, + -1, 194, -1, 123, -1, 167, 105, 194, -1, 167, + 105, 123, -1, -1, 74, 75, 123, 99, 163, 194, + 169, 166, 174, -1, -1, 74, 75, 123, 99, 163, + 123, 170, 166, 174, -1, 74, 75, 123, 100, 163, + 194, 174, -1, 74, 75, 123, 100, 163, 123, 174, + -1, 74, 75, 123, 96, 174, -1, 74, 75, 123, + 80, 123, 174, -1, 74, 75, 171, 172, 174, -1, + 74, 75, 164, 77, 165, 174, -1, 74, 75, 82, + 123, 125, 78, 125, 174, -1, 74, 75, 117, 118, + 124, 119, 101, 123, 121, 122, 174, -1, 74, 75, + 22, 99, 163, 196, 101, 123, 197, 174, -1, 74, + 75, 120, 123, 78, 123, 101, 123, 121, 122, 174, + -1, 194, -1, 123, -1, 171, 105, 194, -1, 171, + 105, 123, -1, -1, 101, 173, -1, 102, 122, -1, + 173, 105, 102, 122, -1, -1, 76, 122, -1, 79, + 76, 122, -1, 81, 123, 99, 163, 194, 174, -1, + 81, 123, 99, 163, 123, 174, -1, 81, 123, 100, + 163, 194, 174, -1, 81, 123, 100, 163, 123, 174, + -1, 82, 123, 125, -1, 83, 125, -1, 83, 84, + -1, 83, 84, 123, -1, 83, 84, 122, -1, 85, + 179, -1, 123, -1, 179, 105, 123, -1, 86, 87, + -1, 86, 87, 102, 122, -1, 86, 87, 101, 18, + 123, -1, 86, 87, 101, 18, 123, 102, 122, -1, + 88, 89, 123, -1, 88, 90, 123, -1, 48, 110, + 123, 123, 123, -1, 48, 111, 123, 123, 123, -1, + 91, 122, -1, 92, 93, -1, 92, 94, 123, -1, + 92, 95, 123, -1, 92, 97, 123, -1, 92, 98, + 123, 126, -1, 95, 106, 149, -1, 94, 106, 149, + -1, 112, 10, 149, -1, -1, 108, 188, 103, 160, + 104, -1, 81, 149, 107, 194, 187, -1, 110, 123, + 123, -1, 113, 123, 115, 124, -1, 113, 123, 114, + 115, 124, -1, 113, 123, 116, 124, -1, 113, 123, + 114, 116, 124, -1, 11, 193, -1, -1, 193, 195, + -1, 49, -1, 50, -1, 51, -1, 52, -1, 53, + -1, 54, -1, 55, -1, 56, -1, 57, -1, 58, + -1, 59, -1, 60, -1, 61, -1, 62, -1, 63, + -1, 64, -1, 65, -1, 66, -1, 67, -1, 68, + -1, 69, -1, 123, -1, 124, -1, 194, -1, 123, + -1, -1, 121, 122, -1 }; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ static const yytype_uint16 yyrline[] = { - 0, 216, 216, 217, 221, 222, 223, 224, 225, 238, - 237, 247, 249, 253, 254, 255, 256, 257, 258, 259, - 260, 273, 277, 284, 291, 297, 304, 311, 318, 331, - 337, 347, 353, 363, 373, 379, 390, 389, 406, 408, - 417, 418, 419, 420, 421, 425, 430, 434, 440, 442, - 461, 462, 471, 488, 487, 495, 494, 502, 504, 508, - 513, 518, 522, 526, 530, 536, 541, 545, 550, 554, - 558, 562, 566, 570, 575, 580, 584, 588, 594, 600, - 605, 610, 615, 619, 623, 629, 635, 649, 670, 677, - 688, 706, 721, 724, 732, 733, 734, 735, 736, 737, - 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, - 761, 768, 774, 781, 787, 794, 800, 808, 814, 841, - 841, 852, 867, 885, 886, 901, 903, 907, 915, 923, - 930, 942, 941, 953, 952, 963, 972, 981, 995, 1003, - 1017, 1032, 1038, 1045, 1051, 1064, 1066, 1070, 1075, 1083, - 1084, 1085, 1096, 1104, 1112, 1120, 1138, 1153, 1160, 1164, - 1170, 1183, 1191, 1199, 1220, 1227, 1234, 1242, 1258, 1264, - 1285, 1293, 1308, 1322, 1326, 1332, 1338, 1364, 1398, 1404, - 1425, 1442, 1442, 1447, 1466, 1491, 1500, 1509, 1518, 1534, - 1537, 1539, 1561, 1562, 1563, 1564, 1565, 1566, 1567, 1568, - 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, - 1579, 1580, 1581, 1589, 1590 + 0, 220, 220, 221, 225, 226, 227, 228, 229, 242, + 241, 251, 253, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 288, 287, 305, 307, 311, 325, 330, 335, + 342, 346, 362, 366, 373, 380, 386, 393, 400, 407, + 420, 426, 436, 442, 452, 462, 468, 479, 478, 495, + 497, 506, 507, 508, 509, 510, 514, 519, 523, 529, + 531, 550, 551, 560, 577, 576, 584, 583, 591, 593, + 597, 602, 607, 611, 615, 619, 623, 629, 634, 638, + 643, 647, 651, 655, 659, 663, 668, 673, 677, 681, + 687, 693, 698, 703, 708, 712, 716, 722, 728, 742, + 763, 770, 781, 799, 814, 817, 825, 826, 827, 828, + 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, + 839, 840, 854, 861, 867, 874, 880, 887, 893, 901, + 907, 934, 934, 945, 960, 978, 979, 994, 996, 1000, + 1008, 1016, 1023, 1035, 1034, 1046, 1045, 1056, 1065, 1074, + 1088, 1096, 1110, 1125, 1142, 1166, 1195, 1225, 1231, 1238, + 1244, 1257, 1259, 1263, 1268, 1276, 1277, 1278, 1289, 1297, + 1305, 1313, 1331, 1346, 1353, 1357, 1363, 1376, 1384, 1392, + 1413, 1420, 1427, 1435, 1451, 1457, 1478, 1486, 1501, 1515, + 1519, 1525, 1531, 1557, 1591, 1597, 1618, 1635, 1635, 1640, + 1659, 1684, 1693, 1702, 1711, 1727, 1730, 1732, 1754, 1755, + 1756, 1757, 1758, 1759, 1760, 1761, 1762, 1763, 1764, 1765, + 1766, 1767, 1768, 1769, 1770, 1771, 1772, 1773, 1774, 1782, + 1783, 1794, 1795, 1803, 1804 }; #endif @@ -911,8 +938,8 @@ static const char *const yytname[] = "T_CITUS_COORDINATOR", "T_CITUS_WORKER", "T_SETUP", "T_TEARDOWN", "T_STEP", "T_SEQUENCE", "T_EQUALS", "T_IMAGE", "T_IMAGE_TARGET", "T_SSL", "T_AUTH", "T_AUTH_METHOD", "T_FORMATION", "T_NUM_SYNC", "T_COORDINATOR", - "T_WORKER", "T_ASYNC", "T_NO_MONITOR", "T_SUSPENDED", "T_LAUNCH", - "T_CREATE", "T_DEFERRED", "T_IMMEDIATE", "T_FALSE", "T_TRUE", + "T_WORKER", "T_ARCHIVER", "T_ASYNC", "T_NO_MONITOR", "T_SUSPENDED", + "T_LAUNCH", "T_CREATE", "T_DEFERRED", "T_IMMEDIATE", "T_FALSE", "T_TRUE", "T_INITIALLY", "T_VOLUME", "T_LISTEN", "T_CITUS_SECONDARY", "T_CANDIDATE_PRIORITY", "T_PORT", "T_PASSWORD", "T_MONITOR_PASSWORD", "T_CITUS_CLUSTER_NAME", "T_DEBIAN_CLUSTER", "T_REPLICATION_QUORUM", @@ -931,22 +958,25 @@ static const char *const yytname[] = "T_DOWN", "T_START", "T_STOP", "T_STOPPED", "T_KILL", "T_INJECT", "T_STATE", "T_ASSIGNED_STATE", "T_IN", "T_GROUP", "T_LBRACE", "T_RBRACE", "T_COMMA", "T_POSTGRES", "T_STAYS", "T_WHILE", "T_THROUGH", "T_SET", - "T_GET", "T_FSM", "T_LOGS", "T_NOT", "T_CONTAINS", "T_MATCHES", - "T_INTEGER", "T_IDENT", "T_STRING", "T_BLOCK", "T_SHELL_ARGS", "$accept", - "spec", "spec_item", "cluster_block", "@1", "cluster_item_list", - "cluster_item", "monitor_line", "image_line", "extension_version_line", - "ssl_line", "auth_line", "formation_block", "@2", "formation_opt_list", - "bare_name", "formation_opt", "node_list", "node_name", "init_node_slot", - "node_line", "@3", "@4", "node_opt_list", "node_opt", "setup_block", - "teardown_block", "named_step", "cmd_block", "cmd_list", "step_cmd", - "exec_cmd", "state_op", "wait_multi_condition", - "wait_multi_condition_list", "opt_passing_through", "pass_state_list", - "wait_cmd", "@5", "@6", "state_name_list", "opt_in_group", "group_items", - "opt_timeout", "assert_cmd", "sql_cmd", "expect_cmd", "promote_cmd", - "promote_list", "perform_cmd", "network_cmd", "nodeini_cmd", "sleep_cmd", - "compose_cmd", "postgres_ctl_cmd", "fsm_step_cmd", "while_body", "@7", + "T_GET", "T_FSM", "T_LOGS", "T_NOT", "T_CONTAINS", "T_MATCHES", "T_WAL", + "T_SEGMENT", "T_ARCHIVED", "T_BASEBACKUP", "T_SLASH", "T_INTEGER", + "T_IDENT", "T_STRING", "T_BLOCK", "T_SHELL_ARGS", "$accept", "spec", + "spec_item", "cluster_block", "@1", "cluster_item_list", "cluster_item", + "archiver_block", "@2", "archiver_opt_list", "archiver_opt", + "monitor_line", "image_line", "extension_version_line", "ssl_line", + "auth_line", "formation_block", "@3", "formation_opt_list", "bare_name", + "formation_opt", "node_list", "node_name", "init_node_slot", "node_line", + "@4", "@5", "node_opt_list", "node_opt", "setup_block", "teardown_block", + "named_step", "cmd_block", "cmd_list", "step_cmd", "exec_cmd", + "state_op", "wait_multi_condition", "wait_multi_condition_list", + "opt_passing_through", "pass_state_list", "wait_cmd", "@6", "@7", + "state_name_list", "opt_in_group", "group_items", "opt_timeout", + "assert_cmd", "sql_cmd", "expect_cmd", "promote_cmd", "promote_list", + "perform_cmd", "network_cmd", "nodeini_cmd", "sleep_cmd", "compose_cmd", + "postgres_ctl_cmd", "fsm_step_cmd", "while_body", "@8", "stays_while_cmd", "set_monitor_cmd", "logs_cmd", "sequence_block", - "sequence_names", "fsm_state", "ident_or_string", 0 + "sequence_names", "fsm_state", "ident_or_string", "wait_state_name", + "opt_wait_group", 0 }; #endif @@ -967,35 +997,37 @@ static const yytype_uint16 yytoknum[] = 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, - 375 + 375, 376, 377, 378, 379, 380, 381 }; # endif /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ static const yytype_uint8 yyr1[] = { - 0, 121, 122, 122, 123, 123, 123, 123, 123, 125, - 124, 126, 126, 127, 127, 127, 127, 127, 127, 127, - 127, 128, 128, 128, 128, 128, 128, 128, 128, 129, - 129, 130, 130, 131, 132, 132, 134, 133, 135, 135, - 136, 136, 136, 136, 136, 137, 137, 137, 138, 138, - 139, 139, 140, 142, 141, 143, 141, 144, 144, 145, - 145, 145, 145, 145, 145, 145, 145, 145, 145, 145, - 145, 145, 145, 145, 145, 145, 145, 145, 145, 145, - 145, 145, 145, 145, 145, 145, 145, 145, 146, 147, - 148, 149, 150, 150, 151, 151, 151, 151, 151, 151, - 151, 151, 151, 151, 151, 151, 151, 151, 151, 151, - 152, 152, 152, 152, 152, 152, 152, 152, 152, 153, - 153, 154, 154, 155, 155, 156, 156, 157, 157, 157, - 157, 159, 158, 160, 158, 158, 158, 158, 158, 158, - 158, 161, 161, 161, 161, 162, 162, 163, 163, 164, - 164, 164, 165, 165, 165, 165, 166, 167, 167, 167, - 167, 168, 169, 169, 170, 170, 170, 170, 171, 171, - 172, 172, 173, 174, 174, 174, 174, 174, 175, 175, - 176, 178, 177, 179, 180, 181, 181, 181, 181, 182, - 183, 183, 184, 184, 184, 184, 184, 184, 184, 184, - 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, - 184, 184, 184, 185, 185 + 0, 127, 128, 128, 129, 129, 129, 129, 129, 131, + 130, 132, 132, 133, 133, 133, 133, 133, 133, 133, + 133, 133, 135, 134, 136, 136, 137, 137, 137, 137, + 137, 137, 138, 138, 138, 138, 138, 138, 138, 138, + 139, 139, 140, 140, 141, 142, 142, 144, 143, 145, + 145, 146, 146, 146, 146, 146, 147, 147, 147, 148, + 148, 149, 149, 150, 152, 151, 153, 151, 154, 154, + 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, + 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, + 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, + 156, 157, 158, 159, 160, 160, 161, 161, 161, 161, + 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, + 161, 161, 162, 162, 162, 162, 162, 162, 162, 162, + 162, 163, 163, 164, 164, 165, 165, 166, 166, 167, + 167, 167, 167, 169, 168, 170, 168, 168, 168, 168, + 168, 168, 168, 168, 168, 168, 168, 171, 171, 171, + 171, 172, 172, 173, 173, 174, 174, 174, 175, 175, + 175, 175, 176, 177, 177, 177, 177, 178, 179, 179, + 180, 180, 180, 180, 181, 181, 182, 182, 183, 184, + 184, 184, 184, 184, 185, 185, 186, 188, 187, 189, + 190, 191, 191, 191, 191, 192, 193, 193, 194, 194, + 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, + 194, 194, 194, 194, 194, 194, 194, 194, 194, 195, + 195, 196, 196, 197, 197 }; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ @@ -1003,26 +1035,28 @@ static const yytype_uint8 yyr2[] = { 0, 2, 1, 2, 1, 1, 1, 1, 1, 0, 5, 0, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 3, 3, 3, 3, 4, 4, 6, 2, - 2, 2, 2, 2, 2, 2, 0, 6, 0, 2, - 1, 1, 1, 1, 1, 1, 2, 2, 0, 2, - 1, 1, 0, 0, 4, 0, 7, 0, 2, 1, - 1, 1, 1, 1, 1, 2, 2, 4, 2, 1, - 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 3, 3, 2, 2, - 3, 3, 0, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 0, 6, 0, 2, 2, 2, 2, 4, + 2, 2, 1, 3, 3, 3, 3, 4, 4, 6, + 2, 2, 2, 2, 2, 2, 2, 0, 6, 0, + 2, 1, 1, 1, 1, 1, 1, 2, 2, 0, + 2, 1, 1, 0, 0, 4, 0, 7, 0, 2, + 1, 1, 1, 1, 1, 1, 1, 2, 2, 4, + 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, + 2, 2, 3, 3, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 3, 2, 3, 2, 3, 2, 3, 2, 1, 1, - 1, 4, 4, 1, 3, 0, 2, 1, 1, 3, - 3, 0, 9, 0, 9, 7, 7, 5, 6, 5, - 6, 1, 1, 3, 3, 0, 2, 2, 4, 0, - 2, 3, 6, 6, 6, 6, 3, 2, 2, 3, - 3, 2, 1, 3, 2, 4, 5, 7, 3, 3, - 5, 5, 2, 2, 3, 3, 3, 4, 3, 3, - 3, 0, 5, 5, 3, 4, 5, 4, 5, 2, - 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 3, 2, 3, 2, 3, 2, 3, 2, + 1, 1, 1, 4, 4, 1, 3, 0, 2, 1, + 1, 3, 3, 0, 9, 0, 9, 7, 7, 5, + 6, 5, 6, 8, 11, 10, 11, 1, 1, 3, + 3, 0, 2, 2, 4, 0, 2, 3, 6, 6, + 6, 6, 3, 2, 2, 3, 3, 2, 1, 3, + 2, 4, 5, 7, 3, 3, 5, 5, 2, 2, + 3, 3, 3, 4, 3, 3, 3, 0, 5, 5, + 3, 4, 5, 4, 5, 2, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 0, 2 }; /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state @@ -1030,290 +1064,322 @@ static const yytype_uint8 yyr2[] = means the default is an error. */ static const yytype_uint8 yydefact[] = { - 0, 0, 0, 0, 0, 190, 0, 2, 4, 5, - 6, 7, 8, 9, 92, 88, 89, 213, 214, 0, - 189, 1, 3, 11, 0, 90, 191, 0, 0, 0, - 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 91, 0, 0, 0, 93, 94, - 95, 96, 97, 98, 99, 100, 101, 109, 102, 103, - 104, 105, 106, 107, 108, 21, 0, 0, 0, 0, - 36, 0, 19, 20, 10, 12, 13, 14, 17, 15, - 16, 18, 0, 0, 111, 113, 115, 117, 0, 51, - 50, 0, 0, 158, 157, 162, 161, 164, 0, 0, - 172, 173, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 30, 29, 33, 34, - 35, 38, 31, 32, 0, 0, 110, 112, 114, 116, - 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, - 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, - 212, 142, 0, 145, 141, 0, 0, 0, 156, 160, - 159, 0, 0, 0, 168, 169, 174, 175, 176, 0, - 50, 179, 178, 184, 180, 0, 0, 0, 23, 24, - 25, 22, 0, 0, 0, 0, 0, 0, 149, 0, - 0, 0, 0, 0, 149, 119, 120, 0, 0, 0, - 163, 0, 165, 177, 0, 0, 185, 187, 26, 27, - 43, 44, 42, 0, 0, 48, 40, 41, 45, 39, - 170, 171, 149, 0, 0, 137, 0, 0, 0, 123, - 149, 0, 146, 144, 143, 139, 149, 149, 149, 149, - 181, 183, 166, 186, 188, 0, 46, 47, 0, 138, - 150, 0, 133, 131, 149, 149, 0, 0, 140, 147, - 0, 153, 152, 155, 154, 0, 0, 28, 0, 37, - 52, 49, 151, 125, 125, 136, 135, 0, 124, 0, - 92, 167, 52, 53, 0, 149, 149, 122, 121, 148, - 0, 55, 57, 128, 126, 127, 134, 132, 182, 0, - 54, 0, 57, 0, 0, 0, 59, 60, 61, 62, - 63, 0, 0, 64, 69, 0, 70, 71, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 58, 130, 129, - 0, 79, 80, 81, 65, 68, 66, 0, 0, 72, - 76, 85, 77, 78, 83, 82, 84, 73, 74, 75, - 56, 0, 86, 87, 67 + 0, 0, 0, 0, 0, 206, 0, 2, 4, 5, + 6, 7, 8, 9, 104, 100, 101, 229, 230, 0, + 205, 1, 3, 11, 0, 102, 207, 0, 0, 0, + 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 103, 0, 0, 0, 105, 106, + 107, 108, 109, 110, 111, 112, 113, 121, 114, 115, + 116, 117, 118, 119, 120, 32, 0, 0, 0, 0, + 47, 0, 0, 20, 21, 10, 12, 19, 13, 14, + 17, 15, 16, 18, 0, 0, 123, 125, 127, 129, + 0, 62, 61, 0, 0, 174, 173, 178, 177, 180, + 0, 0, 188, 189, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 41, 40, + 44, 45, 46, 49, 22, 42, 43, 0, 0, 122, + 124, 126, 128, 0, 208, 209, 210, 211, 212, 213, + 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 0, 0, 0, 158, 0, + 161, 157, 0, 0, 0, 172, 176, 175, 0, 0, + 0, 184, 185, 190, 191, 192, 0, 61, 195, 194, + 200, 196, 0, 0, 0, 34, 35, 36, 33, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 165, 0, 0, 0, 0, 0, 165, 131, 132, 0, + 0, 0, 179, 0, 181, 193, 0, 0, 201, 203, + 37, 38, 54, 55, 53, 0, 0, 59, 51, 52, + 56, 50, 24, 186, 187, 0, 0, 0, 0, 165, + 0, 0, 149, 0, 0, 0, 135, 165, 0, 162, + 160, 159, 151, 165, 165, 165, 165, 197, 199, 182, + 202, 204, 0, 57, 58, 0, 0, 232, 231, 0, + 0, 0, 0, 150, 166, 0, 145, 143, 165, 165, + 0, 0, 152, 163, 0, 169, 168, 171, 170, 0, + 0, 39, 0, 48, 63, 60, 0, 0, 0, 0, + 23, 25, 0, 165, 0, 0, 167, 137, 137, 148, + 147, 0, 136, 0, 104, 183, 63, 64, 26, 30, + 31, 0, 27, 28, 233, 153, 0, 0, 0, 165, + 165, 134, 133, 164, 0, 66, 68, 0, 0, 165, + 0, 0, 140, 138, 139, 146, 144, 198, 0, 65, + 29, 234, 155, 165, 165, 0, 68, 0, 0, 0, + 70, 71, 72, 73, 74, 75, 0, 0, 76, 81, + 0, 82, 83, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 69, 154, 156, 142, 141, 0, 91, 92, + 93, 77, 80, 78, 0, 0, 84, 88, 97, 89, + 90, 95, 94, 96, 85, 86, 87, 67, 0, 98, + 99, 79 }; /* YYDEFGOTO[NTERM-NUM]. */ static const yytype_int16 yydefgoto[] = { - -1, 6, 7, 8, 23, 27, 75, 76, 77, 78, - 79, 80, 81, 121, 184, 218, 219, 248, 91, 283, - 271, 292, 299, 300, 327, 9, 10, 11, 15, 24, - 48, 49, 197, 152, 230, 285, 294, 50, 274, 273, - 153, 194, 232, 225, 51, 52, 53, 54, 96, 55, - 56, 57, 58, 59, 60, 61, 241, 265, 62, 63, - 64, 12, 20, 154, 19 + -1, 6, 7, 8, 23, 27, 76, 77, 192, 266, + 301, 78, 79, 80, 81, 82, 83, 123, 191, 230, + 231, 265, 93, 317, 295, 336, 348, 349, 382, 9, + 10, 11, 15, 24, 48, 49, 209, 159, 247, 329, + 343, 50, 308, 307, 160, 206, 249, 242, 51, 52, + 53, 54, 98, 55, 56, 57, 58, 59, 60, 61, + 258, 289, 62, 63, 64, 12, 20, 161, 19, 269, + 339 }; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ -#define YYPACT_NINF -180 +#define YYPACT_NINF -204 static const yytype_int16 yypact[] = { - 77, -81, -74, -74, -36, -180, 39, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -74, - -36, -180, -180, -180, 442, -180, -180, 9, -17, -59, - -43, -40, -23, 30, -1, -4, -64, 0, 36, 7, - 26, -21, 21, 45, -180, 43, 160, 61, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -7, -20, 69, 94, 95, - -180, -18, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, 96, 98, 118, 119, 120, 121, 140, -180, - 3, 137, 125, -11, -180, -180, 143, 8, 128, 129, - -180, -180, 131, 132, 133, 134, 6, 6, 135, 6, - -24, 136, 138, 161, 139, 31, -180, -180, -180, -180, - -180, -180, -180, -180, 163, 164, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -61, 179, -69, -180, 2, 2, 552, -180, -180, - -180, 165, 265, 168, -180, -180, -180, -180, -180, 188, - -180, -180, -180, -180, -180, 10, 167, 191, -180, -180, - -180, -180, 283, 216, 1, 195, 196, 197, -32, 2, - 2, 198, 215, 169, -32, -180, -180, 210, 239, 211, - -180, 200, -180, -180, 201, 202, -180, -180, 284, -180, - -180, -180, -180, 206, 294, -180, -180, -180, -180, -180, - -180, -180, -32, 208, 250, -180, 280, 309, 228, -180, - -15, 233, 246, -180, -180, -180, -32, -32, -32, -32, - -180, -180, 251, -180, -180, 235, -180, -180, 4, -180, - -180, 238, 275, 279, -32, -32, 2, 198, -180, -180, - 277, -180, -180, -180, -180, 278, 263, -180, 264, -180, - -180, -180, -180, 274, 274, -180, -180, 350, -180, 267, - -180, -180, -180, -180, 379, -32, -32, -180, -180, -180, - 487, -180, -180, -180, 281, -180, -180, -180, -180, 282, - 141, 420, -180, 269, 270, 271, -180, -180, -180, -180, - -180, 104, -12, -180, -180, 272, -180, -180, 276, 303, - 273, 304, 305, 142, 302, 67, 307, -180, -180, -180, - 113, -180, -180, -180, -180, -180, -180, 365, 92, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, 366, -180, -180, -180 + 102, -70, -56, -56, -98, -204, 82, -204, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, -204, -56, + -98, -204, -204, -204, 504, -204, -204, 52, -33, -74, + -63, -57, -51, -18, 7, -20, -66, -2, 21, -6, + 10, 25, 11, 23, -204, 14, 144, 32, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, -204, -7, -29, 34, 36, 37, + -204, 38, -22, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, 39, 40, 60, 61, 62, 63, + 116, -204, 15, 83, 67, -16, -204, -204, 88, 33, + 74, 86, -204, -204, 87, 94, 100, 101, 8, 8, + 104, 8, -27, 105, 89, 106, 108, -3, -204, -204, + -204, -204, -204, -204, -204, -204, -204, 109, 111, -204, + -204, -204, -204, 126, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, -204, 112, 119, 120, -61, 152, + -73, -204, 3, 3, 614, -204, -204, -204, 121, 220, + 133, -204, -204, -204, -204, -204, 130, -204, -204, -204, + -204, -204, 26, 139, 145, -204, -204, -204, -204, 229, + 174, 1, 168, 150, 151, 3, 153, 155, 197, 154, + -39, 3, 3, 157, 180, 235, -39, -204, -204, 256, + 279, 218, -204, 226, -204, -204, 227, 228, -204, -204, + 238, -204, -204, -204, -204, 231, 320, -204, -204, -204, + -204, -204, -204, -204, -204, 331, 276, 236, 233, -39, + 237, 281, -204, 354, 375, 261, -204, -15, 239, 257, + -204, -204, -204, -39, -39, -39, -39, -204, -204, 262, + -204, -204, 241, -204, -204, 5, -5, -204, -204, 265, + 242, 267, 268, -204, -204, 248, 286, 294, -39, -39, + 3, 157, -204, -204, 270, -204, -204, -204, -204, 271, + 251, -204, 252, -204, -204, -204, 253, 349, -14, 16, + -204, -204, 255, -39, 278, 322, -204, 337, 337, -204, + -204, 406, -204, 325, -204, -204, -204, -204, -204, -204, + -204, 422, -204, -204, 328, -204, 329, 330, 450, -39, + -39, -204, -204, -204, 549, -204, -204, 424, 356, -39, + 357, 358, -204, 348, -204, -204, -204, -204, 373, 225, + -204, -204, -204, -39, -39, 481, -204, 359, 360, 361, + -204, -204, -204, -204, -204, -204, 115, -4, -204, -204, + 362, -204, -204, 364, 365, 366, 368, 369, 118, 370, + 22, 367, -204, -204, -204, -204, -204, 179, -204, -204, + -204, -204, -204, -204, 455, 29, -204, -204, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, 460, -204, + -204, -204 }; /* YYPGOTO[NTERM-NUM]. */ static const yytype_int16 yypgoto[] = { - -180, -180, 388, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -105, 114, - -180, -180, -180, 93, -180, -180, -180, -180, 13, 144, - -180, -180, -145, -179, -180, 151, -180, -180, -180, -180, - -180, -180, -180, -171, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -157, 428 + -204, -204, 487, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -107, 181, -204, -204, -204, 140, -204, -204, + -204, -204, 24, 206, -204, -204, -147, -195, -204, 187, + -204, -204, -204, -204, -204, -204, -204, -203, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -164, 501, -204, + -204 }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which number is the opposite. If zero, do what YYDEFACT says. If YYTABLE_NINF, syntax error. */ -#define YYTABLE_NINF -123 +#define YYTABLE_NINF -135 static const yytype_int16 yytable[] = { - 199, 171, 172, 89, 174, 210, 211, 111, 89, 268, - 89, 198, 229, 65, 195, 336, 16, 212, 187, 93, - 213, 13, 66, 235, 67, 68, 69, 70, 14, 112, - 113, 192, 25, 114, 188, 193, 234, 189, 190, 21, - 237, 239, 1, 223, 226, 227, 224, 2, 3, 4, - 5, 249, 71, 72, 73, 94, 182, 214, 84, 258, - 223, 257, 183, 224, 337, 261, 262, 263, 264, 253, - 255, 101, 102, 103, 85, 104, 105, 86, 278, 196, - 1, 17, 18, 275, 276, 2, 3, 4, 5, 175, - 176, 177, 82, 83, 87, 98, 99, 116, 117, 122, - 123, 155, 156, 215, 88, 159, 160, 269, 162, 163, - 115, 277, 74, 92, 296, 297, 90, 95, 216, 217, - 288, 170, 97, 170, 204, 205, 106, 295, 303, 304, - 305, 334, 335, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 100, 270, 329, 315, 316, 317, 318, 319, - 107, 320, 321, 322, 323, 324, 303, 304, 305, 325, - 108, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 109, 344, 345, 315, 316, 317, 318, 319, 110, 320, - 321, 322, 323, 324, 347, 348, 118, 325, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 352, - 353, 119, 120, 124, 326, 125, 350, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 143, 144, 145, 146, 147, 148, 149, 150, 126, 127, - 128, 129, 326, 157, 158, 164, 165, 161, 166, 167, - 168, 169, 173, 178, 179, 191, 181, 151, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 180, - 185, 186, 200, 201, 202, 206, 233, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 143, 144, 145, 146, 147, 148, 149, 150, 203, 207, - 208, 209, 220, 221, 222, 228, 231, 242, 240, 243, - 244, 245, 246, 247, 250, 251, 256, 236, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 259, - 260, -122, 266, 267, 272, -121, 238, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 143, 144, 145, 146, 147, 148, 149, 150, 279, 281, - 280, 282, 284, 289, 302, 301, 331, 332, 333, 338, - 351, 341, 339, 354, 22, 330, 291, 252, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 340, - 346, 342, 343, 349, 290, 286, 254, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 143, 144, 145, 146, 147, 148, 149, 150, 26, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 287, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 28, - 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 29, 30, 31, 32, 33, 0, 0, 0, 0, - 0, 0, 34, 35, 36, 0, 37, 38, 0, 39, - 0, 0, 40, 41, 28, 42, 43, 328, 0, 0, - 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, - 0, 45, 0, 46, 47, 0, 29, 30, 31, 32, - 33, 0, 0, 0, 0, 0, 0, 34, 35, 36, - 0, 37, 38, 0, 39, 0, 0, 40, 41, 0, - 42, 43, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 0, 0, 0, 0, 0, 45, 0, 46, 47, - 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 211, 178, 179, 252, 181, 222, 223, 113, 246, 91, + 292, 91, 91, 296, 320, 207, 210, 224, 95, 199, + 225, 297, 298, 189, 393, 17, 18, 16, 204, 190, + 114, 115, 205, 13, 116, 200, 273, 240, 201, 202, + 241, 251, 299, 25, 282, 254, 256, 14, 235, 86, + 285, 286, 287, 288, 243, 244, 65, 90, 226, 96, + 87, 240, 281, 321, 241, 66, 88, 67, 68, 69, + 70, 268, 89, 394, 71, 309, 310, 84, 85, 277, + 279, 208, 21, 100, 101, 1, 312, 182, 183, 184, + 2, 3, 4, 5, 118, 119, 72, 73, 74, 300, + 325, 125, 126, 94, 227, 1, 166, 167, 99, 293, + 2, 3, 4, 5, 162, 163, 117, 108, 103, 104, + 105, 97, 106, 107, 228, 229, 345, 346, 177, 109, + 92, 177, 102, 311, 169, 170, 352, 110, 133, 322, + 323, 216, 217, 391, 392, 404, 405, 332, 401, 402, + 383, 384, 409, 410, 111, 112, 75, 120, 294, 121, + 122, 124, 127, 128, 344, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, + 149, 150, 151, 152, 153, 154, 129, 130, 131, 132, + 164, 386, 165, 168, 357, 358, 359, 171, 155, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 172, + 173, 186, 370, 371, 372, 373, 374, 174, 375, 376, + 377, 378, 379, 175, 176, 195, 380, 180, 185, 203, + 187, 188, 193, 156, 194, 196, 157, 197, 213, 158, + 357, 358, 359, 198, 212, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 214, 215, 220, 370, 371, + 372, 373, 374, 218, 375, 376, 377, 378, 379, 219, + 221, 232, 380, 233, 234, 238, 262, 239, 236, 237, + 245, 381, 248, 407, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150 + 150, 151, 152, 153, 154, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, + 149, 150, 151, 152, 153, 154, 257, 381, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, + 146, 147, 148, 149, 150, 151, 152, 153, 154, 259, + 264, 260, 261, 263, 270, 271, 272, 275, 250, 274, + 280, 283, 284, -134, 290, 291, 302, 303, 304, 305, + 306, -133, 313, 315, 314, 316, 318, 319, 324, 253, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, + 154, 326, 255, 134, 135, 136, 137, 138, 139, 140, + 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, + 151, 152, 153, 154, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 327, 328, 333, 337, 338, + 340, 341, 350, 355, 267, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, + 149, 150, 151, 152, 153, 154, 356, 276, 351, 353, + 354, 408, 388, 389, 390, 395, 396, 397, 411, 406, + 398, 399, 400, 22, 403, 330, 387, 335, 278, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, + 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, + 334, 26, 0, 0, 0, 0, 0, 0, 0, 331, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, + 154, 0, 28, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 342, 29, 30, 31, 32, 33, 0, + 0, 0, 0, 0, 0, 34, 35, 36, 0, 37, + 38, 0, 39, 0, 0, 40, 41, 28, 42, 43, + 0, 0, 0, 0, 385, 0, 0, 0, 44, 0, + 0, 0, 0, 0, 45, 0, 46, 47, 0, 29, + 30, 31, 32, 33, 0, 0, 0, 0, 0, 0, + 34, 35, 36, 0, 37, 38, 0, 39, 0, 0, + 40, 41, 0, 42, 43, 0, 0, 0, 0, 0, + 0, 0, 0, 347, 0, 0, 0, 0, 0, 45, + 0, 46, 47, 134, 135, 136, 137, 138, 139, 140, + 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, + 151, 152, 153, 154 }; static const yytype_int16 yycheck[] = { - 157, 106, 107, 4, 109, 4, 5, 14, 4, 5, - 4, 156, 191, 4, 12, 27, 3, 16, 79, 83, - 19, 102, 13, 194, 15, 16, 17, 18, 102, 36, - 37, 100, 19, 40, 95, 104, 193, 98, 99, 0, - 197, 198, 3, 75, 189, 190, 78, 8, 9, 10, - 11, 222, 43, 44, 45, 119, 25, 56, 117, 230, - 75, 76, 31, 78, 76, 236, 237, 238, 239, 226, - 227, 92, 93, 94, 117, 96, 97, 117, 257, 77, - 3, 117, 118, 254, 255, 8, 9, 10, 11, 113, - 114, 115, 109, 110, 117, 88, 89, 117, 118, 117, - 118, 98, 99, 102, 74, 116, 117, 103, 100, 101, - 117, 256, 103, 117, 285, 286, 117, 117, 117, 118, - 277, 117, 86, 117, 114, 115, 105, 284, 15, 16, - 17, 27, 28, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 116, 248, 301, 32, 33, 34, 35, 36, - 105, 38, 39, 40, 41, 42, 15, 16, 17, 46, - 117, 20, 21, 22, 23, 24, 25, 26, 27, 28, - 10, 29, 30, 32, 33, 34, 35, 36, 117, 38, - 39, 40, 41, 42, 117, 118, 117, 46, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 117, - 118, 117, 117, 117, 101, 117, 103, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 120, 120, - 120, 120, 101, 106, 119, 117, 117, 104, 117, 117, - 117, 117, 117, 117, 116, 76, 117, 117, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 118, - 117, 117, 117, 18, 116, 118, 117, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 120, 118, - 27, 95, 117, 117, 117, 117, 101, 117, 107, 118, - 118, 37, 116, 29, 116, 75, 98, 117, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 116, - 104, 76, 101, 118, 116, 76, 117, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 101, 116, - 102, 117, 108, 116, 102, 104, 117, 117, 117, 117, - 25, 118, 116, 27, 6, 302, 282, 117, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 116, - 118, 117, 117, 116, 280, 274, 117, 48, 49, 50, + 164, 108, 109, 206, 111, 4, 5, 14, 203, 4, + 5, 4, 4, 18, 28, 12, 163, 16, 84, 80, + 19, 26, 27, 26, 28, 123, 124, 3, 101, 32, + 37, 38, 105, 103, 41, 96, 239, 76, 99, 100, + 79, 205, 47, 19, 247, 209, 210, 103, 195, 123, + 253, 254, 255, 256, 201, 202, 4, 75, 57, 125, + 123, 76, 77, 77, 79, 13, 123, 15, 16, 17, + 18, 235, 123, 77, 22, 278, 279, 110, 111, 243, + 244, 78, 0, 89, 90, 3, 281, 114, 115, 116, + 8, 9, 10, 11, 123, 124, 44, 45, 46, 104, + 303, 123, 124, 123, 103, 3, 122, 123, 87, 104, + 8, 9, 10, 11, 99, 100, 123, 106, 93, 94, + 95, 123, 97, 98, 123, 124, 329, 330, 123, 106, + 123, 123, 122, 280, 101, 102, 339, 123, 22, 123, + 124, 115, 116, 28, 29, 123, 124, 311, 30, 31, + 353, 354, 123, 124, 10, 123, 104, 123, 265, 123, + 123, 123, 123, 123, 328, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 126, 126, 126, 126, + 107, 355, 125, 105, 15, 16, 17, 123, 82, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 123, + 123, 122, 33, 34, 35, 36, 37, 123, 39, 40, + 41, 42, 43, 123, 123, 99, 47, 123, 123, 77, + 124, 123, 123, 117, 123, 123, 120, 118, 18, 123, + 15, 16, 17, 123, 123, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 122, 126, 28, 33, 34, + 35, 36, 37, 124, 39, 40, 41, 42, 43, 124, + 96, 103, 47, 123, 123, 78, 38, 123, 125, 124, + 123, 102, 102, 104, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 108, 102, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 20, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 117, 48, 49, + 61, 62, 63, 64, 65, 66, 67, 68, 69, 123, + 30, 124, 124, 122, 78, 119, 123, 76, 123, 122, + 99, 122, 105, 77, 102, 124, 101, 125, 101, 101, + 122, 77, 102, 122, 103, 123, 123, 28, 123, 123, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 123, 123, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 123, 109, 122, 26, 121, + 121, 121, 28, 105, 123, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 103, 123, 122, 122, + 122, 26, 123, 123, 123, 123, 122, 122, 28, 122, + 124, 123, 123, 6, 124, 308, 356, 316, 123, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 47, - -1, -1, -1, -1, -1, -1, 117, -1, -1, -1, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 314, 20, -1, -1, -1, -1, -1, -1, -1, 123, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, -1, 48, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 69, 70, 71, 72, 73, -1, -1, -1, -1, - -1, -1, 80, 81, 82, -1, 84, 85, -1, 87, - -1, -1, 90, 91, 47, 93, 94, 117, -1, -1, - -1, -1, -1, -1, -1, 103, -1, -1, -1, -1, - -1, 109, -1, 111, 112, -1, 69, 70, 71, 72, - 73, -1, -1, -1, -1, -1, -1, 80, 81, 82, - -1, 84, 85, -1, 87, -1, -1, 90, 91, -1, - 93, 94, -1, -1, -1, -1, -1, -1, -1, -1, - 103, -1, -1, -1, -1, -1, 109, -1, 111, 112, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, - 68 + -1, -1, -1, 123, 70, 71, 72, 73, 74, -1, + -1, -1, -1, -1, -1, 81, 82, 83, -1, 85, + 86, -1, 88, -1, -1, 91, 92, 48, 94, 95, + -1, -1, -1, -1, 123, -1, -1, -1, 104, -1, + -1, -1, -1, -1, 110, -1, 112, 113, -1, 70, + 71, 72, 73, 74, -1, -1, -1, -1, -1, -1, + 81, 82, 83, -1, 85, 86, -1, 88, -1, -1, + 91, 92, -1, 94, 95, -1, -1, -1, -1, -1, + -1, -1, -1, 104, -1, -1, -1, -1, -1, 110, + -1, 112, 113, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69 }; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ static const yytype_uint8 yystos[] = { - 0, 3, 8, 9, 10, 11, 122, 123, 124, 146, - 147, 148, 182, 102, 102, 149, 149, 117, 118, 185, - 183, 0, 123, 125, 150, 149, 185, 126, 47, 69, - 70, 71, 72, 73, 80, 81, 82, 84, 85, 87, - 90, 91, 93, 94, 103, 109, 111, 112, 151, 152, - 158, 165, 166, 167, 168, 170, 171, 172, 173, 174, - 175, 176, 179, 180, 181, 4, 13, 15, 16, 17, - 18, 43, 44, 45, 103, 127, 128, 129, 130, 131, - 132, 133, 109, 110, 117, 117, 117, 117, 74, 4, - 117, 139, 117, 83, 119, 117, 169, 86, 88, 89, - 116, 92, 93, 94, 96, 97, 105, 105, 117, 10, - 117, 14, 36, 37, 40, 117, 117, 118, 117, 117, - 117, 134, 117, 118, 117, 117, 120, 120, 120, 120, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, - 68, 117, 154, 161, 184, 98, 99, 106, 119, 116, - 117, 104, 100, 101, 117, 117, 117, 117, 117, 117, - 117, 139, 139, 117, 139, 113, 114, 115, 117, 116, - 118, 117, 25, 31, 135, 117, 117, 79, 95, 98, - 99, 76, 100, 104, 162, 12, 77, 153, 153, 184, - 117, 18, 116, 120, 114, 115, 118, 118, 27, 95, - 4, 5, 16, 19, 56, 102, 117, 118, 136, 137, - 117, 117, 117, 75, 78, 164, 153, 153, 117, 154, - 155, 101, 163, 117, 184, 164, 117, 184, 117, 184, - 107, 177, 117, 118, 118, 37, 116, 29, 138, 164, - 116, 75, 117, 184, 117, 184, 98, 76, 164, 116, - 104, 164, 164, 164, 164, 178, 101, 118, 5, 103, - 139, 141, 116, 160, 159, 164, 164, 153, 154, 101, - 102, 116, 117, 140, 108, 156, 156, 117, 184, 116, - 150, 140, 142, 117, 157, 184, 164, 164, 103, 143, - 144, 104, 102, 15, 16, 17, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 32, 33, 34, 35, 36, - 38, 39, 40, 41, 42, 46, 101, 145, 117, 184, - 144, 117, 117, 117, 27, 28, 27, 76, 117, 116, - 116, 118, 117, 117, 29, 30, 118, 117, 118, 116, - 103, 25, 117, 118, 27 + 0, 3, 8, 9, 10, 11, 128, 129, 130, 156, + 157, 158, 192, 103, 103, 159, 159, 123, 124, 195, + 193, 0, 129, 131, 160, 159, 195, 132, 48, 70, + 71, 72, 73, 74, 81, 82, 83, 85, 86, 88, + 91, 92, 94, 95, 104, 110, 112, 113, 161, 162, + 168, 175, 176, 177, 178, 180, 181, 182, 183, 184, + 185, 186, 189, 190, 191, 4, 13, 15, 16, 17, + 18, 22, 44, 45, 46, 104, 133, 134, 138, 139, + 140, 141, 142, 143, 110, 111, 123, 123, 123, 123, + 75, 4, 123, 149, 123, 84, 125, 123, 179, 87, + 89, 90, 122, 93, 94, 95, 97, 98, 106, 106, + 123, 10, 123, 14, 37, 38, 41, 123, 123, 124, + 123, 123, 123, 144, 123, 123, 124, 123, 123, 126, + 126, 126, 126, 22, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 82, 117, 120, 123, 164, + 171, 194, 99, 100, 107, 125, 122, 123, 105, 101, + 102, 123, 123, 123, 123, 123, 123, 123, 149, 149, + 123, 149, 114, 115, 116, 123, 122, 124, 123, 26, + 32, 145, 135, 123, 123, 99, 123, 118, 123, 80, + 96, 99, 100, 77, 101, 105, 172, 12, 78, 163, + 163, 194, 123, 18, 122, 126, 115, 116, 124, 124, + 28, 96, 4, 5, 16, 19, 57, 103, 123, 124, + 146, 147, 103, 123, 123, 163, 125, 124, 78, 123, + 76, 79, 174, 163, 163, 123, 164, 165, 102, 173, + 123, 194, 174, 123, 194, 123, 194, 108, 187, 123, + 124, 124, 38, 122, 30, 148, 136, 123, 194, 196, + 78, 119, 123, 174, 122, 76, 123, 194, 123, 194, + 99, 77, 174, 122, 105, 174, 174, 174, 174, 188, + 102, 124, 5, 104, 149, 151, 18, 26, 27, 47, + 104, 137, 101, 125, 101, 101, 122, 170, 169, 174, + 174, 163, 164, 102, 103, 122, 123, 150, 123, 28, + 28, 77, 123, 124, 123, 174, 123, 123, 109, 166, + 166, 123, 194, 122, 160, 150, 152, 26, 121, 197, + 121, 121, 123, 167, 194, 174, 174, 104, 153, 154, + 28, 122, 174, 122, 122, 105, 103, 15, 16, 17, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 33, 34, 35, 36, 37, 39, 40, 41, 42, 43, + 47, 102, 155, 174, 174, 123, 194, 154, 123, 123, + 123, 28, 29, 28, 77, 123, 122, 122, 124, 123, + 123, 30, 31, 124, 123, 124, 122, 104, 26, 123, + 124, 28 }; #define yyerrok (yyerrstatus = 0) @@ -2128,7 +2194,7 @@ yyparse () switch (yyn) { case 9: -#line 238 "test_spec_parse.y" +#line 242 "test_spec_parse.y" { strlcpy(current_spec->cluster.ssl, "self-signed", sizeof(current_spec->cluster.ssl)); @@ -2137,25 +2203,100 @@ yyparse () ;} break; - case 19: -#line 259 "test_spec_parse.y" + case 20: +#line 264 "test_spec_parse.y" { current_spec->cluster.bindSource = true; ;} break; - case 20: -#line 260 "test_spec_parse.y" + case 21: +#line 265 "test_spec_parse.y" { current_spec->cluster.legacyStartup = true; ;} break; - case 21: -#line 274 "test_spec_parse.y" + case 22: +#line 288 "test_spec_parse.y" + { + TestCluster *cl = ¤t_spec->cluster; + + if (cl->archiverCount >= PGAF_MAX_ARCHIVERS) + { + fprintf(stderr, "pgaftest: too many archivers (max %d)\n", + PGAF_MAX_ARCHIVERS); + exit(1); + } + + current_archiver = &cl->archivers[cl->archiverCount++]; + strlcpy(current_archiver->name, (yyvsp[(2) - (2)].str), sizeof(current_archiver->name)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 26: +#line 312 "test_spec_parse.y" + { + if (current_archiver->formationCount >= PGAF_MAX_ARCHIVER_FORMATIONS) + { + fprintf(stderr, + "pgaftest: too many --formation entries for archiver " + "\"%s\" (max %d)\n", + current_archiver->name, PGAF_MAX_ARCHIVER_FORMATIONS); + exit(1); + } + strlcpy(current_archiver->formations[current_archiver->formationCount++], + (yyvsp[(2) - (2)].str), sizeof(current_archiver->formations[0])); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 27: +#line 326 "test_spec_parse.y" + { + strlcpy(current_archiver->region, (yyvsp[(2) - (2)].str), sizeof(current_archiver->region)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 28: +#line 331 "test_spec_parse.y" + { + strlcpy(current_archiver->region, (yyvsp[(2) - (2)].str), sizeof(current_archiver->region)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 29: +#line 336 "test_spec_parse.y" + { + /* bare "create and launch deferred" = both gates, matching + * node_opt's own identical form */ + current_archiver->createDeferred = true; + current_archiver->launchDeferred = true; + ;} + break; + + case 30: +#line 343 "test_spec_parse.y" + { + current_archiver->launchDeferred = true; + ;} + break; + + case 31: +#line 347 "test_spec_parse.y" + { + current_archiver->createDeferred = true; + ;} + break; + + case 32: +#line 363 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; ;} break; - case 22: -#line 278 "test_spec_parse.y" + case 33: +#line 367 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; strlcpy(current_spec->cluster.monitorDebianCluster, (yyvsp[(3) - (3)].str), @@ -2164,8 +2305,8 @@ yyparse () ;} break; - case 23: -#line 285 "test_spec_parse.y" + case 34: +#line 374 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; strlcpy(current_spec->cluster.monitorImageTarget, (yyvsp[(3) - (3)].str), @@ -2174,8 +2315,8 @@ yyparse () ;} break; - case 24: -#line 292 "test_spec_parse.y" + case 35: +#line 381 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; /* monitor port not stored in TestCluster yet; ignore */ @@ -2183,8 +2324,8 @@ yyparse () ;} break; - case 25: -#line 298 "test_spec_parse.y" + case 36: +#line 387 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; strlcpy(current_spec->cluster.monitorPassword, (yyvsp[(3) - (3)].str), @@ -2193,8 +2334,8 @@ yyparse () ;} break; - case 26: -#line 305 "test_spec_parse.y" + case 37: +#line 394 "test_spec_parse.y" { strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (4)].str), sizeof(current_spec->cluster.secondMonitorName)); @@ -2203,8 +2344,8 @@ yyparse () ;} break; - case 27: -#line 312 "test_spec_parse.y" + case 38: +#line 401 "test_spec_parse.y" { strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (4)].str), sizeof(current_spec->cluster.secondMonitorName)); @@ -2213,8 +2354,8 @@ yyparse () ;} break; - case 28: -#line 319 "test_spec_parse.y" + case 39: +#line 408 "test_spec_parse.y" { strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (6)].str), sizeof(current_spec->cluster.secondMonitorName)); @@ -2225,8 +2366,8 @@ yyparse () ;} break; - case 29: -#line 332 "test_spec_parse.y" + case 40: +#line 421 "test_spec_parse.y" { strlcpy(current_spec->cluster.image, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.image)); @@ -2234,8 +2375,8 @@ yyparse () ;} break; - case 30: -#line 338 "test_spec_parse.y" + case 41: +#line 427 "test_spec_parse.y" { strlcpy(current_spec->cluster.image, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.image)); @@ -2243,8 +2384,8 @@ yyparse () ;} break; - case 31: -#line 348 "test_spec_parse.y" + case 42: +#line 437 "test_spec_parse.y" { strlcpy(current_spec->cluster.extensionVersion, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.extensionVersion)); @@ -2252,8 +2393,8 @@ yyparse () ;} break; - case 32: -#line 354 "test_spec_parse.y" + case 43: +#line 443 "test_spec_parse.y" { strlcpy(current_spec->cluster.extensionVersion, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.extensionVersion)); @@ -2261,8 +2402,8 @@ yyparse () ;} break; - case 33: -#line 364 "test_spec_parse.y" + case 44: +#line 453 "test_spec_parse.y" { strlcpy(current_spec->cluster.ssl, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.ssl)); @@ -2270,8 +2411,8 @@ yyparse () ;} break; - case 34: -#line 374 "test_spec_parse.y" + case 45: +#line 463 "test_spec_parse.y" { strlcpy(current_spec->cluster.auth, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.auth)); @@ -2279,8 +2420,8 @@ yyparse () ;} break; - case 35: -#line 380 "test_spec_parse.y" + case 46: +#line 469 "test_spec_parse.y" { strlcpy(current_spec->cluster.auth, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.auth)); @@ -2288,8 +2429,8 @@ yyparse () ;} break; - case 36: -#line 390 "test_spec_parse.y" + case 47: +#line 479 "test_spec_parse.y" { TestCluster *cl = ¤t_spec->cluster; if (cl->formationCount >= PGAF_MAX_FORMATIONS) @@ -2305,65 +2446,65 @@ yyparse () ;} break; - case 40: -#line 417 "test_spec_parse.y" + case 51: +#line 506 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 41: -#line 418 "test_spec_parse.y" + case 52: +#line 507 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 42: -#line 419 "test_spec_parse.y" + case 53: +#line 508 "test_spec_parse.y" { (yyval.str) = strdup("auth"); ;} break; - case 43: -#line 420 "test_spec_parse.y" + case 54: +#line 509 "test_spec_parse.y" { (yyval.str) = strdup("monitor"); ;} break; - case 44: -#line 421 "test_spec_parse.y" + case 55: +#line 510 "test_spec_parse.y" { (yyval.str) = strdup("node"); ;} break; - case 45: -#line 426 "test_spec_parse.y" + case 56: +#line 515 "test_spec_parse.y" { strlcpy(current_formation->name, (yyvsp[(1) - (1)].str), sizeof(current_formation->name)); free((yyvsp[(1) - (1)].str)); ;} break; - case 46: -#line 431 "test_spec_parse.y" + case 57: +#line 520 "test_spec_parse.y" { current_formation->numSync = (yyvsp[(2) - (2)].ival); ;} break; - case 47: -#line 435 "test_spec_parse.y" + case 58: +#line 524 "test_spec_parse.y" { current_formation->disableSecondary = true; ;} break; - case 50: -#line 461 "test_spec_parse.y" + case 61: +#line 550 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 51: -#line 462 "test_spec_parse.y" + case 62: +#line 551 "test_spec_parse.y" { (yyval.str) = strdup("monitor"); ;} break; - case 52: -#line 471 "test_spec_parse.y" + case 63: +#line 560 "test_spec_parse.y" { if (current_formation->nodeCount >= PGAF_MAX_NODES) { @@ -2378,61 +2519,68 @@ yyparse () ;} break; - case 53: -#line 488 "test_spec_parse.y" + case 64: +#line 577 "test_spec_parse.y" { strlcpy(current_node->name, (yyvsp[(1) - (2)].str), sizeof(current_node->name)); free((yyvsp[(1) - (2)].str)); ;} break; - case 55: -#line 495 "test_spec_parse.y" + case 66: +#line 584 "test_spec_parse.y" { strlcpy(current_node->name, (yyvsp[(2) - (3)].str), sizeof(current_node->name)); free((yyvsp[(2) - (3)].str)); ;} break; - case 59: -#line 509 "test_spec_parse.y" + case 70: +#line 598 "test_spec_parse.y" { current_node->kind = NODE_KIND_CITUS_COORDINATOR; current_spec->cluster.withCitus = true; ;} break; - case 60: -#line 514 "test_spec_parse.y" + case 71: +#line 603 "test_spec_parse.y" { current_node->kind = NODE_KIND_CITUS_WORKER; current_spec->cluster.withCitus = true; ;} break; - case 61: -#line 519 "test_spec_parse.y" + case 72: +#line 608 "test_spec_parse.y" + { + current_node->kind = NODE_KIND_ARCHIVER; + ;} + break; + + case 73: +#line 612 "test_spec_parse.y" { current_node->replicationQuorum = false; ;} break; - case 62: -#line 523 "test_spec_parse.y" + case 74: +#line 616 "test_spec_parse.y" { current_node->noMonitor = true; ;} break; - case 63: -#line 527 "test_spec_parse.y" + case 75: +#line 620 "test_spec_parse.y" { current_node->suspended = true; ;} break; - case 64: -#line 531 "test_spec_parse.y" + case 76: +#line 624 "test_spec_parse.y" { /* bare "deferred" = create and launch deferred (both gates) */ current_node->createDeferred = true; @@ -2440,96 +2588,96 @@ yyparse () ;} break; - case 65: -#line 537 "test_spec_parse.y" + case 77: +#line 630 "test_spec_parse.y" { /* "launch deferred" alone = run-deferred only, create immediate */ current_node->launchDeferred = true; ;} break; - case 66: -#line 542 "test_spec_parse.y" + case 78: +#line 635 "test_spec_parse.y" { current_node->createDeferred = true; ;} break; - case 67: -#line 546 "test_spec_parse.y" + case 79: +#line 639 "test_spec_parse.y" { current_node->createDeferred = true; current_node->launchDeferred = true; ;} break; - case 68: -#line 551 "test_spec_parse.y" + case 80: +#line 644 "test_spec_parse.y" { current_node->launchDeferred = false; ;} break; - case 69: -#line 555 "test_spec_parse.y" + case 81: +#line 648 "test_spec_parse.y" { current_node->launchDeferred = false; ;} break; - case 70: -#line 559 "test_spec_parse.y" + case 82: +#line 652 "test_spec_parse.y" { current_node->listen = true; ;} break; - case 71: -#line 563 "test_spec_parse.y" + case 83: +#line 656 "test_spec_parse.y" { current_node->citusSecondary = true; ;} break; - case 72: -#line 567 "test_spec_parse.y" + case 84: +#line 660 "test_spec_parse.y" { current_node->candidatePriority = (yyvsp[(2) - (2)].ival); ;} break; - case 73: -#line 571 "test_spec_parse.y" + case 85: +#line 664 "test_spec_parse.y" { strlcpy(current_node->region, (yyvsp[(2) - (2)].str), sizeof(current_node->region)); free((yyvsp[(2) - (2)].str)); ;} break; - case 74: -#line 576 "test_spec_parse.y" + case 86: +#line 669 "test_spec_parse.y" { strlcpy(current_node->region, (yyvsp[(2) - (2)].str), sizeof(current_node->region)); free((yyvsp[(2) - (2)].str)); ;} break; - case 75: -#line 581 "test_spec_parse.y" + case 87: +#line 674 "test_spec_parse.y" { current_node->group = (yyvsp[(2) - (2)].ival); ;} break; - case 76: -#line 585 "test_spec_parse.y" + case 88: +#line 678 "test_spec_parse.y" { current_node->pgPort = (yyvsp[(2) - (2)].ival); ;} break; - case 77: -#line 589 "test_spec_parse.y" + case 89: +#line 682 "test_spec_parse.y" { strlcpy(current_node->citusClusterName, (yyvsp[(2) - (2)].str), sizeof(current_node->citusClusterName)); @@ -2537,8 +2685,8 @@ yyparse () ;} break; - case 78: -#line 595 "test_spec_parse.y" + case 90: +#line 688 "test_spec_parse.y" { strlcpy(current_node->debianCluster, (yyvsp[(2) - (2)].str), sizeof(current_node->debianCluster)); @@ -2546,46 +2694,46 @@ yyparse () ;} break; - case 79: -#line 601 "test_spec_parse.y" + case 91: +#line 694 "test_spec_parse.y" { strlcpy(current_node->ssl, (yyvsp[(2) - (2)].str), sizeof(current_node->ssl)); free((yyvsp[(2) - (2)].str)); ;} break; - case 80: -#line 606 "test_spec_parse.y" + case 92: +#line 699 "test_spec_parse.y" { strlcpy(current_node->auth, (yyvsp[(2) - (2)].str), sizeof(current_node->auth)); free((yyvsp[(2) - (2)].str)); ;} break; - case 81: -#line 611 "test_spec_parse.y" + case 93: +#line 704 "test_spec_parse.y" { strlcpy(current_node->auth, (yyvsp[(2) - (2)].str), sizeof(current_node->auth)); free((yyvsp[(2) - (2)].str)); ;} break; - case 82: -#line 616 "test_spec_parse.y" + case 94: +#line 709 "test_spec_parse.y" { current_node->replicationQuorum = true; ;} break; - case 83: -#line 620 "test_spec_parse.y" + case 95: +#line 713 "test_spec_parse.y" { current_node->replicationQuorum = false; ;} break; - case 84: -#line 624 "test_spec_parse.y" + case 96: +#line 717 "test_spec_parse.y" { strlcpy(current_node->replicationPassword, (yyvsp[(2) - (2)].str), sizeof(current_node->replicationPassword)); @@ -2593,8 +2741,8 @@ yyparse () ;} break; - case 85: -#line 630 "test_spec_parse.y" + case 97: +#line 723 "test_spec_parse.y" { strlcpy(current_node->monitorPassword, (yyvsp[(2) - (2)].str), sizeof(current_node->monitorPassword)); @@ -2602,8 +2750,8 @@ yyparse () ;} break; - case 86: -#line 636 "test_spec_parse.y" + case 98: +#line 729 "test_spec_parse.y" { /* volume — adds a named Docker volume */ int vi = current_node->volumeCount; @@ -2619,8 +2767,8 @@ yyparse () ;} break; - case 87: -#line 650 "test_spec_parse.y" + case 99: +#line 743 "test_spec_parse.y" { /* volume "/path/with spaces" */ int vi = current_node->volumeCount; @@ -2636,22 +2784,22 @@ yyparse () ;} break; - case 88: -#line 671 "test_spec_parse.y" + case 100: +#line 764 "test_spec_parse.y" { current_spec->setup = (yyvsp[(2) - (2)].step); ;} break; - case 89: -#line 678 "test_spec_parse.y" + case 101: +#line 771 "test_spec_parse.y" { current_spec->teardown = (yyvsp[(2) - (2)].step); ;} break; - case 90: -#line 689 "test_spec_parse.y" + case 102: +#line 782 "test_spec_parse.y" { TestStep *s = (yyvsp[(3) - (3)].step); strncpy(s->name, (yyvsp[(2) - (3)].str), sizeof(s->name) - 1); @@ -2660,8 +2808,8 @@ yyparse () ;} break; - case 91: -#line 707 "test_spec_parse.y" + case 103: +#line 800 "test_spec_parse.y" { /* post-process: CMD_SQL immediately before CMD_EXPECT_ERROR */ for (TestCmd *c = (yyvsp[(2) - (3)].step)->commands; c; c = c->next) @@ -2674,103 +2822,103 @@ yyparse () ;} break; - case 92: -#line 721 "test_spec_parse.y" + case 104: +#line 814 "test_spec_parse.y" { (yyval.step) = make_step(""); ;} break; - case 93: -#line 725 "test_spec_parse.y" + case 105: +#line 818 "test_spec_parse.y" { if ((yyvsp[(2) - (2)].cmd)) append_cmd((yyvsp[(1) - (2)].step), (yyvsp[(2) - (2)].cmd)); (yyval.step) = (yyvsp[(1) - (2)].step); ;} break; - case 94: -#line 732 "test_spec_parse.y" + case 106: +#line 825 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 95: -#line 733 "test_spec_parse.y" + case 107: +#line 826 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 96: -#line 734 "test_spec_parse.y" + case 108: +#line 827 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 97: -#line 735 "test_spec_parse.y" + case 109: +#line 828 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 98: -#line 736 "test_spec_parse.y" + case 110: +#line 829 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 99: -#line 737 "test_spec_parse.y" + case 111: +#line 830 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 100: -#line 738 "test_spec_parse.y" + case 112: +#line 831 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 101: -#line 739 "test_spec_parse.y" + case 113: +#line 832 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 102: -#line 740 "test_spec_parse.y" + case 114: +#line 833 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 103: -#line 741 "test_spec_parse.y" + case 115: +#line 834 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 104: -#line 742 "test_spec_parse.y" + case 116: +#line 835 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 105: -#line 743 "test_spec_parse.y" + case 117: +#line 836 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 106: -#line 744 "test_spec_parse.y" + case 118: +#line 837 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 107: -#line 745 "test_spec_parse.y" + case 119: +#line 838 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 108: -#line 746 "test_spec_parse.y" + case 120: +#line 839 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 109: -#line 747 "test_spec_parse.y" + case 121: +#line 840 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 110: -#line 762 "test_spec_parse.y" + case 122: +#line 855 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2779,8 +2927,8 @@ yyparse () ;} break; - case 111: -#line 769 "test_spec_parse.y" + case 123: +#line 862 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2788,8 +2936,8 @@ yyparse () ;} break; - case 112: -#line 775 "test_spec_parse.y" + case 124: +#line 868 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC_FAILS); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2798,8 +2946,8 @@ yyparse () ;} break; - case 113: -#line 782 "test_spec_parse.y" + case 125: +#line 875 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC_FAILS); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2807,8 +2955,8 @@ yyparse () ;} break; - case 114: -#line 788 "test_spec_parse.y" + case 126: +#line 881 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_RUN); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2817,8 +2965,8 @@ yyparse () ;} break; - case 115: -#line 795 "test_spec_parse.y" + case 127: +#line 888 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_RUN); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2826,8 +2974,8 @@ yyparse () ;} break; - case 116: -#line 801 "test_spec_parse.y" + case 128: +#line 894 "test_spec_parse.y" { /* "pg_autoctl perform failover --formation auth" * EXEC_ARGS returns T_IDENT for first word, T_SHELL_ARGS for rest */ @@ -2837,8 +2985,8 @@ yyparse () ;} break; - case 117: -#line 809 "test_spec_parse.y" + case 129: +#line 902 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_PG_AUTOCTL); strlcpy((yyval.cmd)->args, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->args)); @@ -2846,15 +2994,15 @@ yyparse () ;} break; - case 118: -#line 815 "test_spec_parse.y" + case 130: +#line 908 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_PG_AUTOCTL); ;} break; - case 121: -#line 853 "test_spec_parse.y" + case 133: +#line 946 "test_spec_parse.y" { if (!current_wait_cmd) current_wait_cmd = make_cmd(CMD_WAIT_MULTI); @@ -2871,8 +3019,8 @@ yyparse () ;} break; - case 122: -#line 868 "test_spec_parse.y" + case 134: +#line 961 "test_spec_parse.y" { if (!current_wait_cmd) current_wait_cmd = make_cmd(CMD_WAIT_MULTI); @@ -2889,8 +3037,8 @@ yyparse () ;} break; - case 127: -#line 908 "test_spec_parse.y" + case 139: +#line 1001 "test_spec_parse.y" { /* current_pass_cmd set by the enclosing wait_cmd rule */ if (current_pass_cmd && @@ -2900,8 +3048,8 @@ yyparse () ;} break; - case 128: -#line 916 "test_spec_parse.y" + case 140: +#line 1009 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -2911,8 +3059,8 @@ yyparse () ;} break; - case 129: -#line 924 "test_spec_parse.y" + case 141: +#line 1017 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -2921,8 +3069,8 @@ yyparse () ;} break; - case 130: -#line 931 "test_spec_parse.y" + case 142: +#line 1024 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -2932,16 +3080,16 @@ yyparse () ;} break; - case 131: -#line 942 "test_spec_parse.y" + case 143: +#line 1035 "test_spec_parse.y" { current_pass_cmd = make_cmd(CMD_WAIT_STATE); strlcpy(current_pass_cmd->service, (yyvsp[(3) - (6)].str), sizeof(current_pass_cmd->service)); strlcpy(current_pass_cmd->state, (yyvsp[(6) - (6)].str), sizeof(current_pass_cmd->state)); free((yyvsp[(3) - (6)].str)); ;} break; - case 132: -#line 947 "test_spec_parse.y" + case 144: +#line 1040 "test_spec_parse.y" { current_pass_cmd->timeoutSeconds = (yyvsp[(9) - (9)].ival); (yyval.cmd) = current_pass_cmd; @@ -2949,16 +3097,16 @@ yyparse () ;} break; - case 133: -#line 953 "test_spec_parse.y" + case 145: +#line 1046 "test_spec_parse.y" { current_pass_cmd = make_cmd(CMD_WAIT_STATE); strlcpy(current_pass_cmd->service, (yyvsp[(3) - (6)].str), sizeof(current_pass_cmd->service)); strlcpy(current_pass_cmd->state, (yyvsp[(6) - (6)].str), sizeof(current_pass_cmd->state)); free((yyvsp[(3) - (6)].str)); free((yyvsp[(6) - (6)].str)); ;} break; - case 134: -#line 958 "test_spec_parse.y" + case 146: +#line 1051 "test_spec_parse.y" { current_pass_cmd->timeoutSeconds = (yyvsp[(9) - (9)].ival); (yyval.cmd) = current_pass_cmd; @@ -2966,8 +3114,8 @@ yyparse () ;} break; - case 135: -#line 964 "test_spec_parse.y" + case 147: +#line 1057 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STATE); (yyval.cmd)->kind = CMD_ASSERT_ASSIGNED; @@ -2978,8 +3126,8 @@ yyparse () ;} break; - case 136: -#line 973 "test_spec_parse.y" + case 148: +#line 1066 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STATE); (yyval.cmd)->kind = CMD_ASSERT_ASSIGNED; @@ -2990,8 +3138,8 @@ yyparse () ;} break; - case 137: -#line 982 "test_spec_parse.y" + case 149: +#line 1075 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STOPPED); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3000,8 +3148,8 @@ yyparse () ;} break; - case 138: -#line 996 "test_spec_parse.y" + case 150: +#line 1089 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_LSN); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3011,8 +3159,8 @@ yyparse () ;} break; - case 139: -#line 1004 "test_spec_parse.y" + case 151: +#line 1097 "test_spec_parse.y" { (yyval.cmd) = current_wait_cmd; (yyval.cmd)->timeoutSeconds = (yyvsp[(5) - (5)].ival); @@ -3020,8 +3168,8 @@ yyparse () ;} break; - case 140: -#line 1018 "test_spec_parse.y" + case 152: +#line 1111 "test_spec_parse.y" { (yyval.cmd) = current_wait_cmd; (yyval.cmd)->timeoutSeconds = (yyvsp[(6) - (6)].ival); @@ -3029,8 +3177,82 @@ yyparse () ;} break; - case 141: -#line 1033 "test_spec_parse.y" + case 153: +#line 1126 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_WAIT_SQL); + strlcpy((yyval.cmd)->service, (yyvsp[(4) - (8)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->args, (yyvsp[(5) - (8)].str), sizeof((yyval.cmd)->args)); + strlcpy((yyval.cmd)->expected, (yyvsp[(7) - (8)].str), sizeof((yyval.cmd)->expected)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(8) - (8)].ival); + free((yyvsp[(4) - (8)].str)); free((yyvsp[(5) - (8)].str)); free((yyvsp[(7) - (8)].str)); + ;} + break; + + case 154: +#line 1143 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_WAIT_SQL); + strlcpy((yyval.cmd)->service, "monitor", sizeof((yyval.cmd)->service)); + sformat((yyval.cmd)->args, sizeof((yyval.cmd)->args), + "SELECT pgautofailover.wal_archived('%s', %d, '%s')", + (yyvsp[(8) - (11)].str), (yyvsp[(10) - (11)].ival), (yyvsp[(5) - (11)].str)); + strlcpy((yyval.cmd)->expected, "t", sizeof((yyval.cmd)->expected)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(11) - (11)].ival); + free((yyvsp[(5) - (11)].str)); free((yyvsp[(8) - (11)].str)); + ;} + break; + + case 155: +#line 1167 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_WAIT_SQL); + strlcpy((yyval.cmd)->service, "monitor", sizeof((yyval.cmd)->service)); + if ((yyvsp[(9) - (10)].ival) >= 0) + { + sformat((yyval.cmd)->args, sizeof((yyval.cmd)->args), + "SELECT reportedstate::text FROM pgautofailover.node" + " WHERE nodename LIKE 'archiver-%%' AND formationid = '%s'" + " AND groupid = %d", (yyvsp[(8) - (10)].str), (yyvsp[(9) - (10)].ival)); + } + else + { + sformat((yyval.cmd)->args, sizeof((yyval.cmd)->args), + "SELECT reportedstate::text FROM pgautofailover.node" + " WHERE nodename LIKE 'archiver-%%' AND formationid = '%s'", (yyvsp[(8) - (10)].str)); + } + strlcpy((yyval.cmd)->expected, (yyvsp[(6) - (10)].str), sizeof((yyval.cmd)->expected)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(10) - (10)].ival); + free((yyvsp[(6) - (10)].str)); free((yyvsp[(8) - (10)].str)); + ;} + break; + + case 156: +#line 1196 "test_spec_parse.y" + { + if (strcmp((yyvsp[(4) - (11)].str), "source") != 0 && + strcmp((yyvsp[(4) - (11)].str), "status") != 0 && + strcmp((yyvsp[(4) - (11)].str), "replaymode") != 0) + { + fprintf(stderr, + "pgaftest: line %d: \"wait until basebackup %s ...\" -- " + "unknown property (expected source, status, or replaymode)\n", + pgaf_line_number, (yyvsp[(4) - (11)].str)); + exit(1); + } + (yyval.cmd) = make_cmd(CMD_WAIT_SQL); + strlcpy((yyval.cmd)->service, "monitor", sizeof((yyval.cmd)->service)); + sformat((yyval.cmd)->args, sizeof((yyval.cmd)->args), + "SELECT %s::text FROM pgautofailover.get_latest_basebackup('%s', %d)", + (yyvsp[(4) - (11)].str), (yyvsp[(8) - (11)].str), (yyvsp[(10) - (11)].ival)); + strlcpy((yyval.cmd)->expected, (yyvsp[(6) - (11)].str), sizeof((yyval.cmd)->expected)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(11) - (11)].ival); + free((yyvsp[(4) - (11)].str)); free((yyvsp[(6) - (11)].str)); free((yyvsp[(8) - (11)].str)); + ;} + break; + + case 157: +#line 1226 "test_spec_parse.y" { current_wait_cmd = make_cmd(CMD_WAIT_STATES); strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3038,8 +3260,8 @@ yyparse () ;} break; - case 142: -#line 1039 "test_spec_parse.y" + case 158: +#line 1232 "test_spec_parse.y" { current_wait_cmd = make_cmd(CMD_WAIT_STATES); strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3048,8 +3270,8 @@ yyparse () ;} break; - case 143: -#line 1046 "test_spec_parse.y" + case 159: +#line 1239 "test_spec_parse.y" { if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3057,8 +3279,8 @@ yyparse () ;} break; - case 144: -#line 1052 "test_spec_parse.y" + case 160: +#line 1245 "test_spec_parse.y" { if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3067,39 +3289,39 @@ yyparse () ;} break; - case 147: -#line 1071 "test_spec_parse.y" + case 163: +#line 1264 "test_spec_parse.y" { if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = (yyvsp[(2) - (2)].ival); ;} break; - case 148: -#line 1076 "test_spec_parse.y" + case 164: +#line 1269 "test_spec_parse.y" { if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = (yyvsp[(4) - (4)].ival); ;} break; - case 149: -#line 1083 "test_spec_parse.y" + case 165: +#line 1276 "test_spec_parse.y" { (yyval.ival) = PGAF_TIMEOUT_DEFAULT; ;} break; - case 150: -#line 1084 "test_spec_parse.y" + case 166: +#line 1277 "test_spec_parse.y" { (yyval.ival) = (yyvsp[(2) - (2)].ival); ;} break; - case 151: -#line 1085 "test_spec_parse.y" + case 167: +#line 1278 "test_spec_parse.y" { (yyval.ival) = (yyvsp[(3) - (3)].ival); ;} break; - case 152: -#line 1097 "test_spec_parse.y" + case 168: +#line 1290 "test_spec_parse.y" { (yyval.cmd) = make_cmd((yyvsp[(6) - (6)].ival) > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3109,8 +3331,8 @@ yyparse () ;} break; - case 153: -#line 1105 "test_spec_parse.y" + case 169: +#line 1298 "test_spec_parse.y" { (yyval.cmd) = make_cmd((yyvsp[(6) - (6)].ival) > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3120,8 +3342,8 @@ yyparse () ;} break; - case 154: -#line 1113 "test_spec_parse.y" + case 170: +#line 1306 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_ASSERT_ASSIGNED); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3131,8 +3353,8 @@ yyparse () ;} break; - case 155: -#line 1121 "test_spec_parse.y" + case 171: +#line 1314 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_ASSERT_ASSIGNED); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3142,8 +3364,8 @@ yyparse () ;} break; - case 156: -#line 1139 "test_spec_parse.y" + case 172: +#line 1332 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_SQL); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3152,8 +3374,8 @@ yyparse () ;} break; - case 157: -#line 1154 "test_spec_parse.y" + case 173: +#line 1347 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT); strlcpy((yyval.cmd)->expected, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->expected)); @@ -3162,15 +3384,15 @@ yyparse () ;} break; - case 158: -#line 1161 "test_spec_parse.y" + case 174: +#line 1354 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); ;} break; - case 159: -#line 1165 "test_spec_parse.y" + case 175: +#line 1358 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); strlcpy((yyval.cmd)->state, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->state)); @@ -3178,8 +3400,8 @@ yyparse () ;} break; - case 160: -#line 1171 "test_spec_parse.y" + case 176: +#line 1364 "test_spec_parse.y" { /* SQLSTATE codes like 25006 are all digits, lexed as T_INTEGER */ (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); @@ -3187,16 +3409,16 @@ yyparse () ;} break; - case 161: -#line 1184 "test_spec_parse.y" + case 177: +#line 1377 "test_spec_parse.y" { (yyval.cmd) = current_promote_cmd; current_promote_cmd = NULL; ;} break; - case 162: -#line 1192 "test_spec_parse.y" + case 178: +#line 1385 "test_spec_parse.y" { current_promote_cmd = make_cmd(CMD_PROMOTE); current_promote_cmd->timeoutSeconds = PGAF_TIMEOUT_DEFAULT; @@ -3206,8 +3428,8 @@ yyparse () ;} break; - case 163: -#line 1200 "test_spec_parse.y" + case 179: +#line 1393 "test_spec_parse.y" { if (current_promote_cmd->promoteCount < PGAF_MAX_PROMOTE_NODES) strlcpy(current_promote_cmd->promoteNodes[current_promote_cmd->promoteCount++], @@ -3216,8 +3438,8 @@ yyparse () ;} break; - case 164: -#line 1221 "test_spec_parse.y" + case 180: +#line 1414 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, "default", sizeof((yyval.cmd)->service)); @@ -3226,8 +3448,8 @@ yyparse () ;} break; - case 165: -#line 1228 "test_spec_parse.y" + case 181: +#line 1421 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, "default", sizeof((yyval.cmd)->service)); @@ -3236,8 +3458,8 @@ yyparse () ;} break; - case 166: -#line 1235 "test_spec_parse.y" + case 182: +#line 1428 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, (yyvsp[(5) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3247,8 +3469,8 @@ yyparse () ;} break; - case 167: -#line 1243 "test_spec_parse.y" + case 183: +#line 1436 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, (yyvsp[(5) - (7)].str), sizeof((yyval.cmd)->service)); @@ -3258,8 +3480,8 @@ yyparse () ;} break; - case 168: -#line 1259 "test_spec_parse.y" + case 184: +#line 1452 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NETWORK_OFF); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3267,8 +3489,8 @@ yyparse () ;} break; - case 169: -#line 1265 "test_spec_parse.y" + case 185: +#line 1458 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NETWORK_ON); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3276,8 +3498,8 @@ yyparse () ;} break; - case 170: -#line 1286 "test_spec_parse.y" + case 186: +#line 1479 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NODEINI_SET); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3287,8 +3509,8 @@ yyparse () ;} break; - case 171: -#line 1294 "test_spec_parse.y" + case 187: +#line 1487 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NODEINI_GET); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3298,23 +3520,23 @@ yyparse () ;} break; - case 172: -#line 1309 "test_spec_parse.y" + case 188: +#line 1502 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_SLEEP); (yyval.cmd)->timeoutSeconds = (yyvsp[(2) - (2)].ival); ;} break; - case 173: -#line 1323 "test_spec_parse.y" + case 189: +#line 1516 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_DOWN); ;} break; - case 174: -#line 1327 "test_spec_parse.y" + case 190: +#line 1520 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_START); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3322,8 +3544,8 @@ yyparse () ;} break; - case 175: -#line 1333 "test_spec_parse.y" + case 191: +#line 1526 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_STOP); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3331,8 +3553,8 @@ yyparse () ;} break; - case 176: -#line 1339 "test_spec_parse.y" + case 192: +#line 1532 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_KILL); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3340,8 +3562,8 @@ yyparse () ;} break; - case 177: -#line 1365 "test_spec_parse.y" + case 193: +#line 1558 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_INJECT); strlcpy((yyval.cmd)->expected, (yyvsp[(3) - (4)].str), sizeof((yyval.cmd)->expected)); /* image */ @@ -3366,8 +3588,8 @@ yyparse () ;} break; - case 178: -#line 1399 "test_spec_parse.y" + case 194: +#line 1592 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_STOP_POSTGRES); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3375,8 +3597,8 @@ yyparse () ;} break; - case 179: -#line 1405 "test_spec_parse.y" + case 195: +#line 1598 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_START_POSTGRES); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3384,8 +3606,8 @@ yyparse () ;} break; - case 180: -#line 1426 "test_spec_parse.y" + case 196: +#line 1619 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FSM_STEP); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3393,18 +3615,18 @@ yyparse () ;} break; - case 181: -#line 1442 "test_spec_parse.y" + case 197: +#line 1635 "test_spec_parse.y" { pgaf_next_brace_is_while = 1; ;} break; - case 182: -#line 1443 "test_spec_parse.y" + case 198: +#line 1636 "test_spec_parse.y" { (yyval.step) = (yyvsp[(4) - (5)].step); ;} break; - case 183: -#line 1448 "test_spec_parse.y" + case 199: +#line 1641 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_STAYS_WHILE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3414,8 +3636,8 @@ yyparse () ;} break; - case 184: -#line 1467 "test_spec_parse.y" + case 200: +#line 1660 "test_spec_parse.y" { /* only "set monitor " is supported; $2 must be "monitor" */ if (strcmp((yyvsp[(2) - (3)].str), "monitor") != 0) @@ -3430,8 +3652,8 @@ yyparse () ;} break; - case 185: -#line 1492 "test_spec_parse.y" + case 201: +#line 1685 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (4)].str), sizeof((yyval.cmd)->service)); @@ -3442,8 +3664,8 @@ yyparse () ;} break; - case 186: -#line 1501 "test_spec_parse.y" + case 202: +#line 1694 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3454,8 +3676,8 @@ yyparse () ;} break; - case 187: -#line 1510 "test_spec_parse.y" + case 203: +#line 1703 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (4)].str), sizeof((yyval.cmd)->service)); @@ -3466,8 +3688,8 @@ yyparse () ;} break; - case 188: -#line 1519 "test_spec_parse.y" + case 204: +#line 1712 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3478,8 +3700,8 @@ yyparse () ;} break; - case 191: -#line 1540 "test_spec_parse.y" + case 207: +#line 1733 "test_spec_parse.y" { int i = current_spec->sequenceLength; if (i < PGAF_MAX_SEQ) @@ -3493,124 +3715,144 @@ yyparse () ;} break; - case 192: -#line 1561 "test_spec_parse.y" + case 208: +#line 1754 "test_spec_parse.y" { (yyval.str) = "init"; ;} break; - case 193: -#line 1562 "test_spec_parse.y" + case 209: +#line 1755 "test_spec_parse.y" { (yyval.str) = "single"; ;} break; - case 194: -#line 1563 "test_spec_parse.y" + case 210: +#line 1756 "test_spec_parse.y" { (yyval.str) = "primary"; ;} break; - case 195: -#line 1564 "test_spec_parse.y" + case 211: +#line 1757 "test_spec_parse.y" { (yyval.str) = "wait_primary"; ;} break; - case 196: -#line 1565 "test_spec_parse.y" + case 212: +#line 1758 "test_spec_parse.y" { (yyval.str) = "wait_standby"; ;} break; - case 197: -#line 1566 "test_spec_parse.y" + case 213: +#line 1759 "test_spec_parse.y" { (yyval.str) = "demoted"; ;} break; - case 198: -#line 1567 "test_spec_parse.y" + case 214: +#line 1760 "test_spec_parse.y" { (yyval.str) = "demote_timeout"; ;} break; - case 199: -#line 1568 "test_spec_parse.y" + case 215: +#line 1761 "test_spec_parse.y" { (yyval.str) = "draining"; ;} break; - case 200: -#line 1569 "test_spec_parse.y" + case 216: +#line 1762 "test_spec_parse.y" { (yyval.str) = "secondary"; ;} break; - case 201: -#line 1570 "test_spec_parse.y" + case 217: +#line 1763 "test_spec_parse.y" { (yyval.str) = "catchingup"; ;} break; - case 202: -#line 1571 "test_spec_parse.y" + case 218: +#line 1764 "test_spec_parse.y" { (yyval.str) = "prepare_promotion"; ;} break; - case 203: -#line 1572 "test_spec_parse.y" + case 219: +#line 1765 "test_spec_parse.y" { (yyval.str) = "stop_replication"; ;} break; - case 204: -#line 1573 "test_spec_parse.y" + case 220: +#line 1766 "test_spec_parse.y" { (yyval.str) = "maintenance"; ;} break; - case 205: -#line 1574 "test_spec_parse.y" + case 221: +#line 1767 "test_spec_parse.y" { (yyval.str) = "join_primary"; ;} break; - case 206: -#line 1575 "test_spec_parse.y" + case 222: +#line 1768 "test_spec_parse.y" { (yyval.str) = "apply_settings"; ;} break; - case 207: -#line 1576 "test_spec_parse.y" + case 223: +#line 1769 "test_spec_parse.y" { (yyval.str) = "prepare_maintenance"; ;} break; - case 208: -#line 1577 "test_spec_parse.y" + case 224: +#line 1770 "test_spec_parse.y" { (yyval.str) = "wait_maintenance"; ;} break; - case 209: -#line 1578 "test_spec_parse.y" + case 225: +#line 1771 "test_spec_parse.y" { (yyval.str) = "report_lsn"; ;} break; - case 210: -#line 1579 "test_spec_parse.y" + case 226: +#line 1772 "test_spec_parse.y" { (yyval.str) = "fast_forward"; ;} break; - case 211: -#line 1580 "test_spec_parse.y" + case 227: +#line 1773 "test_spec_parse.y" { (yyval.str) = "join_secondary"; ;} break; - case 212: -#line 1581 "test_spec_parse.y" + case 228: +#line 1774 "test_spec_parse.y" { (yyval.str) = "dropped"; ;} break; - case 213: -#line 1589 "test_spec_parse.y" + case 229: +#line 1782 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 214: -#line 1590 "test_spec_parse.y" + case 230: +#line 1783 "test_spec_parse.y" + { (yyval.str) = (yyvsp[(1) - (1)].str); ;} + break; + + case 231: +#line 1794 "test_spec_parse.y" + { (yyval.str) = strdup((yyvsp[(1) - (1)].str)); ;} + break; + + case 232: +#line 1795 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; + case 233: +#line 1803 "test_spec_parse.y" + { (yyval.ival) = -1; ;} + break; + + case 234: +#line 1804 "test_spec_parse.y" + { (yyval.ival) = (yyvsp[(2) - (2)].ival); ;} + break; + /* Line 1267 of yacc.c. */ -#line 3614 "test_spec_parse.c" +#line 3856 "test_spec_parse.c" default: break; } YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); @@ -3824,7 +4066,98 @@ yyparse () } -#line 1593 "test_spec_parse.y" +#line 1807 "test_spec_parse.y" + + +/* + * fold_archivers_into_formations turns each top-level "archiver { }" + * declaration (TestArchiverNode, cluster->archivers[]) into an ordinary + * TestNode of kind NODE_KIND_ARCHIVER, appended to its own declared + * formation's own node list -- see TestArchiverNode's own comment + * (test_spec.h) for why the *declaration* still needs to be top-level even + * though it ends up represented identically to the older, still-supported + * "archiver nested inside a formation_block" spelling once parsed. Called + * once, right after yyparse() returns, so every caller downstream of + * parse_test_spec() (compose_gen.c included) only ever sees ordinary + * TestNode entries and needs no awareness of TestArchiverNode at all. + * + * cluster->archiverCount is reset to 0 once every entry has been folded, + * so cluster->archivers[] is never a second, stale source of truth for + * the very same nodes now living in cluster->formations[].nodes[]. + */ +static void +fold_archivers_into_formations(TestCluster *cluster) +{ + for (int ai = 0; ai < cluster->archiverCount; ai++) + { + TestArchiverNode *a = &cluster->archivers[ai]; + + if (a->formationCount == 0) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" needs at least one " + "\"formation \" entry\n", a->name); + exit(1); + } + + if (a->formationCount > 1) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" lists %d formations, but " + "pg_autoctl create archiver's own ini-driven bring-up " + "only attaches to one at create time -- declare just " + "\"formation %s\" here and attach the rest (e.g. " + "\"%s\") dynamically once it's running instead, via a " + "direct \"sql monitor { SELECT pgautofailover." + "archiver_add_formation(...) }\" step -- see " + "archiver_multi_formation.pgaf for the pattern\n", + a->name, a->formationCount, a->formations[0], + a->formations[1]); + exit(1); + } + + TestFormation *form = NULL; + + for (int fi = 0; fi < cluster->formationCount; fi++) + { + if (strcmp(cluster->formations[fi].name, a->formations[0]) == 0) + { + form = &cluster->formations[fi]; + break; + } + } + + if (form == NULL) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" attaches to formation " + "\"%s\", which is not declared in this cluster{} " + "block\n", a->name, a->formations[0]); + exit(1); + } + + if (form->nodeCount >= PGAF_MAX_NODES) + { + fprintf(stderr, + "pgaftest: too many nodes in formation \"%s\" (max %d)\n", + form->name, PGAF_MAX_NODES); + exit(1); + } + + TestNode *node = &form->nodes[form->nodeCount++]; + + memset(node, 0, sizeof(*node)); + strlcpy(node->name, a->name, sizeof(node->name)); + node->kind = NODE_KIND_ARCHIVER; + node->candidatePriority = 50; + node->replicationQuorum = true; + strlcpy(node->region, a->region, sizeof(node->region)); + node->createDeferred = a->createDeferred; + node->launchDeferred = a->launchDeferred; + } + + cluster->archiverCount = 0; +} /* ----------------------------------------------------------------------- @@ -3853,6 +4186,8 @@ parse_test_spec(const char *filename) yyparse(); fclose(f); + fold_archivers_into_formations(&spec->cluster); + /* * If the file has no explicit sequence{} block, default to running * steps in declaration order. Populated here (not just in the CI diff --git a/src/bin/pgaftest/test_spec_parse.h b/src/bin/pgaftest/test_spec_parse.h index 0f520e315..a840f1a49 100644 --- a/src/bin/pgaftest/test_spec_parse.h +++ b/src/bin/pgaftest/test_spec_parse.h @@ -58,105 +58,111 @@ T_NUM_SYNC = 274, T_COORDINATOR = 275, T_WORKER = 276, - T_ASYNC = 277, - T_NO_MONITOR = 278, - T_SUSPENDED = 279, - T_LAUNCH = 280, - T_CREATE = 281, - T_DEFERRED = 282, - T_IMMEDIATE = 283, - T_FALSE = 284, - T_TRUE = 285, - T_INITIALLY = 286, - T_VOLUME = 287, - T_LISTEN = 288, - T_CITUS_SECONDARY = 289, - T_CANDIDATE_PRIORITY = 290, - T_PORT = 291, - T_PASSWORD = 292, - T_MONITOR_PASSWORD = 293, - T_CITUS_CLUSTER_NAME = 294, - T_DEBIAN_CLUSTER = 295, - T_REPLICATION_QUORUM = 296, - T_REPLICATION_PASSWORD = 297, - T_EXTENSION_VERSION = 298, - T_BIND_SOURCE = 299, - T_LEGACY_STARTUP = 300, - T_REGION = 301, - T_NODEINI = 302, - T_FS_INIT = 303, - T_FS_SINGLE = 304, - T_FS_PRIMARY = 305, - T_FS_WAIT_PRIMARY = 306, - T_FS_WAIT_STANDBY = 307, - T_FS_DEMOTED = 308, - T_FS_DEMOTE_TIMEOUT = 309, - T_FS_DRAINING = 310, - T_FS_SECONDARY = 311, - T_FS_CATCHINGUP = 312, - T_FS_PREP_PROMOTION = 313, - T_FS_STOP_REPLICATION = 314, - T_FS_MAINTENANCE = 315, - T_FS_JOIN_PRIMARY = 316, - T_FS_APPLY_SETTINGS = 317, - T_FS_PREPARE_MAINTENANCE = 318, - T_FS_WAIT_MAINTENANCE = 319, - T_FS_REPORT_LSN = 320, - T_FS_FAST_FORWARD = 321, - T_FS_JOIN_SECONDARY = 322, - T_FS_DROPPED = 323, - T_EXEC = 324, - T_EXEC_FAILS = 325, - T_RUN = 326, - T_PG_AUTOCTL = 327, - T_WAIT = 328, - T_UNTIL = 329, - T_TIMEOUT = 330, - T_AND = 331, - T_IS = 332, - T_WITH = 333, - T_REPLAYS = 334, - T_ASSERT = 335, - T_SQL = 336, - T_EXPECT = 337, - T_ERROR = 338, - T_PROMOTE = 339, - T_PERFORM = 340, - T_FAILOVER = 341, - T_NETWORK = 342, - T_DISCONNECT = 343, - T_CONNECT = 344, - T_SLEEP = 345, - T_COMPOSE = 346, - T_DOWN = 347, - T_START = 348, - T_STOP = 349, - T_STOPPED = 350, - T_KILL = 351, - T_INJECT = 352, - T_STATE = 353, - T_ASSIGNED_STATE = 354, - T_IN = 355, - T_GROUP = 356, - T_LBRACE = 357, - T_RBRACE = 358, - T_COMMA = 359, - T_POSTGRES = 360, - T_STAYS = 361, - T_WHILE = 362, - T_THROUGH = 363, - T_SET = 364, - T_GET = 365, - T_FSM = 366, - T_LOGS = 367, - T_NOT = 368, - T_CONTAINS = 369, - T_MATCHES = 370, - T_INTEGER = 371, - T_IDENT = 372, - T_STRING = 373, - T_BLOCK = 374, - T_SHELL_ARGS = 375 + T_ARCHIVER = 277, + T_ASYNC = 278, + T_NO_MONITOR = 279, + T_SUSPENDED = 280, + T_LAUNCH = 281, + T_CREATE = 282, + T_DEFERRED = 283, + T_IMMEDIATE = 284, + T_FALSE = 285, + T_TRUE = 286, + T_INITIALLY = 287, + T_VOLUME = 288, + T_LISTEN = 289, + T_CITUS_SECONDARY = 290, + T_CANDIDATE_PRIORITY = 291, + T_PORT = 292, + T_PASSWORD = 293, + T_MONITOR_PASSWORD = 294, + T_CITUS_CLUSTER_NAME = 295, + T_DEBIAN_CLUSTER = 296, + T_REPLICATION_QUORUM = 297, + T_REPLICATION_PASSWORD = 298, + T_EXTENSION_VERSION = 299, + T_BIND_SOURCE = 300, + T_LEGACY_STARTUP = 301, + T_REGION = 302, + T_NODEINI = 303, + T_FS_INIT = 304, + T_FS_SINGLE = 305, + T_FS_PRIMARY = 306, + T_FS_WAIT_PRIMARY = 307, + T_FS_WAIT_STANDBY = 308, + T_FS_DEMOTED = 309, + T_FS_DEMOTE_TIMEOUT = 310, + T_FS_DRAINING = 311, + T_FS_SECONDARY = 312, + T_FS_CATCHINGUP = 313, + T_FS_PREP_PROMOTION = 314, + T_FS_STOP_REPLICATION = 315, + T_FS_MAINTENANCE = 316, + T_FS_JOIN_PRIMARY = 317, + T_FS_APPLY_SETTINGS = 318, + T_FS_PREPARE_MAINTENANCE = 319, + T_FS_WAIT_MAINTENANCE = 320, + T_FS_REPORT_LSN = 321, + T_FS_FAST_FORWARD = 322, + T_FS_JOIN_SECONDARY = 323, + T_FS_DROPPED = 324, + T_EXEC = 325, + T_EXEC_FAILS = 326, + T_RUN = 327, + T_PG_AUTOCTL = 328, + T_WAIT = 329, + T_UNTIL = 330, + T_TIMEOUT = 331, + T_AND = 332, + T_IS = 333, + T_WITH = 334, + T_REPLAYS = 335, + T_ASSERT = 336, + T_SQL = 337, + T_EXPECT = 338, + T_ERROR = 339, + T_PROMOTE = 340, + T_PERFORM = 341, + T_FAILOVER = 342, + T_NETWORK = 343, + T_DISCONNECT = 344, + T_CONNECT = 345, + T_SLEEP = 346, + T_COMPOSE = 347, + T_DOWN = 348, + T_START = 349, + T_STOP = 350, + T_STOPPED = 351, + T_KILL = 352, + T_INJECT = 353, + T_STATE = 354, + T_ASSIGNED_STATE = 355, + T_IN = 356, + T_GROUP = 357, + T_LBRACE = 358, + T_RBRACE = 359, + T_COMMA = 360, + T_POSTGRES = 361, + T_STAYS = 362, + T_WHILE = 363, + T_THROUGH = 364, + T_SET = 365, + T_GET = 366, + T_FSM = 367, + T_LOGS = 368, + T_NOT = 369, + T_CONTAINS = 370, + T_MATCHES = 371, + T_WAL = 372, + T_SEGMENT = 373, + T_ARCHIVED = 374, + T_BASEBACKUP = 375, + T_SLASH = 376, + T_INTEGER = 377, + T_IDENT = 378, + T_STRING = 379, + T_BLOCK = 380, + T_SHELL_ARGS = 381 }; #endif /* Tokens. */ @@ -179,112 +185,118 @@ #define T_NUM_SYNC 274 #define T_COORDINATOR 275 #define T_WORKER 276 -#define T_ASYNC 277 -#define T_NO_MONITOR 278 -#define T_SUSPENDED 279 -#define T_LAUNCH 280 -#define T_CREATE 281 -#define T_DEFERRED 282 -#define T_IMMEDIATE 283 -#define T_FALSE 284 -#define T_TRUE 285 -#define T_INITIALLY 286 -#define T_VOLUME 287 -#define T_LISTEN 288 -#define T_CITUS_SECONDARY 289 -#define T_CANDIDATE_PRIORITY 290 -#define T_PORT 291 -#define T_PASSWORD 292 -#define T_MONITOR_PASSWORD 293 -#define T_CITUS_CLUSTER_NAME 294 -#define T_DEBIAN_CLUSTER 295 -#define T_REPLICATION_QUORUM 296 -#define T_REPLICATION_PASSWORD 297 -#define T_EXTENSION_VERSION 298 -#define T_BIND_SOURCE 299 -#define T_LEGACY_STARTUP 300 -#define T_REGION 301 -#define T_NODEINI 302 -#define T_FS_INIT 303 -#define T_FS_SINGLE 304 -#define T_FS_PRIMARY 305 -#define T_FS_WAIT_PRIMARY 306 -#define T_FS_WAIT_STANDBY 307 -#define T_FS_DEMOTED 308 -#define T_FS_DEMOTE_TIMEOUT 309 -#define T_FS_DRAINING 310 -#define T_FS_SECONDARY 311 -#define T_FS_CATCHINGUP 312 -#define T_FS_PREP_PROMOTION 313 -#define T_FS_STOP_REPLICATION 314 -#define T_FS_MAINTENANCE 315 -#define T_FS_JOIN_PRIMARY 316 -#define T_FS_APPLY_SETTINGS 317 -#define T_FS_PREPARE_MAINTENANCE 318 -#define T_FS_WAIT_MAINTENANCE 319 -#define T_FS_REPORT_LSN 320 -#define T_FS_FAST_FORWARD 321 -#define T_FS_JOIN_SECONDARY 322 -#define T_FS_DROPPED 323 -#define T_EXEC 324 -#define T_EXEC_FAILS 325 -#define T_RUN 326 -#define T_PG_AUTOCTL 327 -#define T_WAIT 328 -#define T_UNTIL 329 -#define T_TIMEOUT 330 -#define T_AND 331 -#define T_IS 332 -#define T_WITH 333 -#define T_REPLAYS 334 -#define T_ASSERT 335 -#define T_SQL 336 -#define T_EXPECT 337 -#define T_ERROR 338 -#define T_PROMOTE 339 -#define T_PERFORM 340 -#define T_FAILOVER 341 -#define T_NETWORK 342 -#define T_DISCONNECT 343 -#define T_CONNECT 344 -#define T_SLEEP 345 -#define T_COMPOSE 346 -#define T_DOWN 347 -#define T_START 348 -#define T_STOP 349 -#define T_STOPPED 350 -#define T_KILL 351 -#define T_INJECT 352 -#define T_STATE 353 -#define T_ASSIGNED_STATE 354 -#define T_IN 355 -#define T_GROUP 356 -#define T_LBRACE 357 -#define T_RBRACE 358 -#define T_COMMA 359 -#define T_POSTGRES 360 -#define T_STAYS 361 -#define T_WHILE 362 -#define T_THROUGH 363 -#define T_SET 364 -#define T_GET 365 -#define T_FSM 366 -#define T_LOGS 367 -#define T_NOT 368 -#define T_CONTAINS 369 -#define T_MATCHES 370 -#define T_INTEGER 371 -#define T_IDENT 372 -#define T_STRING 373 -#define T_BLOCK 374 -#define T_SHELL_ARGS 375 +#define T_ARCHIVER 277 +#define T_ASYNC 278 +#define T_NO_MONITOR 279 +#define T_SUSPENDED 280 +#define T_LAUNCH 281 +#define T_CREATE 282 +#define T_DEFERRED 283 +#define T_IMMEDIATE 284 +#define T_FALSE 285 +#define T_TRUE 286 +#define T_INITIALLY 287 +#define T_VOLUME 288 +#define T_LISTEN 289 +#define T_CITUS_SECONDARY 290 +#define T_CANDIDATE_PRIORITY 291 +#define T_PORT 292 +#define T_PASSWORD 293 +#define T_MONITOR_PASSWORD 294 +#define T_CITUS_CLUSTER_NAME 295 +#define T_DEBIAN_CLUSTER 296 +#define T_REPLICATION_QUORUM 297 +#define T_REPLICATION_PASSWORD 298 +#define T_EXTENSION_VERSION 299 +#define T_BIND_SOURCE 300 +#define T_LEGACY_STARTUP 301 +#define T_REGION 302 +#define T_NODEINI 303 +#define T_FS_INIT 304 +#define T_FS_SINGLE 305 +#define T_FS_PRIMARY 306 +#define T_FS_WAIT_PRIMARY 307 +#define T_FS_WAIT_STANDBY 308 +#define T_FS_DEMOTED 309 +#define T_FS_DEMOTE_TIMEOUT 310 +#define T_FS_DRAINING 311 +#define T_FS_SECONDARY 312 +#define T_FS_CATCHINGUP 313 +#define T_FS_PREP_PROMOTION 314 +#define T_FS_STOP_REPLICATION 315 +#define T_FS_MAINTENANCE 316 +#define T_FS_JOIN_PRIMARY 317 +#define T_FS_APPLY_SETTINGS 318 +#define T_FS_PREPARE_MAINTENANCE 319 +#define T_FS_WAIT_MAINTENANCE 320 +#define T_FS_REPORT_LSN 321 +#define T_FS_FAST_FORWARD 322 +#define T_FS_JOIN_SECONDARY 323 +#define T_FS_DROPPED 324 +#define T_EXEC 325 +#define T_EXEC_FAILS 326 +#define T_RUN 327 +#define T_PG_AUTOCTL 328 +#define T_WAIT 329 +#define T_UNTIL 330 +#define T_TIMEOUT 331 +#define T_AND 332 +#define T_IS 333 +#define T_WITH 334 +#define T_REPLAYS 335 +#define T_ASSERT 336 +#define T_SQL 337 +#define T_EXPECT 338 +#define T_ERROR 339 +#define T_PROMOTE 340 +#define T_PERFORM 341 +#define T_FAILOVER 342 +#define T_NETWORK 343 +#define T_DISCONNECT 344 +#define T_CONNECT 345 +#define T_SLEEP 346 +#define T_COMPOSE 347 +#define T_DOWN 348 +#define T_START 349 +#define T_STOP 350 +#define T_STOPPED 351 +#define T_KILL 352 +#define T_INJECT 353 +#define T_STATE 354 +#define T_ASSIGNED_STATE 355 +#define T_IN 356 +#define T_GROUP 357 +#define T_LBRACE 358 +#define T_RBRACE 359 +#define T_COMMA 360 +#define T_POSTGRES 361 +#define T_STAYS 362 +#define T_WHILE 363 +#define T_THROUGH 364 +#define T_SET 365 +#define T_GET 366 +#define T_FSM 367 +#define T_LOGS 368 +#define T_NOT 369 +#define T_CONTAINS 370 +#define T_MATCHES 371 +#define T_WAL 372 +#define T_SEGMENT 373 +#define T_ARCHIVED 374 +#define T_BASEBACKUP 375 +#define T_SLASH 376 +#define T_INTEGER 377 +#define T_IDENT 378 +#define T_STRING 379 +#define T_BLOCK 380 +#define T_SHELL_ARGS 381 #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED typedef union YYSTYPE -#line 145 "test_spec_parse.y" +#line 146 "test_spec_parse.y" { int ival; char *str; @@ -292,7 +304,7 @@ typedef union YYSTYPE TestCmd *cmd; } /* Line 1529 of yacc.c. */ -#line 296 "test_spec_parse.h" +#line 308 "test_spec_parse.h" YYSTYPE; # define yystype YYSTYPE /* obsolescent; will be withdrawn */ # define YYSTYPE_IS_DECLARED 1 diff --git a/src/bin/pgaftest/test_spec_parse.y b/src/bin/pgaftest/test_spec_parse.y index 956a60711..f4140aadb 100644 --- a/src/bin/pgaftest/test_spec_parse.y +++ b/src/bin/pgaftest/test_spec_parse.y @@ -139,6 +139,7 @@ static TestCmd *current_promote_cmd = NULL; static TestCmd *current_pass_cmd = NULL; /* for opt_passing_through */ static TestFormation *current_formation = NULL; static TestNode *current_node = NULL; +static TestArchiverNode *current_archiver = NULL; %} @@ -156,7 +157,7 @@ static TestNode *current_node = NULL; /* ---- Cluster-body tokens ---- */ %token T_IMAGE T_IMAGE_TARGET T_SSL T_AUTH T_AUTH_METHOD T_FORMATION T_NUM_SYNC -%token T_COORDINATOR T_WORKER T_ASYNC T_NO_MONITOR T_SUSPENDED +%token T_COORDINATOR T_WORKER T_ARCHIVER T_ASYNC T_NO_MONITOR T_SUSPENDED %token T_LAUNCH T_CREATE T_DEFERRED T_IMMEDIATE T_FALSE T_TRUE T_INITIALLY T_VOLUME %token T_LISTEN T_CITUS_SECONDARY T_CANDIDATE_PRIORITY T_PORT T_PASSWORD T_MONITOR_PASSWORD %token T_CITUS_CLUSTER_NAME T_DEBIAN_CLUSTER T_REPLICATION_QUORUM T_REPLICATION_PASSWORD @@ -190,6 +191,7 @@ static TestNode *current_node = NULL; %token T_POSTGRES T_STAYS T_WHILE T_THROUGH T_SET T_GET %token T_FSM %token T_LOGS T_NOT T_CONTAINS T_MATCHES +%token T_WAL T_SEGMENT T_ARCHIVED T_BASEBACKUP T_SLASH /* ---- Tokens with values ---- */ %token T_INTEGER @@ -199,6 +201,7 @@ static TestNode *current_node = NULL; %type ident_or_string %type bare_name %type fsm_state +%type wait_state_name %type node_name %type cmd_block cmd_list %type step_cmd @@ -208,6 +211,7 @@ static TestNode *current_node = NULL; %type fsm_step_cmd %type nodeini_cmd %type opt_timeout +%type opt_wait_group %type while_body %% @@ -256,10 +260,95 @@ cluster_item: | auth_line | extension_version_line | formation_block + | archiver_block | T_BIND_SOURCE { current_spec->cluster.bindSource = true; } | T_LEGACY_STARTUP { current_spec->cluster.legacyStartup = true; } ; +/* + * archiver { formation [formation ...] [region ] } + * + * Top-level, sibling to "monitor" and "formation" -- NOT nested inside a + * formation_block's node_list the way ordinary/coordinator/worker nodes + * are (see TestArchiverNode's own comment in test_spec.h for why: an + * archiver attaches to one or more formations by name, it isn't a member + * of any one of them). May appear more than once, for a cluster with + * several archivers. + * + * Braces are mandatory here (unlike monitor_line's own bare/flat form): + * archiver_opt's own "T_FORMATION T_IDENT" would otherwise be + * indistinguishable, at one token of lookahead, from a brand new + * top-level formation_block starting right after this one (formation_ + * block's own opening is also "T_FORMATION bare_name ...", bare_name + * itself accepting a plain T_IDENT) -- a real shift/reduce ambiguity + * caught while writing this grammar, not a stylistic choice. + */ +archiver_block: + T_ARCHIVER T_IDENT + { + TestCluster *cl = ¤t_spec->cluster; + + if (cl->archiverCount >= PGAF_MAX_ARCHIVERS) + { + fprintf(stderr, "pgaftest: too many archivers (max %d)\n", + PGAF_MAX_ARCHIVERS); + exit(1); + } + + current_archiver = &cl->archivers[cl->archiverCount++]; + strlcpy(current_archiver->name, $2, sizeof(current_archiver->name)); + free($2); + } + T_LBRACE archiver_opt_list T_RBRACE + ; + +archiver_opt_list: + /* empty */ + | archiver_opt_list archiver_opt + ; + +archiver_opt: + T_FORMATION T_IDENT + { + if (current_archiver->formationCount >= PGAF_MAX_ARCHIVER_FORMATIONS) + { + fprintf(stderr, + "pgaftest: too many --formation entries for archiver " + "\"%s\" (max %d)\n", + current_archiver->name, PGAF_MAX_ARCHIVER_FORMATIONS); + exit(1); + } + strlcpy(current_archiver->formations[current_archiver->formationCount++], + $2, sizeof(current_archiver->formations[0])); + free($2); + } + | T_REGION T_IDENT + { + strlcpy(current_archiver->region, $2, sizeof(current_archiver->region)); + free($2); + } + | T_REGION T_STRING + { + strlcpy(current_archiver->region, $2, sizeof(current_archiver->region)); + free($2); + } + | T_CREATE T_AND T_LAUNCH T_DEFERRED + { + /* bare "create and launch deferred" = both gates, matching + * node_opt's own identical form */ + current_archiver->createDeferred = true; + current_archiver->launchDeferred = true; + } + | T_LAUNCH T_DEFERRED + { + current_archiver->launchDeferred = true; + } + | T_CREATE T_DEFERRED + { + current_archiver->createDeferred = true; + } + ; + /* * monitor [port N] * @@ -515,6 +604,10 @@ node_opt: current_node->kind = NODE_KIND_CITUS_WORKER; current_spec->cluster.withCitus = true; } + | T_ARCHIVER + { + current_node->kind = NODE_KIND_ARCHIVER; + } | T_ASYNC { current_node->replicationQuorum = false; @@ -1020,6 +1113,106 @@ wait_cmd: $$->timeoutSeconds = $6; current_wait_cmd = NULL; } + /* + * Generic form: wait until sql { SQL } is { value } [timeout Ns] + * + * Polls an arbitrary scalar SQL expression until its (substring- + * matched, same semantics as `expect { }`) result contains . + * The "wal segment ... archived", "archiver state is ...", and + * "basebackup ... is ..." forms below are all sugar for this at parse + * time -- reach for this directly only when none of those fit. + */ + | T_WAIT T_UNTIL T_SQL T_IDENT T_BLOCK T_IS T_BLOCK opt_timeout + { + $$ = make_cmd(CMD_WAIT_SQL); + strlcpy($$->service, $4, sizeof($$->service)); + strlcpy($$->args, $5, sizeof($$->args)); + strlcpy($$->expected, $7, sizeof($$->expected)); + $$->timeoutSeconds = $8; + free($4); free($5); free($7); + } + /* + * wait until wal segment "" archived in / [timeout Ns] + * + * Sugar for polling pgautofailover.wal_archived(). The segment name is + * quoted (T_STRING) rather than bare: a real segment name is all + * digits, which the lexer's own T_INTEGER rule would otherwise + * swallow (and overflow -- a segment name is 24 digits, an int isn't). + */ + | T_WAIT T_UNTIL T_WAL T_SEGMENT T_STRING T_ARCHIVED T_IN T_IDENT T_SLASH T_INTEGER opt_timeout + { + $$ = make_cmd(CMD_WAIT_SQL); + strlcpy($$->service, "monitor", sizeof($$->service)); + sformat($$->args, sizeof($$->args), + "SELECT pgautofailover.wal_archived('%s', %d, '%s')", + $8, $10, $5); + strlcpy($$->expected, "t", sizeof($$->expected)); + $$->timeoutSeconds = $11; + free($5); free($8); + } + /* + * wait until archiver state is in [/] [timeout Ns] + * + * Sugar for the nodename LIKE 'archiver-%' idiom every multi- + * membership archiver spec needs: archiver_add_formation() (pgautofailover.sql) + * never uses the plain --name given at create-archiver time as an + * ARCHIVING row's own nodename, so the ordinary "wait until + * state is " form (which matches on nodename = $1) can't see these + * rows at all, let alone disambiguate more than one. Group is + * optional: omit it when the formation has exactly one archiver + * membership (the common case, and formationid alone is unambiguous), + * give it to disambiguate a multi-group Citus formation. + */ + | T_WAIT T_UNTIL T_ARCHIVER T_STATE state_op wait_state_name T_IN T_IDENT opt_wait_group opt_timeout + { + $$ = make_cmd(CMD_WAIT_SQL); + strlcpy($$->service, "monitor", sizeof($$->service)); + if ($9 >= 0) + { + sformat($$->args, sizeof($$->args), + "SELECT reportedstate::text FROM pgautofailover.node" + " WHERE nodename LIKE 'archiver-%%' AND formationid = '%s'" + " AND groupid = %d", $8, $9); + } + else + { + sformat($$->args, sizeof($$->args), + "SELECT reportedstate::text FROM pgautofailover.node" + " WHERE nodename LIKE 'archiver-%%' AND formationid = '%s'", $8); + } + strlcpy($$->expected, $6, sizeof($$->expected)); + $$->timeoutSeconds = $10; + free($6); free($8); + } + /* + * wait until basebackup is in / [timeout Ns] + * + * Sugar for polling pgautofailover.get_latest_basebackup(). + * is validated here rather than tokenized: it's the one piece of this + * command that's genuinely open content (a column name), not fixed + * syntax, so a clear parse-time error beats a cryptic runtime SQL one. + */ + | T_WAIT T_UNTIL T_BASEBACKUP T_IDENT T_IS T_IDENT T_IN T_IDENT T_SLASH T_INTEGER opt_timeout + { + if (strcmp($4, "source") != 0 && + strcmp($4, "status") != 0 && + strcmp($4, "replaymode") != 0) + { + fprintf(stderr, + "pgaftest: line %d: \"wait until basebackup %s ...\" -- " + "unknown property (expected source, status, or replaymode)\n", + pgaf_line_number, $4); + exit(1); + } + $$ = make_cmd(CMD_WAIT_SQL); + strlcpy($$->service, "monitor", sizeof($$->service)); + sformat($$->args, sizeof($$->args), + "SELECT %s::text FROM pgautofailover.get_latest_basebackup('%s', %d)", + $4, $8, $10); + strlcpy($$->expected, $6, sizeof($$->expected)); + $$->timeoutSeconds = $11; + free($4); free($6); free($8); + } ; /* @@ -1590,8 +1783,120 @@ ident_or_string: | T_STRING { $$ = $1; } ; +/* + * wait_state_name — a state name for "wait until archiver state is X", + * accepting both known FSM state tokens and bare idents (e.g. "archiving", + * which has no T_FS_* token of its own -- see fsm_state's own list). Always + * returns a heap-owned string so the caller can unconditionally free() it, + * unlike fsm_state itself (whose branches return static literals). + */ +wait_state_name: + fsm_state { $$ = strdup($1); } + | T_IDENT { $$ = $1; } + ; + +/* + * opt_wait_group — optional "/" suffix for "wait until archiver + * state is X in [/]". -1 means "no group filter". + */ +opt_wait_group: + /* empty */ { $$ = -1; } + | T_SLASH T_INTEGER { $$ = $2; } + ; + %% +/* + * fold_archivers_into_formations turns each top-level "archiver { }" + * declaration (TestArchiverNode, cluster->archivers[]) into an ordinary + * TestNode of kind NODE_KIND_ARCHIVER, appended to its own declared + * formation's own node list -- see TestArchiverNode's own comment + * (test_spec.h) for why the *declaration* still needs to be top-level even + * though it ends up represented identically to the older, still-supported + * "archiver nested inside a formation_block" spelling once parsed. Called + * once, right after yyparse() returns, so every caller downstream of + * parse_test_spec() (compose_gen.c included) only ever sees ordinary + * TestNode entries and needs no awareness of TestArchiverNode at all. + * + * cluster->archiverCount is reset to 0 once every entry has been folded, + * so cluster->archivers[] is never a second, stale source of truth for + * the very same nodes now living in cluster->formations[].nodes[]. + */ +static void +fold_archivers_into_formations(TestCluster *cluster) +{ + for (int ai = 0; ai < cluster->archiverCount; ai++) + { + TestArchiverNode *a = &cluster->archivers[ai]; + + if (a->formationCount == 0) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" needs at least one " + "\"formation \" entry\n", a->name); + exit(1); + } + + if (a->formationCount > 1) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" lists %d formations, but " + "pg_autoctl create archiver's own ini-driven bring-up " + "only attaches to one at create time -- declare just " + "\"formation %s\" here and attach the rest (e.g. " + "\"%s\") dynamically once it's running instead, via a " + "direct \"sql monitor { SELECT pgautofailover." + "archiver_add_formation(...) }\" step -- see " + "archiver_multi_formation.pgaf for the pattern\n", + a->name, a->formationCount, a->formations[0], + a->formations[1]); + exit(1); + } + + TestFormation *form = NULL; + + for (int fi = 0; fi < cluster->formationCount; fi++) + { + if (strcmp(cluster->formations[fi].name, a->formations[0]) == 0) + { + form = &cluster->formations[fi]; + break; + } + } + + if (form == NULL) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" attaches to formation " + "\"%s\", which is not declared in this cluster{} " + "block\n", a->name, a->formations[0]); + exit(1); + } + + if (form->nodeCount >= PGAF_MAX_NODES) + { + fprintf(stderr, + "pgaftest: too many nodes in formation \"%s\" (max %d)\n", + form->name, PGAF_MAX_NODES); + exit(1); + } + + TestNode *node = &form->nodes[form->nodeCount++]; + + memset(node, 0, sizeof(*node)); + strlcpy(node->name, a->name, sizeof(node->name)); + node->kind = NODE_KIND_ARCHIVER; + node->candidatePriority = 50; + node->replicationQuorum = true; + strlcpy(node->region, a->region, sizeof(node->region)); + node->createDeferred = a->createDeferred; + node->launchDeferred = a->launchDeferred; + } + + cluster->archiverCount = 0; +} + + /* ----------------------------------------------------------------------- * Public entry point * ----------------------------------------------------------------------- */ @@ -1618,6 +1923,8 @@ parse_test_spec(const char *filename) yyparse(); fclose(f); + fold_archivers_into_formations(&spec->cluster); + /* * If the file has no explicit sequence{} block, default to running * steps in declaration order. Populated here (not just in the CI diff --git a/src/bin/pgaftest/test_spec_scan.c b/src/bin/pgaftest/test_spec_scan.c index 815753697..2d16af03b 100644 --- a/src/bin/pgaftest/test_spec_scan.c +++ b/src/bin/pgaftest/test_spec_scan.c @@ -356,8 +356,8 @@ static void yynoreturn yy_fatal_error ( const char* msg ); (yy_hold_char) = *yy_cp; \ *yy_cp = '\0'; \ (yy_c_buf_p) = yy_cp; -#define YY_NUM_RULES 163 -#define YY_END_OF_BUFFER 164 +#define YY_NUM_RULES 170 +#define YY_END_OF_BUFFER 171 /* This struct is not used in this scanner, but its presence is necessary. */ struct yy_trans_info @@ -365,146 +365,149 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static const flex_int16_t yy_accept[1252] = +static const flex_int16_t yy_accept[1284] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 18, 17, 16, 18, 1, 12, 11, 13, 13, - 13, 13, 13, 13, 15, 163, 21, 20, 163, 19, - 62, 61, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 64, 65, 102, 101, 163, 100, 138, 153, 137, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 155, - 156, 159, 158, 160, 161, 162, 17, 0, 14, 1, - 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, - - 21, 0, 63, 19, 62, 62, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 102, 0, 154, 100, 153, - 153, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 129, 135, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 159, 158, 161, 13, 13, 13, - - 13, 13, 13, 13, 13, 44, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 25, 99, 99, 99, 99, - 99, 99, 134, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 140, 147, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 150, 157, 157, 157, 157, 157, 157, 157, - 157, 105, 157, 146, 157, 157, 112, 157, 157, 157, - - 157, 157, 157, 157, 157, 157, 13, 13, 13, 4, - 13, 13, 9, 13, 99, 99, 27, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 66, 99, 99, 99, 99, - 99, 99, 99, 30, 99, 99, 53, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 43, 99, 99, 99, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 124, 157, 157, 157, 104, 157, 157, 157, 157, 157, - 66, 157, 157, 128, 149, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - - 157, 157, 157, 157, 141, 126, 157, 157, 157, 107, - 157, 136, 13, 13, 13, 13, 7, 13, 99, 33, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 42, 99, 99, 99, 52, 24, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 114, 157, - 157, 157, 157, 157, 157, 133, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - - 157, 157, 157, 157, 121, 125, 130, 142, 157, 157, - 157, 157, 157, 108, 157, 157, 143, 13, 13, 13, - 13, 13, 99, 99, 99, 99, 99, 99, 99, 99, - 39, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 38, 99, 48, - 99, 99, 99, 99, 99, 99, 99, 51, 99, 99, - 99, 67, 99, 99, 99, 99, 47, 99, 99, 99, - 99, 99, 99, 32, 157, 157, 111, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 113, 157, - 157, 157, 157, 148, 157, 157, 157, 157, 157, 157, - - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 67, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 13, 13, 2, 3, 13, 13, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 73, 99, 98, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 22, - 99, 99, 99, 99, 68, 99, 99, 99, 99, 99, - 99, 46, 99, 99, 99, 99, 99, 99, 99, 157, - 157, 157, 157, 157, 122, 120, 157, 157, 157, 73, - 157, 157, 98, 157, 157, 157, 157, 157, 157, 157, - - 157, 157, 157, 152, 118, 123, 157, 116, 157, 157, - 157, 68, 115, 109, 157, 157, 157, 157, 157, 127, - 145, 110, 157, 157, 157, 157, 157, 157, 13, 13, - 10, 8, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 40, 99, 99, 76, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 29, 36, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 157, 157, - 157, 157, 157, 151, 157, 157, 157, 76, 157, 117, - 157, 157, 157, 157, 157, 157, 157, 157, 0, 157, - - 139, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 13, 13, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 28, 99, 41, 45, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 77, 99, 99, 35, 99, 99, 99, 99, 99, 99, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 28, 157, 157, 157, 157, 157, 0, 157, 157, - 157, 157, 157, 157, 157, 77, 157, 157, 157, 157, - 157, 157, 157, 157, 13, 13, 99, 99, 99, 99, - - 99, 78, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 34, - 99, 99, 99, 99, 99, 93, 92, 99, 99, 99, - 99, 99, 99, 99, 99, 157, 157, 157, 157, 78, - 157, 157, 119, 103, 157, 157, 157, 157, 157, 157, - 157, 0, 106, 157, 157, 157, 157, 93, 92, 157, - 157, 157, 157, 157, 157, 157, 157, 13, 13, 99, - 99, 26, 59, 99, 99, 99, 31, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 83, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - - 99, 99, 99, 99, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 83, 0, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 13, 6, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 95, 94, 23, 85, 99, 84, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 70, 72, - 99, 69, 71, 157, 157, 157, 157, 157, 157, 95, - 94, 85, 157, 84, 157, 0, 157, 157, 157, 157, - 157, 157, 157, 70, 72, 157, 69, 71, 13, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 157, 157, 157, 157, 157, 157, 157, 157, - 0, 157, 157, 157, 157, 157, 157, 157, 157, 13, - 87, 86, 99, 99, 99, 55, 75, 74, 99, 97, - 96, 60, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 87, 86, 131, 157, 75, 74, 97, - 96, 0, 157, 157, 157, 157, 157, 157, 157, 157, - 13, 99, 99, 49, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 157, 144, 157, 157, - 157, 157, 157, 157, 157, 157, 13, 99, 99, 99, - - 37, 99, 99, 99, 99, 99, 99, 82, 81, 91, - 90, 157, 157, 157, 157, 157, 82, 81, 91, 90, - 5, 99, 99, 58, 99, 80, 99, 79, 99, 99, - 157, 157, 80, 157, 79, 50, 54, 99, 99, 99, - 56, 132, 157, 157, 89, 88, 99, 89, 88, 57, - 0 + 171, 18, 17, 16, 18, 1, 12, 11, 13, 13, + 13, 13, 13, 13, 15, 170, 21, 20, 170, 19, + 63, 62, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 65, 66, 103, 102, 170, 101, 145, 119, 160, + 144, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 162, 163, 166, 165, 167, 168, 169, 17, 0, + 14, 1, 12, 12, 13, 13, 13, 13, 13, 13, + + 13, 13, 21, 0, 64, 19, 63, 63, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 103, 0, + 161, 101, 160, 160, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 136, 142, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 166, + + 165, 168, 13, 13, 13, 13, 13, 13, 13, 13, + 45, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 25, 100, 100, 100, 100, 100, 100, 141, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 147, 154, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 157, 164, 164, 164, 164, 164, 164, 164, 164, 106, + + 164, 164, 153, 164, 164, 113, 164, 164, 164, 164, + 164, 164, 164, 114, 164, 164, 13, 13, 13, 4, + 13, 13, 9, 13, 100, 100, 100, 27, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 67, 100, 100, 100, + 100, 100, 100, 100, 30, 100, 100, 54, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 44, 100, 100, + 100, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 131, 164, 164, 164, 105, 164, 164, + 164, 164, 164, 67, 164, 164, 135, 156, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 148, 133, + 164, 164, 164, 108, 164, 143, 13, 13, 13, 13, + 7, 13, 100, 100, 34, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 43, 100, + 100, 100, 53, 24, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 121, 164, 164, 164, 164, + + 164, 164, 140, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 128, 132, 137, 149, 164, 164, 164, 164, + 164, 109, 164, 164, 150, 13, 13, 13, 13, 13, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 40, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 39, 100, 49, 100, + 100, 100, 100, 100, 100, 100, 52, 100, 100, 100, + 68, 100, 100, 100, 100, 48, 100, 100, 100, 100, + 100, 100, 32, 164, 164, 164, 112, 164, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 120, + 164, 164, 164, 164, 155, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 68, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 13, 13, 2, 3, + 13, 13, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 74, 100, 99, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 22, 100, 100, 100, 100, 69, 100, 100, + 100, 100, 100, 100, 47, 100, 100, 100, 100, 100, + + 100, 100, 164, 164, 164, 164, 164, 164, 164, 129, + 127, 164, 164, 164, 74, 164, 164, 99, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 159, 125, + 130, 164, 123, 164, 164, 164, 69, 122, 110, 164, + 164, 164, 115, 164, 164, 134, 152, 111, 164, 164, + 164, 164, 164, 164, 13, 13, 10, 8, 100, 100, + 33, 100, 100, 100, 100, 100, 100, 100, 100, 41, + 100, 100, 77, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 29, 37, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + + 100, 100, 100, 100, 100, 164, 164, 116, 118, 164, + 164, 164, 164, 158, 164, 164, 164, 77, 164, 124, + 164, 164, 164, 164, 164, 164, 164, 164, 0, 164, + 146, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 13, 13, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 28, 100, 42, 46, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 78, 100, 100, 36, 100, 100, 100, 100, 100, 100, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + + 164, 164, 28, 164, 164, 164, 164, 164, 0, 164, + 164, 164, 164, 164, 164, 164, 78, 164, 164, 164, + 164, 164, 164, 164, 164, 13, 13, 100, 100, 100, + 100, 100, 79, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 35, 100, 100, 100, 100, 100, 94, 93, 100, 100, + 100, 100, 100, 100, 100, 100, 164, 164, 164, 117, + 164, 79, 164, 164, 126, 104, 164, 164, 164, 164, + 164, 164, 164, 0, 107, 164, 164, 164, 164, 94, + 93, 164, 164, 164, 164, 164, 164, 164, 164, 13, + + 13, 100, 100, 26, 60, 100, 100, 100, 31, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 84, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 84, 0, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 13, 6, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 96, 95, 23, 86, 100, 85, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 71, 73, 100, 70, 72, 164, 164, 164, 164, 164, + + 164, 96, 95, 86, 164, 85, 164, 0, 164, 164, + 164, 164, 164, 164, 164, 71, 73, 164, 70, 72, + 13, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 164, 164, 164, 164, 164, 164, + 164, 164, 0, 164, 164, 164, 164, 164, 164, 164, + 164, 13, 88, 87, 100, 100, 100, 56, 76, 75, + 100, 98, 97, 61, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 88, 87, 138, 164, 76, + 75, 98, 97, 0, 164, 164, 164, 164, 164, 164, + + 164, 164, 13, 100, 100, 50, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 164, 151, + 164, 164, 164, 164, 164, 164, 164, 164, 13, 100, + 100, 100, 38, 100, 100, 100, 100, 100, 100, 83, + 82, 92, 91, 164, 164, 164, 164, 164, 83, 82, + 92, 91, 5, 100, 100, 59, 100, 81, 100, 80, + 100, 100, 164, 164, 81, 164, 80, 51, 55, 100, + 100, 100, 57, 139, 164, 164, 90, 89, 100, 90, + 89, 58, 0 } ; static const YY_CHAR yy_ec[256] = @@ -513,16 +516,16 @@ static const YY_CHAR yy_ec[256] = 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 5, 6, 1, 1, 1, 1, 1, - 1, 1, 1, 7, 8, 1, 1, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 1, 1, 1, - 10, 1, 1, 1, 11, 11, 11, 11, 11, 11, - 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, - 11, 11, 12, 11, 11, 11, 11, 11, 11, 11, - 1, 1, 1, 1, 13, 1, 14, 15, 16, 17, - - 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - 38, 11, 39, 1, 40, 1, 1, 1, 1, 1, + 1, 1, 1, 7, 8, 1, 9, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 1, 1, 1, + 11, 1, 1, 1, 12, 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 12, 12, 13, 12, 12, 12, 12, 12, 12, 12, + 1, 1, 1, 1, 14, 1, 15, 16, 17, 18, + + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + 39, 12, 40, 1, 41, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -539,610 +542,629 @@ static const YY_CHAR yy_ec[256] = 1, 1, 1, 1, 1 } ; -static const YY_CHAR yy_meta[41] = +static const YY_CHAR yy_meta[42] = { 0, - 1, 2, 3, 1, 1, 1, 1, 4, 4, 1, + 1, 2, 3, 1, 1, 1, 1, 4, 1, 4, + 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 1, 1 + 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, + 1 } ; -static const flex_int16_t yy_base[1265] = +static const flex_int16_t yy_base[1297] = { 0, - 0, 0, 40, 0, 80, 0, 119, 121, 1344, 1343, - 1345, 1348, 123, 1348, 1339, 0, 117, 1348, 0, 106, - 1315, 1314, 112, 1323, 1348, 1348, 130, 1348, 1335, 0, - 124, 1348, 0, 108, 1317, 124, 123, 1301, 125, 1306, - 117, 1308, 143, 134, 130, 145, 1317, 145, 1303, 1305, - 146, 1348, 1348, 164, 1348, 1327, 0, 1348, 138, 1348, - 0, 153, 155, 153, 155, 173, 171, 161, 1303, 1308, - 1301, 1314, 172, 189, 176, 186, 174, 1300, 177, 1348, - 1348, 0, 1324, 1348, 0, 1348, 210, 1320, 1348, 0, - 206, 1348, 0, 1291, 1289, 1295, 1304, 191, 1302, 1305, - - 221, 1313, 1348, 0, 217, 1348, 0, 1300, 1287, 1277, - 1281, 1286, 195, 1279, 1283, 1292, 215, 199, 1276, 207, - 1277, 1279, 217, 1284, 1283, 1270, 1283, 1270, 1279, 1273, - 189, 1273, 1266, 1266, 215, 215, 1280, 1268, 1269, 1265, - 1260, 1257, 1265, 1267, 1257, 238, 1282, 1348, 0, 236, - 1348, 0, 1269, 1256, 1252, 219, 224, 1257, 1250, 1245, - 233, 1249, 235, 233, 1248, 1252, 1244, 1248, 234, 0, - 1253, 1249, 1253, 236, 1239, 237, 1239, 1239, 1256, 1236, - 244, 1238, 1239, 243, 1238, 1246, 1238, 249, 1231, 1235, - 1227, 1237, 1236, 1224, 0, 1254, 0, 1221, 1222, 1231, - - 1234, 1217, 1216, 1220, 1217, 0, 1222, 1219, 1224, 1227, - 1226, 1226, 1207, 1209, 1225, 1216, 1219, 1208, 1213, 1205, - 1215, 1200, 1198, 1204, 1195, 1208, 1209, 1193, 1198, 1197, - 1209, 1189, 1194, 1198, 1193, 1200, 1209, 1184, 1182, 1185, - 1187, 1190, 246, 1183, 1190, 0, 1180, 1179, 1189, 1172, - 1172, 1180, 0, 1178, 257, 1185, 1185, 1171, 251, 1171, - 1182, 1170, 1174, 1166, 1166, 1177, 1174, 1166, 1157, 1163, - 0, 0, 1154, 1154, 1168, 1158, 1159, 1151, 1155, 1165, - 1144, 1161, 0, 1146, 1158, 1162, 1142, 1145, 1147, 1146, - 255, 0, 1143, 0, 1150, 1151, 0, 254, 1139, 1138, - - 1138, 1147, 1142, 1130, 1137, 1140, 1128, 1126, 1125, 0, - 1139, 1127, 0, 1138, 1116, 1137, 1144, 1143, 1128, 1128, - 1116, 1130, 1113, 1131, 1113, 1110, 1115, 1112, 1113, 1121, - 273, 1124, 1108, 1118, 1118, 1112, 280, 1117, 1116, 1113, - 1097, 1096, 1100, 0, 1095, 1090, 0, 1111, 1110, 1095, - 1100, 1090, 1093, 1094, 281, 1100, 0, 1091, 282, 1098, - 1077, 1083, 1093, 1090, 1090, 1082, 1091, 1094, 1074, 1078, - 0, 1078, 1075, 1072, 1094, 1085, 1072, 283, 1085, 1069, - 0, 1081, 289, 0, 0, 1063, 1074, 1066, 1071, 1070, - 1063, 1056, 1069, 1074, 1073, 1058, 1071, 1053, 1056, 1057, - - 1052, 1047, 1061, 1046, 0, 290, 1043, 1048, 1050, 291, - 1056, 0, 1065, 1054, 1043, 1043, 0, 1041, 292, 0, - 1042, 1035, 1049, 1043, 1056, 1041, 1044, 1034, 1029, 1041, - 1036, 1039, 1024, 0, 1036, 1035, 1020, 0, 1044, 1029, - 1036, 277, 279, 1028, 1010, 1020, 1028, 1017, 1017, 1005, - 1014, 1010, 1009, 1012, 1022, 1004, 1019, 1017, 1003, 1002, - 1014, 1004, 1012, 286, 288, 998, 308, 995, 1000, 1009, - 1003, 992, 1007, 1000, 1003, 993, 997, 1000, 0, 998, - 983, 980, 995, 994, 979, 0, 978, 293, 294, 992, - 991, 977, 980, 979, 974, 971, 972, 971, 970, 967, - - 961, 965, 980, 978, 0, 0, 0, 0, 964, 963, - 975, 972, 957, 0, 298, 302, 0, 297, 959, 958, - 972, 951, 954, 953, 966, 955, 968, 954, 313, 953, - 0, 971, 960, 324, 950, 959, 953, 946, 945, 950, - 938, 956, 944, 937, 949, 935, 947, 0, 956, 0, - 936, 931, 939, 933, 928, 940, 919, 0, 942, 327, - 941, 0, 936, 935, 935, 934, 0, 936, 918, 915, - 933, 915, 912, 0, 912, 911, 0, 924, 927, 913, - 921, 905, 910, 330, 909, 908, 917, 919, 0, 914, - 903, 902, 907, 0, 897, 909, 895, 907, 897, 891, - - 898, 899, 900, 893, 890, 899, 898, 877, 896, 881, - 331, 898, 0, 893, 892, 892, 887, 874, 892, 874, - 871, 889, 871, 868, 872, 871, 0, 0, 880, 870, - 878, 877, 861, 859, 859, 871, 865, 871, 874, 871, - 869, 852, 851, 0, 863, 0, 854, 850, 849, 851, - 864, 844, 851, 853, 858, 851, 856, 839, 856, 861, - 835, 851, 849, 338, 0, 832, 839, 838, 831, 832, - 831, 0, 841, 836, 835, 842, 833, 832, 839, 834, - 832, 829, 812, 131, 0, 0, 140, 203, 227, 0, - 255, 262, 0, 279, 276, 305, 311, 320, 327, 334, - - 330, 337, 340, 0, 0, 0, 353, 0, 342, 327, - 352, 0, 0, 0, 336, 337, 332, 335, 337, 0, - 0, 0, 345, 346, 355, 348, 349, 358, 345, 343, - 0, 0, 342, 343, 356, 347, 361, 346, 347, 366, - 350, 359, 0, 363, 364, 0, 360, 352, 353, 363, - 360, 374, 355, 368, 367, 370, 369, 365, 372, 371, - 373, 0, 0, 376, 377, 382, 375, 376, 371, 385, - 386, 395, 386, 388, 388, 389, 391, 391, 386, 387, - 413, 404, 389, 0, 402, 403, 410, 0, 402, 0, - 392, 393, 403, 405, 404, 407, 406, 408, 434, 406, - - 0, 414, 415, 410, 413, 408, 422, 423, 422, 424, - 424, 425, 427, 427, 424, 432, 424, 425, 431, 444, - 453, 433, 431, 436, 437, 432, 442, 443, 462, 457, - 458, 0, 453, 0, 0, 460, 448, 462, 450, 464, - 463, 466, 450, 468, 452, 470, 454, 458, 460, 461, - 0, 467, 468, 0, 458, 478, 476, 461, 481, 479, - 464, 465, 467, 492, 472, 476, 477, 471, 473, 492, - 493, 0, 494, 482, 496, 484, 496, 492, 489, 501, - 485, 503, 487, 492, 493, 0, 499, 500, 490, 510, - 508, 493, 513, 511, 512, 512, 509, 510, 516, 516, - - 506, 0, 503, 510, 507, 507, 522, 523, 507, 512, - 513, 527, 515, 530, 517, 532, 519, 533, 520, 0, - 531, 526, 533, 528, 530, 0, 0, 542, 543, 542, - 530, 547, 545, 533, 550, 544, 545, 535, 540, 0, - 552, 553, 0, 0, 541, 542, 543, 558, 545, 560, - 560, 548, 0, 558, 553, 560, 555, 0, 0, 568, - 569, 568, 556, 573, 571, 559, 576, 570, 562, 567, - 568, 0, 0, 565, 579, 581, 0, 566, 572, 573, - 584, 586, 587, 572, 568, 593, 570, 595, 577, 0, - 579, 585, 587, 587, 589, 608, 603, 604, 592, 582, - - 583, 595, 585, 586, 598, 599, 613, 597, 601, 602, - 614, 615, 595, 620, 597, 622, 0, 609, 611, 613, - 613, 615, 628, 629, 617, 607, 608, 620, 610, 611, - 623, 0, 631, 632, 631, 623, 641, 638, 623, 624, - 628, 0, 0, 0, 0, 629, 0, 630, 628, 627, - 631, 637, 633, 639, 639, 637, 638, 658, 0, 0, - 659, 0, 0, 654, 655, 643, 655, 644, 645, 0, - 0, 0, 649, 0, 650, 648, 650, 656, 652, 658, - 654, 655, 675, 0, 0, 676, 0, 0, 677, 660, - 661, 666, 687, 665, 666, 665, 666, 668, 663, 664, - - 674, 676, 687, 673, 689, 675, 695, 676, 689, 690, - 686, 687, 683, 684, 699, 690, 686, 687, 683, 684, - 703, 706, 692, 708, 694, 706, 707, 703, 704, 699, - 0, 0, 702, 707, 697, 0, 0, 0, 714, 0, - 0, 0, 706, 711, 717, 713, 719, 710, 715, 716, - 717, 730, 731, 0, 0, 0, 717, 0, 0, 0, - 0, 728, 723, 729, 725, 731, 726, 727, 740, 741, - 730, 737, 746, 0, 733, 745, 749, 736, 751, 738, - 735, 737, 742, 743, 753, 754, 751, 1348, 760, 747, - 762, 749, 751, 752, 762, 763, 751, 750, 758, 758, - - 0, 759, 760, 761, 762, 754, 757, 0, 0, 0, - 0, 759, 766, 767, 768, 769, 0, 0, 0, 0, - 0, 759, 780, 0, 783, 0, 784, 0, 773, 776, - 765, 788, 0, 789, 0, 0, 0, 788, 789, 777, - 0, 0, 791, 792, 0, 0, 794, 0, 0, 0, - 1348, 811, 815, 819, 823, 822, 827, 831, 830, 835, - 839, 838, 843, 847 + 0, 0, 41, 0, 82, 0, 122, 124, 1377, 1376, + 1378, 1381, 126, 1381, 1372, 0, 119, 1381, 0, 108, + 1347, 1346, 114, 1355, 1381, 1381, 133, 1381, 1368, 0, + 126, 1381, 0, 110, 1349, 126, 125, 1333, 131, 1338, + 123, 1340, 146, 138, 127, 139, 1349, 147, 1335, 1337, + 148, 1381, 1381, 170, 1381, 1360, 0, 1381, 1381, 163, + 1381, 0, 155, 1349, 149, 170, 152, 171, 160, 170, + 1334, 1339, 1332, 1345, 172, 190, 175, 189, 184, 1331, + 202, 1381, 1381, 0, 1356, 1381, 0, 1381, 193, 1352, + 1381, 0, 203, 1381, 0, 1322, 1320, 1326, 1335, 187, + + 1333, 1336, 224, 1345, 1381, 0, 217, 1381, 0, 1331, + 1318, 1330, 1307, 1311, 1316, 201, 1309, 1313, 1322, 217, + 216, 1306, 206, 1307, 1309, 219, 1314, 1313, 1300, 1313, + 1300, 1309, 1303, 230, 1303, 1296, 1296, 224, 219, 1310, + 1298, 1299, 1295, 1290, 1287, 1295, 1297, 1287, 249, 1313, + 1381, 0, 242, 1381, 0, 1299, 1286, 1298, 1281, 1280, + 226, 214, 1285, 1278, 1273, 241, 1277, 238, 236, 1276, + 1280, 1272, 1276, 238, 0, 1281, 1277, 1281, 240, 1267, + 246, 1267, 1267, 1284, 1264, 248, 1266, 1267, 255, 1266, + 1274, 1266, 263, 1259, 1263, 1255, 258, 1265, 1253, 0, + + 1284, 0, 1250, 1251, 1260, 1263, 1246, 1245, 1249, 1246, + 0, 1251, 1254, 1247, 1252, 1255, 1254, 1254, 1235, 1237, + 1253, 1244, 1247, 1236, 1241, 1233, 1243, 1228, 1226, 1232, + 1223, 1236, 1237, 1221, 1226, 1225, 1237, 1217, 1222, 1226, + 1221, 1228, 1238, 1212, 1210, 1213, 1215, 1218, 257, 1211, + 1218, 0, 1208, 1207, 1217, 1200, 1200, 1208, 0, 1206, + 1209, 268, 1211, 1211, 1211, 1197, 245, 1197, 1208, 1196, + 1200, 1192, 1192, 1203, 1200, 1192, 1183, 1189, 0, 0, + 1180, 1180, 1194, 1184, 1185, 1177, 1181, 1191, 1170, 1187, + 0, 1172, 1184, 1188, 1168, 1171, 1173, 1172, 259, 0, + + 1169, 1170, 0, 1175, 1176, 0, 261, 1164, 1163, 1163, + 1172, 1167, 1155, 0, 1162, 1165, 1153, 1151, 1150, 0, + 1164, 1152, 0, 1163, 1141, 1156, 1161, 1169, 1168, 1152, + 1152, 1140, 1154, 1137, 1155, 1137, 1134, 1139, 1136, 1137, + 1145, 282, 1148, 1132, 1142, 1142, 1136, 289, 1141, 1140, + 1137, 1121, 1120, 1124, 0, 1119, 1114, 0, 1135, 1134, + 1119, 1124, 1114, 1117, 1118, 290, 1124, 0, 1115, 291, + 1122, 1101, 1116, 1106, 1116, 1120, 1112, 1112, 1104, 1113, + 1116, 1096, 1100, 0, 1100, 1097, 1094, 1117, 1107, 1094, + 293, 1107, 1091, 0, 1103, 294, 0, 0, 1085, 1096, + + 1088, 1093, 1092, 1085, 1078, 1091, 1096, 1095, 1080, 1093, + 1075, 1078, 1086, 1078, 1073, 1068, 1082, 1067, 0, 298, + 1064, 1069, 1071, 301, 1077, 0, 1087, 1075, 1064, 1064, + 0, 1062, 302, 1054, 0, 1062, 1055, 1069, 1063, 1077, + 1061, 1064, 1054, 1049, 1061, 1056, 1059, 1044, 0, 1056, + 1055, 1040, 0, 1065, 1049, 1056, 281, 288, 1048, 1030, + 1040, 1048, 1037, 1037, 1025, 1034, 1030, 1029, 1032, 1042, + 1024, 1039, 1037, 1023, 1022, 1034, 1024, 1032, 292, 296, + 1018, 316, 1013, 1014, 1019, 1031, 1027, 1021, 1010, 1025, + 1018, 1021, 1011, 1015, 1018, 0, 1016, 1001, 998, 1013, + + 1012, 997, 0, 996, 301, 302, 1010, 1009, 995, 998, + 997, 992, 989, 990, 989, 988, 985, 979, 983, 998, + 987, 995, 0, 0, 0, 0, 981, 980, 992, 989, + 974, 0, 306, 310, 0, 310, 976, 975, 989, 968, + 971, 970, 983, 982, 971, 984, 970, 321, 969, 0, + 988, 976, 334, 966, 975, 969, 962, 961, 966, 954, + 972, 960, 953, 965, 951, 963, 0, 973, 0, 952, + 947, 955, 949, 944, 956, 935, 0, 958, 336, 957, + 0, 952, 951, 951, 950, 0, 952, 934, 931, 949, + 931, 928, 0, 928, 927, 940, 0, 939, 940, 941, + + 927, 935, 919, 924, 337, 923, 922, 931, 933, 0, + 928, 917, 916, 921, 0, 911, 923, 909, 921, 911, + 905, 912, 913, 914, 907, 904, 913, 912, 891, 910, + 895, 345, 912, 892, 0, 906, 905, 905, 900, 887, + 905, 887, 884, 902, 884, 881, 885, 884, 0, 0, + 893, 883, 891, 890, 876, 873, 871, 871, 883, 877, + 883, 886, 883, 881, 864, 863, 0, 875, 0, 866, + 862, 861, 863, 876, 856, 863, 865, 870, 863, 868, + 850, 864, 870, 113, 158, 196, 348, 0, 224, 239, + 240, 262, 283, 287, 0, 322, 323, 326, 342, 335, + + 337, 346, 344, 345, 347, 348, 342, 334, 348, 0, + 0, 337, 337, 338, 0, 354, 353, 0, 352, 344, + 345, 346, 351, 358, 365, 360, 367, 370, 0, 0, + 0, 384, 0, 372, 357, 383, 0, 0, 0, 366, + 367, 362, 0, 365, 366, 0, 0, 0, 375, 376, + 385, 378, 379, 388, 375, 373, 0, 0, 372, 373, + 0, 386, 377, 391, 376, 377, 396, 380, 389, 0, + 393, 394, 0, 390, 382, 383, 393, 390, 404, 385, + 398, 397, 400, 399, 395, 402, 401, 403, 0, 0, + 406, 407, 412, 405, 406, 401, 415, 416, 425, 416, + + 418, 418, 419, 421, 421, 416, 417, 0, 0, 444, + 418, 435, 420, 0, 433, 434, 441, 0, 433, 0, + 423, 424, 434, 436, 435, 438, 437, 439, 466, 437, + 0, 445, 446, 441, 444, 439, 453, 454, 453, 455, + 455, 456, 458, 458, 455, 463, 455, 456, 462, 475, + 485, 464, 462, 467, 468, 463, 472, 474, 494, 488, + 489, 0, 484, 0, 0, 491, 479, 493, 481, 495, + 494, 497, 481, 499, 483, 501, 485, 489, 491, 492, + 0, 498, 499, 0, 489, 509, 507, 492, 512, 510, + 495, 496, 498, 502, 525, 504, 508, 509, 503, 505, + + 524, 525, 0, 526, 514, 528, 516, 528, 524, 521, + 533, 517, 535, 519, 524, 525, 0, 531, 532, 522, + 542, 540, 525, 545, 543, 544, 544, 541, 542, 548, + 548, 538, 0, 535, 542, 539, 539, 554, 555, 539, + 544, 545, 559, 547, 562, 549, 564, 551, 565, 552, + 0, 563, 558, 565, 560, 562, 0, 0, 574, 575, + 574, 562, 579, 577, 565, 582, 576, 577, 567, 0, + 572, 0, 584, 585, 0, 0, 573, 574, 575, 590, + 577, 592, 592, 580, 0, 590, 585, 592, 587, 0, + 0, 600, 601, 600, 588, 605, 603, 591, 608, 602, + + 594, 599, 600, 0, 0, 597, 611, 613, 0, 598, + 604, 605, 616, 618, 619, 604, 600, 625, 602, 627, + 609, 0, 611, 617, 619, 619, 621, 641, 635, 636, + 624, 614, 615, 627, 617, 618, 630, 631, 645, 629, + 633, 634, 646, 647, 627, 652, 629, 654, 0, 641, + 643, 645, 645, 647, 660, 661, 649, 639, 640, 652, + 642, 643, 655, 0, 663, 664, 663, 655, 673, 670, + 655, 656, 660, 0, 0, 0, 0, 661, 0, 662, + 660, 659, 663, 669, 665, 671, 671, 669, 670, 690, + 0, 0, 691, 0, 0, 686, 687, 675, 687, 676, + + 677, 0, 0, 0, 681, 0, 682, 680, 682, 688, + 684, 690, 686, 687, 707, 0, 0, 708, 0, 0, + 709, 692, 693, 698, 720, 697, 698, 697, 698, 700, + 695, 696, 706, 708, 719, 705, 721, 707, 727, 708, + 721, 722, 718, 719, 715, 716, 731, 722, 718, 719, + 715, 716, 735, 738, 724, 740, 726, 738, 739, 735, + 736, 731, 0, 0, 734, 739, 729, 0, 0, 0, + 746, 0, 0, 0, 738, 743, 749, 745, 751, 742, + 747, 748, 749, 762, 763, 0, 0, 0, 749, 0, + 0, 0, 0, 760, 755, 761, 757, 763, 758, 759, + + 772, 773, 762, 769, 778, 0, 765, 777, 781, 768, + 783, 770, 767, 769, 774, 775, 785, 786, 783, 1381, + 792, 779, 794, 781, 783, 784, 794, 795, 783, 782, + 790, 790, 0, 791, 792, 793, 794, 786, 789, 0, + 0, 0, 0, 791, 798, 799, 800, 801, 0, 0, + 0, 0, 0, 791, 812, 0, 815, 0, 816, 0, + 805, 808, 797, 820, 0, 821, 0, 0, 0, 820, + 821, 809, 0, 0, 823, 824, 0, 0, 826, 0, + 0, 0, 1381, 844, 848, 852, 856, 855, 860, 864, + 863, 868, 872, 871, 876, 880 + } ; -static const flex_int16_t yy_def[1265] = +static const flex_int16_t yy_def[1297] = { 0, - 1251, 1, 1251, 3, 1251, 5, 1252, 1252, 1253, 1253, - 1251, 1251, 1251, 1251, 1254, 1255, 1251, 1251, 1256, 1256, - 1256, 1256, 1256, 1256, 1251, 1251, 1251, 1251, 1257, 1258, - 1251, 1251, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1251, 1251, 1251, 1251, 1260, 1261, 1251, 1251, 1251, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1251, - 1251, 1263, 1251, 1251, 1264, 1251, 1251, 1254, 1251, 1255, - 1251, 1251, 1256, 1256, 1256, 1256, 1256, 1256, 1256, 1256, - - 1251, 1257, 1251, 1258, 1251, 1251, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1251, 1260, 1251, 1261, 1251, - 1251, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1263, 1251, 1264, 1256, 1256, 1256, - - 1256, 1256, 1256, 1256, 1256, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1256, 1256, 1256, - 1256, 1256, 1256, 1256, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1256, 1256, 1256, 1256, 1256, 1256, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1256, 1256, - 1256, 1256, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1256, 1256, 1256, 1256, 1256, 1256, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1256, - 1256, 1256, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1251, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1256, 1256, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1251, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1256, 1256, 1259, 1259, 1259, 1259, - - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1251, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1256, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - - 1259, 1259, 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1251, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1256, 1256, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1251, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1251, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1256, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1251, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1256, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1262, 1251, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1259, 1259, 1259, - - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1256, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1262, 1262, 1262, 1262, 1262, 1259, 1259, 1259, 1259, 1259, - 1259, 1262, 1262, 1262, 1259, 1259, 1259, 1262, 1262, 1259, - 0, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251 + 1283, 1, 1283, 3, 1283, 5, 1284, 1284, 1285, 1285, + 1283, 1283, 1283, 1283, 1286, 1287, 1283, 1283, 1288, 1288, + 1288, 1288, 1288, 1288, 1283, 1283, 1283, 1283, 1289, 1290, + 1283, 1283, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1283, 1283, 1283, 1283, 1292, 1293, 1283, 1283, 1283, + 1283, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1283, 1283, 1295, 1283, 1283, 1296, 1283, 1283, 1286, + 1283, 1287, 1283, 1283, 1288, 1288, 1288, 1288, 1288, 1288, + + 1288, 1288, 1283, 1289, 1283, 1290, 1283, 1283, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1283, 1292, + 1283, 1293, 1283, 1283, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1295, + + 1283, 1296, 1288, 1288, 1288, 1288, 1288, 1288, 1288, 1288, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1288, 1288, 1288, 1288, + 1288, 1288, 1288, 1288, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1288, 1288, 1288, 1288, + 1288, 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1288, 1288, 1288, 1288, 1288, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1288, 1288, 1288, 1288, + 1288, 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + + 1291, 1291, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1288, 1288, 1288, 1288, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + + 1291, 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1283, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1288, 1288, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1283, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1288, 1288, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1283, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1288, + + 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1283, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1288, 1288, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1283, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1283, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1283, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1294, 1283, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1288, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1294, 1294, 1294, 1294, 1294, 1291, 1291, 1291, + 1291, 1291, 1291, 1294, 1294, 1294, 1291, 1291, 1291, 1294, + 1294, 1291, 0, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283, 1283, 1283, 1283, 1283 + } ; -static const flex_int16_t yy_nxt[1389] = +static const flex_int16_t yy_nxt[1423] = { 0, - 12, 13, 14, 13, 15, 16, 12, 12, 17, 18, - 19, 19, 19, 19, 19, 20, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 21, 22, 19, 19, 19, - 19, 23, 24, 19, 19, 19, 19, 19, 25, 12, - 26, 27, 28, 27, 29, 30, 26, 26, 31, 32, - 33, 33, 33, 34, 35, 36, 37, 38, 39, 40, - 33, 41, 42, 33, 43, 44, 45, 33, 46, 33, - 47, 48, 49, 33, 50, 51, 33, 33, 52, 53, - 26, 54, 55, 54, 56, 57, 58, 26, 59, 60, - 61, 61, 61, 62, 61, 63, 64, 65, 66, 67, - - 61, 68, 69, 70, 71, 72, 73, 61, 74, 61, - 75, 76, 77, 78, 61, 79, 61, 61, 80, 81, - 83, 84, 83, 84, 87, 91, 87, 94, 92, 98, - 95, 101, 105, 101, 108, 106, 109, 113, 120, 110, - 117, 111, 123, 124, 99, 114, 150, 129, 92, 151, - 783, 115, 121, 118, 116, 106, 126, 131, 133, 144, - 127, 130, 137, 132, 128, 146, 138, 146, 156, 151, - 158, 784, 134, 145, 159, 135, 139, 140, 141, 153, - 160, 154, 157, 161, 155, 162, 164, 169, 167, 175, - 192, 163, 170, 182, 189, 190, 235, 193, 194, 176, - - 165, 168, 177, 184, 166, 236, 178, 185, 179, 183, - 186, 87, 219, 87, 91, 187, 180, 92, 188, 181, - 202, 211, 101, 203, 101, 105, 220, 212, 106, 216, - 226, 222, 240, 217, 242, 785, 241, 92, 223, 146, - 218, 146, 227, 243, 150, 256, 263, 151, 106, 258, - 259, 257, 266, 282, 268, 274, 275, 279, 293, 786, - 264, 288, 298, 267, 269, 289, 299, 151, 280, 283, - 351, 290, 787, 352, 362, 294, 300, 367, 363, 397, - 435, 788, 398, 368, 402, 436, 403, 442, 459, 464, - 483, 404, 443, 460, 465, 484, 488, 509, 515, 523, - - 789, 489, 510, 516, 524, 544, 790, 546, 545, 461, - 547, 568, 625, 571, 569, 575, 572, 570, 511, 573, - 576, 595, 597, 619, 596, 598, 620, 622, 637, 621, - 623, 642, 626, 624, 667, 791, 643, 688, 715, 668, - 644, 792, 689, 716, 638, 764, 690, 793, 794, 795, - 765, 796, 797, 798, 799, 799, 799, 800, 801, 802, - 804, 805, 806, 807, 803, 808, 809, 810, 811, 812, - 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, - 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, - 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, - - 843, 844, 846, 848, 845, 847, 849, 850, 851, 852, - 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, - 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, - 873, 874, 875, 876, 877, 799, 799, 799, 879, 880, - 882, 884, 881, 883, 885, 886, 887, 888, 889, 890, - 891, 892, 893, 894, 895, 896, 897, 898, 899, 900, - 901, 902, 903, 904, 905, 906, 878, 907, 908, 909, - 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, + 12, 13, 14, 13, 15, 16, 12, 12, 12, 17, + 18, 19, 19, 19, 19, 19, 20, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 21, 22, 19, 19, + 19, 19, 23, 24, 19, 19, 19, 19, 19, 25, + 12, 26, 27, 28, 27, 29, 30, 26, 26, 26, + 31, 32, 33, 33, 33, 34, 35, 36, 37, 38, + 39, 40, 33, 41, 42, 33, 43, 44, 45, 33, + 46, 33, 47, 48, 49, 33, 50, 51, 33, 33, + 52, 53, 26, 54, 55, 54, 56, 57, 58, 26, + 59, 60, 61, 62, 62, 62, 63, 64, 65, 66, + + 67, 68, 69, 62, 70, 71, 72, 73, 74, 75, + 62, 76, 62, 77, 78, 79, 80, 62, 81, 62, + 62, 82, 83, 85, 86, 85, 86, 89, 93, 89, + 96, 94, 100, 97, 103, 107, 103, 110, 108, 111, + 116, 112, 113, 120, 114, 123, 788, 101, 117, 126, + 127, 94, 132, 136, 118, 134, 121, 119, 108, 124, + 129, 135, 147, 161, 130, 140, 133, 137, 131, 141, + 138, 149, 153, 149, 789, 154, 148, 162, 172, 142, + 143, 144, 156, 167, 157, 169, 158, 159, 163, 168, + 180, 173, 164, 187, 89, 154, 89, 174, 165, 170, + + 181, 166, 175, 171, 182, 194, 195, 189, 183, 188, + 184, 190, 93, 790, 191, 94, 197, 207, 185, 192, + 208, 186, 193, 198, 199, 103, 107, 103, 217, 108, + 225, 228, 222, 232, 218, 94, 223, 241, 229, 248, + 266, 267, 246, 224, 226, 233, 247, 242, 249, 108, + 149, 153, 149, 264, 154, 271, 274, 793, 276, 265, + 282, 283, 287, 290, 794, 795, 296, 275, 277, 272, + 297, 301, 380, 288, 154, 302, 298, 307, 381, 291, + 313, 308, 362, 314, 410, 363, 374, 411, 303, 450, + 375, 309, 416, 796, 417, 451, 457, 474, 479, 418, + + 500, 505, 458, 475, 480, 527, 501, 506, 533, 541, + 563, 528, 797, 564, 534, 542, 798, 565, 587, 476, + 566, 588, 590, 594, 589, 591, 647, 529, 592, 595, + 616, 618, 641, 617, 619, 642, 644, 660, 643, 645, + 799, 665, 646, 690, 713, 800, 648, 666, 801, 691, + 714, 667, 740, 661, 715, 791, 802, 803, 741, 804, + 805, 792, 806, 807, 808, 810, 811, 812, 813, 814, + 815, 816, 817, 818, 819, 820, 821, 822, 809, 823, + 824, 825, 826, 827, 828, 829, 829, 829, 830, 831, + 832, 834, 835, 836, 837, 838, 833, 839, 840, 841, + + 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, + 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, + 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, + 872, 873, 874, 876, 878, 875, 877, 879, 880, 881, + 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, + 892, 893, 894, 895, 896, 897, 898, 899, 900, 901, + 902, 903, 904, 905, 906, 907, 908, 829, 829, 829, + 910, 911, 913, 915, 912, 914, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, - 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, - - 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, - 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, - 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, - 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, - 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, - 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, - 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, - 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, - 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, - 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, - - 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049, - 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, - 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, - 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, - 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, - 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, - 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, - 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, - 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, - 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, - - 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, - 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159, - 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, - 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, - 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, - 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, - 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, - 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, - 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, - 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, - - 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, 1248, 1249, - 1250, 82, 82, 82, 82, 85, 85, 85, 85, 88, - 88, 88, 88, 90, 90, 93, 90, 102, 102, 102, - 102, 104, 104, 107, 104, 147, 147, 147, 147, 149, - 149, 152, 149, 195, 782, 781, 195, 197, 197, 780, - 197, 779, 778, 777, 776, 775, 774, 773, 772, 771, - 770, 769, 768, 767, 766, 763, 762, 761, 760, 759, + 930, 931, 932, 933, 934, 935, 936, 937, 938, 909, + + 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, + 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, + 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, + 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, + 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, + 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, + 999, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, + 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, + 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, + 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, + + 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, + 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, + 1059, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, + 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1078, + 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, + 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, + 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, + 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, + 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, + 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, + + 1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, + 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, + 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, + 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, + 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, + 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, + 1199, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, + 1209, 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, + 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, + 1229, 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, + + 1239, 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, 1248, + 1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258, + 1259, 1260, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, + 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, + 1279, 1280, 1281, 1282, 84, 84, 84, 84, 87, 87, + 87, 87, 90, 90, 90, 90, 92, 92, 95, 92, + 104, 104, 104, 104, 106, 106, 109, 106, 150, 150, + 150, 150, 152, 152, 155, 152, 200, 787, 786, 200, + 202, 202, 785, 202, 784, 783, 782, 781, 780, 779, + 778, 777, 776, 775, 774, 773, 772, 771, 770, 769, + + 768, 767, 766, 765, 764, 763, 762, 761, 760, 759, 758, 757, 756, 755, 754, 753, 752, 751, 750, 749, - 748, 747, 746, 745, 744, 743, 742, 741, 740, 739, - 738, 737, 736, 735, 734, 733, 732, 731, 730, 729, - - 728, 727, 726, 725, 724, 723, 722, 721, 720, 719, - 718, 717, 714, 713, 712, 711, 710, 709, 708, 707, - 706, 705, 704, 703, 702, 701, 700, 699, 698, 697, - 696, 695, 694, 693, 692, 691, 687, 686, 685, 684, - 683, 682, 681, 680, 679, 678, 677, 676, 675, 674, - 673, 672, 671, 670, 669, 666, 665, 664, 663, 662, - 661, 660, 659, 658, 657, 656, 655, 654, 653, 652, - 651, 650, 649, 648, 647, 646, 645, 641, 640, 639, - 636, 635, 634, 633, 632, 631, 630, 629, 628, 627, - 618, 617, 616, 615, 614, 613, 612, 611, 610, 609, - - 608, 607, 606, 605, 604, 603, 602, 601, 600, 599, - 594, 593, 592, 591, 590, 589, 588, 587, 586, 585, - 584, 583, 582, 581, 580, 579, 578, 577, 574, 567, - 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, - 556, 555, 554, 553, 552, 551, 550, 549, 548, 543, - 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, - 532, 531, 530, 529, 528, 527, 526, 525, 522, 521, - 520, 519, 518, 517, 514, 513, 512, 508, 507, 506, - 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, - 495, 494, 493, 492, 491, 490, 487, 486, 485, 482, - - 481, 480, 479, 478, 477, 476, 475, 474, 473, 472, - 471, 470, 469, 468, 467, 466, 463, 462, 458, 457, - 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, - 446, 445, 444, 441, 440, 439, 438, 437, 434, 433, + 748, 747, 746, 745, 744, 743, 742, 739, 738, 737, + 736, 735, 734, 733, 732, 731, 730, 729, 728, 727, + 726, 725, 724, 723, 722, 721, 720, 719, 718, 717, + 716, 712, 711, 710, 709, 708, 707, 706, 705, 704, + 703, 702, 701, 700, 699, 698, 697, 696, 695, 694, + 693, 692, 689, 688, 687, 686, 685, 684, 683, 682, + 681, 680, 679, 678, 677, 676, 675, 674, 673, 672, + 671, 670, 669, 668, 664, 663, 662, 659, 658, 657, + + 656, 655, 654, 653, 652, 651, 650, 649, 640, 639, + 638, 637, 636, 635, 634, 633, 632, 631, 630, 629, + 628, 627, 626, 625, 624, 623, 622, 621, 620, 615, + 614, 613, 612, 611, 610, 609, 608, 607, 606, 605, + 604, 603, 602, 601, 600, 599, 598, 597, 596, 593, + 586, 585, 584, 583, 582, 581, 580, 579, 578, 577, + 576, 575, 574, 573, 572, 571, 570, 569, 568, 567, + 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, + 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, + 540, 539, 538, 537, 536, 535, 532, 531, 530, 526, + + 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, + 515, 514, 513, 512, 511, 510, 509, 508, 507, 504, + 503, 502, 499, 498, 497, 496, 495, 494, 493, 492, + 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, + 481, 478, 477, 473, 472, 471, 470, 469, 468, 467, + 466, 465, 464, 463, 462, 461, 460, 459, 456, 455, + 454, 453, 452, 449, 448, 447, 446, 445, 444, 443, + 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, - 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, - 412, 411, 410, 409, 408, 407, 406, 405, 401, 400, - 399, 396, 395, 394, 393, 392, 391, 390, 389, 388, - 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, - 377, 376, 375, 374, 373, 372, 371, 370, 369, 366, - - 365, 364, 361, 360, 359, 358, 357, 356, 355, 354, - 353, 350, 349, 348, 347, 346, 345, 344, 343, 342, + 422, 421, 420, 419, 415, 414, 413, 412, 409, 408, + + 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, + 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, + 387, 386, 385, 384, 383, 382, 379, 378, 377, 376, + 373, 372, 371, 370, 369, 368, 367, 366, 365, 364, + 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, + 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, - 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, - 311, 310, 309, 308, 307, 196, 306, 305, 304, 303, - 302, 301, 297, 296, 295, 292, 291, 287, 286, 285, - 284, 281, 278, 277, 276, 273, 272, 271, 270, 265, - 262, 261, 260, 255, 254, 253, 148, 252, 251, 250, - 249, 248, 247, 246, 245, 244, 239, 238, 237, 234, - - 233, 232, 231, 230, 229, 228, 225, 224, 221, 215, - 214, 213, 210, 209, 208, 207, 206, 103, 205, 204, - 201, 200, 199, 198, 89, 196, 191, 174, 173, 172, - 171, 148, 143, 142, 136, 125, 122, 119, 112, 103, - 100, 97, 96, 89, 1251, 86, 86, 11, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251 + 321, 320, 319, 318, 317, 201, 316, 315, 312, 311, + 310, 306, 305, 304, 300, 299, 295, 294, 293, 292, + + 289, 286, 285, 284, 281, 280, 279, 278, 273, 270, + 269, 268, 263, 262, 261, 260, 259, 151, 258, 257, + 256, 255, 254, 253, 252, 251, 250, 245, 244, 243, + 240, 239, 238, 237, 236, 235, 234, 231, 230, 227, + 221, 220, 219, 216, 215, 214, 213, 212, 211, 105, + 210, 209, 206, 205, 204, 203, 91, 201, 196, 179, + 178, 177, 176, 160, 151, 146, 145, 139, 128, 125, + 122, 115, 105, 102, 99, 98, 91, 1283, 88, 88, + 11, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283 } ; -static const flex_int16_t yy_chk[1389] = +static const flex_int16_t yy_chk[1423] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 7, 7, 8, 8, 13, 17, 13, 20, 17, 23, - 20, 27, 31, 27, 34, 31, 34, 36, 39, 34, - 37, 34, 41, 41, 23, 36, 59, 44, 17, 59, - 684, 36, 39, 37, 36, 31, 43, 45, 46, 51, - 43, 44, 48, 45, 43, 54, 48, 54, 63, 59, - 64, 687, 46, 51, 64, 46, 48, 48, 48, 62, - 64, 62, 63, 64, 62, 65, 66, 68, 67, 73, - 79, 65, 68, 75, 77, 77, 131, 79, 79, 73, - - 66, 67, 74, 76, 66, 131, 74, 76, 74, 75, - 76, 87, 118, 87, 91, 76, 74, 91, 76, 74, - 98, 113, 101, 98, 101, 105, 118, 113, 105, 117, - 123, 120, 135, 117, 136, 688, 135, 91, 120, 146, - 117, 146, 123, 136, 150, 156, 161, 150, 105, 157, - 157, 156, 163, 176, 164, 169, 169, 174, 184, 689, - 161, 181, 188, 163, 164, 181, 188, 150, 174, 176, - 243, 181, 691, 243, 255, 184, 188, 259, 255, 291, - 331, 692, 291, 259, 298, 331, 298, 337, 355, 359, - 378, 298, 337, 355, 359, 378, 383, 406, 410, 419, - - 694, 383, 406, 410, 419, 442, 695, 443, 442, 355, - 443, 464, 518, 465, 464, 467, 465, 464, 406, 465, - 467, 488, 489, 515, 488, 489, 515, 516, 529, 515, - 516, 534, 518, 516, 560, 696, 534, 584, 611, 560, - 534, 697, 584, 611, 529, 664, 584, 698, 699, 700, - 664, 701, 702, 703, 707, 707, 707, 709, 710, 711, - 715, 716, 717, 718, 711, 719, 723, 724, 725, 726, - 727, 728, 729, 730, 733, 734, 735, 736, 737, 738, - 739, 740, 741, 742, 744, 745, 747, 748, 749, 750, - 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, - - 761, 764, 765, 766, 764, 765, 767, 768, 769, 770, - 771, 772, 773, 774, 775, 776, 777, 778, 779, 780, - 781, 782, 783, 785, 786, 787, 789, 791, 792, 793, - 794, 795, 796, 797, 798, 799, 799, 799, 800, 802, - 803, 804, 802, 803, 805, 806, 807, 808, 809, 810, - 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, - 821, 822, 823, 824, 825, 826, 799, 827, 828, 829, - 830, 831, 833, 836, 837, 838, 839, 840, 841, 842, - 843, 844, 845, 846, 847, 848, 849, 850, 852, 853, - 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, - - 865, 866, 867, 868, 869, 870, 871, 873, 874, 875, - 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, - 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, - 897, 898, 899, 900, 901, 903, 904, 905, 906, 907, - 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, - 918, 919, 921, 922, 923, 924, 925, 928, 929, 930, - 931, 932, 933, 934, 935, 936, 937, 938, 939, 941, - 942, 945, 946, 947, 948, 949, 950, 951, 952, 954, - 955, 956, 957, 960, 961, 962, 963, 964, 965, 966, - 967, 968, 969, 970, 971, 974, 975, 976, 978, 979, - - 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, - 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000, - 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, - 1011, 1012, 1013, 1014, 1015, 1016, 1018, 1019, 1020, 1021, - 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, - 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1046, - 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1055, 1056, - 1057, 1058, 1061, 1064, 1065, 1066, 1067, 1068, 1069, 1073, - 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1086, - 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, - - 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, - 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, - 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, - 1129, 1130, 1133, 1134, 1135, 1139, 1143, 1144, 1145, 1146, - 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1157, 1162, 1163, - 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, - 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, 1184, - 1185, 1186, 1187, 1189, 1190, 1191, 1192, 1193, 1194, 1195, - 1196, 1197, 1198, 1199, 1200, 1202, 1203, 1204, 1205, 1206, - 1207, 1212, 1213, 1214, 1215, 1216, 1222, 1223, 1225, 1227, - - 1229, 1230, 1231, 1232, 1234, 1238, 1239, 1240, 1243, 1244, - 1247, 1252, 1252, 1252, 1252, 1253, 1253, 1253, 1253, 1254, - 1254, 1254, 1254, 1255, 1255, 1256, 1255, 1257, 1257, 1257, - 1257, 1258, 1258, 1259, 1258, 1260, 1260, 1260, 1260, 1261, - 1261, 1262, 1261, 1263, 683, 682, 1263, 1264, 1264, 681, - 1264, 680, 679, 678, 677, 676, 675, 674, 673, 671, - 670, 669, 668, 667, 666, 663, 662, 661, 660, 659, - 658, 657, 656, 655, 654, 653, 652, 651, 650, 649, - 648, 647, 645, 643, 642, 641, 640, 639, 638, 637, - 636, 635, 634, 633, 632, 631, 630, 629, 626, 625, - - 624, 623, 622, 621, 620, 619, 618, 617, 616, 615, - 614, 612, 610, 609, 608, 607, 606, 605, 604, 603, - 602, 601, 600, 599, 598, 597, 596, 595, 593, 592, - 591, 590, 588, 587, 586, 585, 583, 582, 581, 580, - 579, 578, 576, 575, 573, 572, 571, 570, 569, 568, - 566, 565, 564, 563, 561, 559, 557, 556, 555, 554, - 553, 552, 551, 549, 547, 546, 545, 544, 543, 542, - 541, 540, 539, 538, 537, 536, 535, 533, 532, 530, - 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, - 513, 512, 511, 510, 509, 504, 503, 502, 501, 500, - - 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, - 487, 485, 484, 483, 482, 481, 480, 478, 477, 476, - 475, 474, 473, 472, 471, 470, 469, 468, 466, 463, - 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, - 452, 451, 450, 449, 448, 447, 446, 445, 444, 441, - 440, 439, 437, 436, 435, 433, 432, 431, 430, 429, - 428, 427, 426, 425, 424, 423, 422, 421, 418, 416, - 415, 414, 413, 411, 409, 408, 407, 404, 403, 402, - 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, - 391, 390, 389, 388, 387, 386, 382, 380, 379, 377, - - 376, 375, 374, 373, 372, 370, 369, 368, 367, 366, - 365, 364, 363, 362, 361, 360, 358, 356, 354, 353, - 352, 351, 350, 349, 348, 346, 345, 343, 342, 341, - 340, 339, 338, 336, 335, 334, 333, 332, 330, 329, - 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, - 318, 317, 316, 315, 314, 312, 311, 309, 308, 307, - 306, 305, 304, 303, 302, 301, 300, 299, 296, 295, - 293, 290, 289, 288, 287, 286, 285, 284, 282, 281, - 280, 279, 278, 277, 276, 275, 274, 273, 270, 269, - 268, 267, 266, 265, 264, 263, 262, 261, 260, 258, - - 257, 256, 254, 252, 251, 250, 249, 248, 247, 245, - 244, 242, 241, 240, 239, 238, 237, 236, 235, 234, - 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, - 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, - 213, 212, 211, 210, 209, 208, 207, 205, 204, 203, - 202, 201, 200, 199, 198, 196, 194, 193, 192, 191, - 190, 189, 187, 186, 185, 183, 182, 180, 179, 178, - 177, 175, 173, 172, 171, 168, 167, 166, 165, 162, - 160, 159, 158, 155, 154, 153, 147, 145, 144, 143, - 142, 141, 140, 139, 138, 137, 134, 133, 132, 130, - - 129, 128, 127, 126, 125, 124, 122, 121, 119, 116, - 115, 114, 112, 111, 110, 109, 108, 102, 100, 99, - 97, 96, 95, 94, 88, 83, 78, 72, 71, 70, - 69, 56, 50, 49, 47, 42, 40, 38, 35, 29, - 24, 22, 21, 15, 11, 10, 9, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251 + 5, 5, 5, 7, 7, 8, 8, 13, 17, 13, + 20, 17, 23, 20, 27, 31, 27, 34, 31, 34, + 36, 34, 34, 37, 34, 39, 684, 23, 36, 41, + 41, 17, 44, 46, 36, 45, 37, 36, 31, 39, + 43, 45, 51, 65, 43, 48, 44, 46, 43, 48, + 46, 54, 60, 54, 685, 60, 51, 65, 69, 48, + 48, 48, 63, 67, 63, 68, 63, 63, 66, 67, + 75, 69, 66, 77, 89, 60, 89, 70, 66, 68, + + 75, 66, 70, 68, 76, 79, 79, 78, 76, 77, + 76, 78, 93, 686, 78, 93, 81, 100, 76, 78, + 100, 76, 78, 81, 81, 103, 107, 103, 116, 107, + 121, 123, 120, 126, 116, 93, 120, 134, 123, 139, + 162, 162, 138, 120, 121, 126, 138, 134, 139, 107, + 149, 153, 149, 161, 153, 166, 168, 689, 169, 161, + 174, 174, 179, 181, 690, 691, 186, 168, 169, 166, + 186, 189, 267, 179, 153, 189, 186, 193, 267, 181, + 197, 193, 249, 197, 299, 249, 262, 299, 189, 342, + 262, 193, 307, 692, 307, 342, 348, 366, 370, 307, + + 391, 396, 348, 366, 370, 420, 391, 396, 424, 433, + 457, 420, 693, 457, 424, 433, 694, 458, 479, 366, + 458, 479, 480, 482, 479, 480, 536, 420, 480, 482, + 505, 506, 533, 505, 506, 533, 534, 548, 533, 534, + 696, 553, 534, 579, 605, 697, 536, 553, 698, 579, + 605, 553, 632, 548, 605, 687, 699, 700, 632, 701, + 702, 687, 703, 704, 705, 706, 707, 708, 709, 712, + 713, 714, 716, 717, 719, 720, 721, 722, 705, 723, + 724, 725, 726, 727, 728, 732, 732, 732, 734, 735, + 736, 740, 741, 742, 744, 745, 736, 749, 750, 751, + + 752, 753, 754, 755, 756, 759, 760, 762, 763, 764, + 765, 766, 767, 768, 769, 771, 772, 774, 775, 776, + 777, 778, 779, 780, 781, 782, 783, 784, 785, 786, + 787, 788, 791, 792, 793, 791, 792, 794, 795, 796, + 797, 798, 799, 800, 801, 802, 803, 804, 805, 806, + 807, 810, 811, 812, 813, 815, 816, 817, 819, 821, + 822, 823, 824, 825, 826, 827, 828, 829, 829, 829, + 830, 832, 833, 834, 832, 833, 835, 836, 837, 838, + 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, + 849, 850, 851, 852, 853, 854, 855, 856, 857, 829, + + 858, 859, 860, 861, 863, 866, 867, 868, 869, 870, + 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, + 882, 883, 885, 886, 887, 888, 889, 890, 891, 892, + 893, 894, 895, 896, 897, 898, 899, 900, 901, 902, + 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, + 914, 915, 916, 918, 919, 920, 921, 922, 923, 924, + 925, 926, 927, 928, 929, 930, 931, 932, 934, 935, + 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, + 946, 947, 948, 949, 950, 952, 953, 954, 955, 956, + 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, + + 969, 971, 973, 974, 977, 978, 979, 980, 981, 982, + 983, 984, 986, 987, 988, 989, 992, 993, 994, 995, + 996, 997, 998, 999, 1000, 1001, 1002, 1003, 1006, 1007, + 1008, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, + 1019, 1020, 1021, 1023, 1024, 1025, 1026, 1027, 1028, 1029, + 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, + 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1050, + 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, + 1061, 1062, 1063, 1065, 1066, 1067, 1068, 1069, 1070, 1071, + 1072, 1073, 1078, 1080, 1081, 1082, 1083, 1084, 1085, 1086, + + 1087, 1087, 1088, 1089, 1090, 1093, 1096, 1097, 1098, 1099, + 1100, 1101, 1105, 1107, 1108, 1109, 1110, 1111, 1112, 1113, + 1114, 1115, 1118, 1121, 1122, 1123, 1124, 1125, 1126, 1127, + 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, + 1138, 1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, + 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, + 1158, 1159, 1160, 1161, 1162, 1165, 1166, 1167, 1171, 1175, + 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, + 1189, 1194, 1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202, + 1203, 1204, 1205, 1207, 1208, 1209, 1210, 1211, 1212, 1213, + + 1214, 1215, 1216, 1217, 1218, 1219, 1221, 1222, 1223, 1224, + 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1234, 1235, + 1236, 1237, 1238, 1239, 1244, 1245, 1246, 1247, 1248, 1254, + 1255, 1257, 1259, 1261, 1262, 1263, 1264, 1266, 1270, 1271, + 1272, 1275, 1276, 1279, 1284, 1284, 1284, 1284, 1285, 1285, + 1285, 1285, 1286, 1286, 1286, 1286, 1287, 1287, 1288, 1287, + 1289, 1289, 1289, 1289, 1290, 1290, 1291, 1290, 1292, 1292, + 1292, 1292, 1293, 1293, 1294, 1293, 1295, 683, 682, 1295, + 1296, 1296, 681, 1296, 680, 679, 678, 677, 676, 675, + 674, 673, 672, 671, 670, 668, 666, 665, 664, 663, + + 662, 661, 660, 659, 658, 657, 656, 655, 654, 653, + 652, 651, 648, 647, 646, 645, 644, 643, 642, 641, + 640, 639, 638, 637, 636, 634, 633, 631, 630, 629, + 628, 627, 626, 625, 624, 623, 622, 621, 620, 619, + 618, 617, 616, 614, 613, 612, 611, 609, 608, 607, + 606, 604, 603, 602, 601, 600, 599, 598, 596, 595, + 594, 592, 591, 590, 589, 588, 587, 585, 584, 583, + 582, 580, 578, 576, 575, 574, 573, 572, 571, 570, + 568, 566, 565, 564, 563, 562, 561, 560, 559, 558, + 557, 556, 555, 554, 552, 551, 549, 547, 546, 545, + + 544, 543, 542, 541, 540, 539, 538, 537, 531, 530, + 529, 528, 527, 522, 521, 520, 519, 518, 517, 516, + 515, 514, 513, 512, 511, 510, 509, 508, 507, 504, + 502, 501, 500, 499, 498, 497, 495, 494, 493, 492, + 491, 490, 489, 488, 487, 486, 485, 484, 483, 481, + 478, 477, 476, 475, 474, 473, 472, 471, 470, 469, + 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, + 456, 455, 454, 452, 451, 450, 448, 447, 446, 445, + 444, 443, 442, 441, 440, 439, 438, 437, 436, 434, + 432, 430, 429, 428, 427, 425, 423, 422, 421, 418, + + 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, + 407, 406, 405, 404, 403, 402, 401, 400, 399, 395, + 393, 392, 390, 389, 388, 387, 386, 385, 383, 382, + 381, 380, 379, 378, 377, 376, 375, 374, 373, 372, + 371, 369, 367, 365, 364, 363, 362, 361, 360, 359, + 357, 356, 354, 353, 352, 351, 350, 349, 347, 346, + 345, 344, 343, 341, 340, 339, 338, 337, 336, 335, + 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, + 324, 322, 321, 319, 318, 317, 316, 315, 313, 312, + 311, 310, 309, 308, 305, 304, 302, 301, 298, 297, + + 296, 295, 294, 293, 292, 290, 289, 288, 287, 286, + 285, 284, 283, 282, 281, 278, 277, 276, 275, 274, + 273, 272, 271, 270, 269, 268, 266, 265, 264, 263, + 261, 260, 258, 257, 256, 255, 254, 253, 251, 250, + 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, + 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, + 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, + 218, 217, 216, 215, 214, 213, 212, 210, 209, 208, + 207, 206, 205, 204, 203, 201, 199, 198, 196, 195, + 194, 192, 191, 190, 188, 187, 185, 184, 183, 182, + + 180, 178, 177, 176, 173, 172, 171, 170, 167, 165, + 164, 163, 160, 159, 158, 157, 156, 150, 148, 147, + 146, 145, 144, 143, 142, 141, 140, 137, 136, 135, + 133, 132, 131, 130, 129, 128, 127, 125, 124, 122, + 119, 118, 117, 115, 114, 113, 112, 111, 110, 104, + 102, 101, 99, 98, 97, 96, 90, 85, 80, 74, + 73, 72, 71, 64, 56, 50, 49, 47, 42, 40, + 38, 35, 29, 24, 22, 21, 15, 11, 10, 9, + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283 } ; static yy_state_type yy_last_accepting_state; @@ -1237,9 +1259,9 @@ static char *pgaf_strdup(const char *s) * call flex's static input() function. */ static void pgaf_read_raw_block(void); -#line 1240 "test_spec_scan.c" +#line 1262 "test_spec_scan.c" -#line 1242 "test_spec_scan.c" +#line 1264 "test_spec_scan.c" #define INITIAL 0 #define CLUSTER_BODY 1 @@ -1461,7 +1483,7 @@ YY_DECL #line 89 "test_spec_scan.l" -#line 1464 "test_spec_scan.c" +#line 1486 "test_spec_scan.c" while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ { @@ -1488,13 +1510,13 @@ YY_DECL while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1252 ) + if ( yy_current_state >= 1284 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; ++yy_cp; } - while ( yy_current_state != 1251 ); + while ( yy_current_state != 1283 ); yy_cp = (yy_last_accepting_cpos); yy_current_state = (yy_last_accepting_state); @@ -1709,177 +1731,182 @@ YY_RULE_SETUP case 33: YY_RULE_SETUP #line 164 "test_spec_scan.l" -{ return T_ASYNC; } +{ return T_ARCHIVER; } YY_BREAK case 34: YY_RULE_SETUP #line 165 "test_spec_scan.l" -{ return T_NO_MONITOR; } +{ return T_ASYNC; } YY_BREAK case 35: YY_RULE_SETUP #line 166 "test_spec_scan.l" -{ return T_SUSPENDED; } +{ return T_NO_MONITOR; } YY_BREAK case 36: YY_RULE_SETUP #line 167 "test_spec_scan.l" -{ return T_PASSWORD; } +{ return T_SUSPENDED; } YY_BREAK case 37: YY_RULE_SETUP #line 168 "test_spec_scan.l" -{ return T_MONITOR_PASSWORD; } +{ return T_PASSWORD; } YY_BREAK case 38: YY_RULE_SETUP #line 169 "test_spec_scan.l" -{ return T_LAUNCH; } +{ return T_MONITOR_PASSWORD; } YY_BREAK case 39: YY_RULE_SETUP #line 170 "test_spec_scan.l" -{ return T_CREATE; } +{ return T_LAUNCH; } YY_BREAK case 40: YY_RULE_SETUP #line 171 "test_spec_scan.l" -{ return T_DEFERRED; } +{ return T_CREATE; } YY_BREAK case 41: YY_RULE_SETUP #line 172 "test_spec_scan.l" -{ return T_IMMEDIATE; } +{ return T_DEFERRED; } YY_BREAK case 42: YY_RULE_SETUP #line 173 "test_spec_scan.l" -{ return T_FALSE; } +{ return T_IMMEDIATE; } YY_BREAK case 43: YY_RULE_SETUP #line 174 "test_spec_scan.l" -{ return T_TRUE; } +{ return T_FALSE; } YY_BREAK case 44: YY_RULE_SETUP #line 175 "test_spec_scan.l" -{ return T_AND; } +{ return T_TRUE; } YY_BREAK case 45: YY_RULE_SETUP #line 176 "test_spec_scan.l" -{ return T_INITIALLY; } +{ return T_AND; } YY_BREAK case 46: YY_RULE_SETUP #line 177 "test_spec_scan.l" -{ return T_STOPPED; } +{ return T_INITIALLY; } YY_BREAK case 47: YY_RULE_SETUP #line 178 "test_spec_scan.l" -{ return T_VOLUME; } +{ return T_STOPPED; } YY_BREAK case 48: YY_RULE_SETUP #line 179 "test_spec_scan.l" -{ return T_LISTEN; } +{ return T_VOLUME; } YY_BREAK case 49: YY_RULE_SETUP #line 180 "test_spec_scan.l" -{ return T_CITUS_SECONDARY; } +{ return T_LISTEN; } YY_BREAK case 50: YY_RULE_SETUP #line 181 "test_spec_scan.l" -{ return T_CANDIDATE_PRIORITY; } +{ return T_CITUS_SECONDARY; } YY_BREAK case 51: YY_RULE_SETUP #line 182 "test_spec_scan.l" -{ return T_REGION; } +{ return T_CANDIDATE_PRIORITY; } YY_BREAK case 52: YY_RULE_SETUP #line 183 "test_spec_scan.l" -{ return T_GROUP; } +{ return T_REGION; } YY_BREAK case 53: YY_RULE_SETUP #line 184 "test_spec_scan.l" -{ return T_PORT; } +{ return T_GROUP; } YY_BREAK case 54: YY_RULE_SETUP #line 185 "test_spec_scan.l" -{ return T_CITUS_CLUSTER_NAME; } +{ return T_PORT; } YY_BREAK case 55: YY_RULE_SETUP #line 186 "test_spec_scan.l" -{ return T_DEBIAN_CLUSTER; } +{ return T_CITUS_CLUSTER_NAME; } YY_BREAK case 56: YY_RULE_SETUP #line 187 "test_spec_scan.l" -{ return T_REPLICATION_QUORUM; } +{ return T_DEBIAN_CLUSTER; } YY_BREAK case 57: YY_RULE_SETUP #line 188 "test_spec_scan.l" -{ return T_REPLICATION_PASSWORD; } +{ return T_REPLICATION_QUORUM; } YY_BREAK case 58: YY_RULE_SETUP #line 189 "test_spec_scan.l" -{ return T_EXTENSION_VERSION; } +{ return T_REPLICATION_PASSWORD; } YY_BREAK case 59: YY_RULE_SETUP #line 190 "test_spec_scan.l" -{ return T_BIND_SOURCE; } +{ return T_EXTENSION_VERSION; } YY_BREAK case 60: YY_RULE_SETUP #line 191 "test_spec_scan.l" -{ return T_LEGACY_STARTUP; } +{ return T_BIND_SOURCE; } YY_BREAK case 61: YY_RULE_SETUP -#line 193 "test_spec_scan.l" -{ return T_EQUALS; } +#line 192 "test_spec_scan.l" +{ return T_LEGACY_STARTUP; } YY_BREAK case 62: YY_RULE_SETUP -#line 195 "test_spec_scan.l" +#line 194 "test_spec_scan.l" +{ return T_EQUALS; } + YY_BREAK +case 63: +YY_RULE_SETUP +#line 196 "test_spec_scan.l" { yylval.ival = atoi(yytext); return T_INTEGER; } YY_BREAK -case 63: -/* rule 63 can match eol */ +case 64: +/* rule 64 can match eol */ YY_RULE_SETUP -#line 200 "test_spec_scan.l" +#line 201 "test_spec_scan.l" { yytext[yyleng - 1] = '\0'; yylval.str = pgaf_strdup(yytext + 1); return T_STRING; } YY_BREAK -case 64: +case 65: YY_RULE_SETUP -#line 206 "test_spec_scan.l" +#line 207 "test_spec_scan.l" { pgaf_cluster_depth++; return T_LBRACE; } YY_BREAK -case 65: +case 66: YY_RULE_SETUP -#line 211 "test_spec_scan.l" +#line 212 "test_spec_scan.l" { pgaf_cluster_depth--; if (pgaf_cluster_depth == 0) @@ -1887,25 +1914,20 @@ YY_RULE_SETUP return T_RBRACE; } YY_BREAK -case 66: -YY_RULE_SETUP -#line 218 "test_spec_scan.l" -{ return T_FS_INIT; } - YY_BREAK case 67: YY_RULE_SETUP #line 219 "test_spec_scan.l" -{ return T_FS_SINGLE; } +{ return T_FS_INIT; } YY_BREAK case 68: YY_RULE_SETUP #line 220 "test_spec_scan.l" -{ return T_FS_PRIMARY; } +{ return T_FS_SINGLE; } YY_BREAK case 69: YY_RULE_SETUP #line 221 "test_spec_scan.l" -{ return T_FS_WAIT_PRIMARY; } +{ return T_FS_PRIMARY; } YY_BREAK case 70: YY_RULE_SETUP @@ -1915,7 +1937,7 @@ YY_RULE_SETUP case 71: YY_RULE_SETUP #line 223 "test_spec_scan.l" -{ return T_FS_WAIT_STANDBY; } +{ return T_FS_WAIT_PRIMARY; } YY_BREAK case 72: YY_RULE_SETUP @@ -1925,12 +1947,12 @@ YY_RULE_SETUP case 73: YY_RULE_SETUP #line 225 "test_spec_scan.l" -{ return T_FS_DEMOTED; } +{ return T_FS_WAIT_STANDBY; } YY_BREAK case 74: YY_RULE_SETUP #line 226 "test_spec_scan.l" -{ return T_FS_DEMOTE_TIMEOUT; } +{ return T_FS_DEMOTED; } YY_BREAK case 75: YY_RULE_SETUP @@ -1940,22 +1962,22 @@ YY_RULE_SETUP case 76: YY_RULE_SETUP #line 228 "test_spec_scan.l" -{ return T_FS_DRAINING; } +{ return T_FS_DEMOTE_TIMEOUT; } YY_BREAK case 77: YY_RULE_SETUP #line 229 "test_spec_scan.l" -{ return T_FS_SECONDARY; } +{ return T_FS_DRAINING; } YY_BREAK case 78: YY_RULE_SETUP #line 230 "test_spec_scan.l" -{ return T_FS_CATCHINGUP; } +{ return T_FS_SECONDARY; } YY_BREAK case 79: YY_RULE_SETUP #line 231 "test_spec_scan.l" -{ return T_FS_PREP_PROMOTION; } +{ return T_FS_CATCHINGUP; } YY_BREAK case 80: YY_RULE_SETUP @@ -1965,7 +1987,7 @@ YY_RULE_SETUP case 81: YY_RULE_SETUP #line 233 "test_spec_scan.l" -{ return T_FS_STOP_REPLICATION; } +{ return T_FS_PREP_PROMOTION; } YY_BREAK case 82: YY_RULE_SETUP @@ -1975,12 +1997,12 @@ YY_RULE_SETUP case 83: YY_RULE_SETUP #line 235 "test_spec_scan.l" -{ return T_FS_MAINTENANCE; } +{ return T_FS_STOP_REPLICATION; } YY_BREAK case 84: YY_RULE_SETUP #line 236 "test_spec_scan.l" -{ return T_FS_JOIN_PRIMARY; } +{ return T_FS_MAINTENANCE; } YY_BREAK case 85: YY_RULE_SETUP @@ -1990,7 +2012,7 @@ YY_RULE_SETUP case 86: YY_RULE_SETUP #line 238 "test_spec_scan.l" -{ return T_FS_APPLY_SETTINGS; } +{ return T_FS_JOIN_PRIMARY; } YY_BREAK case 87: YY_RULE_SETUP @@ -2000,7 +2022,7 @@ YY_RULE_SETUP case 88: YY_RULE_SETUP #line 240 "test_spec_scan.l" -{ return T_FS_PREPARE_MAINTENANCE; } +{ return T_FS_APPLY_SETTINGS; } YY_BREAK case 89: YY_RULE_SETUP @@ -2010,7 +2032,7 @@ YY_RULE_SETUP case 90: YY_RULE_SETUP #line 242 "test_spec_scan.l" -{ return T_FS_WAIT_MAINTENANCE; } +{ return T_FS_PREPARE_MAINTENANCE; } YY_BREAK case 91: YY_RULE_SETUP @@ -2020,7 +2042,7 @@ YY_RULE_SETUP case 92: YY_RULE_SETUP #line 244 "test_spec_scan.l" -{ return T_FS_REPORT_LSN; } +{ return T_FS_WAIT_MAINTENANCE; } YY_BREAK case 93: YY_RULE_SETUP @@ -2030,7 +2052,7 @@ YY_RULE_SETUP case 94: YY_RULE_SETUP #line 246 "test_spec_scan.l" -{ return T_FS_FAST_FORWARD; } +{ return T_FS_REPORT_LSN; } YY_BREAK case 95: YY_RULE_SETUP @@ -2040,7 +2062,7 @@ YY_RULE_SETUP case 96: YY_RULE_SETUP #line 248 "test_spec_scan.l" -{ return T_FS_JOIN_SECONDARY; } +{ return T_FS_FAST_FORWARD; } YY_BREAK case 97: YY_RULE_SETUP @@ -2050,304 +2072,339 @@ YY_RULE_SETUP case 98: YY_RULE_SETUP #line 250 "test_spec_scan.l" -{ return T_FS_DROPPED; } +{ return T_FS_JOIN_SECONDARY; } YY_BREAK case 99: YY_RULE_SETUP -#line 252 "test_spec_scan.l" +#line 251 "test_spec_scan.l" +{ return T_FS_DROPPED; } + YY_BREAK +case 100: +YY_RULE_SETUP +#line 253 "test_spec_scan.l" { yylval.str = pgaf_strdup(yytext); return T_IDENT; } YY_BREAK -case 100: -YY_RULE_SETUP -#line 257 "test_spec_scan.l" -{ /* comment */ } - YY_BREAK case 101: -/* rule 101 can match eol */ YY_RULE_SETUP #line 258 "test_spec_scan.l" -{ pgaf_line_number++; } +{ /* comment */ } YY_BREAK case 102: +/* rule 102 can match eol */ YY_RULE_SETUP #line 259 "test_spec_scan.l" -{ /* whitespace */ } +{ pgaf_line_number++; } YY_BREAK case 103: YY_RULE_SETUP -#line 261 "test_spec_scan.l" -{ BEGIN(EXEC_ARGS); return T_EXEC_FAILS; } +#line 260 "test_spec_scan.l" +{ /* whitespace */ } YY_BREAK case 104: YY_RULE_SETUP #line 262 "test_spec_scan.l" -{ BEGIN(EXEC_ARGS); return T_EXEC; } +{ BEGIN(EXEC_ARGS); return T_EXEC_FAILS; } YY_BREAK case 105: YY_RULE_SETUP #line 263 "test_spec_scan.l" -{ BEGIN(EXEC_ARGS); return T_RUN; } +{ BEGIN(EXEC_ARGS); return T_EXEC; } YY_BREAK case 106: YY_RULE_SETUP #line 264 "test_spec_scan.l" -{ BEGIN(EXEC_ARGS); return T_PG_AUTOCTL; } +{ BEGIN(EXEC_ARGS); return T_RUN; } YY_BREAK case 107: YY_RULE_SETUP -#line 266 "test_spec_scan.l" -{ return T_WAIT; } +#line 265 "test_spec_scan.l" +{ BEGIN(EXEC_ARGS); return T_PG_AUTOCTL; } YY_BREAK case 108: YY_RULE_SETUP #line 267 "test_spec_scan.l" -{ return T_UNTIL; } +{ return T_WAIT; } YY_BREAK case 109: YY_RULE_SETUP #line 268 "test_spec_scan.l" -{ return T_REPLAYS; } +{ return T_UNTIL; } YY_BREAK case 110: YY_RULE_SETUP #line 269 "test_spec_scan.l" -{ return T_TIMEOUT; } +{ return T_REPLAYS; } YY_BREAK case 111: YY_RULE_SETUP #line 270 "test_spec_scan.l" -{ return T_ASSERT; } +{ return T_TIMEOUT; } YY_BREAK case 112: YY_RULE_SETUP #line 271 "test_spec_scan.l" -{ return T_SQL; } +{ return T_ASSERT; } YY_BREAK case 113: YY_RULE_SETUP #line 272 "test_spec_scan.l" -{ return T_EXPECT; } +{ return T_SQL; } YY_BREAK case 114: YY_RULE_SETUP #line 273 "test_spec_scan.l" -{ return T_ERROR; } +{ return T_WAL; } YY_BREAK case 115: YY_RULE_SETUP #line 274 "test_spec_scan.l" -{ return T_PROMOTE; } +{ return T_SEGMENT; } YY_BREAK case 116: YY_RULE_SETUP #line 275 "test_spec_scan.l" -{ return T_PERFORM; } +{ return T_ARCHIVED; } YY_BREAK case 117: YY_RULE_SETUP #line 276 "test_spec_scan.l" -{ return T_FAILOVER; } +{ return T_BASEBACKUP; } YY_BREAK case 118: YY_RULE_SETUP #line 277 "test_spec_scan.l" -{ return T_NETWORK; } +{ return T_ARCHIVER; } YY_BREAK case 119: YY_RULE_SETUP #line 278 "test_spec_scan.l" -{ return T_DISCONNECT; } +{ return T_SLASH; } YY_BREAK case 120: YY_RULE_SETUP #line 279 "test_spec_scan.l" -{ return T_CONNECT; } +{ return T_EXPECT; } YY_BREAK case 121: YY_RULE_SETUP #line 280 "test_spec_scan.l" -{ return T_SLEEP; } +{ return T_ERROR; } YY_BREAK case 122: YY_RULE_SETUP #line 281 "test_spec_scan.l" -{ return T_COMPOSE; } +{ return T_PROMOTE; } YY_BREAK case 123: YY_RULE_SETUP #line 282 "test_spec_scan.l" -{ return T_NODEINI; } +{ return T_PERFORM; } YY_BREAK case 124: YY_RULE_SETUP #line 283 "test_spec_scan.l" -{ return T_DOWN; } +{ return T_FAILOVER; } YY_BREAK case 125: YY_RULE_SETUP #line 284 "test_spec_scan.l" -{ return T_START; } +{ return T_NETWORK; } YY_BREAK case 126: YY_RULE_SETUP #line 285 "test_spec_scan.l" -{ return T_STOP; } +{ return T_DISCONNECT; } YY_BREAK case 127: YY_RULE_SETUP #line 286 "test_spec_scan.l" -{ return T_STOPPED; } +{ return T_CONNECT; } YY_BREAK case 128: YY_RULE_SETUP #line 287 "test_spec_scan.l" -{ return T_KILL; } +{ return T_SLEEP; } YY_BREAK case 129: YY_RULE_SETUP #line 288 "test_spec_scan.l" -{ return T_IN; } +{ return T_COMPOSE; } YY_BREAK case 130: YY_RULE_SETUP #line 289 "test_spec_scan.l" -{ return T_STATE; } +{ return T_NODEINI; } YY_BREAK case 131: YY_RULE_SETUP #line 290 "test_spec_scan.l" -{ return T_ASSIGNED_STATE; } +{ return T_DOWN; } YY_BREAK case 132: YY_RULE_SETUP #line 291 "test_spec_scan.l" -{ return T_CANDIDATE_PRIORITY; } +{ return T_START; } YY_BREAK case 133: YY_RULE_SETUP #line 292 "test_spec_scan.l" -{ return T_GROUP; } +{ return T_STOP; } YY_BREAK case 134: YY_RULE_SETUP #line 293 "test_spec_scan.l" -{ return T_AND; } +{ return T_STOPPED; } YY_BREAK case 135: YY_RULE_SETUP #line 294 "test_spec_scan.l" -{ return T_IS; } +{ return T_KILL; } YY_BREAK case 136: YY_RULE_SETUP #line 295 "test_spec_scan.l" -{ return T_WITH; } +{ return T_IN; } YY_BREAK case 137: YY_RULE_SETUP #line 296 "test_spec_scan.l" -{ return T_EQUALS; } +{ return T_STATE; } YY_BREAK case 138: YY_RULE_SETUP #line 297 "test_spec_scan.l" -{ return T_COMMA; } +{ return T_ASSIGNED_STATE; } YY_BREAK case 139: YY_RULE_SETUP #line 298 "test_spec_scan.l" -{ return T_POSTGRES; } +{ return T_CANDIDATE_PRIORITY; } YY_BREAK case 140: YY_RULE_SETUP #line 299 "test_spec_scan.l" -{ return T_FSM; } +{ return T_GROUP; } YY_BREAK case 141: YY_RULE_SETUP #line 300 "test_spec_scan.l" -{ return T_STEP; } +{ return T_AND; } YY_BREAK case 142: YY_RULE_SETUP #line 301 "test_spec_scan.l" -{ return T_STAYS; } +{ return T_IS; } YY_BREAK case 143: YY_RULE_SETUP #line 302 "test_spec_scan.l" -{ return T_WHILE; } +{ return T_WITH; } YY_BREAK case 144: -/* rule 144 can match eol */ YY_RULE_SETUP #line 303 "test_spec_scan.l" -{ return T_THROUGH; } +{ return T_EQUALS; } YY_BREAK case 145: YY_RULE_SETUP #line 304 "test_spec_scan.l" -{ return T_THROUGH; } +{ return T_COMMA; } YY_BREAK case 146: YY_RULE_SETUP #line 305 "test_spec_scan.l" -{ return T_SET; } +{ return T_POSTGRES; } YY_BREAK case 147: YY_RULE_SETUP #line 306 "test_spec_scan.l" -{ return T_GET; } +{ return T_FSM; } YY_BREAK case 148: YY_RULE_SETUP #line 307 "test_spec_scan.l" -{ BEGIN(EXEC_ARGS); return T_INJECT; } +{ return T_STEP; } YY_BREAK case 149: YY_RULE_SETUP #line 308 "test_spec_scan.l" -{ return T_LOGS; } +{ return T_STAYS; } YY_BREAK case 150: YY_RULE_SETUP #line 309 "test_spec_scan.l" -{ return T_NOT; } +{ return T_WHILE; } YY_BREAK case 151: +/* rule 151 can match eol */ YY_RULE_SETUP #line 310 "test_spec_scan.l" -{ return T_CONTAINS; } +{ return T_THROUGH; } YY_BREAK case 152: YY_RULE_SETUP #line 311 "test_spec_scan.l" -{ return T_MATCHES; } +{ return T_THROUGH; } YY_BREAK case 153: YY_RULE_SETUP +#line 312 "test_spec_scan.l" +{ return T_SET; } + YY_BREAK +case 154: +YY_RULE_SETUP #line 313 "test_spec_scan.l" +{ return T_GET; } + YY_BREAK +case 155: +YY_RULE_SETUP +#line 314 "test_spec_scan.l" +{ BEGIN(EXEC_ARGS); return T_INJECT; } + YY_BREAK +case 156: +YY_RULE_SETUP +#line 315 "test_spec_scan.l" +{ return T_LOGS; } + YY_BREAK +case 157: +YY_RULE_SETUP +#line 316 "test_spec_scan.l" +{ return T_NOT; } + YY_BREAK +case 158: +YY_RULE_SETUP +#line 317 "test_spec_scan.l" +{ return T_CONTAINS; } + YY_BREAK +case 159: +YY_RULE_SETUP +#line 318 "test_spec_scan.l" +{ return T_MATCHES; } + YY_BREAK +case 160: +YY_RULE_SETUP +#line 320 "test_spec_scan.l" { yylval.ival = atoi(yytext); return T_INTEGER; } YY_BREAK -case 154: -/* rule 154 can match eol */ +case 161: +/* rule 161 can match eol */ YY_RULE_SETUP -#line 318 "test_spec_scan.l" +#line 325 "test_spec_scan.l" { yytext[yyleng - 1] = '\0'; yylval.str = pgaf_strdup(yytext + 1); return T_STRING; } YY_BREAK -case 155: +case 162: YY_RULE_SETUP -#line 324 "test_spec_scan.l" +#line 331 "test_spec_scan.l" { if (pgaf_next_brace_is_while) { pgaf_next_brace_is_while = 0; @@ -2358,9 +2415,9 @@ YY_RULE_SETUP return T_BLOCK; } YY_BREAK -case 156: +case 163: YY_RULE_SETUP -#line 334 "test_spec_scan.l" +#line 341 "test_spec_scan.l" { if (pgaf_step_brace_depth > 0) { pgaf_step_brace_depth--; @@ -2371,40 +2428,40 @@ YY_RULE_SETUP return T_RBRACE; } YY_BREAK -case 157: +case 164: YY_RULE_SETUP -#line 344 "test_spec_scan.l" +#line 351 "test_spec_scan.l" { yylval.str = pgaf_strdup(yytext); return T_IDENT; } YY_BREAK -case 158: +case 165: YY_RULE_SETUP -#line 349 "test_spec_scan.l" +#line 356 "test_spec_scan.l" { /* skip whitespace before service name */ } YY_BREAK -case 159: +case 166: YY_RULE_SETUP -#line 351 "test_spec_scan.l" +#line 358 "test_spec_scan.l" { yylval.str = pgaf_strdup(yytext); BEGIN(EXEC_ARGS_REST); return T_IDENT; } YY_BREAK -case 160: -/* rule 160 can match eol */ +case 167: +/* rule 167 can match eol */ YY_RULE_SETUP -#line 357 "test_spec_scan.l" +#line 364 "test_spec_scan.l" { pgaf_line_number++; BEGIN(STEP_BODY); } YY_BREAK -case 161: +case 168: YY_RULE_SETUP -#line 362 "test_spec_scan.l" +#line 369 "test_spec_scan.l" { char *p = yytext; while (*p == ' ' || *p == '\t') p++; @@ -2413,21 +2470,21 @@ YY_RULE_SETUP return T_SHELL_ARGS; } YY_BREAK -case 162: -/* rule 162 can match eol */ +case 169: +/* rule 169 can match eol */ YY_RULE_SETUP -#line 370 "test_spec_scan.l" +#line 377 "test_spec_scan.l" { pgaf_line_number++; BEGIN(STEP_BODY); } YY_BREAK -case 163: +case 170: YY_RULE_SETUP -#line 375 "test_spec_scan.l" +#line 382 "test_spec_scan.l" ECHO; YY_BREAK -#line 2430 "test_spec_scan.c" +#line 2487 "test_spec_scan.c" case YY_STATE_EOF(INITIAL): case YY_STATE_EOF(CLUSTER_BODY): case YY_STATE_EOF(STEP_BODY): @@ -2729,7 +2786,7 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1252 ) + if ( yy_current_state >= 1284 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; @@ -2757,11 +2814,11 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1252 ) + if ( yy_current_state >= 1284 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; - yy_is_jam = (yy_current_state == 1251); + yy_is_jam = (yy_current_state == 1283); return yy_is_jam ? 0 : yy_current_state; } @@ -3400,7 +3457,7 @@ void yyfree (void * ptr ) #define YYTABLES_NAME "yytables" -#line 375 "test_spec_scan.l" +#line 382 "test_spec_scan.l" static void diff --git a/src/bin/pgaftest/test_spec_scan.l b/src/bin/pgaftest/test_spec_scan.l index c456617fe..51b06120f 100644 --- a/src/bin/pgaftest/test_spec_scan.l +++ b/src/bin/pgaftest/test_spec_scan.l @@ -161,6 +161,7 @@ static void pgaf_read_raw_block(void); "coordinator" { return T_COORDINATOR; } "worker" { return T_WORKER; } +"archiver" { return T_ARCHIVER; } "async" { return T_ASYNC; } "no-monitor" { return T_NO_MONITOR; } "suspended" { return T_SUSPENDED; } @@ -269,6 +270,12 @@ static void pgaf_read_raw_block(void); "timeout" { return T_TIMEOUT; } "assert" { return T_ASSERT; } "sql" { return T_SQL; } +"wal" { return T_WAL; } +"segment" { return T_SEGMENT; } +"archived" { return T_ARCHIVED; } +"basebackup" { return T_BASEBACKUP; } +"archiver" { return T_ARCHIVER; } +"/" { return T_SLASH; } "expect" { return T_EXPECT; } "error" { return T_ERROR; } "promote" { return T_PROMOTE; } diff --git a/src/monitor/expected/archiving_schema.out b/src/monitor/expected/archiving_schema.out new file mode 100644 index 000000000..30d2b2135 --- /dev/null +++ b/src/monitor/expected/archiving_schema.out @@ -0,0 +1,360 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Regression tests for the Archiving & Disaster Recovery schema and its +-- monitor API (milestone 1: schema + monitor API only -- no +-- service_archiver process involved, everything here is exercised via +-- direct SQL calls against the schema alone). See +-- ~/dev/temp/archiving-disaster-recovery.md for the full design. +\x on +-- A dedicated formation, like every other test in this schedule: 'default' +-- is the seed formation CREATE EXTENSION itself creates, and by this point +-- in regress_schedule it may already have real nodes registered into it by +-- earlier tests, so it's the one name this file must NOT reuse. The +-- 'default' basebackup_policy row (also a CREATE EXTENSION seed) is shared +-- on purpose: this file's own focus is exercising it, not creating another. +-- Two ordinary nodes stand in for a group's primary+secondary, inserted +-- directly rather than through register_node()/node_active(): the ordinary +-- node FSM has its own dedicated coverage elsewhere, this file's own focus +-- is the archiver schema layered on top of it. +SELECT pgautofailover.create_formation('archiving_test', 'pgsql', 'postgres', + true, 1); +-[ RECORD 1 ]----+------------------------------------ +create_formation | (archiving_test,pgsql,postgres,t,1) + +INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, sysidentifier, + goalstate, reportedstate) +VALUES ('archiving_test', 0, 'node1', 'node1.local', 5432, 111, + 'primary', 'primary'), + ('archiving_test', 0, 'node2', 'node2.local', 5432, 111, + 'secondary', 'secondary'); +-- ── register_archiver ──────────────────────────────────────────────────── +SELECT pgautofailover.register_archiver('archiver1', 'archiver1.local') + AS archiverid \gset +SELECT archiverid, archivername, hostname, region, basebackuppolicyid, + autoregister, maxresidentreplay + FROM pgautofailover.archiver; +-[ RECORD 1 ]------+---------------- +archiverid | 1 +archivername | archiver1 +hostname | archiver1.local +region | default +basebackuppolicyid | 1 +autoregister | t +maxresidentreplay | 1 + +-- the mandatory 'local' storage target is created in the same call +SELECT archiverstorageid, archiverid, storagemethod, storagepath, rcloneconfigid + FROM pgautofailover.archiver_storage; +-[ RECORD 1 ]-----+------ +archiverstorageid | 1 +archiverid | 1 +storagemethod | local +storagepath | +rcloneconfigid | + +-- ── archiver_add_formation: the budget setup's own fan-out ───────────────── +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test'); +-[ RECORD 1 ]----------+--- +archiver_add_formation | 40 + +SELECT nodeid, formationid, groupid, nodename, nodehost, nodeport, + goalstate, reportedstate, haspgdata + FROM pgautofailover.node + WHERE haspgdata = false; +-[ RECORD 1 ]-+---------------- +nodeid | 40 +formationid | archiving_test +groupid | 0 +nodename | archiver-1-0 +nodehost | archiver1.local +nodeport | 0 +goalstate | wait_standby +reportedstate | wait_standby +haspgdata | f + +SELECT archivernodeid, archiverid, kind, nodeid + FROM pgautofailover.archiver_node + WHERE kind = 'wal-receiver'; +-[ RECORD 1 ]--+------------- +archivernodeid | 1 +archiverid | 1 +kind | wal-receiver +nodeid | 40 + +SELECT nodeid FROM pgautofailover.node + WHERE formationid = 'archiving_test' AND groupid = 0 AND haspgdata = false \gset +-- calling archiver_add_formation() again for the same (archiver, formation) +-- must be a safe no-op -- no error, no duplicate node/archiver_node rows -- +-- since a real archiver's own reconciler calls this periodically to pick up +-- newly-added groups (e.g. a Citus formation growing a worker), not just +-- once at creation time +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test'); +(0 rows) + +SELECT count(*) AS should_still_be_one FROM pgautofailover.node + WHERE formationid = 'archiving_test' AND groupid = 0 AND haspgdata = false; +-[ RECORD 1 ]-------+-- +should_still_be_one | 1 + +-- ── list_archiver_memberships: what an archiver process discovers ────────── +SELECT * FROM pgautofailover.list_archiver_memberships(:archiverid); +-[ RECORD 1 ]--+--------------- +formation_id | archiving_test +group_id | 0 +node_id | 40 +reported_state | wait_standby +goal_state | wait_standby + +-- a second formation attached to the same archiver shows up alongside the +-- first -- this is the multi-membership case: one archiver, several +-- (formation, group) rows, each its own WAL stream and base-backup schedule +SELECT pgautofailover.create_formation('archiving_test_2', 'pgsql', 'postgres', + true, 1); +-[ RECORD 1 ]----+-------------------------------------- +create_formation | (archiving_test_2,pgsql,postgres,t,1) + +INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, sysidentifier, + goalstate, reportedstate) +VALUES ('archiving_test_2', 0, 'node3', 'node3.local', 5432, 222, + 'primary', 'primary'); +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test_2'); +-[ RECORD 1 ]----------+--- +archiver_add_formation | 43 + +SELECT formation_id, group_id + FROM pgautofailover.list_archiver_memberships(:archiverid) + ORDER BY formation_id; +-[ RECORD 1 ]+----------------- +formation_id | archiving_test +group_id | 0 +-[ RECORD 2 ]+----------------- +formation_id | archiving_test_2 +group_id | 0 + +SELECT pgautofailover.archiver_remove_formation(:archiverid, 'archiving_test_2'); +-[ RECORD 1 ]-------------+- +archiver_remove_formation | + +-- a second archiver serving the same formation/group shares the same +-- (nodehost, nodeport) = (its own hostname, 0) with the first -- the +-- node_nodehost_nodeport_haspgdata_idx partial unique index (scoped to +-- haspgdata rows only) must not reject this. Registered with an explicit, +-- distinct region from archiver1's own default -- this is the intended +-- shape for geographically-redundant DR coverage of the same formation +-- (see archiver.region's own comment); get_archivers() below must surface +-- both regions distinctly. +SELECT pgautofailover.register_archiver('archiver2', 'archiver1.local', + region => 'eu-west') + AS archiverid2 \gset +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid2, 'archiving_test'); +-[ RECORD 1 ]----------+--- +archiver_add_formation | 44 + +SELECT archiver_id, archiver_name, region + FROM pgautofailover.get_archivers('archiving_test') + ORDER BY archiver_id; +-[ RECORD 1 ]-+---------- +archiver_id | 1 +archiver_name | archiver1 +region | default +-[ RECORD 2 ]-+---------- +archiver_id | 2 +archiver_name | archiver2 +region | eu-west + +-- ── WAL capture confirmation: wal_archived() / report_wal_received() ─────── +SELECT pgautofailover.report_wal_received( + :nodeid, '000000010000000000000001', '0/1000000'); +-[ RECORD 1 ]-------+- +report_wal_received | + +-- default archiver_quorum is 1: a single archiver's report already satisfies it +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); +-[ RECORD 1 ]+-- +wal_archived | t + +-- bump the formation-wide default to 2: the same segment, reported by only +-- one archiver, no longer satisfies quorum +SELECT pgautofailover.set_archiver_policy('archiving_test', NULL, 2, NULL, NULL); +-[ RECORD 1 ]-------+- +set_archiver_policy | + +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); +-[ RECORD 1 ]+-- +wal_archived | f + +-- a group-specific override takes precedence over the formation-wide default +SELECT pgautofailover.set_archiver_policy('archiving_test', 0, 1, NULL, NULL); +-[ RECORD 1 ]-------+- +set_archiver_policy | + +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 0); +-[ RECORD 1 ]-------------+-- +archiverquorum | 1 +basebackuppolicyid | +replicationquorumeligible | f + +-- group 1 has no override of its own: falls back to the formation default (2) +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 1); +-[ RECORD 1 ]-------------+-- +archiverquorum | 2 +basebackuppolicyid | +replicationquorumeligible | f + +-- ── base backup lifecycle ─────────────────────────────────────────────────── +SELECT pgautofailover.report_basebackup_started( + :archiverid, 'archiving_test', 0, 'base_20260804', 1, '0/500000', 'live') + AS basebackupid \gset +SELECT pgautofailover.report_basebackup_completed( + :basebackupid, '0/1000000', 123456789, + '/var/lib/pgaf-archiver/backups/base_20260804'); +-[ RECORD 1 ]---------------+- +report_basebackup_completed | + +SELECT basebackupid, status, startlsn, endlsn, sizebytes + FROM pgautofailover.basebackup; +-[ RECORD 1 ]+---------- +basebackupid | 1 +status | complete +startlsn | 0/500000 +endlsn | 0/1000000 +sizebytes | 123456789 + +SELECT basebackupid, formationid, groupid, status + FROM pgautofailover.get_latest_basebackup('archiving_test', 0); +-[ RECORD 1 ]+--------------- +basebackupid | 1 +formationid | archiving_test +groupid | 0 +status | complete + +-- nothing to prune yet: the captured segment's LSN isn't older than this +-- backup's own startlsn +SELECT pgautofailover.prune_archiver_wal('archiving_test', 0); +-[ RECORD 1 ]------+-- +prune_archiver_wal | 0 + +-- report_basebackup_deleted() marks status='deleted' (never a real DELETE) +-- and prunes -- with no 'complete' backup left for this group, there's no +-- anchor point to replay forward from, so nothing prunes either +SELECT pgautofailover.report_basebackup_deleted(:basebackupid); +-[ RECORD 1 ]-------------+- +report_basebackup_deleted | + +SELECT basebackupid, status, deletedat IS NOT NULL AS was_deleted + FROM pgautofailover.basebackup; +-[ RECORD 1 ]+-------- +basebackupid | 1 +status | deleted +was_deleted | t + +-- ── rclone_config + archiver_storage ───────────────────────────────────── +SELECT pgautofailover.create_rclone_config( + 'minio-test', '[minio]' || chr(10) || 'type = s3') + AS rcloneconfigid \gset +SELECT pgautofailover.archiver_add_storage(:archiverid, 'minio-test') + AS archiverstorageid \gset +SELECT archiverstorageid, storagemethod, rcloneconfigid + FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid + ORDER BY archiverstorageid; +-[ RECORD 1 ]-----+------- +archiverstorageid | 1 +storagemethod | local +rcloneconfigid | +-[ RECORD 2 ]-----+------- +archiverstorageid | 3 +storagemethod | rclone +rcloneconfigid | 1 + +-- the mandatory local target cannot be removed +SELECT archiverstorageid AS local_storageid FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid AND storagemethod = 'local' \gset +SELECT pgautofailover.archiver_remove_storage(:local_storageid); +ERROR: archiver_storage 1 does not exist, or is the mandatory local target +CONTEXT: PL/pgSQL function pgautofailover.archiver_remove_storage(bigint) line 8 at RAISE +-- the non-local target can be +SELECT pgautofailover.archiver_remove_storage(:archiverstorageid); +-[ RECORD 1 ]-----------+- +archiver_remove_storage | + +SELECT count(*) AS remaining_storage_targets FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid; +-[ RECORD 1 ]-------------+-- +remaining_storage_targets | 1 + +-- ── warm-standby archiver_node + maxresidentreplay cap ────────────────────── +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby', + NULL, NULL, 'archiving_test', 0, 'continuous') + AS archivernodeid1 \gset +-- default maxresidentreplay is 1: a second resident warm-standby on the +-- same archiver must be refused +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby2', + NULL, NULL, 'archiving_test', 0, 'continuous'); +ERROR: archiver 1 is already at its maxresidentreplay cap (1) +CONTEXT: PL/pgSQL function pgautofailover.create_archiver_node(bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,integer,pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) line 18 at RAISE +-- ── PITR lifecycle ─────────────────────────────────────────────────────── +SELECT pgautofailover.create_archiver_node( + :archiverid, 'pitr', '/var/lib/pgaf-archiver/pitr-recovery', + NULL, NULL, NULL, NULL, NULL, NULL, 'restoring') + AS pitrnodeid \gset +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'create', + '{"restore_target_time": "2026-08-04 00:00:00+00"}'::jsonb, + NULL, NULL, 'not paused'); +-[ RECORD 1 ]------+- +report_pitr_status | + +SELECT pgautofailover.set_archiver_node_pitr_status(:pitrnodeid, 'paused'); +-[ RECORD 1 ]-----------------+- +set_archiver_node_pitr_status | + +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'status', NULL, '0/900000'::pg_lsn, '2026-08-04 00:00:05+00', 'paused'); +-[ RECORD 1 ]------+- +report_pitr_status | + +SELECT archivernodeid, archiverid, pitrstatus, lastoperation, + observedlsn, observedpausestate + FROM pgautofailover.pitr_node_status; +-[ RECORD 1 ]------+--------- +archivernodeid | 5 +archiverid | 1 +pitrstatus | paused +lastoperation | status +observedlsn | 0/900000 +observedpausestate | paused + +-- ── PITR command queue: pops and clears exactly once ──────────────────────── +SELECT pgautofailover.pitr_queue_command(:pitrnodeid, 'promote', NULL); +-[ RECORD 1 ]------+- +pitr_queue_command | + +SELECT pgautofailover.pitr_next_command(:pitrnodeid); +-[ RECORD 1 ]-----+-------- +pitr_next_command | promote + +SELECT pgautofailover.pitr_next_command(:pitrnodeid); +-[ RECORD 1 ]-----+----- +pitr_next_command | none + +-- ── archiver_remove_formation cleans up the ARCHIVING node row ────────────── +SELECT pgautofailover.archiver_remove_formation(:archiverid, 'archiving_test'); +-[ RECORD 1 ]-------------+- +archiver_remove_formation | + +SELECT count(*) AS should_be_zero FROM pgautofailover.node + WHERE haspgdata = false AND nodeid = :nodeid; +-[ RECORD 1 ]--+-- +should_be_zero | 0 + +SELECT count(*) AS should_also_be_zero FROM pgautofailover.archiver_node + WHERE archiverid = :archiverid AND kind = 'wal-receiver'; +-[ RECORD 1 ]-------+-- +should_also_be_zero | 0 + diff --git a/src/monitor/expected/candidate_count_gate.out b/src/monitor/expected/candidate_count_gate.out index 9cfa786d7..f2abbeb9c 100644 --- a/src/monitor/expected/candidate_count_gate.out +++ b/src/monitor/expected/candidate_count_gate.out @@ -279,112 +279,112 @@ RESET pgautofailover.startup_grace_period; -- neither is a stable value to pin in this file's own expected output. SELECT reportedstate, goalstate, rule_pos, rule_section, description FROM pgautofailover.last_events('ccg_test', count => 100); --[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 1 ]-+---------------------------------------------------------------------------------------------- reportedstate | init goalstate | single rule_pos | 209 rule_section | early_checks description | alone in group, candidate-eligible -> single --[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 2 ]-+---------------------------------------------------------------------------------------------- reportedstate | single goalstate | single rule_pos | rule_section | description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "single" --[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 3 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "wait_standby" --[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 4 ]-+---------------------------------------------------------------------------------------------- reportedstate | single goalstate | wait_primary rule_pos | 401 rule_section | primary_node description | primary alone, another node reached wait_standby -> wait_primary --[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 5 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | wait_primary rule_pos | rule_section | description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "wait_primary" --[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 6 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup rule_pos | 315 rule_section | reporting_node description | wait_standby, primary converged wait/join_primary -> catchingup --[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 7 ]-+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "catchingup" --[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 8 ]-+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | secondary rule_pos | 321 rule_section | reporting_node description | caught up, same TLI as primary, within sync threshold -> secondary --[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 9 ]-+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | secondary rule_pos | rule_section | description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "secondary" --[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 10 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | primary rule_pos | 411 rule_section | primary_node description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 11 ]+---------------------------------------------------------------------------------------------- reportedstate | primary goalstate | primary rule_pos | rule_section | description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "primary" --[ RECORD 12 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 12 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "wait_standby" --[ RECORD 13 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 13 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup rule_pos | 317 rule_section | reporting_node description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings --[ RECORD 14 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 14 ]+---------------------------------------------------------------------------------------------- reportedstate | primary goalstate | apply_settings rule_pos | 317 rule_section | reporting_node description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings --[ RECORD 15 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 15 ]+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "catchingup" --[ RECORD 16 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 16 ]+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "secondary" --[ RECORD 17 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 17 ]+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | report_lsn rule_pos | 367 rule_section | reporting_node -description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) --[ RECORD 18 ]+-------------------------------------------------------------------------------------------- +description | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) +-[ RECORD 18 ]+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | report_lsn rule_pos | 367 rule_section | reporting_node -description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) +description | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index 467daebb8..04031b3b2 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 177 + 182 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 177 + 182 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index f0040eb11..75d7899cb 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -417,7 +417,7 @@ section_path | reporting_node.from_context active_node_current_state | report_lsn other_node_current_state | wait_primary, join_primary candidate_node_current_state | -active_node_conditions | +active_node_conditions | hasPgData=true other_node_conditions | isHealthy=true candidate_node_conditions | group_conditions | @@ -432,7 +432,7 @@ section_path | reporting_node.from_context active_node_current_state | report_lsn other_node_current_state | primary candidate_node_current_state | -active_node_conditions | +active_node_conditions | hasPgData=true other_node_conditions | isHealthy=true candidate_node_conditions | group_conditions | @@ -477,7 +477,7 @@ section_path | reporting_node.from_context active_node_current_state | wait_standby other_node_current_state | wait_primary, join_primary candidate_node_current_state | -active_node_conditions | +active_node_conditions | hasPgData=true other_node_conditions | candidate_node_conditions | group_conditions | @@ -492,7 +492,7 @@ section_path | reporting_node.from_context active_node_current_state | wait_standby other_node_current_state | primary candidate_node_current_state | -active_node_conditions | replicationQuorum=true +active_node_conditions | replicationQuorum=true, hasPgData=true other_node_conditions | candidate_node_conditions | group_conditions | @@ -507,7 +507,7 @@ section_path | reporting_node.from_context active_node_current_state | wait_standby other_node_current_state | primary candidate_node_current_state | -active_node_conditions | replicationQuorum=false +active_node_conditions | replicationQuorum=false, hasPgData=true other_node_conditions | candidate_node_conditions | group_conditions | @@ -852,7 +852,7 @@ section_path | reporting_node.ms_failover.candidate_join active_node_current_state | report_lsn other_node_current_state | candidate_node_current_state | -active_node_conditions | +active_node_conditions | hasPgData=true other_node_conditions | candidate_node_conditions | isReadyToStreamWAL=true group_conditions | candidatePromotionInProgress=true @@ -864,7 +864,7 @@ comment | MS-failover: activeNode in report_lsn, failover c pos | 367 section | reporting_node section_path | reporting_node.ms_failover.candidate_fanout -active_node_current_state | secondary, catchingup +active_node_current_state | secondary, catchingup, archiving other_node_current_state | candidate_node_current_state | active_node_conditions | @@ -874,7 +874,7 @@ group_conditions | inMSFailoverCluster=true active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f -comment | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) +comment | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) -[ RECORD 58 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 369 section | reporting_node @@ -1071,6 +1071,96 @@ other_node_assigned_state | maintenance has_extra_action | f comment | nodesCount>2, primary unhealthy, converged prepare_maintenance -> primary maintenance -[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 394 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | report_lsn +other_node_current_state | single, wait_primary, join_primary +candidate_node_current_state | +active_node_conditions | hasPgData=false +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | archiving +other_node_assigned_state | +has_extra_action | f +comment | archiver mirror of pos 307: report_lsn, primary converged single/wait/join_primary, healthy -> archiving +-[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 395 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | report_lsn +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | hasPgData=false +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | archiving +other_node_assigned_state | +has_extra_action | f +comment | archiver mirror of pos 309: report_lsn, primary converged primary, healthy -> archiving +-[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 396 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | wait_standby +other_node_current_state | single, wait_primary, join_primary +candidate_node_current_state | +active_node_conditions | hasPgData=false +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | archiving +other_node_assigned_state | +has_extra_action | f +comment | archiver mirror of pos 315: wait_standby, primary converged single/wait/join_primary -> archiving +-[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 397 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | wait_standby +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | replicationQuorum=true, hasPgData=false +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | archiving +other_node_assigned_state | apply_settings +has_extra_action | f +comment | archiver mirror of pos 317: wait_standby (quorum member), primary converged primary -> archiving + apply_settings +-[ RECORD 75 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 398 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | wait_standby +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | replicationQuorum=false, hasPgData=false +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | archiving +other_node_assigned_state | +has_extra_action | f +comment | archiver mirror of pos 319: wait_standby (not a quorum member), primary converged primary -> archiving +-[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 399 +section | reporting_node +section_path | reporting_node.ms_failover.candidate_join +active_node_current_state | report_lsn +other_node_current_state | +candidate_node_current_state | +active_node_conditions | hasPgData=false +other_node_conditions | +candidate_node_conditions | isReadyToStreamWAL=true +group_conditions | candidatePromotionInProgress=true +active_node_assigned_state | archiving +other_node_assigned_state | +has_extra_action | f +comment | archiver mirror of pos 365: MS-failover, activeNode in report_lsn, failover candidate ready to stream WAL -> archiving (no join_secondary detour -- see pos 365's own comment) +-[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 401 section | primary_node section_path | primary_node @@ -1085,7 +1175,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | primary alone, another node reached wait_standby -> wait_primary --[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 403 section | primary_node section_path | primary_node @@ -1100,7 +1190,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | all nodes async, zero secondaries -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 405 section | primary_node section_path | primary_node @@ -1115,7 +1205,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | all nodes async, >=1 secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 80 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 407 section | primary_node section_path | primary_node @@ -1130,7 +1220,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 75 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 81 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 409 section | primary_node section_path | primary_node @@ -1145,7 +1235,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 82 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 411 section | primary_node section_path | primary_node @@ -1160,7 +1250,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 83 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 413 section | primary_node section_path | primary_node @@ -1175,7 +1265,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, both zero -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 84 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 415 section | primary_node section_path | primary_node @@ -1190,7 +1280,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 85 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 417 section | primary_node section_path | primary_node @@ -1205,7 +1295,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 80 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 86 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 419 section | primary_node section_path | primary_node @@ -1220,7 +1310,7 @@ active_node_assigned_state | other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/wait_primary/apply_settings, no other condition applies -> no-op besides the unhealthy-secondary fan-out to catchingup --[ RECORD 81 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 87 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 421 section | primary_node section_path | primary_node diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 90060dbeb..20eebeb31 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -54,6 +54,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; apply_settings | demote_timeout apply_settings | demoted apply_settings | join_primary + archiving | report_lsn catchingup | single catchingup | demote_timeout catchingup | demoted @@ -126,6 +127,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; report_lsn | prepare_promotion report_lsn | fast_forward report_lsn | join_secondary + report_lsn | archiving secondary | single secondary | demote_timeout secondary | demoted @@ -154,7 +156,8 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; wait_primary | join_primary wait_primary | apply_settings wait_standby | catchingup -(108 rows) + wait_standby | archiving +(111 rows) -- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A @@ -176,7 +179,12 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- summary row right before its own detail rows, as a header. -- -- Expected result: empty. Every MonitorFSM[] rule currently has a matching --- KeeperFSM[] row for every current_state it can assign a transition from. +-- KeeperFSM[] row for every current_state it can assign a transition from +-- -- including the pos 367/396/397/398 archiver-related edges (Archiving & +-- Disaster Recovery design, milestone 2): KeeperFSM[]'s own +-- WAIT_STANDBY_STATE/ARCHIVING_STATE/REPORT_LSN_STATE rows +-- (fsm_init_archiver/fsm_archiver_report_lsn/fsm_archiver_follow_new_primary, +-- fsm.c/fsm_transition.c) close this milestone's own gap. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos @@ -244,11 +252,12 @@ SELECT k.current_state, k.assigned_state report_lsn | prepare_promotion report_lsn | fast_forward report_lsn | join_secondary + report_lsn | archiving secondary | wait_standby secondary | maintenance secondary | wait_maintenance wait_primary | join_primary wait_primary | apply_settings -(23 rows) +(24 rows) DROP TABLE keeper_fsm_edges; diff --git a/src/monitor/expected/pg19/expected/archiving_schema.out b/src/monitor/expected/pg19/expected/archiving_schema.out new file mode 100644 index 000000000..c6acb9bba --- /dev/null +++ b/src/monitor/expected/pg19/expected/archiving_schema.out @@ -0,0 +1,360 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Regression tests for the Archiving & Disaster Recovery schema and its +-- monitor API (milestone 1: schema + monitor API only -- no +-- service_archiver process involved, everything here is exercised via +-- direct SQL calls against the schema alone). See +-- ~/dev/temp/archiving-disaster-recovery.md for the full design. +\x on +-- A dedicated formation, like every other test in this schedule: 'default' +-- is the seed formation CREATE EXTENSION itself creates, and by this point +-- in regress_schedule it may already have real nodes registered into it by +-- earlier tests, so it's the one name this file must NOT reuse. The +-- 'default' basebackup_policy row (also a CREATE EXTENSION seed) is shared +-- on purpose: this file's own focus is exercising it, not creating another. +-- Two ordinary nodes stand in for a group's primary+secondary, inserted +-- directly rather than through register_node()/node_active(): the ordinary +-- node FSM has its own dedicated coverage elsewhere, this file's own focus +-- is the archiver schema layered on top of it. +SELECT pgautofailover.create_formation('archiving_test', 'pgsql', 'postgres', + true, 1); +-[ RECORD 1 ]----+------------------------------------ +create_formation | (archiving_test,pgsql,postgres,t,1) + +INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, sysidentifier, + goalstate, reportedstate) +VALUES ('archiving_test', 0, 'node1', 'node1.local', 5432, 111, + 'primary', 'primary'), + ('archiving_test', 0, 'node2', 'node2.local', 5432, 111, + 'secondary', 'secondary'); +-- ── register_archiver ──────────────────────────────────────────────────── +SELECT pgautofailover.register_archiver('archiver1', 'archiver1.local') + AS archiverid \gset +SELECT archiverid, archivername, hostname, region, basebackuppolicyid, + autoregister, maxresidentreplay + FROM pgautofailover.archiver; +-[ RECORD 1 ]------+---------------- +archiverid | 1 +archivername | archiver1 +hostname | archiver1.local +region | default +basebackuppolicyid | 1 +autoregister | t +maxresidentreplay | 1 + +-- the mandatory 'local' storage target is created in the same call +SELECT archiverstorageid, archiverid, storagemethod, storagepath, rcloneconfigid + FROM pgautofailover.archiver_storage; +-[ RECORD 1 ]-----+------ +archiverstorageid | 1 +archiverid | 1 +storagemethod | local +storagepath | +rcloneconfigid | + +-- ── archiver_add_formation: the budget setup's own fan-out ───────────────── +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test'); +-[ RECORD 1 ]----------+--- +archiver_add_formation | 40 + +SELECT nodeid, formationid, groupid, nodename, nodehost, nodeport, + goalstate, reportedstate, haspgdata + FROM pgautofailover.node + WHERE haspgdata = false; +-[ RECORD 1 ]-+---------------- +nodeid | 40 +formationid | archiving_test +groupid | 0 +nodename | archiver-1-0 +nodehost | archiver1.local +nodeport | 0 +goalstate | wait_standby +reportedstate | wait_standby +haspgdata | f + +SELECT archivernodeid, archiverid, kind, nodeid + FROM pgautofailover.archiver_node + WHERE kind = 'wal-receiver'; +-[ RECORD 1 ]--+------------- +archivernodeid | 1 +archiverid | 1 +kind | wal-receiver +nodeid | 40 + +SELECT nodeid FROM pgautofailover.node + WHERE formationid = 'archiving_test' AND groupid = 0 AND haspgdata = false \gset +-- calling archiver_add_formation() again for the same (archiver, formation) +-- must be a safe no-op -- no error, no duplicate node/archiver_node rows -- +-- since a real archiver's own reconciler calls this periodically to pick up +-- newly-added groups (e.g. a Citus formation growing a worker), not just +-- once at creation time +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test'); +(0 rows) + +SELECT count(*) AS should_still_be_one FROM pgautofailover.node + WHERE formationid = 'archiving_test' AND groupid = 0 AND haspgdata = false; +-[ RECORD 1 ]-------+-- +should_still_be_one | 1 + +-- ── list_archiver_memberships: what an archiver process discovers ────────── +SELECT * FROM pgautofailover.list_archiver_memberships(:archiverid); +-[ RECORD 1 ]--+--------------- +formation_id | archiving_test +group_id | 0 +node_id | 40 +reported_state | wait_standby +goal_state | wait_standby + +-- a second formation attached to the same archiver shows up alongside the +-- first -- this is the multi-membership case: one archiver, several +-- (formation, group) rows, each its own WAL stream and base-backup schedule +SELECT pgautofailover.create_formation('archiving_test_2', 'pgsql', 'postgres', + true, 1); +-[ RECORD 1 ]----+-------------------------------------- +create_formation | (archiving_test_2,pgsql,postgres,t,1) + +INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, sysidentifier, + goalstate, reportedstate) +VALUES ('archiving_test_2', 0, 'node3', 'node3.local', 5432, 222, + 'primary', 'primary'); +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test_2'); +-[ RECORD 1 ]----------+--- +archiver_add_formation | 43 + +SELECT formation_id, group_id + FROM pgautofailover.list_archiver_memberships(:archiverid) + ORDER BY formation_id; +-[ RECORD 1 ]+----------------- +formation_id | archiving_test +group_id | 0 +-[ RECORD 2 ]+----------------- +formation_id | archiving_test_2 +group_id | 0 + +SELECT pgautofailover.archiver_remove_formation(:archiverid, 'archiving_test_2'); +-[ RECORD 1 ]-------------+- +archiver_remove_formation | + +-- a second archiver serving the same formation/group shares the same +-- (nodehost, nodeport) = (its own hostname, 0) with the first -- the +-- node_nodehost_nodeport_haspgdata_idx partial unique index (scoped to +-- haspgdata rows only) must not reject this. Registered with an explicit, +-- distinct region from archiver1's own default -- this is the intended +-- shape for geographically-redundant DR coverage of the same formation +-- (see archiver.region's own comment); get_archivers() below must surface +-- both regions distinctly. +SELECT pgautofailover.register_archiver('archiver2', 'archiver1.local', + region => 'eu-west') + AS archiverid2 \gset +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid2, 'archiving_test'); +-[ RECORD 1 ]----------+--- +archiver_add_formation | 44 + +SELECT archiver_id, archiver_name, region + FROM pgautofailover.get_archivers('archiving_test') + ORDER BY archiver_id; +-[ RECORD 1 ]-+---------- +archiver_id | 1 +archiver_name | archiver1 +region | default +-[ RECORD 2 ]-+---------- +archiver_id | 2 +archiver_name | archiver2 +region | eu-west + +-- ── WAL capture confirmation: wal_archived() / report_wal_received() ─────── +SELECT pgautofailover.report_wal_received( + :nodeid, '000000010000000000000001', '0/1000000'); +-[ RECORD 1 ]-------+- +report_wal_received | + +-- default archiver_quorum is 1: a single archiver's report already satisfies it +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); +-[ RECORD 1 ]+-- +wal_archived | t + +-- bump the formation-wide default to 2: the same segment, reported by only +-- one archiver, no longer satisfies quorum +SELECT pgautofailover.set_archiver_policy('archiving_test', NULL, 2, NULL, NULL); +-[ RECORD 1 ]-------+- +set_archiver_policy | + +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); +-[ RECORD 1 ]+-- +wal_archived | f + +-- a group-specific override takes precedence over the formation-wide default +SELECT pgautofailover.set_archiver_policy('archiving_test', 0, 1, NULL, NULL); +-[ RECORD 1 ]-------+- +set_archiver_policy | + +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 0); +-[ RECORD 1 ]-------------+-- +archiverquorum | 1 +basebackuppolicyid | +replicationquorumeligible | f + +-- group 1 has no override of its own: falls back to the formation default (2) +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 1); +-[ RECORD 1 ]-------------+-- +archiverquorum | 2 +basebackuppolicyid | +replicationquorumeligible | f + +-- ── base backup lifecycle ─────────────────────────────────────────────────── +SELECT pgautofailover.report_basebackup_started( + :archiverid, 'archiving_test', 0, 'base_20260804', 1, '0/500000', 'live') + AS basebackupid \gset +SELECT pgautofailover.report_basebackup_completed( + :basebackupid, '0/1000000', 123456789, + '/var/lib/pgaf-archiver/backups/base_20260804'); +-[ RECORD 1 ]---------------+- +report_basebackup_completed | + +SELECT basebackupid, status, startlsn, endlsn, sizebytes + FROM pgautofailover.basebackup; +-[ RECORD 1 ]+----------- +basebackupid | 1 +status | complete +startlsn | 0/00500000 +endlsn | 0/01000000 +sizebytes | 123456789 + +SELECT basebackupid, formationid, groupid, status + FROM pgautofailover.get_latest_basebackup('archiving_test', 0); +-[ RECORD 1 ]+--------------- +basebackupid | 1 +formationid | archiving_test +groupid | 0 +status | complete + +-- nothing to prune yet: the captured segment's LSN isn't older than this +-- backup's own startlsn +SELECT pgautofailover.prune_archiver_wal('archiving_test', 0); +-[ RECORD 1 ]------+-- +prune_archiver_wal | 0 + +-- report_basebackup_deleted() marks status='deleted' (never a real DELETE) +-- and prunes -- with no 'complete' backup left for this group, there's no +-- anchor point to replay forward from, so nothing prunes either +SELECT pgautofailover.report_basebackup_deleted(:basebackupid); +-[ RECORD 1 ]-------------+- +report_basebackup_deleted | + +SELECT basebackupid, status, deletedat IS NOT NULL AS was_deleted + FROM pgautofailover.basebackup; +-[ RECORD 1 ]+-------- +basebackupid | 1 +status | deleted +was_deleted | t + +-- ── rclone_config + archiver_storage ───────────────────────────────────── +SELECT pgautofailover.create_rclone_config( + 'minio-test', '[minio]' || chr(10) || 'type = s3') + AS rcloneconfigid \gset +SELECT pgautofailover.archiver_add_storage(:archiverid, 'minio-test') + AS archiverstorageid \gset +SELECT archiverstorageid, storagemethod, rcloneconfigid + FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid + ORDER BY archiverstorageid; +-[ RECORD 1 ]-----+------- +archiverstorageid | 1 +storagemethod | local +rcloneconfigid | +-[ RECORD 2 ]-----+------- +archiverstorageid | 3 +storagemethod | rclone +rcloneconfigid | 1 + +-- the mandatory local target cannot be removed +SELECT archiverstorageid AS local_storageid FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid AND storagemethod = 'local' \gset +SELECT pgautofailover.archiver_remove_storage(:local_storageid); +ERROR: archiver_storage 1 does not exist, or is the mandatory local target +CONTEXT: PL/pgSQL function pgautofailover.archiver_remove_storage(bigint) line 8 at RAISE +-- the non-local target can be +SELECT pgautofailover.archiver_remove_storage(:archiverstorageid); +-[ RECORD 1 ]-----------+- +archiver_remove_storage | + +SELECT count(*) AS remaining_storage_targets FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid; +-[ RECORD 1 ]-------------+-- +remaining_storage_targets | 1 + +-- ── warm-standby archiver_node + maxresidentreplay cap ────────────────────── +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby', + NULL, NULL, 'archiving_test', 0, 'continuous') + AS archivernodeid1 \gset +-- default maxresidentreplay is 1: a second resident warm-standby on the +-- same archiver must be refused +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby2', + NULL, NULL, 'archiving_test', 0, 'continuous'); +ERROR: archiver 1 is already at its maxresidentreplay cap (1) +CONTEXT: PL/pgSQL function pgautofailover.create_archiver_node(bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,integer,pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) line 18 at RAISE +-- ── PITR lifecycle ─────────────────────────────────────────────────────── +SELECT pgautofailover.create_archiver_node( + :archiverid, 'pitr', '/var/lib/pgaf-archiver/pitr-recovery', + NULL, NULL, NULL, NULL, NULL, NULL, 'restoring') + AS pitrnodeid \gset +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'create', + '{"restore_target_time": "2026-08-04 00:00:00+00"}'::jsonb, + NULL, NULL, 'not paused'); +-[ RECORD 1 ]------+- +report_pitr_status | + +SELECT pgautofailover.set_archiver_node_pitr_status(:pitrnodeid, 'paused'); +-[ RECORD 1 ]-----------------+- +set_archiver_node_pitr_status | + +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'status', NULL, '0/900000'::pg_lsn, '2026-08-04 00:00:05+00', 'paused'); +-[ RECORD 1 ]------+- +report_pitr_status | + +SELECT archivernodeid, archiverid, pitrstatus, lastoperation, + observedlsn, observedpausestate + FROM pgautofailover.pitr_node_status; +-[ RECORD 1 ]------+----------- +archivernodeid | 5 +archiverid | 1 +pitrstatus | paused +lastoperation | status +observedlsn | 0/00900000 +observedpausestate | paused + +-- ── PITR command queue: pops and clears exactly once ──────────────────────── +SELECT pgautofailover.pitr_queue_command(:pitrnodeid, 'promote', NULL); +-[ RECORD 1 ]------+- +pitr_queue_command | + +SELECT pgautofailover.pitr_next_command(:pitrnodeid); +-[ RECORD 1 ]-----+-------- +pitr_next_command | promote + +SELECT pgautofailover.pitr_next_command(:pitrnodeid); +-[ RECORD 1 ]-----+----- +pitr_next_command | none + +-- ── archiver_remove_formation cleans up the ARCHIVING node row ────────────── +SELECT pgautofailover.archiver_remove_formation(:archiverid, 'archiving_test'); +-[ RECORD 1 ]-------------+- +archiver_remove_formation | + +SELECT count(*) AS should_be_zero FROM pgautofailover.node + WHERE haspgdata = false AND nodeid = :nodeid; +-[ RECORD 1 ]--+-- +should_be_zero | 0 + +SELECT count(*) AS should_also_be_zero FROM pgautofailover.archiver_node + WHERE archiverid = :archiverid AND kind = 'wal-receiver'; +-[ RECORD 1 ]-------+-- +should_also_be_zero | 0 + diff --git a/src/monitor/expected/stale_primary_report.out b/src/monitor/expected/stale_primary_report.out index cc40f4d87..c9f7456d3 100644 --- a/src/monitor/expected/stale_primary_report.out +++ b/src/monitor/expected/stale_primary_report.out @@ -319,112 +319,112 @@ reportedstate | secondary -- neither is a stable value to pin in this file's own expected output. SELECT reportedstate, goalstate, rule_pos, rule_section, description FROM pgautofailover.last_events('spr_test', count => 100); --[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 1 ]-+---------------------------------------------------------------------------------------------- reportedstate | init goalstate | single rule_pos | 209 rule_section | early_checks description | alone in group, candidate-eligible -> single --[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 2 ]-+---------------------------------------------------------------------------------------------- reportedstate | single goalstate | single rule_pos | rule_section | description | New state is reported by node 18 "spr_p" (spr_p:5432): "single" --[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 3 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "wait_standby" --[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 4 ]-+---------------------------------------------------------------------------------------------- reportedstate | single goalstate | wait_primary rule_pos | 401 rule_section | primary_node description | primary alone, another node reached wait_standby -> wait_primary --[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 5 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | wait_primary rule_pos | rule_section | description | New state is reported by node 18 "spr_p" (spr_p:5432): "wait_primary" --[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 6 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup rule_pos | 315 rule_section | reporting_node description | wait_standby, primary converged wait/join_primary -> catchingup --[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 7 ]-+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "catchingup" --[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 8 ]-+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | secondary rule_pos | 321 rule_section | reporting_node description | caught up, same TLI as primary, within sync threshold -> secondary --[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 9 ]-+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | secondary rule_pos | rule_section | description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "secondary" --[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 10 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | primary rule_pos | 411 rule_section | primary_node description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 11 ]+---------------------------------------------------------------------------------------------- reportedstate | primary goalstate | primary rule_pos | rule_section | description | New state is reported by node 18 "spr_p" (spr_p:5432): "primary" --[ RECORD 12 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 12 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "wait_standby" --[ RECORD 13 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 13 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup rule_pos | 317 rule_section | reporting_node description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings --[ RECORD 14 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 14 ]+---------------------------------------------------------------------------------------------- reportedstate | primary goalstate | apply_settings rule_pos | 317 rule_section | reporting_node description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings --[ RECORD 15 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 15 ]+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "catchingup" --[ RECORD 16 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 16 ]+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "secondary" --[ RECORD 17 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 17 ]+---------------------------------------------------------------------------------------------- reportedstate | primary goalstate | secondary rule_pos | rule_section | description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "primary" --[ RECORD 18 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 18 ]+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | report_lsn rule_pos | 367 rule_section | reporting_node -description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) +description | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 33310e12d..9e751f429 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -292,6 +292,27 @@ static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY = { REPLICATION_STATE_JOIN_PRIMARY), }; +/* + * FSM_WAIT_OR_JOIN_PRIMARY plus SINGLE -- used only by the archiver mirror + * rows (pos 394/396), never by their ordinary hasPgData=true siblings (pos + * 307/315): a real secondary joining a lone primary always first bumps that + * primary from SINGLE to WAIT_PRIMARY (pos 401, "primary alone, another node + * reached wait_standby"), so pos 307/315 never actually need to match SINGLE + * themselves. An archiver attaching to a lone primary is different -- since + * BuildForPrimaryNodeNodeActiveContext excludes archiver rows from ever + * triggering that same pos 401 bump (an archiver isn't a quorum-eligible + * node kind, see that function's own comment), the primary legitimately + * stays SINGLE the entire time the archiver is only being watched by it. + * Without SINGLE in this set, an archiver attached to a genuinely + * single-node formation could never leave WAIT_STANDBY/REPORT_LSN at all. + */ +static const NodeStatePattern FSM_SINGLE_OR_WAIT_OR_JOIN_PRIMARY = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_SINGLE, + REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_JOIN_PRIMARY), +}; + /* * the "primary role" states MONITOR_FSM_SECTION_PRIMARY_NODE's own rows * match against -- a different three-element set from @@ -487,6 +508,7 @@ typedef struct NodeStatus bool isCitusWorkerGroup; bool replicationQuorum; bool isComparableToReferenceTli; + bool hasPgData; } NodeStatus; typedef struct NodeStatusPattern @@ -510,6 +532,12 @@ typedef struct NodeStatusPattern BoolPattern replicationQuorum; BoolPattern isComparableToReferenceTli; BoolPattern unreachableFromDemoteTimeout; + + /* + * true for every ordinary Postgres node; false only for an ARCHIVING + * membership row. See AutoFailoverNode.hasPgData's own comment. + */ + BoolPattern hasPgData; } NodeStatusPattern; static void @@ -535,6 +563,7 @@ BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *stat status->candidateEligible = node->candidatePriority > 0; status->isCitusWorkerGroup = IsCitusFormation(ctx->formation) && node->groupId > 0; status->replicationQuorum = node->replicationQuorum; + status->hasPgData = node->hasPgData; } @@ -667,7 +696,8 @@ NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) BoolMatchesPattern(status->isComparableToReferenceTli, pattern->isComparableToReferenceTli) && BoolMatchesPattern(unreachableFromDemoteTimeout, - pattern->unreachableFromDemoteTimeout); + pattern->unreachableFromDemoteTimeout) && + BoolMatchesPattern(status->hasPgData, pattern->hasPgData); } @@ -1895,12 +1925,26 @@ BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *prim /* * BuildForPrimaryNodeNodeActiveContext computes every fact SectionPrimaryNode - * (MonitorFSM[]'s pos 401-421 rows) needs: it loops over every other node in - * the primary's group, using the same OtherNodeIsDueForCatchingUp() test - * OtherNodesDueForCatchingUp() (above) uses for its own fan-out, to derive - * the group-level counts (replicationQuorumCount, secondaryNodesCount, - * secondaryQuorumNodesCount) and the anyOtherNodeWaitingStandby flag those - * rows match against. + * (MonitorFSM[]'s pos 401-421 rows) needs: it loops over every other *real* + * (hasPgData) node in the primary's group, using the same OtherNodeIsDueFor + * CatchingUp() test OtherNodesDueForCatchingUp() (above) uses for its own + * fan-out, to derive the group-level counts (replicationQuorumCount, + * secondaryNodesCount, secondaryQuorumNodesCount) and the anyOtherNode + * WaitingStandby flag those rows match against. + * + * An ARCHIVING node is skipped entirely here (see the hasPgData check inside + * the loop below): it is never a real Postgres secondary participating in + * synchronous-replication quorum, and it can never reach reported SECONDARY + * state. Counting it like an ordinary node would let it single-handedly + * block this primary's own SINGLE -> WAIT_PRIMARY -> PRIMARY progression -- + * anyOtherNodeWaitingStandby would fire (pos 401) the moment the archiver's + * own bootstrap briefly passes through WAIT_STANDBY, bumping the primary off + * SINGLE, and it could then never reach PRIMARY since secondaryQuorumNodes + * Count could never legitimately drop to zero via an archiver's own reported + * state. Same hasPgData-based exclusion this file's own REPORTING_NODE + * section already applies for a different purpose (pos 365/399's own + * comment) -- an archiver simply isn't a quorum-eligible node kind, in + * either section. */ static void BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, @@ -1918,11 +1962,10 @@ BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, */ List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); - int otherNodesCount = list_length(otherNodesGroupList); - int replicationQuorumCount = otherNodesCount; - int secondaryNodesCount = otherNodesCount; - int secondaryQuorumNodesCount = otherNodesCount; + int replicationQuorumCount = 0; + int secondaryNodesCount = 0; + int secondaryQuorumNodesCount = 0; ListCell *nodeCell = NULL; @@ -1930,6 +1973,16 @@ BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, { AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); + if (!otherNode->hasPgData) + { + /* an ARCHIVING row -- see this function's own header comment */ + continue; + } + + ++replicationQuorumCount; + ++secondaryNodesCount; + ++secondaryQuorumNodesCount; + if (OtherNodeIsDueForCatchingUp(ctx, otherNode)) { --secondaryNodesCount; @@ -2720,26 +2773,36 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade" }, - /* report_lsn, primary converged wait/join_primary, healthy */ + /* + * report_lsn, primary converged wait/join_primary, healthy -- hasPgData + * = BOOL_TRUE restricts this to ordinary nodes now that pos 394 (below, + * in the archiver mirror cluster appended after pos 393) is the + * hasPgData = BOOL_FALSE sibling assigning ARCHIVING instead of + * SECONDARY; the two are mutually exclusive on hasPgData alone, so + * their relative order doesn't matter. + */ { .pos = 307, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, - .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY, .isHealthy = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), .comment = "report_lsn, primary converged wait/join_primary, healthy -> secondary" }, - /* report_lsn, primary converged primary, healthy */ + /* report_lsn, primary converged primary, healthy -- see pos 307's own + * comment on hasPgData; pos 395 is this row's archiver mirror. */ { .pos = 309, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, - .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY), .isHealthy = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), @@ -2771,39 +2834,51 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "report_lsn or fast_forward, continuing an already-started failover -> " "MS-failover cascade" }, - /* wait_standby, primary converged wait/join_primary */ + /* + * wait_standby, primary converged wait/join_primary -- hasPgData = + * BOOL_TRUE restricts this to ordinary nodes; pos 396 is the + * hasPgData = BOOL_FALSE sibling assigning ARCHIVING (see pos 307's + * own comment on why order between the two doesn't matter). + */ { .pos = 315, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, - .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY) }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .hasPgData = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "wait_standby, primary converged wait/join_primary -> catchingup" }, - /* wait_standby (quorum member), primary converged primary */ + /* wait_standby (quorum member), primary converged primary -- see pos + * 315's own comment on hasPgData; pos 397 is this row's archiver + * mirror. */ { .pos = 317, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), - .replicationQuorum = BOOL_TRUE }, + .replicationQuorum = BOOL_TRUE, + .hasPgData = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .otherNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), .comment = "wait_standby (quorum member), primary converged primary -> " "catchingup + apply_settings" }, - /* wait_standby (not a quorum member), primary converged primary */ + /* wait_standby (not a quorum member), primary converged primary -- see + * pos 315's own comment on hasPgData; pos 398 is this row's archiver + * mirror. */ { .pos = 319, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), - .replicationQuorum = BOOL_FALSE }, + .replicationQuorum = BOOL_FALSE, + .hasPgData = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = @@ -3190,6 +3265,21 @@ static const MonitorFSMTransition MonitorFSM[] = { /* * MS-failover: candidate ready to stream WAL -> follower joins as secondary */ + + /* + * hasPgData = BOOL_TRUE restricts this to ordinary nodes now that pos + * 399 (in the archiver mirror cluster, below) is the hasPgData = + * BOOL_FALSE sibling assigning ARCHIVING directly instead of the + * intermediate JOIN_SECONDARY -> SECONDARY dance an ARCHIVING row has + * no real Postgres to actually perform (its client-side transition + * function, fsm_checkpoint_and_stop_postgres, unconditionally fails + * for a haspgdata=false node): REPORT_LSN_STATE -> ARCHIVING_STATE is + * already a real, working transition on its own (fsm_archiver_follow_ + * new_primary, exercised by archiver_wal_capture.pgaf's own failover + * test), so there's no need for an archiver to ever pass through + * JOIN_SECONDARY_STATE at all -- unlike SECONDARY, ARCHIVING isn't + * gated on the primary having fully converged first. + */ { .pos = 365, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -3197,7 +3287,8 @@ static const MonitorFSMTransition MonitorFSM[] = { MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN }, .conditions = { .candidatePromotionInProgress = BOOL_TRUE }, - .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_TRUE }, .candidateNode = { .isReadyToStreamWAL = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_JOIN_SECONDARY), .comment = @@ -3239,15 +3330,17 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, .reportedStates = STATES( REPLICATION_STATE_SECONDARY, - REPLICATION_STATE_CATCHINGUP), + REPLICATION_STATE_CATCHINGUP, + REPLICATION_STATE_ARCHIVING), .assignedStates = STATES( REPLICATION_STATE_SECONDARY, - REPLICATION_STATE_CATCHINGUP) } + REPLICATION_STATE_CATCHINGUP, + REPLICATION_STATE_ARCHIVING) } }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), .comment = - "MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn " - "(1 of 4)" }, + "MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> " + "report_lsn (1 of 4)" }, { .pos = 369, .sectionPath = { @@ -3512,6 +3605,103 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "nodesCount>2, primary unhealthy, converged prepare_maintenance -> " "primary maintenance" }, + /* + * Archiver mirror cluster: the hasPgData = BOOL_FALSE siblings of pos + * 307/309/315/317/319/365 above, assigning ARCHIVING instead of + * SECONDARY/CATCHINGUP/JOIN_SECONDARY for an ARCHIVING membership row. + * Appended here rather than interleaved next to each one, for the same + * reason the MS-failover cluster above is appended rather than + * renumbered into the ordinary rows: pos 307/309/315/317/319/365 are + * numbered every 2 with no room between consecutive pairs for 6 more + * rows, and since hasPgData makes each pair mutually exclusive, their + * relative array order doesn't affect first-match-wins correctness -- + * see each of those rows' own comment for the exact pairing. Pos 399 + * is the one exception to "sectionPath'd under REPORTING_NODE/ + * FROM_CONTEXT, like their siblings": it mirrors pos 365, which lives + * under the MS-failover cluster's own sectionPath, so it must too -- + * sectionPath is what the dispatcher actually matches evaluation + * context against, not physical position in this array. + */ + { .pos = 394, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_SINGLE_OR_WAIT_OR_JOIN_PRIMARY, + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .comment = "archiver mirror of pos 307: report_lsn, primary converged " + "single/wait/join_primary, healthy -> archiving" }, + + { .pos = 395, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY), + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .comment = "archiver mirror of pos 309: report_lsn, primary converged " + "primary, healthy -> archiving" }, + + { .pos = 396, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .hasPgData = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_SINGLE_OR_WAIT_OR_JOIN_PRIMARY }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .comment = "archiver mirror of pos 315: wait_standby, primary converged " + "single/wait/join_primary -> archiving" }, + + { .pos = 397, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .replicationQuorum = BOOL_TRUE, + .hasPgData = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), + .comment = "archiver mirror of pos 317: wait_standby (quorum member), " + "primary converged primary -> archiving + apply_settings" }, + + { .pos = 398, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .replicationQuorum = BOOL_FALSE, + .hasPgData = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .comment = "archiver mirror of pos 319: wait_standby (not a quorum member), " + "primary converged primary -> archiving" }, + + { .pos = 399, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN + }, + .conditions = { .candidatePromotionInProgress = BOOL_TRUE }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_FALSE }, + .candidateNode = { .isReadyToStreamWAL = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .comment = "archiver mirror of pos 365: MS-failover, activeNode in report_lsn, " + "failover candidate ready to stream WAL -> archiving (no " + "join_secondary detour -- see pos 365's own comment)" }, + /* * --- the PRIMARY_NODE section (sectionPath[0] == * MONITOR_FSM_SECTION_PRIMARY_NODE, pos 401 onward): the declarative @@ -4316,6 +4506,7 @@ NodeStatusPatternConditionsText(const NodeStatusPattern *pattern, bool *isNull) pattern->isComparableToReferenceTli); APPEND_BOOL_CONDITION(&buf, "unreachableFromDemoteTimeout", pattern->unreachableFromDemoteTimeout); + APPEND_BOOL_CONDITION(&buf, "hasPgData", pattern->hasPgData); if (buf.len == 0) { @@ -6242,8 +6433,9 @@ BuildCandidateList(GroupStateContext *ctx, List *nodesGroupList, ListCell *nodeCell = NULL; List *candidateNodesGroupList = NIL; - List *secondaryStates = list_make2_int(REPLICATION_STATE_SECONDARY, - REPLICATION_STATE_CATCHINGUP); + List *secondaryStates = list_make3_int(REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP, + REPLICATION_STATE_ARCHIVING); foreach(nodeCell, nodesGroupList) { diff --git a/src/monitor/keeper_fsm_edges.json b/src/monitor/keeper_fsm_edges.json index 20ab42ac9..9a81aea70 100644 --- a/src/monitor/keeper_fsm_edges.json +++ b/src/monitor/keeper_fsm_edges.json @@ -307,6 +307,18 @@ "current": "wait_standby", "assigned": "catchingup" }, + { + "current": "wait_standby", + "assigned": "archiving" + }, + { + "current": "archiving", + "assigned": "report_lsn" + }, + { + "current": "report_lsn", + "assigned": "archiving" + }, { "current": "demoted", "assigned": "catchingup" diff --git a/src/monitor/node_active_protocol.c b/src/monitor/node_active_protocol.c index a1e22db01..d96f66cf1 100644 --- a/src/monitor/node_active_protocol.c +++ b/src/monitor/node_active_protocol.c @@ -524,8 +524,7 @@ NodeActive(char *formationId, AutoFailoverNodeState *currentNodeState) * Report the current state. The state might not have changed, but in * that case we still update the last report time. */ - ReportAutoFailoverNodeState(pgAutoFailoverNode->nodeHost, - pgAutoFailoverNode->nodePort, + ReportAutoFailoverNodeState(pgAutoFailoverNode->nodeId, currentNodeState->replicationState, currentNodeState->pgIsRunning, currentNodeState->pgsrSyncState, diff --git a/src/monitor/node_metadata.c b/src/monitor/node_metadata.c index 2b182ec70..143ca57e8 100644 --- a/src/monitor/node_metadata.c +++ b/src/monitor/node_metadata.c @@ -180,6 +180,23 @@ TupleToAutoFailoverNode(TupleDesc tupleDescriptor, HeapTuple heapTuple) Anum_pgautofailover_node_replication_stall_since, tupleDescriptor, &stallIsNull); + /* + * haspgdata is looked up by name, not by the Anum_ constant every other + * field here uses: this function is also called against a "RETURNING + * node.*" tuple descriptor (health_check_metadata.c), which reflects + * the table's true physical column order -- pg_versionnum/pg_version/ + * pg_versionstring/citus_version were appended between + * replication_stall_since and haspgdata by an earlier migration but + * were never added to AUTO_FAILOVER_NODE_TABLE_ALL_COLUMNS, so + * haspgdata's physical position (28) and its position in that + * explicit column list (24) genuinely differ. SPI_fnumber resolves the + * real attnum against whichever tupdesc was actually passed in, so + * this works correctly for both callers. + */ + int hasPgDataAttNum = SPI_fnumber(tupleDescriptor, "haspgdata"); + Datum hasPgData = heap_getattr(heapTuple, hasPgDataAttNum, + tupleDescriptor, &isNull); + Oid goalStateOid = DatumGetObjectId(goalState); Oid reportedStateOid = DatumGetObjectId(reportedState); @@ -214,6 +231,7 @@ TupleToAutoFailoverNode(TupleDesc tupleDescriptor, HeapTuple heapTuple) regionIsNull ? "" : TextDatumGetCString(region); pgAutoFailoverNode->replicationStallSince = stallIsNull ? 0 : DatumGetTimestampTz(replicationStallSince); + pgAutoFailoverNode->hasPgData = DatumGetBool(hasPgData); return pgAutoFailoverNode; } @@ -1574,9 +1592,18 @@ SetNodeGoalState(AutoFailoverNode *pgAutoFailoverNode, * a node. * * We use SPI to automatically handle triggers, function calls, etc. + * + * Scoped by nodeid, not (nodehost, nodeport): an ARCHIVING row's nodeport + * is a permanent 0 sentinel and its nodehost is the owning archiver's own + * hostname, both identical across every (formation, group) membership of + * the same archiver identity (see archiver_add_formation()'s own comment + * on this, pgautofailover.sql). Scoping on that pair used to make any one + * membership's routine report blindly overwrite reportedstate on every + * other membership sharing the same archiver -- nodeid is the one column + * that's actually unique per row. */ void -ReportAutoFailoverNodeState(char *nodeHost, int nodePort, +ReportAutoFailoverNodeState(int64 nodeId, ReplicationState reportedState, bool pgIsRunning, SyncState pgSyncState, int reportedTLI, @@ -1591,8 +1618,7 @@ ReportAutoFailoverNodeState(char *nodeHost, int nodePort, TEXTOID, /* pg_stat_replication.sync_state */ INT4OID, /* reportedtli */ LSNOID, /* reportedlsn */ - TEXTOID, /* nodehost */ - INT4OID /* nodeport */ + INT8OID /* nodeid */ }; Datum argValues[] = { @@ -1601,8 +1627,7 @@ ReportAutoFailoverNodeState(char *nodeHost, int nodePort, CStringGetTextDatum(SyncStateToString(pgSyncState)), /* sync_state */ Int32GetDatum(reportedTLI), /* reportedtli */ LSNGetDatum(reportedLSN), /* reportedlsn */ - CStringGetTextDatum(nodeHost), /* nodehost */ - Int32GetDatum(nodePort) /* nodeport */ + Int64GetDatum(nodeId) /* nodeid */ }; const int argCount = sizeof(argValues) / sizeof(argValues[0]); @@ -1627,7 +1652,7 @@ ReportAutoFailoverNodeState(char *nodeHost, int nodePort, " THEN COALESCE(replication_stall_since, now()) " " ELSE NULL " "END " - "WHERE nodehost = $6 AND nodeport = $7"; + "WHERE nodeid = $6"; SPI_connect(); diff --git a/src/monitor/node_metadata.h b/src/monitor/node_metadata.h index 0d336ca61..7c4ca50b9 100644 --- a/src/monitor/node_metadata.h +++ b/src/monitor/node_metadata.h @@ -50,6 +50,7 @@ #define Anum_pgautofailover_node_nodecluster 21 #define Anum_pgautofailover_node_region 22 #define Anum_pgautofailover_node_replication_stall_since 23 +#define Anum_pgautofailover_node_haspgdata 24 #define AUTO_FAILOVER_NODE_TABLE_ALL_COLUMNS \ "formationid, " \ @@ -74,7 +75,8 @@ "replicationquorum, " \ "nodecluster, " \ "region, " \ - "replication_stall_since" + "replication_stall_since, " \ + "haspgdata" #define SELECT_ALL_FROM_AUTO_FAILOVER_NODE_TABLE \ @@ -139,6 +141,13 @@ typedef struct AutoFailoverNode char *nodeCluster; char *region; TimestampTz replicationStallSince; /* 0 = not stalled */ + + /* + * true for every ordinary Postgres node; false only for an ARCHIVING + * membership row (a pg_receivewal client, no PGDATA, no postmaster to + * manage). See pgautofailover.sql's own comment on the haspgdata column. + */ + bool hasPgData; } AutoFailoverNode; @@ -220,7 +229,7 @@ extern int AddAutoFailoverNode(char *formationId, extern void SetNodeGoalState(AutoFailoverNode *pgAutoFailoverNode, ReplicationState goalState, const char *message); -extern void ReportAutoFailoverNodeState(char *nodeHost, int nodePort, +extern void ReportAutoFailoverNodeState(int64 nodeId, ReplicationState reportedState, bool pgIsRunning, SyncState pgSyncState, diff --git a/src/monitor/pgautofailover--2.2--2.3.sql b/src/monitor/pgautofailover--2.2--2.3.sql index 871e6b83a..79e1cd243 100644 --- a/src/monitor/pgautofailover--2.2--2.3.sql +++ b/src/monitor/pgautofailover--2.2--2.3.sql @@ -689,3 +689,1643 @@ with last_events as ) select * from last_events order by eventtime, eventid; $$; + + +-- +-- Archiving & Disaster Recovery, milestone 1: schema + monitor API only +-- (#TODO -- update with the actual PR number once opened). See +-- ~/dev/temp/archiving-disaster-recovery.md for the full design, and +-- pgautofailover.sql's own comments on each object below (this mirrors +-- that file's DDL, applied incrementally to an existing 2.2 install +-- instead of as part of a fresh CREATE EXTENSION). +-- + +-- New terminal state for a pgautofailover.node row representing an +-- ARCHIVING membership (see haspgdata below) rather than an ordinary +-- Postgres instance. Safe to add live: nothing in this script uses the +-- new value in the same transaction it's added in. +ALTER TYPE pgautofailover.replication_state ADD VALUE 'archiving'; + +-- true for every ordinary Postgres node (its own PGDATA, promotable); +-- false only for an ARCHIVING membership row (a pg_receivewal client, +-- no PGDATA, no postmaster to manage). See archiving-disaster-recovery +-- design: this single boolean is what candidate_priority enforcement, +-- keeper_ensure_current_state's liveness check, and the FAST_FORWARD +-- source-selection branch all key off, instead of a third node-kind +-- value -- a cascading follower is still haspgdata = true, and a +-- future proxy never becomes a pgautofailover.node row at all. +ALTER TABLE pgautofailover.node + ADD COLUMN IF NOT EXISTS haspgdata bool NOT NULL DEFAULT true; + +-- The old "any nodehost:port can only be a unique node in the system" +-- constraint (unconditional UNIQUE (nodehost, nodeport), added in +-- 1.5--1.6) can't hold for ARCHIVING rows: one archiver's (hostname, 0) +-- pair is deliberately shared across every group it serves. Replace it +-- with the same partial unique index pgautofailover.sql's fresh-install +-- table definition uses, scoped to haspgdata rows only. +-- +-- The live constraint name is node_nodehost_nodeport_key1, not the +-- "expected" node_nodehost_nodeport_key: both 1.3--1.4 and 1.5--1.6 +-- separately recreate pgautofailover.node with their own unnamed +-- UNIQUE (nodehost, nodeport), so by the time a real install reaches +-- 2.2 (via the only upgrade path that exists -- there is no standalone +-- "--2.2.sql", so even a "fresh" VERSION '2.2' install runs this same +-- incremental chain from 1.0), Postgres has already disambiguated the +-- second one with a "1" suffix. Verified empirically against a real +-- 1.0 -> ... -> 2.2 -> 2.3 upgrade, not guessed from naming convention. +ALTER TABLE pgautofailover.node + DROP CONSTRAINT IF EXISTS node_nodehost_nodeport_key1; + +CREATE UNIQUE INDEX IF NOT EXISTS node_nodehost_nodeport_haspgdata_idx + ON pgautofailover.node (nodehost, nodeport) + WHERE haspgdata; + +-- +-- +-- Archiving & Disaster Recovery: schema for the Archiver process identity, +-- ARCHIVING node memberships, base-backup policy/history, and PITR. +-- See ~/dev/temp/archiving-disaster-recovery.md for the full design. +-- +-- Milestone 1 (schema + monitor API only): every function here is plain +-- plpgsql/SQL, callable directly with no service_archiver process running +-- -- the pgaftest coverage for this milestone exercises these functions +-- via direct SQL calls against a plain cluster. +-- + +CREATE TYPE pgautofailover.storage_method + AS ENUM ('local', 'rclone'); + +CREATE TYPE pgautofailover.basebackup_source + AS ENUM ('live', 'replay'); + +CREATE TYPE pgautofailover.basebackup_replay_mode + AS ENUM ('volatile', 'persistent'); + +CREATE TYPE pgautofailover.basebackup_cache + AS ENUM ('local', 'none'); + +CREATE TYPE pgautofailover.basebackup_status + AS ENUM ('in_progress', 'complete', 'failed', 'deleted'); + -- 'deleted' is what makes basebackup a full history rather + -- than just a live catalog + +-- shared or per-archiver base-backup production/retention policy +CREATE TABLE pgautofailover.basebackup_policy + ( + basebackuppolicyid bigserial PRIMARY KEY, + policyname text UNIQUE, + + source pgautofailover.basebackup_source + NOT NULL DEFAULT 'replay', + replaymode pgautofailover.basebackup_replay_mode + DEFAULT 'volatile', + cache pgautofailover.basebackup_cache + NOT NULL DEFAULT 'local', + + -- strong, ready-to-use-as-is defaults -- nightly, 3 days retention + frequency interval NOT NULL DEFAULT '24 hours', + maxcount int NOT NULL DEFAULT 3, + maxage interval NOT NULL DEFAULT '3 days', + onpromotion bool NOT NULL DEFAULT true, + + -- backpressure: cap on simultaneous base-backup production jobs, + -- per archiver, per referencing policy + concurrency int NOT NULL DEFAULT 1, + + CHECK (source <> 'replay' OR replaymode IS NOT NULL), + CHECK (concurrency >= 1) + ); + +INSERT INTO pgautofailover.basebackup_policy (policyname) VALUES ('default'); + +-- the physical Archiver entity: one row per archiver host/process +CREATE TABLE pgautofailover.archiver + ( + archiverid bigserial PRIMARY KEY, + archivername text NOT NULL, + hostname text NOT NULL, + createdat timestamptz NOT NULL DEFAULT now(), + + -- same convention as pgautofailover.node.region: a free-form label for + -- the data-centre or availability zone this archiver runs in, purely + -- informational (get_archivers()'s own consumers, e.g. pg_autoctl + -- watch, may display it) -- set at registration time via + -- register_archiver()'s own region parameter, never inferred. Multiple + -- archivers can attach to the very same formation (archiver_add_ + -- formation() names each ARCHIVING node row after its own archiverid, + -- so two different archivers never collide there) -- distinct regions + -- is the expected shape for geographically-redundant DR coverage of + -- one formation, and archiver_policy's own archiverquorum column + -- already anticipates requiring more than one archiver's confirmation. + region text not null default 'default', + + basebackuppolicyid bigint NOT NULL + REFERENCES pgautofailover.basebackup_policy (basebackuppolicyid), + + autoregister bool NOT NULL DEFAULT true, + + -- cap on resident 'warm-standby' archiver_node rows (either cadence) + -- this host is allowed to keep running at once + maxresidentreplay int NOT NULL DEFAULT 1, + + -- storage stats for the archiver's own PGDATA (walcache + basebackups, + -- same root -- see service_archiver_serve.c's own header comment on + -- why an archiver has no other pgdata to speak of), reported + -- periodically by service_archiver_loop(); NULL until the first report. + -- usedbytes is this archiver's own footprint (directory_size() over its + -- whole pgdata); freebytes is the containing filesystem's available + -- space (statvfs's f_bavail, "available to a non-privileged process" -- + -- the number that actually predicts whether the next base backup or + -- WAL segment fits, not f_bfree's superuser-reserved total). + usedbytes bigint, + freebytes bigint, + + lastreporttime timestamptz, + + UNIQUE (archivername), + CHECK (maxresidentreplay >= 0), + CHECK (usedbytes IS NULL OR usedbytes >= 0), + CHECK (freebytes IS NULL OR freebytes >= 0) + ); + +-- a named, shareable rclone remote configuration -- the literal contents +-- of an rclone config file (real INI format, exactly as rclone itself +-- reads it: https://rclone.org/docs/#config-file). `config` should hold +-- only the non-secret, architectural half of an rclone remote (type, +-- provider, endpoint, region, acl, and a `type = alias` remote baking in +-- the bucket/prefix) -- credentials belong in the archiver process's own +-- environment (RCLONE_CONFIG__), never in this column, which +-- is backed up and readable by anyone with SQL access to the monitor. +CREATE TABLE pgautofailover.rclone_config + ( + rcloneconfigid bigserial PRIMARY KEY, + name text UNIQUE NOT NULL, + config text NOT NULL, + createdat timestamptz NOT NULL DEFAULT now() + ); + +-- 1-N: an archiver's storage targets. Exactly one 'local' row always +-- exists (the mandatory default); adding cloud storage means adding one +-- or more 'rclone' rows, each an independent push target, each +-- referencing a (possibly shared) rclone_config row +CREATE TABLE pgautofailover.archiver_storage + ( + archiverstorageid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + storagemethod pgautofailover.storage_method NOT NULL, + + storagepath text, -- 'local' only: override the default topdir path + rcloneconfigid bigint REFERENCES pgautofailover.rclone_config (rcloneconfigid), + -- 'rclone' only: which named config this target uses + + createdat timestamptz NOT NULL DEFAULT now(), + + CHECK (storagemethod <> 'local' OR rcloneconfigid IS NULL), + CHECK (storagemethod <> 'rclone' OR rcloneconfigid IS NOT NULL) + ); + +CREATE UNIQUE INDEX archiver_storage_one_local + ON pgautofailover.archiver_storage (archiverid) + WHERE storagemethod = 'local'; + +-- formation-granularity attachment. Only holds explicit rows for the +-- restricted case -- when autoregister is true this table isn't consulted +CREATE TABLE pgautofailover.archiver_formation + ( + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + formationid text NOT NULL REFERENCES pgautofailover.formation (formationid) + ON DELETE CASCADE, + attachedat timestamptz NOT NULL DEFAULT now(), + + PRIMARY KEY (archiverid, formationid) + ); + +-- policy override, resolved formation-default then group-specific; +-- groupid IS NULL means "the formation-wide default for this archiver" +CREATE TABLE pgautofailover.archiver_policy + ( + formationid text NOT NULL REFERENCES pgautofailover.formation (formationid) + ON DELETE CASCADE, + groupid int, + archiverquorum int NOT NULL DEFAULT 1, + basebackuppolicyid bigint + REFERENCES pgautofailover.basebackup_policy (basebackuppolicyid), + replicationquorumeligible bool NOT NULL DEFAULT false + ); + +-- A plain UNIQUE (formationid, groupid) constraint would not actually +-- enforce "at most one formation-wide default row": Postgres treats every +-- NULL groupid as distinct from every other NULL for uniqueness purposes, +-- so two formation-wide rows for the same formation would never conflict. +-- coalesce(groupid, -1) normalizes NULL to a real, comparable value +-- instead -- -1 is safe as a stand-in since groupid is otherwise always +-- >= 0. set_archiver_policy's own ON CONFLICT targets this index. +CREATE UNIQUE INDEX archiver_policy_formation_group_idx + ON pgautofailover.archiver_policy (formationid, coalesce(groupid, -1)); + +-- one row per base backup taken by any archiver -- full history, not just +-- a live catalog: rows are never deleted by retention, only marked +-- status = 'deleted'; get_latest_basebackup filters on status = 'complete' +CREATE TABLE pgautofailover.basebackup + ( + basebackupid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + formationid text NOT NULL, + groupid int NOT NULL, + label text NOT NULL, + timeline int NOT NULL, + startlsn pg_lsn NOT NULL, + endlsn pg_lsn, + + period tstzrange NOT NULL DEFAULT tstzrange(now(), NULL), + + -- snapshot of how this specific backup was produced, independent of + -- whatever basebackup_policy says *now* + source pgautofailover.basebackup_source NOT NULL, + replaymode pgautofailover.basebackup_replay_mode, + + sizebytes bigint, + storagelocation text NOT NULL, -- local path, or object-storage URI + status pgautofailover.basebackup_status + NOT NULL DEFAULT 'in_progress', + deletedat timestamptz + ); + +CREATE INDEX basebackup_group_idx + ON pgautofailover.basebackup (formationid, groupid, lower(period) DESC); + +-- remote-side sync/prune tracking, one row per (basebackup, remote +-- storage target) -- a single backup can sync to several remotes +CREATE TABLE pgautofailover.basebackup_storage + ( + basebackupid bigint NOT NULL REFERENCES pgautofailover.basebackup (basebackupid) + ON DELETE CASCADE, + archiverstorageid bigint NOT NULL REFERENCES pgautofailover.archiver_storage (archiverstorageid) + ON DELETE CASCADE, + + syncedat timestamptz, + remotelocation text, + deletedat timestamptz, + + PRIMARY KEY (basebackupid, archiverstorageid) + ); + +-- one row per (archiver, WAL segment) durably captured -- the real +-- backing store wal_archived() queries. +-- +-- PRIMARY KEY is (formationid, groupid, walfilename, archiverid) -- the +-- hot path is wal_archived()'s lookup across every archiver holding %f +-- for this group, so this ordering makes it a direct index range scan. +-- +-- FILLFACTOR 20: traffic is INSERT + DELETE, never UPDATE, but is +-- continuous and high-throughput -- a low fillfactor spreads rows across +-- more pages, reducing buffer-lock contention between concurrently +-- inserting archivers and easing autovacuum on a table that's never +-- write-quiet. +CREATE TABLE pgautofailover.archiver_wal + ( + formationid text NOT NULL, + groupid int NOT NULL, + walfilename text NOT NULL, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + + lsn pg_lsn NOT NULL, + receivedat timestamptz NOT NULL DEFAULT now(), + + PRIMARY KEY (formationid, groupid, walfilename, archiverid) + ) WITH (fillfactor = 20); + +CREATE TYPE pgautofailover.archiver_node_kind + AS ENUM ('wal-receiver', 'warm-standby', 'pitr'); + -- 'staging' anticipated for a later, not-yet-designed feature + -- (periodic dev/test environments refreshed from the archiver) + +CREATE TYPE pgautofailover.archiver_node_cadence + AS ENUM ('continuous', 'scheduled'); + -- 'manual' considered (operator-driven "advance only when I say so"), + -- not added yet -- same one-value-enum-addition cost as 'staging' + +CREATE TYPE pgautofailover.pitr_status + AS ENUM ('restoring', 'paused', 'registered', 'discarded'); + +-- every concrete Postgres instance an archiver hosts, derives, or is +-- otherwise associated with, beyond the archiver process itself +CREATE TABLE pgautofailover.archiver_node + ( + archivernodeid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + kind pgautofailover.archiver_node_kind NOT NULL, + + -- placement, uniform across every kind: NULL = colocated (local file + -- reads, zero network); non-NULL = a separate node (remote fetch) + hostname text, + pgdata text NOT NULL, + + -- 'wal-receiver' only: which ARCHIVING row this instance backs. + -- ON DELETE CASCADE: the ARCHIVING node row can be removed through + -- more than one path (this schema's own archiver_remove_formation, + -- or the ordinary pgautofailover.remove_node() every other node type + -- already goes through) -- cascading here means every path safely + -- cleans up this row too, instead of only the one this schema + -- controls directly. + nodeid bigint REFERENCES pgautofailover.node (nodeid) + ON DELETE CASCADE, + + -- 'warm-standby' only: which group's WAL cache this instance replays + formationid text REFERENCES pgautofailover.formation (formationid), + groupid int, + + -- 'warm-standby' only: continuous (chases the primary continuously, + -- eligible for nodecluster read exposure) or scheduled (advances only + -- at basebackup_policy.frequency's cadence, paused via + -- recovery_target_action = pause in between) + cadence pgautofailover.archiver_node_cadence, + + -- 'warm-standby' + cadence = 'continuous' only: opt-in read-only + -- exposure. Enforced by CHECK, not just CLI convention -- a + -- 'scheduled' instance is stale by up to a full frequency between + -- cycles and must never be reachable as an ordinary read-replica + -- connection string without that caveat + nodecluster text, + + -- 'pitr' only: lifecycle (restoring -> paused -> registered/discarded) + pitrstatus pgautofailover.pitr_status, + + createdat timestamptz NOT NULL DEFAULT now(), + + CHECK (kind <> 'wal-receiver' OR nodeid IS NOT NULL), + CHECK (kind = 'wal-receiver' OR nodeid IS NULL), + CHECK (kind <> 'warm-standby' + OR (formationid IS NOT NULL AND groupid IS NOT NULL AND cadence IS NOT NULL)), + CHECK (kind = 'warm-standby' + OR (formationid IS NULL AND groupid IS NULL AND cadence IS NULL)), + CHECK (nodecluster IS NULL OR (kind = 'warm-standby' AND cadence = 'continuous')), + CHECK (kind = 'pitr' OR pitrstatus IS NULL) + ); + +CREATE TYPE pgautofailover.pitr_operation + AS ENUM ('create', 'status', 'retarget', 'resume', 'promote', + 'register', 'discard'); + +-- every PITR operation, recorded -- not just current status +CREATE TABLE pgautofailover.pitr_history + ( + pitrhistoryid bigserial PRIMARY KEY, + archivernodeid bigint NOT NULL + REFERENCES pgautofailover.archiver_node (archivernodeid) + ON DELETE CASCADE, + operation pgautofailover.pitr_operation NOT NULL, + occurredat timestamptz NOT NULL DEFAULT now(), + + requestedspec jsonb, -- what was asked for + observedlsn pg_lsn, -- what Postgres actually reported afterward + observedtimestamp timestamptz, + observedpausestate text, -- verbatim: 'not paused'/'pause requested'/'paused' + + note text + ); + +CREATE INDEX pitr_history_node_idx + ON pgautofailover.pitr_history (archivernodeid, occurredat); + +CREATE VIEW pgautofailover.pitr_node_status AS + SELECT n.archivernodeid, n.archiverid, n.hostname, n.pgdata, + n.pitrstatus, h.operation AS lastoperation, + h.observedlsn, h.observedtimestamp, h.observedpausestate, + h.occurredat AS lastupdatedat + FROM pgautofailover.archiver_node n + LEFT JOIN LATERAL ( + SELECT * FROM pgautofailover.pitr_history + WHERE archivernodeid = n.archivernodeid + ORDER BY occurredat DESC LIMIT 1 + ) h ON true + WHERE n.kind = 'pitr'; + +-- opt-in monitor-mediated PITR command queue, for the headless, +-- no-interactive-access deployment shape only (pg_autoctl node run +-- against a node.ini declaring kind = pitr) +CREATE TYPE pgautofailover.pitr_command + AS ENUM ('none', 'retarget', 'pause', 'resume', 'promote', + 'register', 'discard'); + +CREATE TABLE pgautofailover.pitr_pending_command + ( + archivernodeid bigint PRIMARY KEY + REFERENCES pgautofailover.archiver_node (archivernodeid) + ON DELETE CASCADE, + command pgautofailover.pitr_command NOT NULL DEFAULT 'none', + commandspec jsonb, + queuedat timestamptz NOT NULL DEFAULT now() + ); + + +-- +-- Functions +-- + +CREATE FUNCTION pgautofailover.create_basebackup_policy + ( + IN policyname text, + IN policyspec jsonb + ) +RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.basebackup_policy + (policyname, source, replaymode, cache, + frequency, maxcount, maxage, onpromotion, concurrency) + SELECT policyname, + coalesce((policyspec->>'source')::pgautofailover.basebackup_source, + 'replay'), + coalesce((policyspec->>'replaymode')::pgautofailover.basebackup_replay_mode, + 'volatile'), + coalesce((policyspec->>'cache')::pgautofailover.basebackup_cache, + 'local'), + coalesce((policyspec->>'frequency')::interval, '24 hours'), + coalesce((policyspec->>'maxcount')::int, 3), + coalesce((policyspec->>'maxage')::interval, '3 days'), + coalesce((policyspec->>'onpromotion')::bool, true), + coalesce((policyspec->>'concurrency')::int, 1) + RETURNING basebackuppolicyid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_basebackup_policy(text,jsonb) + is 'create a named, shareable base-backup production/retention policy'; + +grant execute on function + pgautofailover.create_basebackup_policy(text,jsonb) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_basebackup_policy + ( + IN policyname text, + IN policyspec jsonb + ) +RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup_policy + SET source = coalesce((policyspec->>'source')::pgautofailover.basebackup_source, source), + replaymode = coalesce((policyspec->>'replaymode')::pgautofailover.basebackup_replay_mode, replaymode), + cache = coalesce((policyspec->>'cache')::pgautofailover.basebackup_cache, cache), + frequency = coalesce((policyspec->>'frequency')::interval, frequency), + maxcount = coalesce((policyspec->>'maxcount')::int, maxcount), + maxage = coalesce((policyspec->>'maxage')::interval, maxage), + onpromotion = coalesce((policyspec->>'onpromotion')::bool, onpromotion), + concurrency = coalesce((policyspec->>'concurrency')::int, concurrency) + WHERE basebackup_policy.policyname = set_basebackup_policy.policyname; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup_policy "%" does not exist', policyname; + END IF; +END; +$$; + +comment on function pgautofailover.set_basebackup_policy(text,jsonb) + is 'update an existing named base-backup production/retention policy'; + +grant execute on function + pgautofailover.set_basebackup_policy(text,jsonb) + to autoctl_node; + +CREATE FUNCTION pgautofailover.get_basebackup_policy(policyname text) + RETURNS pgautofailover.basebackup_policy LANGUAGE sql STRICT SECURITY DEFINER +AS $$ + SELECT * FROM pgautofailover.basebackup_policy + WHERE basebackup_policy.policyname = get_basebackup_policy.policyname; +$$; + +comment on function pgautofailover.get_basebackup_policy(text) + is 'fetch a named base-backup production/retention policy'; + +grant execute on function pgautofailover.get_basebackup_policy(text) + to autoctl_node; + +-- creates the physical Archiver entity plus its mandatory 'local' +-- archiver_storage row. basebackuppolicyid NULL resolves to 'default'. +-- rcloneconfigname, when given, also attaches an additional 'rclone' row +-- referencing that existing, already-created rclone_config -- the +-- one-command way to "start a new archiver with the same shared rclone +-- setup" another archiver already uses; omit it to start local-only and +-- attach storage later via archiver_add_storage +CREATE FUNCTION pgautofailover.register_archiver + ( + archivername text, hostname text, + storagepath text DEFAULT NULL, + basebackuppolicyid bigint DEFAULT NULL, + autoregister bool DEFAULT true, + maxresidentreplay int DEFAULT 1, + rcloneconfigname text DEFAULT NULL, + region text DEFAULT 'default' + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_archiverid bigint; + resolved_policyid bigint; +BEGIN + resolved_policyid := coalesce( + basebackuppolicyid, + (SELECT p.basebackuppolicyid + FROM pgautofailover.basebackup_policy p + WHERE p.policyname = 'default')); + + INSERT INTO pgautofailover.archiver + (archivername, hostname, basebackuppolicyid, + autoregister, maxresidentreplay, region) + VALUES (archivername, hostname, resolved_policyid, + autoregister, maxresidentreplay, + coalesce(register_archiver.region, 'default')) + RETURNING archiverid INTO new_archiverid; + + INSERT INTO pgautofailover.archiver_storage + (archiverid, storagemethod, storagepath) + VALUES (new_archiverid, 'local', storagepath); + + IF rcloneconfigname IS NOT NULL THEN + PERFORM pgautofailover.archiver_add_storage(new_archiverid, rcloneconfigname); + END IF; + + RETURN new_archiverid; +END; +$$; + +comment on function pgautofailover.register_archiver(text,text,text,bigint,bool,int,text,text) + is 'register a new Archiver process identity, with its mandatory local storage target'; + +grant execute on function + pgautofailover.register_archiver(text,text,text,bigint,bool,int,text,text) + to autoctl_node; + +-- periodic storage heartbeat: usedbytes/freebytes/lastreporttime all move +-- together, from the same service_archiver_loop() tick (service_archiver.c) +-- that already reports this archiver's captured-WAL LSN. +CREATE FUNCTION pgautofailover.report_archiver_storage + (archiverid bigint, usedbytes bigint, freebytes bigint) + RETURNS void LANGUAGE sql SECURITY DEFINER +AS $$ + UPDATE pgautofailover.archiver + SET usedbytes = report_archiver_storage.usedbytes, + freebytes = report_archiver_storage.freebytes, + lastreporttime = now() + WHERE archiver.archiverid = report_archiver_storage.archiverid; +$$; + +comment on function pgautofailover.report_archiver_storage(bigint,bigint,bigint) + is 'record an archiver''s own reported disk usage and free space'; + +grant execute on function + pgautofailover.report_archiver_storage(bigint,bigint,bigint) + to autoctl_node; + +-- one row per archiver attached to formationid, with its FSM state (the +-- 'wal-receiver' archiver_node row created by archiver_add_formation, one +-- per group -- a multi-group formation returns one row per (archiver, +-- group)). Used by `pg_autoctl watch`'s own archivers section. +CREATE FUNCTION pgautofailover.get_archivers + ( + IN formationid text default 'default', + OUT archiver_id bigint, + OUT archiver_name text, + OUT hostname text, + OUT region text, + OUT used_bytes bigint, + OUT free_bytes bigint, + OUT last_report_time timestamptz, + OUT node_id bigint, + OUT reported_state pgautofailover.replication_state, + OUT goal_state pgautofailover.replication_state + ) +RETURNS SETOF record LANGUAGE SQL STRICT SECURITY DEFINER +AS $$ + SELECT a.archiverid, a.archivername, a.hostname, a.region, + a.usedbytes, a.freebytes, a.lastreporttime, + n.nodeid, n.reportedstate, n.goalstate + FROM pgautofailover.archiver a + JOIN pgautofailover.archiver_formation af + ON af.archiverid = a.archiverid + AND af.formationid = get_archivers.formationid + LEFT JOIN pgautofailover.archiver_node an + ON an.archiverid = a.archiverid AND an.kind = 'wal-receiver' + LEFT JOIN pgautofailover.node n + ON n.nodeid = an.nodeid AND n.formationid = get_archivers.formationid + ORDER BY a.archiverid; +$$; + +comment on function pgautofailover.get_archivers(text) + is 'list the archivers attached to a formation, with storage stats and FSM state'; + +grant execute on function pgautofailover.get_archivers(text) + to autoctl_node; + +-- named, shareable rclone config objects -- see rclone_config above for +-- what belongs in `config` (architecture only, never credentials) +CREATE FUNCTION pgautofailover.create_rclone_config(name text, config text) + RETURNS bigint -- rcloneconfigid + LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.rclone_config (name, config) + VALUES (name, config) + RETURNING rcloneconfigid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_rclone_config(text,text) + is 'register a named, shareable rclone remote configuration'; + +grant execute on function pgautofailover.create_rclone_config(text,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_rclone_config(name text, config text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.rclone_config AS rc + SET config = set_rclone_config.config + WHERE rc.name = set_rclone_config.name; + + IF NOT FOUND THEN + RAISE EXCEPTION 'rclone_config "%" does not exist', name; + END IF; +END; +$$; + +comment on function pgautofailover.set_rclone_config(text,text) + is 'update the content of an existing named rclone configuration -- every archiver referencing it picks up the change'; + +grant execute on function pgautofailover.set_rclone_config(text,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.get_rclone_config(name text) + RETURNS pgautofailover.rclone_config LANGUAGE sql STRICT +AS $$ + SELECT * FROM pgautofailover.rclone_config AS rc + WHERE rc.name = get_rclone_config.name; +$$; + +comment on function pgautofailover.get_rclone_config(text) + is 'fetch a named rclone configuration''s raw content'; + +grant execute on function pgautofailover.get_rclone_config(text) + to autoctl_node; + +-- attaches an archiver to an existing, already-named rclone_config row +-- (the sharing path -- several archivers' archiver_storage rows can +-- reference the same rcloneconfigid at once, edit the config once via +-- set_rclone_config and every referencing archiver picks it up) +CREATE FUNCTION pgautofailover.archiver_add_storage + (archiverid bigint, rcloneconfigname text) + RETURNS bigint -- archiverstorageid + LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + resolved_rcloneconfigid bigint; + new_id bigint; +BEGIN + SELECT rc.rcloneconfigid INTO resolved_rcloneconfigid + FROM pgautofailover.rclone_config rc + WHERE rc.name = rcloneconfigname; + + IF resolved_rcloneconfigid IS NULL THEN + RAISE EXCEPTION 'rclone_config "%" does not exist', rcloneconfigname; + END IF; + + INSERT INTO pgautofailover.archiver_storage + (archiverid, storagemethod, rcloneconfigid) + VALUES (archiverid, 'rclone', resolved_rcloneconfigid) + RETURNING archiverstorageid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.archiver_add_storage(bigint,text) + is 'attach an additional rclone storage target to an archiver, referencing an existing named rclone_config'; + +grant execute on function pgautofailover.archiver_add_storage(bigint,text) + to autoctl_node; + +-- detaches only; the referenced rclone_config row is untouched and +-- keeps serving any other archiver still referencing it +CREATE FUNCTION pgautofailover.archiver_remove_storage(archiverstorageid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.archiver_storage AS a_s + WHERE a_s.archiverstorageid = archiver_remove_storage.archiverstorageid + AND a_s.storagemethod <> 'local'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_storage % does not exist, or is the mandatory local target', + archiverstorageid; + END IF; +END; +$$; + +comment on function pgautofailover.archiver_remove_storage(bigint) + is 'detach a non-local storage target from an archiver (the local target cannot be removed)'; + +grant execute on function pgautofailover.archiver_remove_storage(bigint) + to autoctl_node; + +-- fans out to one CREATE of a pgautofailover.node row (haspgdata = +-- false) per group currently in formationid +-- Parameters are prefixed in_* here (unlike this file's usual +-- function-qualified-reference convention): ON CONFLICT's own target +-- column list can't be schema/function-qualified at all (that syntax +-- only accepts bare column names or ON CONSTRAINT), so a same-named +-- parameter would still be genuinely ambiguous there even when every +-- other clause in this function could disambiguate it. +CREATE FUNCTION pgautofailover.archiver_add_formation + (in_archiverid bigint, in_formationid text) + RETURNS SETOF bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + grp record; + new_nodeid bigint; +BEGIN + INSERT INTO pgautofailover.archiver_formation (archiverid, formationid) + VALUES (in_archiverid, in_formationid) + ON CONFLICT (archiverid, formationid) DO NOTHING; + + FOR grp IN + SELECT DISTINCT n.groupid + FROM pgautofailover.node n + WHERE n.formationid = in_formationid + LOOP + new_nodeid := NULL; + + -- nodeport = 0 is a permanent sentinel, not an M1 stopgap: an + -- ARCHIVING row has no postmaster of its own to be reachable on, + -- so nodehost:nodeport isn't a connectable address here the way + -- it is for every haspgdata row -- see node_nodehost_nodeport_ + -- haspgdata_idx's own comment, which is exactly why that unique + -- index is scoped to haspgdata rows only. reportedstate starts at + -- 'wait_standby', same as any freshly-registered node -- it only + -- reaches 'archiving' once a real keeper's pg_receivewal is + -- actually running. + -- + -- ON CONFLICT DO NOTHING on (formationid, nodename): this + -- function must be safe to call again for a formation some of + -- whose groups are already attached -- an operator re-running it + -- on purpose, or the archiver's own reconciler picking up a + -- newly-added Citus worker group -- without failing on every + -- group that was already covered by an earlier call. A skipped + -- insert leaves new_nodeid NULL (no row returned), handled below. + INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, + goalstate, reportedstate, haspgdata, candidatepriority, + replicationquorum) + VALUES (in_formationid, grp.groupid, + 'archiver-' || in_archiverid || '-' || grp.groupid, + (SELECT a.hostname FROM pgautofailover.archiver a + WHERE a.archiverid = in_archiverid), + 0, + 'wait_standby', 'wait_standby', false, 0, false) + ON CONFLICT (formationid, nodename) DO NOTHING + RETURNING nodeid INTO new_nodeid; + + IF new_nodeid IS NULL THEN + -- this group was already attached by an earlier call -- + -- nothing new to report for it, and archiver_node already + -- has its row from that earlier call too. + CONTINUE; + END IF; + + INSERT INTO pgautofailover.archiver_node + (archiverid, kind, pgdata, nodeid) + VALUES (in_archiverid, 'wal-receiver', + '', new_nodeid); + + RETURN NEXT new_nodeid; + END LOOP; + + RETURN; +END; +$$; + +comment on function pgautofailover.archiver_add_formation(bigint,text) + is 'attach an archiver to every group of a formation, creating one lightweight ARCHIVING node row per group not already attached'; + +grant execute on function pgautofailover.archiver_add_formation(bigint,text) + to autoctl_node; + +-- one row per (formation, group) an archiver currently holds a +-- 'wal-receiver' membership in, across every formation it is attached +-- to -- what an archiver process itself calls, at startup and +-- periodically thereafter, to discover the full set of WAL streams and +-- base-backup schedules it is responsible for running (see +-- service_archiver_reconciler.c). Unlike get_archivers() above (scoped +-- to one formation, for `pg_autoctl watch`'s own display), this is +-- scoped to one archiver, across all its formations. +CREATE FUNCTION pgautofailover.list_archiver_memberships + ( + IN archiverid bigint, + OUT formation_id text, + OUT group_id int, + OUT node_id bigint, + OUT reported_state pgautofailover.replication_state, + OUT goal_state pgautofailover.replication_state + ) +RETURNS SETOF record LANGUAGE SQL STRICT SECURITY DEFINER +AS $$ + SELECT n.formationid, n.groupid, n.nodeid, n.reportedstate, n.goalstate + FROM pgautofailover.archiver_node an + JOIN pgautofailover.node n ON n.nodeid = an.nodeid + WHERE an.archiverid = list_archiver_memberships.archiverid + AND an.kind = 'wal-receiver' + ORDER BY n.formationid, n.groupid; +$$; + +comment on function pgautofailover.list_archiver_memberships(bigint) + is 'list every (formation, group) membership an archiver currently belongs to, across every formation it is attached to'; + +grant execute on function pgautofailover.list_archiver_memberships(bigint) + to autoctl_node; + +-- Deleting the node row is enough: archiver_node.nodeid's own +-- ON DELETE CASCADE removes the matching wal-receiver archiver_node row +-- automatically (see that column's own comment). +CREATE FUNCTION pgautofailover.archiver_remove_formation + (archiverid bigint, formationid text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.node n + WHERE n.formationid = archiver_remove_formation.formationid + AND n.nodeid IN (SELECT an.nodeid + FROM pgautofailover.archiver_node an + WHERE an.archiverid = archiver_remove_formation.archiverid + AND an.kind = 'wal-receiver'); + + DELETE FROM pgautofailover.archiver_formation af + WHERE af.archiverid = archiver_remove_formation.archiverid + AND af.formationid = archiver_remove_formation.formationid; +END; +$$; + +comment on function pgautofailover.archiver_remove_formation(bigint,text) + is 'detach an archiver from a formation, removing its ARCHIVING node row in every group'; + +grant execute on function pgautofailover.archiver_remove_formation(bigint,text) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified, even inside an +-- expression like coalesce(groupid, -1)) forces this naming here. +CREATE FUNCTION pgautofailover.set_archiver_policy + ( + in_formationid text, in_groupid int DEFAULT NULL, + in_archiverquorum int DEFAULT NULL, + in_basebackuppolicyid bigint DEFAULT NULL, + in_replicationquorumeligible bool DEFAULT NULL + ) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.archiver_policy + (formationid, groupid, archiverquorum, + basebackuppolicyid, replicationquorumeligible) + VALUES (in_formationid, in_groupid, + coalesce(in_archiverquorum, 1), + in_basebackuppolicyid, + coalesce(in_replicationquorumeligible, false)) + ON CONFLICT (formationid, (coalesce(groupid, -1))) DO UPDATE + SET archiverquorum = coalesce(EXCLUDED.archiverquorum, + pgautofailover.archiver_policy.archiverquorum), + basebackuppolicyid = coalesce(EXCLUDED.basebackuppolicyid, + pgautofailover.archiver_policy.basebackuppolicyid), + replicationquorumeligible = coalesce(EXCLUDED.replicationquorumeligible, + pgautofailover.archiver_policy.replicationquorumeligible); +END; +$$; + +comment on function pgautofailover.set_archiver_policy(text,int,int,bigint,bool) + is 'set (or override) archiver_quorum/basebackup policy/replication-quorum eligibility for a formation, or one of its groups'; + +grant execute on function + pgautofailover.set_archiver_policy(text,int,int,bigint,bool) + to autoctl_node; + +-- resolves group-specific override first, then the formation-wide +-- (groupid IS NULL) default, then this schema's own hardcoded defaults. +-- Deliberately plpgsql, not a single SQL query: an earlier draft tried to +-- express the three-way fallback as one UNION ALL ... LIMIT 1 query, but +-- UNION ALL has no ordering guarantee across its branches, so LIMIT 1 +-- could just as easily return the formation-wide or hardcoded default +-- even when a group-specific override exists. Sequential SELECT INTO ... +-- IF FOUND is unambiguous. +CREATE FUNCTION pgautofailover.get_archiver_policy(formationid text, groupid int) + RETURNS TABLE (archiverquorum int, basebackuppolicyid bigint, + replicationquorumeligible bool) + LANGUAGE plpgsql STABLE +AS $$ +BEGIN + RETURN QUERY + SELECT ap.archiverquorum, ap.basebackuppolicyid, ap.replicationquorumeligible + FROM pgautofailover.archiver_policy ap + WHERE ap.formationid = get_archiver_policy.formationid + AND ap.groupid = get_archiver_policy.groupid; + + IF FOUND THEN + RETURN; + END IF; + + RETURN QUERY + SELECT ap.archiverquorum, ap.basebackuppolicyid, ap.replicationquorumeligible + FROM pgautofailover.archiver_policy ap + WHERE ap.formationid = get_archiver_policy.formationid + AND ap.groupid IS NULL; + + IF FOUND THEN + RETURN; + END IF; + + RETURN QUERY + SELECT 1, p.basebackuppolicyid, false + FROM pgautofailover.basebackup_policy p + WHERE p.policyname = 'default'; +END; +$$; + +comment on function pgautofailover.get_archiver_policy(text,int) + is 'resolve archiver policy for (formation, group): group override, else formation default, else this schema''s own defaults'; + +grant execute on function pgautofailover.get_archiver_policy(text,int) + to autoctl_node; + +-- one round trip from the archiver-basebackup side: resolves the +-- basebackup_policy row that applies to (formation, group) via get_ +-- archiver_policy() above, then flattens its interval columns to plain +-- integer seconds -- easy time_t arithmetic on the C side, no interval- +-- text parsing needed. SECURITY DEFINER: reads archiver_policy/ +-- basebackup_policy directly, both created (like every table in this +-- milestone's own schema) after the blanket "GRANT SELECT ON ALL TABLES" +-- near the top of this file, so autoctl_node has no direct grant on +-- either -- same class of gap already hit (and fixed) twice for wal_ +-- archived()/get_latest_basebackup(). +CREATE FUNCTION pgautofailover.get_basebackup_policy_for_group + ( + formationid text, + groupid int, + OUT policyname text, + OUT source pgautofailover.basebackup_source, + OUT replaymode pgautofailover.basebackup_replay_mode, + OUT cache pgautofailover.basebackup_cache, + OUT frequency_seconds int, + OUT maxcount int, + OUT maxage_seconds int, + OUT onpromotion bool, + OUT concurrency int + ) + RETURNS record LANGUAGE plpgsql STABLE SECURITY DEFINER +AS $$ +DECLARE + ap record; +BEGIN + SELECT * INTO ap + FROM pgautofailover.get_archiver_policy( + get_basebackup_policy_for_group.formationid, + get_basebackup_policy_for_group.groupid); + + SELECT p.policyname, p.source, p.replaymode, p.cache, + extract(epoch FROM p.frequency)::int, + p.maxcount, + extract(epoch FROM p.maxage)::int, + p.onpromotion, p.concurrency + INTO policyname, source, replaymode, cache, frequency_seconds, + maxcount, maxage_seconds, onpromotion, concurrency + FROM pgautofailover.basebackup_policy p + WHERE p.basebackuppolicyid = ap.basebackuppolicyid; +END; +$$; + +comment on function pgautofailover.get_basebackup_policy_for_group(text,int) + is 'resolve the full base-backup production/retention policy for (formation, group), intervals flattened to seconds'; + +grant execute on function pgautofailover.get_basebackup_policy_for_group(text,int) + to autoctl_node; + +-- the archive_command confirmation check: true iff at least +-- archiver_quorum distinct archivers have durably reported %f +CREATE FUNCTION pgautofailover.wal_archived + (formationid text, groupid int, walfilename text) + RETURNS bool + LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT count(DISTINCT aw.archiverid) >= + (SELECT archiverquorum + FROM pgautofailover.get_archiver_policy(wal_archived.formationid, + wal_archived.groupid)) + FROM pgautofailover.archiver_wal aw + WHERE aw.formationid = wal_archived.formationid + AND aw.groupid = wal_archived.groupid + AND aw.walfilename = wal_archived.walfilename; +$$; + +comment on function pgautofailover.wal_archived(text,int,text) + is 'archive_command confirmation check: has segment %f already landed durably on archiver_quorum archiver(s)?'; + +grant execute on function pgautofailover.wal_archived(text,int,text) + to autoctl_node; + +-- inserts into archiver_wal (idempotent on conflict) +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.report_wal_received + (in_nodeid bigint, in_walfilename text, in_lsn pg_lsn) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + target record; +BEGIN + SELECT n.formationid, n.groupid, an.archiverid + INTO target + FROM pgautofailover.archiver_node an + JOIN pgautofailover.node n ON n.nodeid = an.nodeid + WHERE an.nodeid = in_nodeid + AND an.kind = 'wal-receiver'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'node % is not an ARCHIVING wal-receiver node', in_nodeid; + END IF; + + INSERT INTO pgautofailover.archiver_wal + (formationid, groupid, walfilename, archiverid, lsn) + VALUES (target.formationid, target.groupid, in_walfilename, target.archiverid, in_lsn) + ON CONFLICT (formationid, groupid, walfilename, archiverid) DO NOTHING; +END; +$$; + +comment on function pgautofailover.report_wal_received(bigint,text,pg_lsn) + is 'reports a WAL segment durably captured by an ARCHIVING node'; + +grant execute on function pgautofailover.report_wal_received(bigint,text,pg_lsn) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_started + ( + archiverid bigint, formationid text, groupid int, + label text, timeline int, startlsn pg_lsn, + source pgautofailover.basebackup_source, + replaymode pgautofailover.basebackup_replay_mode DEFAULT NULL + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.basebackup + (archiverid, formationid, groupid, label, timeline, startlsn, + source, replaymode, storagelocation, status) + VALUES (archiverid, formationid, groupid, label, timeline, startlsn, + source, replaymode, '', 'in_progress') + RETURNING basebackupid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.report_basebackup_started + (bigint,text,int,text,int,pg_lsn,pgautofailover.basebackup_source,pgautofailover.basebackup_replay_mode) + is 'records the start of a new base-backup production job'; + +grant execute on function + pgautofailover.report_basebackup_started + (bigint,text,int,text,int,pg_lsn,pgautofailover.basebackup_source,pgautofailover.basebackup_replay_mode) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_completed + (basebackupid bigint, endlsn pg_lsn, sizebytes bigint, storagelocation text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup AS bb + SET endlsn = report_basebackup_completed.endlsn, + sizebytes = report_basebackup_completed.sizebytes, + storagelocation = report_basebackup_completed.storagelocation, + status = 'complete', + period = tstzrange(lower(bb.period), now()) + WHERE bb.basebackupid = report_basebackup_completed.basebackupid; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup % does not exist', basebackupid; + END IF; +END; +$$; + +comment on function pgautofailover.report_basebackup_completed(bigint,pg_lsn,bigint,text) + is 'records the successful completion of a base-backup production job'; + +grant execute on function + pgautofailover.report_basebackup_completed(bigint,pg_lsn,bigint,text) + to autoctl_node; + +-- marks the basebackup row deleted (never a real DELETE), then prunes +-- any archiver_wal rows this group no longer needs to retain +CREATE FUNCTION pgautofailover.report_basebackup_deleted(basebackupid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + bb record; +BEGIN + UPDATE pgautofailover.basebackup AS b + SET status = 'deleted', deletedat = now() + WHERE b.basebackupid = report_basebackup_deleted.basebackupid + RETURNING b.formationid, b.groupid INTO bb; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup % does not exist', basebackupid; + END IF; + + PERFORM pgautofailover.prune_archiver_wal(bb.formationid, bb.groupid); +END; +$$; + +comment on function pgautofailover.report_basebackup_deleted(bigint) + is 'marks a base backup deleted (retains history) and prunes any archiver_wal rows no group backup needs anymore'; + +grant execute on function pgautofailover.report_basebackup_deleted(bigint) + to autoctl_node; + +-- deletes every archiver_wal row for (formationid, groupid) older than +-- the earliest still-'complete' basebackup's startlsn, across every +-- archiver holding a copy. When no 'complete' backup remains for this +-- group, nothing is pruned -- there is no anchor point to replay forward +-- from, so every captured segment is still needed. +CREATE FUNCTION pgautofailover.prune_archiver_wal(formationid text, groupid int) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + oldest_startlsn pg_lsn; + deleted_count bigint; +BEGIN + SELECT min(b.startlsn) INTO oldest_startlsn + FROM pgautofailover.basebackup b + WHERE b.formationid = prune_archiver_wal.formationid + AND b.groupid = prune_archiver_wal.groupid + AND b.status = 'complete'; + + IF oldest_startlsn IS NULL THEN + RETURN 0; + END IF; + + WITH deleted AS ( + DELETE FROM pgautofailover.archiver_wal aw + WHERE aw.formationid = prune_archiver_wal.formationid + AND aw.groupid = prune_archiver_wal.groupid + AND aw.lsn < oldest_startlsn + RETURNING 1 + ) + SELECT count(*) INTO deleted_count FROM deleted; + + RETURN deleted_count; +END; +$$; + +comment on function pgautofailover.prune_archiver_wal(text,int) + is 'deletes archiver_wal rows for (formation, group) older than the oldest still-complete base backup''s startlsn'; + +grant execute on function pgautofailover.prune_archiver_wal(text,int) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.report_basebackup_synced + (in_basebackupid bigint, in_archiverstorageid bigint, in_remotelocation text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.basebackup_storage + (basebackupid, archiverstorageid, syncedat, remotelocation) + VALUES (in_basebackupid, in_archiverstorageid, now(), in_remotelocation) + ON CONFLICT (basebackupid, archiverstorageid) DO UPDATE + SET syncedat = now(), + remotelocation = EXCLUDED.remotelocation; +END; +$$; + +comment on function pgautofailover.report_basebackup_synced(bigint,bigint,text) + is 'records a successful cold-storage sync of a base backup to one storage target'; + +grant execute on function + pgautofailover.report_basebackup_synced(bigint,bigint,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_remote_deleted + (basebackupid bigint, archiverstorageid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup_storage AS bs + SET deletedat = now() + WHERE bs.basebackupid = report_basebackup_remote_deleted.basebackupid + AND bs.archiverstorageid = report_basebackup_remote_deleted.archiverstorageid; +END; +$$; + +comment on function pgautofailover.report_basebackup_remote_deleted(bigint,bigint) + is 'records that a base backup''s remote copy on one storage target has been pruned'; + +grant execute on function + pgautofailover.report_basebackup_remote_deleted(bigint,bigint) + to autoctl_node; + +-- filters status = 'complete' only. SECURITY DEFINER matches every other +-- autoctl_node-callable helper reading a table that role has no direct +-- SELECT grant on (e.g. archiver_add_formation) -- autoctl_node is only +-- ever granted EXECUTE on the function, never SELECT on pgautofailover. +-- basebackup itself. +-- +-- preferred_source (default NULL, meaning "any") exists for service_ +-- archiver_serve.c's own routes-file refresh: a 'replay' backup promotes a +-- throwaway extracted copy, which genuinely puts it on a *later* timeline +-- than whatever the archiver's own walcache has actually captured (which +-- only ever advances on the real primary's timeline) -- serving that pair +-- together breaks a real pg_basebackup's own timeline consistency check +-- (receivelog.c). Since a 'live' backup is taken directly from the +-- actively-followed primary, it always shares the walcache's timeline by +-- construction; passing preferred_source = 'live' is how the routes +-- refresh asks for one specifically, rather than "whatever is newest +-- regardless of type". +CREATE FUNCTION pgautofailover.get_latest_basebackup + ( + formationid text, + groupid int, + preferred_source pgautofailover.basebackup_source default NULL + ) + RETURNS pgautofailover.basebackup LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT * FROM pgautofailover.basebackup b + WHERE b.formationid = get_latest_basebackup.formationid + AND b.groupid = get_latest_basebackup.groupid + AND b.status = 'complete' + AND (get_latest_basebackup.preferred_source IS NULL + OR b.source = get_latest_basebackup.preferred_source) + ORDER BY lower(b.period) DESC + LIMIT 1; +$$; + +comment on function pgautofailover.get_latest_basebackup + (text,int,pgautofailover.basebackup_source) + is 'fetch the most recent complete base backup for (formation, group), optionally filtered to one source'; + +grant execute on function pgautofailover.get_latest_basebackup + (text,int,pgautofailover.basebackup_source) + to autoctl_node; + +-- every 'complete' base backup for (formation, group), newest first -- +-- what service_archiver_basebackup.c's own retention pass (maxcount/ +-- maxage) walks to decide what to keep vs. prune, and what a future `pg_ +-- autoctl show basebackup` would list. basebackupid/storagelocation are +-- what report_basebackup_deleted()/an actual directory removal need; +-- startedat_epoch (extract(epoch from lower(period))) is plain integer +-- seconds for the same reason get_basebackup_policy_for_group() flattens +-- its own interval columns -- easy time_t arithmetic, no timestamptz-text +-- parsing on the C side. +CREATE FUNCTION pgautofailover.list_basebackups + ( + formationid text, + groupid int, + OUT basebackupid bigint, + OUT label text, + OUT storagelocation text, + OUT startedat_epoch bigint + ) + RETURNS SETOF record LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT b.basebackupid, b.label, b.storagelocation, + extract(epoch FROM lower(b.period))::bigint + FROM pgautofailover.basebackup b + WHERE b.formationid = list_basebackups.formationid + AND b.groupid = list_basebackups.groupid + AND b.status = 'complete' + ORDER BY lower(b.period) DESC; +$$; + +comment on function pgautofailover.list_basebackups(text,int) + is 'list complete base backups for (formation, group), newest first -- retention/inventory'; + +grant execute on function pgautofailover.list_basebackups(text,int) + to autoctl_node; + +-- an archiving node has no sysidentifier of its own (haspgdata = false, +-- see that column's own comment): it never runs a real Postgres instance +-- to report one. Every other node in the group shares the same physical +-- cluster's identifier, so any one of them answers for the whole group -- +-- needed by pg_walsender's own IDENTIFY_SYSTEM response (cmd_identify_ +-- system.c) so a real standby streaming from the archiver doesn't reject +-- it with "database system identifier differs between the primary and +-- standby". +CREATE FUNCTION pgautofailover.get_group_system_identifier + (formationid text, groupid int) + RETURNS bigint LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT sysidentifier + FROM pgautofailover.node + WHERE node.formationid = get_group_system_identifier.formationid + AND node.groupid = get_group_system_identifier.groupid + AND sysidentifier IS NOT NULL + AND sysidentifier != 0 + LIMIT 1; +$$; + +comment on function pgautofailover.get_group_system_identifier(text,int) + is 'the Postgres system identifier shared by every node in a group, for an archiving node (which has none of its own) to serve via IDENTIFY_SYSTEM'; + +grant execute on function pgautofailover.get_group_system_identifier(text,int) + to autoctl_node; + +-- `create postgres --from-archiver` needs the ARCHIVING row itself, not +-- get_most_advanced_standby()'s election-only pool: that function filters +-- on reportedstate = 'report_lsn', a transient state a group's ARCHIVING +-- node only visits during a FAST_FORWARD election, never during its normal +-- steady-state operation (reportedstate = 'archiving'). node_port is the +-- port == 0 sentinel documented on get_most_advanced_standby's own C +-- caller (keeper_get_most_advanced_standby, keeper.c) -- resolving it to +-- the archiver's real pg_walsender serve port is this milestone's C +-- caller's job too, same pattern. +CREATE FUNCTION pgautofailover.get_archiver_node + ( + IN formationid text default 'default', + IN groupid int default 0, + OUT node_id bigint, + OUT node_name text, + OUT node_host text, + OUT node_port int, + OUT node_lsn pg_lsn, + OUT node_is_primary bool + ) +RETURNS SETOF record LANGUAGE SQL STRICT +AS $$ + select nodeid, nodename, nodehost, nodeport, reportedlsn, false + from pgautofailover.node + where formationid = $1 + and groupid = $2 + and reportedstate = 'archiving' + order by nodeid + limit 1; +$$; + +comment on function pgautofailover.get_archiver_node(text,int) + is 'fetch the ARCHIVING node for (formation, group), for create postgres --from-archiver to bootstrap from'; + +grant execute on function pgautofailover.get_archiver_node(text,int) + to autoctl_node; + +-- for kind = 'warm-standby': raises if the owning archiver is already at +-- its maxresidentreplay cap +CREATE FUNCTION pgautofailover.create_archiver_node + ( + archiverid bigint, + kind pgautofailover.archiver_node_kind, + pgdata text, + hostname text DEFAULT NULL, + nodeid bigint DEFAULT NULL, -- required iff kind = 'wal-receiver' + formationid text DEFAULT NULL, -- required iff kind = 'warm-standby' + groupid int DEFAULT NULL, -- required iff kind = 'warm-standby' + cadence pgautofailover.archiver_node_cadence DEFAULT NULL, + nodecluster text DEFAULT NULL, -- only for 'warm-standby' + cadence = 'continuous' + pitrstatus pgautofailover.pitr_status DEFAULT NULL + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + residentcount int; + maxresident int; + new_id bigint; +BEGIN + IF kind = 'warm-standby' THEN + SELECT a.maxresidentreplay INTO maxresident + FROM pgautofailover.archiver a + WHERE a.archiverid = create_archiver_node.archiverid; + + SELECT count(*) INTO residentcount + FROM pgautofailover.archiver_node an + WHERE an.archiverid = create_archiver_node.archiverid + AND an.kind = 'warm-standby'; + + IF residentcount >= maxresident THEN + RAISE EXCEPTION + 'archiver % is already at its maxresidentreplay cap (%)', + archiverid, maxresident; + END IF; + END IF; + + INSERT INTO pgautofailover.archiver_node + (archiverid, kind, pgdata, hostname, nodeid, + formationid, groupid, cadence, nodecluster, pitrstatus) + VALUES (archiverid, kind, pgdata, hostname, nodeid, + formationid, groupid, cadence, nodecluster, pitrstatus) + RETURNING archivernodeid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_archiver_node + (bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,int, + pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) + is 'registers a concrete Postgres instance an archiver hosts, derives, or is otherwise associated with'; + +grant execute on function + pgautofailover.create_archiver_node + (bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,int, + pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) + to autoctl_node; + +CREATE FUNCTION pgautofailover.remove_archiver_node(archivernodeid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.archiver_node an + WHERE an.archivernodeid = remove_archiver_node.archivernodeid; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_node % does not exist', archivernodeid; + END IF; +END; +$$; + +comment on function pgautofailover.remove_archiver_node(bigint) + is 'removes an archiver_node row'; + +grant execute on function pgautofailover.remove_archiver_node(bigint) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_archiver_node_pitr_status + (archivernodeid bigint, pitrstatus pgautofailover.pitr_status) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.archiver_node AS an + SET pitrstatus = set_archiver_node_pitr_status.pitrstatus + WHERE an.archivernodeid = set_archiver_node_pitr_status.archivernodeid + AND an.kind = 'pitr'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_node % does not exist, or is not kind = pitr', + archivernodeid; + END IF; +END; +$$; + +comment on function pgautofailover.set_archiver_node_pitr_status(bigint,pgautofailover.pitr_status) + is 'updates a PITR archiver_node''s lifecycle status'; + +grant execute on function + pgautofailover.set_archiver_node_pitr_status(bigint,pgautofailover.pitr_status) + to autoctl_node; + +-- pushed by the local pg_autoctl pitr CLI immediately after acting +-- locally -- never blocks or gates the local action on this succeeding +CREATE FUNCTION pgautofailover.report_pitr_status + ( + archivernodeid bigint, operation pgautofailover.pitr_operation, + requestedspec jsonb, + observedlsn pg_lsn, observedtimestamp timestamptz, + observedpausestate text, note text DEFAULT NULL + ) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.pitr_history + (archivernodeid, operation, requestedspec, + observedlsn, observedtimestamp, observedpausestate, note) + VALUES (archivernodeid, operation, requestedspec, + observedlsn, observedtimestamp, observedpausestate, note); +END; +$$; + +comment on function pgautofailover.report_pitr_status + (bigint,pgautofailover.pitr_operation,jsonb,pg_lsn,timestamptz,text,text) + is 'records one PITR operation''s outcome -- a best-effort report, never gating the local action it follows'; + +grant execute on function + pgautofailover.report_pitr_status + (bigint,pgautofailover.pitr_operation,jsonb,pg_lsn,timestamptz,text,text) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.pitr_queue_command + (in_archivernodeid bigint, in_command pgautofailover.pitr_command, + in_commandspec jsonb DEFAULT NULL) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.pitr_pending_command + (archivernodeid, command, commandspec) + VALUES (in_archivernodeid, in_command, in_commandspec) + ON CONFLICT (archivernodeid) DO UPDATE + SET command = EXCLUDED.command, + commandspec = EXCLUDED.commandspec, + queuedat = now(); +END; +$$; + +comment on function pgautofailover.pitr_queue_command(bigint,pgautofailover.pitr_command,jsonb) + is 'queues a PITR command for a monitor-mediated (kind = pitr, pg_autoctl node run) agent to pick up'; + +grant execute on function + pgautofailover.pitr_queue_command(bigint,pgautofailover.pitr_command,jsonb) + to autoctl_node; + +-- returns the pending command and resets the queue slot to 'none' in the +-- same call -- an agent polling this never processes the same command twice +-- Reads the pending command, then clears it, as two separate statements: +-- UPDATE ... RETURNING always reflects the row *after* the update is +-- applied, so folding the reset into the same RETURNING clause that reads +-- the command would always report back the very 'none' this function just +-- set, never the command that was actually queued. FOR UPDATE locks the +-- row across both statements, so a concurrent caller for the same +-- archivernodeid still can't observe or consume the same command twice. +CREATE FUNCTION pgautofailover.pitr_next_command(in_archivernodeid bigint) + RETURNS pgautofailover.pitr_command LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + next_command pgautofailover.pitr_command; +BEGIN + SELECT pc.command INTO next_command + FROM pgautofailover.pitr_pending_command pc + WHERE pc.archivernodeid = in_archivernodeid + FOR UPDATE; + + IF next_command IS NULL OR next_command = 'none' THEN + RETURN 'none'; + END IF; + + UPDATE pgautofailover.pitr_pending_command AS pc + SET command = 'none', commandspec = NULL + WHERE pc.archivernodeid = in_archivernodeid; + + RETURN next_command; +END; +$$; + +comment on function pgautofailover.pitr_next_command(bigint) + is 'pops and clears the next queued PITR command for an agent to act on'; + +grant execute on function pgautofailover.pitr_next_command(bigint) + to autoctl_node; + + +-- +-- Archiving & Disaster Recovery, milestone 2: monitor-side FSM support for +-- the ARCHIVING state (group_state_machine.c). Loosen +-- system_identifier_is_null_at_init_only to also allow a NULL sysidentifier +-- in 'archiving' and 'report_lsn': an ARCHIVING row (haspgdata = false) has +-- no PGDATA of its own, ever, so it never acquires a real sysidentifier -- +-- see pgautofailover.sql's own comment on this constraint for why +-- 'report_lsn' is safe to loosen too. +-- +-- reportedstate::text IN (...) here, not reportedstate IN (...): this +-- script's own earlier "ALTER TYPE ... ADD VALUE 'archiving'" added that +-- label in this same transaction (ALTER EXTENSION ... UPDATE runs the whole +-- upgrade script as one transaction), and Postgres refuses to cast a string +-- literal to a not-yet-committed enum value ("unsafe use of new value... +-- must be committed before they can be used") -- casting the *column* +-- to text instead of the literals to the enum sidesteps that restriction +-- entirely, since reportedstate's own stored value is already a valid, +-- fully-committed enum datum by the time this constraint ever evaluates it. +-- pgautofailover.sql's fresh-install CHECK constraint doesn't need this: a +-- freshly CREATE TYPE'd enum has 'archiving' as a member from the start, +-- never added mid-transaction, so the ordinary enum-typed comparison there +-- is unaffected. +-- + +ALTER TABLE pgautofailover.node + DROP CONSTRAINT system_identifier_is_null_at_init_only; + +ALTER TABLE pgautofailover.node + ADD CONSTRAINT system_identifier_is_null_at_init_only + CHECK ( + ( + sysidentifier IS NULL + AND reportedstate::text + IN ( + 'init', + 'wait_standby', + 'catchingup', + 'dropped', + 'archiving', + 'report_lsn' + ) + ) + OR sysidentifier IS NOT NULL + ); diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index 8545bf38c..5d4e93cd3 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -41,7 +41,8 @@ CREATE TYPE pgautofailover.replication_state 'report_lsn', 'fast_forward', 'join_secondary', - 'dropped' + 'dropped', + 'archiving' ); CREATE TABLE pgautofailover.formation @@ -135,10 +136,18 @@ CREATE TABLE pgautofailover.node pg_versionstring text, citus_version text, + -- true for every ordinary Postgres node (its own PGDATA, promotable); + -- false only for an ARCHIVING membership row (a pg_receivewal client, + -- no PGDATA, no postmaster to manage). See archiving-disaster-recovery + -- design: this single boolean is what candidate_priority enforcement, + -- keeper_ensure_current_state's liveness check, and the FAST_FORWARD + -- source-selection branch all key off, instead of a third node-kind + -- value -- a cascading follower is still haspgdata = true, and a + -- future proxy never becomes a pgautofailover.node row at all. + haspgdata bool NOT NULL DEFAULT true, + -- node names must be unique in a given formation UNIQUE (formationid, nodename), - -- any nodehost:port can only be a unique node in the system - UNIQUE (nodehost, nodeport), -- -- The EXCLUDE constraint only allows the same sysidentifier for all the -- nodes in the same group. The system_identifier is a property that is @@ -149,6 +158,15 @@ CREATE TABLE pgautofailover.node -- primary server from scratch, because we have not done pg_ctl initdb -- at the time we call the register_node() function. -- + -- 'archiving' and 'report_lsn' are also allowed here: an ARCHIVING row + -- (haspgdata = false) has no PGDATA of its own, ever, so it never + -- acquires a real sysidentifier -- and 'report_lsn' is a state it + -- legitimately reaches too, pulled into elections the same as + -- SECONDARY/CATCHINGUP (see haspgdata's own comment). Loosening this + -- CHECK to also permit NULL in 'report_lsn' doesn't hide anything for + -- ordinary nodes: by the time a real node ever reaches report_lsn its + -- own bootstrap sequence has long since given it a real sysidentifier. + -- CONSTRAINT system_identifier_is_null_at_init_only CHECK ( ( @@ -158,7 +176,9 @@ CREATE TABLE pgautofailover.node 'init', 'wait_standby', 'catchingup', - 'dropped' + 'dropped', + 'archiving', + 'report_lsn' ) ) OR sysidentifier IS NOT NULL @@ -176,6 +196,17 @@ CREATE TABLE pgautofailover.node -- we expect few rows and lots of UPDATE, let's benefit from HOT WITH (fillfactor = 25); +-- any nodehost:port can only be a unique real Postgres node in the +-- system -- scoped to haspgdata rows only: an ARCHIVING membership row +-- has no listening postmaster of its own to be unique about (nodehost is +-- its owning archiver's hostname, nodeport is the 0 sentinel -- see +-- haspgdata's own comment above), and the same archiver legitimately +-- backs one row per group it serves, all sharing that same (nodehost, 0) +-- pair. +CREATE UNIQUE INDEX node_nodehost_nodeport_haspgdata_idx + ON pgautofailover.node (nodehost, nodeport) + WHERE haspgdata; + -- Mirrors group_state_machine.h's MonitorFSMSection: which of the three -- real control-flow regions of the monitor's declarative dispatch table -- (MonitorFSM[] in group_state_machine.c) a rule belongs to. See dump_fsm() @@ -1292,6 +1323,1549 @@ comment on function pgautofailover.formation_settings(text) is 'get the current replication settings a formation'; -- +-- +-- Archiving & Disaster Recovery: schema for the Archiver process identity, +-- ARCHIVING node memberships, base-backup policy/history, and PITR. +-- See ~/dev/temp/archiving-disaster-recovery.md for the full design. +-- +-- Milestone 1 (schema + monitor API only): every function here is plain +-- plpgsql/SQL, callable directly with no service_archiver process running +-- -- the pgaftest coverage for this milestone exercises these functions +-- via direct SQL calls against a plain cluster. +-- + +CREATE TYPE pgautofailover.storage_method + AS ENUM ('local', 'rclone'); + +CREATE TYPE pgautofailover.basebackup_source + AS ENUM ('live', 'replay'); + +CREATE TYPE pgautofailover.basebackup_replay_mode + AS ENUM ('volatile', 'persistent'); + +CREATE TYPE pgautofailover.basebackup_cache + AS ENUM ('local', 'none'); + +CREATE TYPE pgautofailover.basebackup_status + AS ENUM ('in_progress', 'complete', 'failed', 'deleted'); + -- 'deleted' is what makes basebackup a full history rather + -- than just a live catalog + +-- shared or per-archiver base-backup production/retention policy +CREATE TABLE pgautofailover.basebackup_policy + ( + basebackuppolicyid bigserial PRIMARY KEY, + policyname text UNIQUE, + + source pgautofailover.basebackup_source + NOT NULL DEFAULT 'replay', + replaymode pgautofailover.basebackup_replay_mode + DEFAULT 'volatile', + cache pgautofailover.basebackup_cache + NOT NULL DEFAULT 'local', + + -- strong, ready-to-use-as-is defaults -- nightly, 3 days retention + frequency interval NOT NULL DEFAULT '24 hours', + maxcount int NOT NULL DEFAULT 3, + maxage interval NOT NULL DEFAULT '3 days', + onpromotion bool NOT NULL DEFAULT true, + + -- backpressure: cap on simultaneous base-backup production jobs, + -- per archiver, per referencing policy + concurrency int NOT NULL DEFAULT 1, + + CHECK (source <> 'replay' OR replaymode IS NOT NULL), + CHECK (concurrency >= 1) + ); + +INSERT INTO pgautofailover.basebackup_policy (policyname) VALUES ('default'); + +-- the physical Archiver entity: one row per archiver host/process +CREATE TABLE pgautofailover.archiver + ( + archiverid bigserial PRIMARY KEY, + archivername text NOT NULL, + hostname text NOT NULL, + createdat timestamptz NOT NULL DEFAULT now(), + + -- same convention as pgautofailover.node.region: a free-form label for + -- the data-centre or availability zone this archiver runs in, purely + -- informational (get_archivers()'s own consumers, e.g. pg_autoctl + -- watch, may display it) -- set at registration time via + -- register_archiver()'s own region parameter, never inferred. Multiple + -- archivers can attach to the very same formation (archiver_add_ + -- formation() names each ARCHIVING node row after its own archiverid, + -- so two different archivers never collide there) -- distinct regions + -- is the expected shape for geographically-redundant DR coverage of + -- one formation, and archiver_policy's own archiverquorum column + -- already anticipates requiring more than one archiver's confirmation. + region text not null default 'default', + + basebackuppolicyid bigint NOT NULL + REFERENCES pgautofailover.basebackup_policy (basebackuppolicyid), + + autoregister bool NOT NULL DEFAULT true, + + -- cap on resident 'warm-standby' archiver_node rows (either cadence) + -- this host is allowed to keep running at once + maxresidentreplay int NOT NULL DEFAULT 1, + + -- storage stats for the archiver's own PGDATA (walcache + basebackups, + -- same root -- see service_archiver_serve.c's own header comment on + -- why an archiver has no other pgdata to speak of), reported + -- periodically by service_archiver_loop(); NULL until the first report. + -- usedbytes is this archiver's own footprint (directory_size() over its + -- whole pgdata); freebytes is the containing filesystem's available + -- space (statvfs's f_bavail, "available to a non-privileged process" -- + -- the number that actually predicts whether the next base backup or + -- WAL segment fits, not f_bfree's superuser-reserved total). + usedbytes bigint, + freebytes bigint, + + lastreporttime timestamptz, + + UNIQUE (archivername), + CHECK (maxresidentreplay >= 0), + CHECK (usedbytes IS NULL OR usedbytes >= 0), + CHECK (freebytes IS NULL OR freebytes >= 0) + ); + +-- a named, shareable rclone remote configuration -- the literal contents +-- of an rclone config file (real INI format, exactly as rclone itself +-- reads it: https://rclone.org/docs/#config-file). `config` should hold +-- only the non-secret, architectural half of an rclone remote (type, +-- provider, endpoint, region, acl, and a `type = alias` remote baking in +-- the bucket/prefix) -- credentials belong in the archiver process's own +-- environment (RCLONE_CONFIG__), never in this column, which +-- is backed up and readable by anyone with SQL access to the monitor. +CREATE TABLE pgautofailover.rclone_config + ( + rcloneconfigid bigserial PRIMARY KEY, + name text UNIQUE NOT NULL, + config text NOT NULL, + createdat timestamptz NOT NULL DEFAULT now() + ); + +-- 1-N: an archiver's storage targets. Exactly one 'local' row always +-- exists (the mandatory default); adding cloud storage means adding one +-- or more 'rclone' rows, each an independent push target, each +-- referencing a (possibly shared) rclone_config row +CREATE TABLE pgautofailover.archiver_storage + ( + archiverstorageid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + storagemethod pgautofailover.storage_method NOT NULL, + + storagepath text, -- 'local' only: override the default topdir path + rcloneconfigid bigint REFERENCES pgautofailover.rclone_config (rcloneconfigid), + -- 'rclone' only: which named config this target uses + + createdat timestamptz NOT NULL DEFAULT now(), + + CHECK (storagemethod <> 'local' OR rcloneconfigid IS NULL), + CHECK (storagemethod <> 'rclone' OR rcloneconfigid IS NOT NULL) + ); + +CREATE UNIQUE INDEX archiver_storage_one_local + ON pgautofailover.archiver_storage (archiverid) + WHERE storagemethod = 'local'; + +-- formation-granularity attachment. Only holds explicit rows for the +-- restricted case -- when autoregister is true this table isn't consulted +CREATE TABLE pgautofailover.archiver_formation + ( + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + formationid text NOT NULL REFERENCES pgautofailover.formation (formationid) + ON DELETE CASCADE, + attachedat timestamptz NOT NULL DEFAULT now(), + + PRIMARY KEY (archiverid, formationid) + ); + +-- policy override, resolved formation-default then group-specific; +-- groupid IS NULL means "the formation-wide default for this archiver" +CREATE TABLE pgautofailover.archiver_policy + ( + formationid text NOT NULL REFERENCES pgautofailover.formation (formationid) + ON DELETE CASCADE, + groupid int, + archiverquorum int NOT NULL DEFAULT 1, + basebackuppolicyid bigint + REFERENCES pgautofailover.basebackup_policy (basebackuppolicyid), + replicationquorumeligible bool NOT NULL DEFAULT false + ); + +-- A plain UNIQUE (formationid, groupid) constraint would not actually +-- enforce "at most one formation-wide default row": Postgres treats every +-- NULL groupid as distinct from every other NULL for uniqueness purposes, +-- so two formation-wide rows for the same formation would never conflict. +-- coalesce(groupid, -1) normalizes NULL to a real, comparable value +-- instead -- -1 is safe as a stand-in since groupid is otherwise always +-- >= 0. set_archiver_policy's own ON CONFLICT targets this index. +CREATE UNIQUE INDEX archiver_policy_formation_group_idx + ON pgautofailover.archiver_policy (formationid, coalesce(groupid, -1)); + +-- one row per base backup taken by any archiver -- full history, not just +-- a live catalog: rows are never deleted by retention, only marked +-- status = 'deleted'; get_latest_basebackup filters on status = 'complete' +CREATE TABLE pgautofailover.basebackup + ( + basebackupid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + formationid text NOT NULL, + groupid int NOT NULL, + label text NOT NULL, + timeline int NOT NULL, + startlsn pg_lsn NOT NULL, + endlsn pg_lsn, + + period tstzrange NOT NULL DEFAULT tstzrange(now(), NULL), + + -- snapshot of how this specific backup was produced, independent of + -- whatever basebackup_policy says *now* + source pgautofailover.basebackup_source NOT NULL, + replaymode pgautofailover.basebackup_replay_mode, + + sizebytes bigint, + storagelocation text NOT NULL, -- local path, or object-storage URI + status pgautofailover.basebackup_status + NOT NULL DEFAULT 'in_progress', + deletedat timestamptz + ); + +CREATE INDEX basebackup_group_idx + ON pgautofailover.basebackup (formationid, groupid, lower(period) DESC); + +-- remote-side sync/prune tracking, one row per (basebackup, remote +-- storage target) -- a single backup can sync to several remotes +CREATE TABLE pgautofailover.basebackup_storage + ( + basebackupid bigint NOT NULL REFERENCES pgautofailover.basebackup (basebackupid) + ON DELETE CASCADE, + archiverstorageid bigint NOT NULL REFERENCES pgautofailover.archiver_storage (archiverstorageid) + ON DELETE CASCADE, + + syncedat timestamptz, + remotelocation text, + deletedat timestamptz, + + PRIMARY KEY (basebackupid, archiverstorageid) + ); + +-- one row per (archiver, WAL segment) durably captured -- the real +-- backing store wal_archived() queries. +-- +-- PRIMARY KEY is (formationid, groupid, walfilename, archiverid) -- the +-- hot path is wal_archived()'s lookup across every archiver holding %f +-- for this group, so this ordering makes it a direct index range scan. +-- +-- FILLFACTOR 20: traffic is INSERT + DELETE, never UPDATE, but is +-- continuous and high-throughput -- a low fillfactor spreads rows across +-- more pages, reducing buffer-lock contention between concurrently +-- inserting archivers and easing autovacuum on a table that's never +-- write-quiet. +CREATE TABLE pgautofailover.archiver_wal + ( + formationid text NOT NULL, + groupid int NOT NULL, + walfilename text NOT NULL, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + + lsn pg_lsn NOT NULL, + receivedat timestamptz NOT NULL DEFAULT now(), + + PRIMARY KEY (formationid, groupid, walfilename, archiverid) + ) WITH (fillfactor = 20); + +CREATE TYPE pgautofailover.archiver_node_kind + AS ENUM ('wal-receiver', 'warm-standby', 'pitr'); + -- 'staging' anticipated for a later, not-yet-designed feature + -- (periodic dev/test environments refreshed from the archiver) + +CREATE TYPE pgautofailover.archiver_node_cadence + AS ENUM ('continuous', 'scheduled'); + -- 'manual' considered (operator-driven "advance only when I say so"), + -- not added yet -- same one-value-enum-addition cost as 'staging' + +CREATE TYPE pgautofailover.pitr_status + AS ENUM ('restoring', 'paused', 'registered', 'discarded'); + +-- every concrete Postgres instance an archiver hosts, derives, or is +-- otherwise associated with, beyond the archiver process itself +CREATE TABLE pgautofailover.archiver_node + ( + archivernodeid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + kind pgautofailover.archiver_node_kind NOT NULL, + + -- placement, uniform across every kind: NULL = colocated (local file + -- reads, zero network); non-NULL = a separate node (remote fetch) + hostname text, + pgdata text NOT NULL, + + -- 'wal-receiver' only: which ARCHIVING row this instance backs. + -- ON DELETE CASCADE: the ARCHIVING node row can be removed through + -- more than one path (this schema's own archiver_remove_formation, + -- or the ordinary pgautofailover.remove_node() every other node type + -- already goes through) -- cascading here means every path safely + -- cleans up this row too, instead of only the one this schema + -- controls directly. + nodeid bigint REFERENCES pgautofailover.node (nodeid) + ON DELETE CASCADE, + + -- 'warm-standby' only: which group's WAL cache this instance replays + formationid text REFERENCES pgautofailover.formation (formationid), + groupid int, + + -- 'warm-standby' only: continuous (chases the primary continuously, + -- eligible for nodecluster read exposure) or scheduled (advances only + -- at basebackup_policy.frequency's cadence, paused via + -- recovery_target_action = pause in between) + cadence pgautofailover.archiver_node_cadence, + + -- 'warm-standby' + cadence = 'continuous' only: opt-in read-only + -- exposure. Enforced by CHECK, not just CLI convention -- a + -- 'scheduled' instance is stale by up to a full frequency between + -- cycles and must never be reachable as an ordinary read-replica + -- connection string without that caveat + nodecluster text, + + -- 'pitr' only: lifecycle (restoring -> paused -> registered/discarded) + pitrstatus pgautofailover.pitr_status, + + createdat timestamptz NOT NULL DEFAULT now(), + + CHECK (kind <> 'wal-receiver' OR nodeid IS NOT NULL), + CHECK (kind = 'wal-receiver' OR nodeid IS NULL), + CHECK (kind <> 'warm-standby' + OR (formationid IS NOT NULL AND groupid IS NOT NULL AND cadence IS NOT NULL)), + CHECK (kind = 'warm-standby' + OR (formationid IS NULL AND groupid IS NULL AND cadence IS NULL)), + CHECK (nodecluster IS NULL OR (kind = 'warm-standby' AND cadence = 'continuous')), + CHECK (kind = 'pitr' OR pitrstatus IS NULL) + ); + +CREATE TYPE pgautofailover.pitr_operation + AS ENUM ('create', 'status', 'retarget', 'resume', 'promote', + 'register', 'discard'); + +-- every PITR operation, recorded -- not just current status +CREATE TABLE pgautofailover.pitr_history + ( + pitrhistoryid bigserial PRIMARY KEY, + archivernodeid bigint NOT NULL + REFERENCES pgautofailover.archiver_node (archivernodeid) + ON DELETE CASCADE, + operation pgautofailover.pitr_operation NOT NULL, + occurredat timestamptz NOT NULL DEFAULT now(), + + requestedspec jsonb, -- what was asked for + observedlsn pg_lsn, -- what Postgres actually reported afterward + observedtimestamp timestamptz, + observedpausestate text, -- verbatim: 'not paused'/'pause requested'/'paused' + + note text + ); + +CREATE INDEX pitr_history_node_idx + ON pgautofailover.pitr_history (archivernodeid, occurredat); + +CREATE VIEW pgautofailover.pitr_node_status AS + SELECT n.archivernodeid, n.archiverid, n.hostname, n.pgdata, + n.pitrstatus, h.operation AS lastoperation, + h.observedlsn, h.observedtimestamp, h.observedpausestate, + h.occurredat AS lastupdatedat + FROM pgautofailover.archiver_node n + LEFT JOIN LATERAL ( + SELECT * FROM pgautofailover.pitr_history + WHERE archivernodeid = n.archivernodeid + ORDER BY occurredat DESC LIMIT 1 + ) h ON true + WHERE n.kind = 'pitr'; + +-- opt-in monitor-mediated PITR command queue, for the headless, +-- no-interactive-access deployment shape only (pg_autoctl node run +-- against a node.ini declaring kind = pitr) +CREATE TYPE pgautofailover.pitr_command + AS ENUM ('none', 'retarget', 'pause', 'resume', 'promote', + 'register', 'discard'); + +CREATE TABLE pgautofailover.pitr_pending_command + ( + archivernodeid bigint PRIMARY KEY + REFERENCES pgautofailover.archiver_node (archivernodeid) + ON DELETE CASCADE, + command pgautofailover.pitr_command NOT NULL DEFAULT 'none', + commandspec jsonb, + queuedat timestamptz NOT NULL DEFAULT now() + ); + + +-- +-- Functions +-- + +CREATE FUNCTION pgautofailover.create_basebackup_policy + ( + IN policyname text, + IN policyspec jsonb + ) +RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.basebackup_policy + (policyname, source, replaymode, cache, + frequency, maxcount, maxage, onpromotion, concurrency) + SELECT policyname, + coalesce((policyspec->>'source')::pgautofailover.basebackup_source, + 'replay'), + coalesce((policyspec->>'replaymode')::pgautofailover.basebackup_replay_mode, + 'volatile'), + coalesce((policyspec->>'cache')::pgautofailover.basebackup_cache, + 'local'), + coalesce((policyspec->>'frequency')::interval, '24 hours'), + coalesce((policyspec->>'maxcount')::int, 3), + coalesce((policyspec->>'maxage')::interval, '3 days'), + coalesce((policyspec->>'onpromotion')::bool, true), + coalesce((policyspec->>'concurrency')::int, 1) + RETURNING basebackuppolicyid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_basebackup_policy(text,jsonb) + is 'create a named, shareable base-backup production/retention policy'; + +grant execute on function + pgautofailover.create_basebackup_policy(text,jsonb) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_basebackup_policy + ( + IN policyname text, + IN policyspec jsonb + ) +RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup_policy + SET source = coalesce((policyspec->>'source')::pgautofailover.basebackup_source, source), + replaymode = coalesce((policyspec->>'replaymode')::pgautofailover.basebackup_replay_mode, replaymode), + cache = coalesce((policyspec->>'cache')::pgautofailover.basebackup_cache, cache), + frequency = coalesce((policyspec->>'frequency')::interval, frequency), + maxcount = coalesce((policyspec->>'maxcount')::int, maxcount), + maxage = coalesce((policyspec->>'maxage')::interval, maxage), + onpromotion = coalesce((policyspec->>'onpromotion')::bool, onpromotion), + concurrency = coalesce((policyspec->>'concurrency')::int, concurrency) + WHERE basebackup_policy.policyname = set_basebackup_policy.policyname; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup_policy "%" does not exist', policyname; + END IF; +END; +$$; + +comment on function pgautofailover.set_basebackup_policy(text,jsonb) + is 'update an existing named base-backup production/retention policy'; + +grant execute on function + pgautofailover.set_basebackup_policy(text,jsonb) + to autoctl_node; + +CREATE FUNCTION pgautofailover.get_basebackup_policy(policyname text) + RETURNS pgautofailover.basebackup_policy LANGUAGE sql STRICT SECURITY DEFINER +AS $$ + SELECT * FROM pgautofailover.basebackup_policy + WHERE basebackup_policy.policyname = get_basebackup_policy.policyname; +$$; + +comment on function pgautofailover.get_basebackup_policy(text) + is 'fetch a named base-backup production/retention policy'; + +grant execute on function pgautofailover.get_basebackup_policy(text) + to autoctl_node; + +-- creates the physical Archiver entity plus its mandatory 'local' +-- archiver_storage row. basebackuppolicyid NULL resolves to 'default'. +-- rcloneconfigname, when given, also attaches an additional 'rclone' row +-- referencing that existing, already-created rclone_config -- the +-- one-command way to "start a new archiver with the same shared rclone +-- setup" another archiver already uses; omit it to start local-only and +-- attach storage later via archiver_add_storage +CREATE FUNCTION pgautofailover.register_archiver + ( + archivername text, hostname text, + storagepath text DEFAULT NULL, + basebackuppolicyid bigint DEFAULT NULL, + autoregister bool DEFAULT true, + maxresidentreplay int DEFAULT 1, + rcloneconfigname text DEFAULT NULL, + region text DEFAULT 'default' + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_archiverid bigint; + resolved_policyid bigint; +BEGIN + resolved_policyid := coalesce( + basebackuppolicyid, + (SELECT p.basebackuppolicyid + FROM pgautofailover.basebackup_policy p + WHERE p.policyname = 'default')); + + INSERT INTO pgautofailover.archiver + (archivername, hostname, basebackuppolicyid, + autoregister, maxresidentreplay, region) + VALUES (archivername, hostname, resolved_policyid, + autoregister, maxresidentreplay, + coalesce(register_archiver.region, 'default')) + RETURNING archiverid INTO new_archiverid; + + INSERT INTO pgautofailover.archiver_storage + (archiverid, storagemethod, storagepath) + VALUES (new_archiverid, 'local', storagepath); + + IF rcloneconfigname IS NOT NULL THEN + PERFORM pgautofailover.archiver_add_storage(new_archiverid, rcloneconfigname); + END IF; + + RETURN new_archiverid; +END; +$$; + +comment on function pgautofailover.register_archiver(text,text,text,bigint,bool,int,text,text) + is 'register a new Archiver process identity, with its mandatory local storage target'; + +grant execute on function + pgautofailover.register_archiver(text,text,text,bigint,bool,int,text,text) + to autoctl_node; + +-- periodic storage heartbeat: usedbytes/freebytes/lastreporttime all move +-- together, from the same service_archiver_loop() tick (service_archiver.c) +-- that already reports this archiver's captured-WAL LSN. +CREATE FUNCTION pgautofailover.report_archiver_storage + (archiverid bigint, usedbytes bigint, freebytes bigint) + RETURNS void LANGUAGE sql SECURITY DEFINER +AS $$ + UPDATE pgautofailover.archiver + SET usedbytes = report_archiver_storage.usedbytes, + freebytes = report_archiver_storage.freebytes, + lastreporttime = now() + WHERE archiver.archiverid = report_archiver_storage.archiverid; +$$; + +comment on function pgautofailover.report_archiver_storage(bigint,bigint,bigint) + is 'record an archiver''s own reported disk usage and free space'; + +grant execute on function + pgautofailover.report_archiver_storage(bigint,bigint,bigint) + to autoctl_node; + +-- one row per archiver attached to formationid, with its FSM state (the +-- 'wal-receiver' archiver_node row created by archiver_add_formation, one +-- per group -- a multi-group formation returns one row per (archiver, +-- group)). Used by `pg_autoctl watch`'s own archivers section. +CREATE FUNCTION pgautofailover.get_archivers + ( + IN formationid text default 'default', + OUT archiver_id bigint, + OUT archiver_name text, + OUT hostname text, + OUT region text, + OUT used_bytes bigint, + OUT free_bytes bigint, + OUT last_report_time timestamptz, + OUT node_id bigint, + OUT reported_state pgautofailover.replication_state, + OUT goal_state pgautofailover.replication_state + ) +RETURNS SETOF record LANGUAGE SQL STRICT SECURITY DEFINER +AS $$ + SELECT a.archiverid, a.archivername, a.hostname, a.region, + a.usedbytes, a.freebytes, a.lastreporttime, + n.nodeid, n.reportedstate, n.goalstate + FROM pgautofailover.archiver a + JOIN pgautofailover.archiver_formation af + ON af.archiverid = a.archiverid + AND af.formationid = get_archivers.formationid + LEFT JOIN pgautofailover.archiver_node an + ON an.archiverid = a.archiverid AND an.kind = 'wal-receiver' + LEFT JOIN pgautofailover.node n + ON n.nodeid = an.nodeid AND n.formationid = get_archivers.formationid + ORDER BY a.archiverid; +$$; + +comment on function pgautofailover.get_archivers(text) + is 'list the archivers attached to a formation, with storage stats and FSM state'; + +grant execute on function pgautofailover.get_archivers(text) + to autoctl_node; + +-- named, shareable rclone config objects -- see rclone_config above for +-- what belongs in `config` (architecture only, never credentials) +CREATE FUNCTION pgautofailover.create_rclone_config(name text, config text) + RETURNS bigint -- rcloneconfigid + LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.rclone_config (name, config) + VALUES (name, config) + RETURNING rcloneconfigid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_rclone_config(text,text) + is 'register a named, shareable rclone remote configuration'; + +grant execute on function pgautofailover.create_rclone_config(text,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_rclone_config(name text, config text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.rclone_config AS rc + SET config = set_rclone_config.config + WHERE rc.name = set_rclone_config.name; + + IF NOT FOUND THEN + RAISE EXCEPTION 'rclone_config "%" does not exist', name; + END IF; +END; +$$; + +comment on function pgautofailover.set_rclone_config(text,text) + is 'update the content of an existing named rclone configuration -- every archiver referencing it picks up the change'; + +grant execute on function pgautofailover.set_rclone_config(text,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.get_rclone_config(name text) + RETURNS pgautofailover.rclone_config LANGUAGE sql STRICT +AS $$ + SELECT * FROM pgautofailover.rclone_config AS rc + WHERE rc.name = get_rclone_config.name; +$$; + +comment on function pgautofailover.get_rclone_config(text) + is 'fetch a named rclone configuration''s raw content'; + +grant execute on function pgautofailover.get_rclone_config(text) + to autoctl_node; + +-- attaches an archiver to an existing, already-named rclone_config row +-- (the sharing path -- several archivers' archiver_storage rows can +-- reference the same rcloneconfigid at once, edit the config once via +-- set_rclone_config and every referencing archiver picks it up) +CREATE FUNCTION pgautofailover.archiver_add_storage + (archiverid bigint, rcloneconfigname text) + RETURNS bigint -- archiverstorageid + LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + resolved_rcloneconfigid bigint; + new_id bigint; +BEGIN + SELECT rc.rcloneconfigid INTO resolved_rcloneconfigid + FROM pgautofailover.rclone_config rc + WHERE rc.name = rcloneconfigname; + + IF resolved_rcloneconfigid IS NULL THEN + RAISE EXCEPTION 'rclone_config "%" does not exist', rcloneconfigname; + END IF; + + INSERT INTO pgautofailover.archiver_storage + (archiverid, storagemethod, rcloneconfigid) + VALUES (archiverid, 'rclone', resolved_rcloneconfigid) + RETURNING archiverstorageid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.archiver_add_storage(bigint,text) + is 'attach an additional rclone storage target to an archiver, referencing an existing named rclone_config'; + +grant execute on function pgautofailover.archiver_add_storage(bigint,text) + to autoctl_node; + +-- detaches only; the referenced rclone_config row is untouched and +-- keeps serving any other archiver still referencing it +CREATE FUNCTION pgautofailover.archiver_remove_storage(archiverstorageid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.archiver_storage AS a_s + WHERE a_s.archiverstorageid = archiver_remove_storage.archiverstorageid + AND a_s.storagemethod <> 'local'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_storage % does not exist, or is the mandatory local target', + archiverstorageid; + END IF; +END; +$$; + +comment on function pgautofailover.archiver_remove_storage(bigint) + is 'detach a non-local storage target from an archiver (the local target cannot be removed)'; + +grant execute on function pgautofailover.archiver_remove_storage(bigint) + to autoctl_node; + +-- fans out to one CREATE of a pgautofailover.node row (haspgdata = +-- false) per group currently in formationid +-- Parameters are prefixed in_* here (unlike this file's usual +-- function-qualified-reference convention): ON CONFLICT's own target +-- column list can't be schema/function-qualified at all (that syntax +-- only accepts bare column names or ON CONSTRAINT), so a same-named +-- parameter would still be genuinely ambiguous there even when every +-- other clause in this function could disambiguate it. +CREATE FUNCTION pgautofailover.archiver_add_formation + (in_archiverid bigint, in_formationid text) + RETURNS SETOF bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + grp record; + new_nodeid bigint; +BEGIN + INSERT INTO pgautofailover.archiver_formation (archiverid, formationid) + VALUES (in_archiverid, in_formationid) + ON CONFLICT (archiverid, formationid) DO NOTHING; + + FOR grp IN + SELECT DISTINCT n.groupid + FROM pgautofailover.node n + WHERE n.formationid = in_formationid + LOOP + new_nodeid := NULL; + + -- nodeport = 0 is a permanent sentinel, not an M1 stopgap: an + -- ARCHIVING row has no postmaster of its own to be reachable on, + -- so nodehost:nodeport isn't a connectable address here the way + -- it is for every haspgdata row -- see node_nodehost_nodeport_ + -- haspgdata_idx's own comment, which is exactly why that unique + -- index is scoped to haspgdata rows only. reportedstate starts at + -- 'wait_standby', same as any freshly-registered node -- it only + -- reaches 'archiving' once a real keeper's pg_receivewal is + -- actually running. + -- + -- ON CONFLICT DO NOTHING on (formationid, nodename): this + -- function must be safe to call again for a formation some of + -- whose groups are already attached -- an operator re-running it + -- on purpose, or the archiver's own reconciler picking up a + -- newly-added Citus worker group -- without failing on every + -- group that was already covered by an earlier call. A skipped + -- insert leaves new_nodeid NULL (no row returned), handled below. + INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, + goalstate, reportedstate, haspgdata, candidatepriority, + replicationquorum) + VALUES (in_formationid, grp.groupid, + 'archiver-' || in_archiverid || '-' || grp.groupid, + (SELECT a.hostname FROM pgautofailover.archiver a + WHERE a.archiverid = in_archiverid), + 0, + 'wait_standby', 'wait_standby', false, 0, false) + ON CONFLICT (formationid, nodename) DO NOTHING + RETURNING nodeid INTO new_nodeid; + + IF new_nodeid IS NULL THEN + -- this group was already attached by an earlier call -- + -- nothing new to report for it, and archiver_node already + -- has its row from that earlier call too. + CONTINUE; + END IF; + + INSERT INTO pgautofailover.archiver_node + (archiverid, kind, pgdata, nodeid) + VALUES (in_archiverid, 'wal-receiver', + '', new_nodeid); + + RETURN NEXT new_nodeid; + END LOOP; + + RETURN; +END; +$$; + +comment on function pgautofailover.archiver_add_formation(bigint,text) + is 'attach an archiver to every group of a formation, creating one lightweight ARCHIVING node row per group not already attached'; + +grant execute on function pgautofailover.archiver_add_formation(bigint,text) + to autoctl_node; + +-- one row per (formation, group) an archiver currently holds a +-- 'wal-receiver' membership in, across every formation it is attached +-- to -- what an archiver process itself calls, at startup and +-- periodically thereafter, to discover the full set of WAL streams and +-- base-backup schedules it is responsible for running (see +-- service_archiver_reconciler.c). Unlike get_archivers() above (scoped +-- to one formation, for `pg_autoctl watch`'s own display), this is +-- scoped to one archiver, across all its formations. +CREATE FUNCTION pgautofailover.list_archiver_memberships + ( + IN archiverid bigint, + OUT formation_id text, + OUT group_id int, + OUT node_id bigint, + OUT reported_state pgautofailover.replication_state, + OUT goal_state pgautofailover.replication_state + ) +RETURNS SETOF record LANGUAGE SQL STRICT SECURITY DEFINER +AS $$ + SELECT n.formationid, n.groupid, n.nodeid, n.reportedstate, n.goalstate + FROM pgautofailover.archiver_node an + JOIN pgautofailover.node n ON n.nodeid = an.nodeid + WHERE an.archiverid = list_archiver_memberships.archiverid + AND an.kind = 'wal-receiver' + ORDER BY n.formationid, n.groupid; +$$; + +comment on function pgautofailover.list_archiver_memberships(bigint) + is 'list every (formation, group) membership an archiver currently belongs to, across every formation it is attached to'; + +grant execute on function pgautofailover.list_archiver_memberships(bigint) + to autoctl_node; + +-- Deleting the node row is enough: archiver_node.nodeid's own +-- ON DELETE CASCADE removes the matching wal-receiver archiver_node row +-- automatically (see that column's own comment). +CREATE FUNCTION pgautofailover.archiver_remove_formation + (archiverid bigint, formationid text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.node n + WHERE n.formationid = archiver_remove_formation.formationid + AND n.nodeid IN (SELECT an.nodeid + FROM pgautofailover.archiver_node an + WHERE an.archiverid = archiver_remove_formation.archiverid + AND an.kind = 'wal-receiver'); + + DELETE FROM pgautofailover.archiver_formation af + WHERE af.archiverid = archiver_remove_formation.archiverid + AND af.formationid = archiver_remove_formation.formationid; +END; +$$; + +comment on function pgautofailover.archiver_remove_formation(bigint,text) + is 'detach an archiver from a formation, removing its ARCHIVING node row in every group'; + +grant execute on function pgautofailover.archiver_remove_formation(bigint,text) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified, even inside an +-- expression like coalesce(groupid, -1)) forces this naming here. +CREATE FUNCTION pgautofailover.set_archiver_policy + ( + in_formationid text, in_groupid int DEFAULT NULL, + in_archiverquorum int DEFAULT NULL, + in_basebackuppolicyid bigint DEFAULT NULL, + in_replicationquorumeligible bool DEFAULT NULL + ) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.archiver_policy + (formationid, groupid, archiverquorum, + basebackuppolicyid, replicationquorumeligible) + VALUES (in_formationid, in_groupid, + coalesce(in_archiverquorum, 1), + in_basebackuppolicyid, + coalesce(in_replicationquorumeligible, false)) + ON CONFLICT (formationid, (coalesce(groupid, -1))) DO UPDATE + SET archiverquorum = coalesce(EXCLUDED.archiverquorum, + pgautofailover.archiver_policy.archiverquorum), + basebackuppolicyid = coalesce(EXCLUDED.basebackuppolicyid, + pgautofailover.archiver_policy.basebackuppolicyid), + replicationquorumeligible = coalesce(EXCLUDED.replicationquorumeligible, + pgautofailover.archiver_policy.replicationquorumeligible); +END; +$$; + +comment on function pgautofailover.set_archiver_policy(text,int,int,bigint,bool) + is 'set (or override) archiver_quorum/basebackup policy/replication-quorum eligibility for a formation, or one of its groups'; + +grant execute on function + pgautofailover.set_archiver_policy(text,int,int,bigint,bool) + to autoctl_node; + +-- resolves group-specific override first, then the formation-wide +-- (groupid IS NULL) default, then this schema's own hardcoded defaults. +-- Deliberately plpgsql, not a single SQL query: an earlier draft tried to +-- express the three-way fallback as one UNION ALL ... LIMIT 1 query, but +-- UNION ALL has no ordering guarantee across its branches, so LIMIT 1 +-- could just as easily return the formation-wide or hardcoded default +-- even when a group-specific override exists. Sequential SELECT INTO ... +-- IF FOUND is unambiguous. +CREATE FUNCTION pgautofailover.get_archiver_policy(formationid text, groupid int) + RETURNS TABLE (archiverquorum int, basebackuppolicyid bigint, + replicationquorumeligible bool) + LANGUAGE plpgsql STABLE +AS $$ +BEGIN + RETURN QUERY + SELECT ap.archiverquorum, ap.basebackuppolicyid, ap.replicationquorumeligible + FROM pgautofailover.archiver_policy ap + WHERE ap.formationid = get_archiver_policy.formationid + AND ap.groupid = get_archiver_policy.groupid; + + IF FOUND THEN + RETURN; + END IF; + + RETURN QUERY + SELECT ap.archiverquorum, ap.basebackuppolicyid, ap.replicationquorumeligible + FROM pgautofailover.archiver_policy ap + WHERE ap.formationid = get_archiver_policy.formationid + AND ap.groupid IS NULL; + + IF FOUND THEN + RETURN; + END IF; + + RETURN QUERY + SELECT 1, p.basebackuppolicyid, false + FROM pgautofailover.basebackup_policy p + WHERE p.policyname = 'default'; +END; +$$; + +comment on function pgautofailover.get_archiver_policy(text,int) + is 'resolve archiver policy for (formation, group): group override, else formation default, else this schema''s own defaults'; + +grant execute on function pgautofailover.get_archiver_policy(text,int) + to autoctl_node; + +-- one round trip from the archiver-basebackup side: resolves the +-- basebackup_policy row that applies to (formation, group) via get_ +-- archiver_policy() above, then flattens its interval columns to plain +-- integer seconds -- easy time_t arithmetic on the C side, no interval- +-- text parsing needed. SECURITY DEFINER: reads archiver_policy/ +-- basebackup_policy directly, both created (like every table in this +-- milestone's own schema) after the blanket "GRANT SELECT ON ALL TABLES" +-- near the top of this file, so autoctl_node has no direct grant on +-- either -- same class of gap already hit (and fixed) twice for wal_ +-- archived()/get_latest_basebackup(). +CREATE FUNCTION pgautofailover.get_basebackup_policy_for_group + ( + formationid text, + groupid int, + OUT policyname text, + OUT source pgautofailover.basebackup_source, + OUT replaymode pgautofailover.basebackup_replay_mode, + OUT cache pgautofailover.basebackup_cache, + OUT frequency_seconds int, + OUT maxcount int, + OUT maxage_seconds int, + OUT onpromotion bool, + OUT concurrency int + ) + RETURNS record LANGUAGE plpgsql STABLE SECURITY DEFINER +AS $$ +DECLARE + ap record; +BEGIN + SELECT * INTO ap + FROM pgautofailover.get_archiver_policy( + get_basebackup_policy_for_group.formationid, + get_basebackup_policy_for_group.groupid); + + SELECT p.policyname, p.source, p.replaymode, p.cache, + extract(epoch FROM p.frequency)::int, + p.maxcount, + extract(epoch FROM p.maxage)::int, + p.onpromotion, p.concurrency + INTO policyname, source, replaymode, cache, frequency_seconds, + maxcount, maxage_seconds, onpromotion, concurrency + FROM pgautofailover.basebackup_policy p + WHERE p.basebackuppolicyid = ap.basebackuppolicyid; +END; +$$; + +comment on function pgautofailover.get_basebackup_policy_for_group(text,int) + is 'resolve the full base-backup production/retention policy for (formation, group), intervals flattened to seconds'; + +grant execute on function pgautofailover.get_basebackup_policy_for_group(text,int) + to autoctl_node; + +-- the archive_command confirmation check: true iff at least +-- archiver_quorum distinct archivers have durably reported %f +CREATE FUNCTION pgautofailover.wal_archived + (formationid text, groupid int, walfilename text) + RETURNS bool + LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT count(DISTINCT aw.archiverid) >= + (SELECT archiverquorum + FROM pgautofailover.get_archiver_policy(wal_archived.formationid, + wal_archived.groupid)) + FROM pgautofailover.archiver_wal aw + WHERE aw.formationid = wal_archived.formationid + AND aw.groupid = wal_archived.groupid + AND aw.walfilename = wal_archived.walfilename; +$$; + +comment on function pgautofailover.wal_archived(text,int,text) + is 'archive_command confirmation check: has segment %f already landed durably on archiver_quorum archiver(s)?'; + +grant execute on function pgautofailover.wal_archived(text,int,text) + to autoctl_node; + +-- inserts into archiver_wal (idempotent on conflict) +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.report_wal_received + (in_nodeid bigint, in_walfilename text, in_lsn pg_lsn) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + target record; +BEGIN + SELECT n.formationid, n.groupid, an.archiverid + INTO target + FROM pgautofailover.archiver_node an + JOIN pgautofailover.node n ON n.nodeid = an.nodeid + WHERE an.nodeid = in_nodeid + AND an.kind = 'wal-receiver'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'node % is not an ARCHIVING wal-receiver node', in_nodeid; + END IF; + + INSERT INTO pgautofailover.archiver_wal + (formationid, groupid, walfilename, archiverid, lsn) + VALUES (target.formationid, target.groupid, in_walfilename, target.archiverid, in_lsn) + ON CONFLICT (formationid, groupid, walfilename, archiverid) DO NOTHING; +END; +$$; + +comment on function pgautofailover.report_wal_received(bigint,text,pg_lsn) + is 'reports a WAL segment durably captured by an ARCHIVING node'; + +grant execute on function pgautofailover.report_wal_received(bigint,text,pg_lsn) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_started + ( + archiverid bigint, formationid text, groupid int, + label text, timeline int, startlsn pg_lsn, + source pgautofailover.basebackup_source, + replaymode pgautofailover.basebackup_replay_mode DEFAULT NULL + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.basebackup + (archiverid, formationid, groupid, label, timeline, startlsn, + source, replaymode, storagelocation, status) + VALUES (archiverid, formationid, groupid, label, timeline, startlsn, + source, replaymode, '', 'in_progress') + RETURNING basebackupid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.report_basebackup_started + (bigint,text,int,text,int,pg_lsn,pgautofailover.basebackup_source,pgautofailover.basebackup_replay_mode) + is 'records the start of a new base-backup production job'; + +grant execute on function + pgautofailover.report_basebackup_started + (bigint,text,int,text,int,pg_lsn,pgautofailover.basebackup_source,pgautofailover.basebackup_replay_mode) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_completed + (basebackupid bigint, endlsn pg_lsn, sizebytes bigint, storagelocation text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup AS bb + SET endlsn = report_basebackup_completed.endlsn, + sizebytes = report_basebackup_completed.sizebytes, + storagelocation = report_basebackup_completed.storagelocation, + status = 'complete', + period = tstzrange(lower(bb.period), now()) + WHERE bb.basebackupid = report_basebackup_completed.basebackupid; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup % does not exist', basebackupid; + END IF; +END; +$$; + +comment on function pgautofailover.report_basebackup_completed(bigint,pg_lsn,bigint,text) + is 'records the successful completion of a base-backup production job'; + +grant execute on function + pgautofailover.report_basebackup_completed(bigint,pg_lsn,bigint,text) + to autoctl_node; + +-- marks the basebackup row deleted (never a real DELETE), then prunes +-- any archiver_wal rows this group no longer needs to retain +CREATE FUNCTION pgautofailover.report_basebackup_deleted(basebackupid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + bb record; +BEGIN + UPDATE pgautofailover.basebackup AS b + SET status = 'deleted', deletedat = now() + WHERE b.basebackupid = report_basebackup_deleted.basebackupid + RETURNING b.formationid, b.groupid INTO bb; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup % does not exist', basebackupid; + END IF; + + PERFORM pgautofailover.prune_archiver_wal(bb.formationid, bb.groupid); +END; +$$; + +comment on function pgautofailover.report_basebackup_deleted(bigint) + is 'marks a base backup deleted (retains history) and prunes any archiver_wal rows no group backup needs anymore'; + +grant execute on function pgautofailover.report_basebackup_deleted(bigint) + to autoctl_node; + +-- deletes every archiver_wal row for (formationid, groupid) older than +-- the earliest still-'complete' basebackup's startlsn, across every +-- archiver holding a copy. When no 'complete' backup remains for this +-- group, nothing is pruned -- there is no anchor point to replay forward +-- from, so every captured segment is still needed. +CREATE FUNCTION pgautofailover.prune_archiver_wal(formationid text, groupid int) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + oldest_startlsn pg_lsn; + deleted_count bigint; +BEGIN + SELECT min(b.startlsn) INTO oldest_startlsn + FROM pgautofailover.basebackup b + WHERE b.formationid = prune_archiver_wal.formationid + AND b.groupid = prune_archiver_wal.groupid + AND b.status = 'complete'; + + IF oldest_startlsn IS NULL THEN + RETURN 0; + END IF; + + WITH deleted AS ( + DELETE FROM pgautofailover.archiver_wal aw + WHERE aw.formationid = prune_archiver_wal.formationid + AND aw.groupid = prune_archiver_wal.groupid + AND aw.lsn < oldest_startlsn + RETURNING 1 + ) + SELECT count(*) INTO deleted_count FROM deleted; + + RETURN deleted_count; +END; +$$; + +comment on function pgautofailover.prune_archiver_wal(text,int) + is 'deletes archiver_wal rows for (formation, group) older than the oldest still-complete base backup''s startlsn'; + +grant execute on function pgautofailover.prune_archiver_wal(text,int) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.report_basebackup_synced + (in_basebackupid bigint, in_archiverstorageid bigint, in_remotelocation text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.basebackup_storage + (basebackupid, archiverstorageid, syncedat, remotelocation) + VALUES (in_basebackupid, in_archiverstorageid, now(), in_remotelocation) + ON CONFLICT (basebackupid, archiverstorageid) DO UPDATE + SET syncedat = now(), + remotelocation = EXCLUDED.remotelocation; +END; +$$; + +comment on function pgautofailover.report_basebackup_synced(bigint,bigint,text) + is 'records a successful cold-storage sync of a base backup to one storage target'; + +grant execute on function + pgautofailover.report_basebackup_synced(bigint,bigint,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_remote_deleted + (basebackupid bigint, archiverstorageid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup_storage AS bs + SET deletedat = now() + WHERE bs.basebackupid = report_basebackup_remote_deleted.basebackupid + AND bs.archiverstorageid = report_basebackup_remote_deleted.archiverstorageid; +END; +$$; + +comment on function pgautofailover.report_basebackup_remote_deleted(bigint,bigint) + is 'records that a base backup''s remote copy on one storage target has been pruned'; + +grant execute on function + pgautofailover.report_basebackup_remote_deleted(bigint,bigint) + to autoctl_node; + +-- filters status = 'complete' only. SECURITY DEFINER matches every other +-- autoctl_node-callable helper reading a table that role has no direct +-- SELECT grant on (e.g. archiver_add_formation) -- autoctl_node is only +-- ever granted EXECUTE on the function, never SELECT on pgautofailover. +-- basebackup itself. +-- +-- preferred_source (default NULL, meaning "any") exists for service_ +-- archiver_serve.c's own routes-file refresh: a 'replay' backup promotes a +-- throwaway extracted copy, which genuinely puts it on a *later* timeline +-- than whatever the archiver's own walcache has actually captured (which +-- only ever advances on the real primary's timeline) -- serving that pair +-- together breaks a real pg_basebackup's own timeline consistency check +-- (receivelog.c). Since a 'live' backup is taken directly from the +-- actively-followed primary, it always shares the walcache's timeline by +-- construction; passing preferred_source = 'live' is how the routes +-- refresh asks for one specifically, rather than "whatever is newest +-- regardless of type". +CREATE FUNCTION pgautofailover.get_latest_basebackup + ( + formationid text, + groupid int, + preferred_source pgautofailover.basebackup_source default NULL + ) + RETURNS pgautofailover.basebackup LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT * FROM pgautofailover.basebackup b + WHERE b.formationid = get_latest_basebackup.formationid + AND b.groupid = get_latest_basebackup.groupid + AND b.status = 'complete' + AND (get_latest_basebackup.preferred_source IS NULL + OR b.source = get_latest_basebackup.preferred_source) + ORDER BY lower(b.period) DESC + LIMIT 1; +$$; + +comment on function pgautofailover.get_latest_basebackup + (text,int,pgautofailover.basebackup_source) + is 'fetch the most recent complete base backup for (formation, group), optionally filtered to one source'; + +grant execute on function pgautofailover.get_latest_basebackup + (text,int,pgautofailover.basebackup_source) + to autoctl_node; + +-- every 'complete' base backup for (formation, group), newest first -- +-- what service_archiver_basebackup.c's own retention pass (maxcount/ +-- maxage) walks to decide what to keep vs. prune, and what a future `pg_ +-- autoctl show basebackup` would list. basebackupid/storagelocation are +-- what report_basebackup_deleted()/an actual directory removal need; +-- startedat_epoch (extract(epoch from lower(period))) is plain integer +-- seconds for the same reason get_basebackup_policy_for_group() flattens +-- its own interval columns -- easy time_t arithmetic, no timestamptz-text +-- parsing on the C side. +CREATE FUNCTION pgautofailover.list_basebackups + ( + formationid text, + groupid int, + OUT basebackupid bigint, + OUT label text, + OUT storagelocation text, + OUT startedat_epoch bigint + ) + RETURNS SETOF record LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT b.basebackupid, b.label, b.storagelocation, + extract(epoch FROM lower(b.period))::bigint + FROM pgautofailover.basebackup b + WHERE b.formationid = list_basebackups.formationid + AND b.groupid = list_basebackups.groupid + AND b.status = 'complete' + ORDER BY lower(b.period) DESC; +$$; + +comment on function pgautofailover.list_basebackups(text,int) + is 'list complete base backups for (formation, group), newest first -- retention/inventory'; + +grant execute on function pgautofailover.list_basebackups(text,int) + to autoctl_node; + +-- an archiving node has no sysidentifier of its own (haspgdata = false, +-- see that column's own comment): it never runs a real Postgres instance +-- to report one. Every other node in the group shares the same physical +-- cluster's identifier, so any one of them answers for the whole group -- +-- needed by pg_walsender's own IDENTIFY_SYSTEM response (cmd_identify_ +-- system.c) so a real standby streaming from the archiver doesn't reject +-- it with "database system identifier differs between the primary and +-- standby". +CREATE FUNCTION pgautofailover.get_group_system_identifier + (formationid text, groupid int) + RETURNS bigint LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT sysidentifier + FROM pgautofailover.node + WHERE node.formationid = get_group_system_identifier.formationid + AND node.groupid = get_group_system_identifier.groupid + AND sysidentifier IS NOT NULL + AND sysidentifier != 0 + LIMIT 1; +$$; + +comment on function pgautofailover.get_group_system_identifier(text,int) + is 'the Postgres system identifier shared by every node in a group, for an archiving node (which has none of its own) to serve via IDENTIFY_SYSTEM'; + +grant execute on function pgautofailover.get_group_system_identifier(text,int) + to autoctl_node; + +-- `create postgres --from-archiver` needs the ARCHIVING row itself, not +-- get_most_advanced_standby()'s election-only pool: that function filters +-- on reportedstate = 'report_lsn', a transient state a group's ARCHIVING +-- node only visits during a FAST_FORWARD election, never during its normal +-- steady-state operation (reportedstate = 'archiving'). node_port is the +-- port == 0 sentinel documented on get_most_advanced_standby's own C +-- caller (keeper_get_most_advanced_standby, keeper.c) -- resolving it to +-- the archiver's real pg_walsender serve port is this milestone's C +-- caller's job too, same pattern. +CREATE FUNCTION pgautofailover.get_archiver_node + ( + IN formationid text default 'default', + IN groupid int default 0, + OUT node_id bigint, + OUT node_name text, + OUT node_host text, + OUT node_port int, + OUT node_lsn pg_lsn, + OUT node_is_primary bool + ) +RETURNS SETOF record LANGUAGE SQL STRICT +AS $$ + select nodeid, nodename, nodehost, nodeport, reportedlsn, false + from pgautofailover.node + where formationid = $1 + and groupid = $2 + and reportedstate = 'archiving' + order by nodeid + limit 1; +$$; + +comment on function pgautofailover.get_archiver_node(text,int) + is 'fetch the ARCHIVING node for (formation, group), for create postgres --from-archiver to bootstrap from'; + +grant execute on function pgautofailover.get_archiver_node(text,int) + to autoctl_node; + +-- for kind = 'warm-standby': raises if the owning archiver is already at +-- its maxresidentreplay cap +CREATE FUNCTION pgautofailover.create_archiver_node + ( + archiverid bigint, + kind pgautofailover.archiver_node_kind, + pgdata text, + hostname text DEFAULT NULL, + nodeid bigint DEFAULT NULL, -- required iff kind = 'wal-receiver' + formationid text DEFAULT NULL, -- required iff kind = 'warm-standby' + groupid int DEFAULT NULL, -- required iff kind = 'warm-standby' + cadence pgautofailover.archiver_node_cadence DEFAULT NULL, + nodecluster text DEFAULT NULL, -- only for 'warm-standby' + cadence = 'continuous' + pitrstatus pgautofailover.pitr_status DEFAULT NULL + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + residentcount int; + maxresident int; + new_id bigint; +BEGIN + IF kind = 'warm-standby' THEN + SELECT a.maxresidentreplay INTO maxresident + FROM pgautofailover.archiver a + WHERE a.archiverid = create_archiver_node.archiverid; + + SELECT count(*) INTO residentcount + FROM pgautofailover.archiver_node an + WHERE an.archiverid = create_archiver_node.archiverid + AND an.kind = 'warm-standby'; + + IF residentcount >= maxresident THEN + RAISE EXCEPTION + 'archiver % is already at its maxresidentreplay cap (%)', + archiverid, maxresident; + END IF; + END IF; + + INSERT INTO pgautofailover.archiver_node + (archiverid, kind, pgdata, hostname, nodeid, + formationid, groupid, cadence, nodecluster, pitrstatus) + VALUES (archiverid, kind, pgdata, hostname, nodeid, + formationid, groupid, cadence, nodecluster, pitrstatus) + RETURNING archivernodeid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_archiver_node + (bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,int, + pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) + is 'registers a concrete Postgres instance an archiver hosts, derives, or is otherwise associated with'; + +grant execute on function + pgautofailover.create_archiver_node + (bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,int, + pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) + to autoctl_node; + +CREATE FUNCTION pgautofailover.remove_archiver_node(archivernodeid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.archiver_node an + WHERE an.archivernodeid = remove_archiver_node.archivernodeid; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_node % does not exist', archivernodeid; + END IF; +END; +$$; + +comment on function pgautofailover.remove_archiver_node(bigint) + is 'removes an archiver_node row'; + +grant execute on function pgautofailover.remove_archiver_node(bigint) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_archiver_node_pitr_status + (archivernodeid bigint, pitrstatus pgautofailover.pitr_status) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.archiver_node AS an + SET pitrstatus = set_archiver_node_pitr_status.pitrstatus + WHERE an.archivernodeid = set_archiver_node_pitr_status.archivernodeid + AND an.kind = 'pitr'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_node % does not exist, or is not kind = pitr', + archivernodeid; + END IF; +END; +$$; + +comment on function pgautofailover.set_archiver_node_pitr_status(bigint,pgautofailover.pitr_status) + is 'updates a PITR archiver_node''s lifecycle status'; + +grant execute on function + pgautofailover.set_archiver_node_pitr_status(bigint,pgautofailover.pitr_status) + to autoctl_node; + +-- pushed by the local pg_autoctl pitr CLI immediately after acting +-- locally -- never blocks or gates the local action on this succeeding +CREATE FUNCTION pgautofailover.report_pitr_status + ( + archivernodeid bigint, operation pgautofailover.pitr_operation, + requestedspec jsonb, + observedlsn pg_lsn, observedtimestamp timestamptz, + observedpausestate text, note text DEFAULT NULL + ) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.pitr_history + (archivernodeid, operation, requestedspec, + observedlsn, observedtimestamp, observedpausestate, note) + VALUES (archivernodeid, operation, requestedspec, + observedlsn, observedtimestamp, observedpausestate, note); +END; +$$; + +comment on function pgautofailover.report_pitr_status + (bigint,pgautofailover.pitr_operation,jsonb,pg_lsn,timestamptz,text,text) + is 'records one PITR operation''s outcome -- a best-effort report, never gating the local action it follows'; + +grant execute on function + pgautofailover.report_pitr_status + (bigint,pgautofailover.pitr_operation,jsonb,pg_lsn,timestamptz,text,text) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.pitr_queue_command + (in_archivernodeid bigint, in_command pgautofailover.pitr_command, + in_commandspec jsonb DEFAULT NULL) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.pitr_pending_command + (archivernodeid, command, commandspec) + VALUES (in_archivernodeid, in_command, in_commandspec) + ON CONFLICT (archivernodeid) DO UPDATE + SET command = EXCLUDED.command, + commandspec = EXCLUDED.commandspec, + queuedat = now(); +END; +$$; + +comment on function pgautofailover.pitr_queue_command(bigint,pgautofailover.pitr_command,jsonb) + is 'queues a PITR command for a monitor-mediated (kind = pitr, pg_autoctl node run) agent to pick up'; + +grant execute on function + pgautofailover.pitr_queue_command(bigint,pgautofailover.pitr_command,jsonb) + to autoctl_node; + +-- returns the pending command and resets the queue slot to 'none' in the +-- same call -- an agent polling this never processes the same command twice +-- Reads the pending command, then clears it, as two separate statements: +-- UPDATE ... RETURNING always reflects the row *after* the update is +-- applied, so folding the reset into the same RETURNING clause that reads +-- the command would always report back the very 'none' this function just +-- set, never the command that was actually queued. FOR UPDATE locks the +-- row across both statements, so a concurrent caller for the same +-- archivernodeid still can't observe or consume the same command twice. +CREATE FUNCTION pgautofailover.pitr_next_command(in_archivernodeid bigint) + RETURNS pgautofailover.pitr_command LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + next_command pgautofailover.pitr_command; +BEGIN + SELECT pc.command INTO next_command + FROM pgautofailover.pitr_pending_command pc + WHERE pc.archivernodeid = in_archivernodeid + FOR UPDATE; + + IF next_command IS NULL OR next_command = 'none' THEN + RETURN 'none'; + END IF; + + UPDATE pgautofailover.pitr_pending_command AS pc + SET command = 'none', commandspec = NULL + WHERE pc.archivernodeid = in_archivernodeid; + + RETURN next_command; +END; +$$; + +comment on function pgautofailover.pitr_next_command(bigint) + is 'pops and clears the next queued PITR command for an agent to act on'; + +grant execute on function pgautofailover.pitr_next_command(bigint) + to autoctl_node; + -- Testing-only functions, not granted to autoctl_node: they let -- regression/isolation tests hold the monitor's own LockFormation()/ -- LockNodeGroup() locks explicitly, and simulate a health-check-worker diff --git a/src/monitor/regress_schedule b/src/monitor/regress_schedule index 4d2ca7bdb..60551f7c9 100644 --- a/src/monitor/regress_schedule +++ b/src/monitor/regress_schedule @@ -46,6 +46,7 @@ test: lock_and_fetch_migration test: timeline_fork_detection test: failover_candidate_leaves_secondary test: cluster_init_failover_rule_attribution +test: archiving_schema test: dummy_update test: drop_extension test: upgrade diff --git a/src/monitor/replication_state.c b/src/monitor/replication_state.c index f1191176e..c69d8c181 100644 --- a/src/monitor/replication_state.c +++ b/src/monitor/replication_state.c @@ -256,6 +256,11 @@ ReplicationStateGetName(ReplicationState replicationState) return "dropped"; } + case REPLICATION_STATE_ARCHIVING: + { + return "archiving"; + } + default: { ereport(ERROR, diff --git a/src/monitor/replication_state.h b/src/monitor/replication_state.h index 040aeec8c..060281f04 100644 --- a/src/monitor/replication_state.h +++ b/src/monitor/replication_state.h @@ -40,7 +40,8 @@ typedef enum ReplicationState REPLICATION_STATE_FAST_FORWARD = 18, REPLICATION_STATE_JOIN_SECONDARY = 19, REPLICATION_STATE_DROPPED = 20, - REPLICATION_STATE_UNKNOWN = 21 + REPLICATION_STATE_ARCHIVING = 21, + REPLICATION_STATE_UNKNOWN = 22 } ReplicationState; diff --git a/src/monitor/sql/archiving_schema.sql b/src/monitor/sql/archiving_schema.sql new file mode 100644 index 000000000..e8b6565f6 --- /dev/null +++ b/src/monitor/sql/archiving_schema.sql @@ -0,0 +1,229 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Regression tests for the Archiving & Disaster Recovery schema and its +-- monitor API (milestone 1: schema + monitor API only -- no +-- service_archiver process involved, everything here is exercised via +-- direct SQL calls against the schema alone). See +-- ~/dev/temp/archiving-disaster-recovery.md for the full design. + +\x on + +-- A dedicated formation, like every other test in this schedule: 'default' +-- is the seed formation CREATE EXTENSION itself creates, and by this point +-- in regress_schedule it may already have real nodes registered into it by +-- earlier tests, so it's the one name this file must NOT reuse. The +-- 'default' basebackup_policy row (also a CREATE EXTENSION seed) is shared +-- on purpose: this file's own focus is exercising it, not creating another. +-- Two ordinary nodes stand in for a group's primary+secondary, inserted +-- directly rather than through register_node()/node_active(): the ordinary +-- node FSM has its own dedicated coverage elsewhere, this file's own focus +-- is the archiver schema layered on top of it. +SELECT pgautofailover.create_formation('archiving_test', 'pgsql', 'postgres', + true, 1); + +INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, sysidentifier, + goalstate, reportedstate) +VALUES ('archiving_test', 0, 'node1', 'node1.local', 5432, 111, + 'primary', 'primary'), + ('archiving_test', 0, 'node2', 'node2.local', 5432, 111, + 'secondary', 'secondary'); + +-- ── register_archiver ──────────────────────────────────────────────────── + +SELECT pgautofailover.register_archiver('archiver1', 'archiver1.local') + AS archiverid \gset + +SELECT archiverid, archivername, hostname, region, basebackuppolicyid, + autoregister, maxresidentreplay + FROM pgautofailover.archiver; + +-- the mandatory 'local' storage target is created in the same call +SELECT archiverstorageid, archiverid, storagemethod, storagepath, rcloneconfigid + FROM pgautofailover.archiver_storage; + +-- ── archiver_add_formation: the budget setup's own fan-out ───────────────── + +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test'); + +SELECT nodeid, formationid, groupid, nodename, nodehost, nodeport, + goalstate, reportedstate, haspgdata + FROM pgautofailover.node + WHERE haspgdata = false; + +SELECT archivernodeid, archiverid, kind, nodeid + FROM pgautofailover.archiver_node + WHERE kind = 'wal-receiver'; + +SELECT nodeid FROM pgautofailover.node + WHERE formationid = 'archiving_test' AND groupid = 0 AND haspgdata = false \gset + +-- calling archiver_add_formation() again for the same (archiver, formation) +-- must be a safe no-op -- no error, no duplicate node/archiver_node rows -- +-- since a real archiver's own reconciler calls this periodically to pick up +-- newly-added groups (e.g. a Citus formation growing a worker), not just +-- once at creation time +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test'); + +SELECT count(*) AS should_still_be_one FROM pgautofailover.node + WHERE formationid = 'archiving_test' AND groupid = 0 AND haspgdata = false; + +-- ── list_archiver_memberships: what an archiver process discovers ────────── + +SELECT * FROM pgautofailover.list_archiver_memberships(:archiverid); + +-- a second formation attached to the same archiver shows up alongside the +-- first -- this is the multi-membership case: one archiver, several +-- (formation, group) rows, each its own WAL stream and base-backup schedule +SELECT pgautofailover.create_formation('archiving_test_2', 'pgsql', 'postgres', + true, 1); +INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, sysidentifier, + goalstate, reportedstate) +VALUES ('archiving_test_2', 0, 'node3', 'node3.local', 5432, 222, + 'primary', 'primary'); +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test_2'); +SELECT formation_id, group_id + FROM pgautofailover.list_archiver_memberships(:archiverid) + ORDER BY formation_id; + +SELECT pgautofailover.archiver_remove_formation(:archiverid, 'archiving_test_2'); + +-- a second archiver serving the same formation/group shares the same +-- (nodehost, nodeport) = (its own hostname, 0) with the first -- the +-- node_nodehost_nodeport_haspgdata_idx partial unique index (scoped to +-- haspgdata rows only) must not reject this. Registered with an explicit, +-- distinct region from archiver1's own default -- this is the intended +-- shape for geographically-redundant DR coverage of the same formation +-- (see archiver.region's own comment); get_archivers() below must surface +-- both regions distinctly. +SELECT pgautofailover.register_archiver('archiver2', 'archiver1.local', + region => 'eu-west') + AS archiverid2 \gset +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid2, 'archiving_test'); + +SELECT archiver_id, archiver_name, region + FROM pgautofailover.get_archivers('archiving_test') + ORDER BY archiver_id; + +-- ── WAL capture confirmation: wal_archived() / report_wal_received() ─────── + +SELECT pgautofailover.report_wal_received( + :nodeid, '000000010000000000000001', '0/1000000'); + +-- default archiver_quorum is 1: a single archiver's report already satisfies it +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); + +-- bump the formation-wide default to 2: the same segment, reported by only +-- one archiver, no longer satisfies quorum +SELECT pgautofailover.set_archiver_policy('archiving_test', NULL, 2, NULL, NULL); +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); + +-- a group-specific override takes precedence over the formation-wide default +SELECT pgautofailover.set_archiver_policy('archiving_test', 0, 1, NULL, NULL); +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 0); +-- group 1 has no override of its own: falls back to the formation default (2) +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 1); + +-- ── base backup lifecycle ─────────────────────────────────────────────────── + +SELECT pgautofailover.report_basebackup_started( + :archiverid, 'archiving_test', 0, 'base_20260804', 1, '0/500000', 'live') + AS basebackupid \gset + +SELECT pgautofailover.report_basebackup_completed( + :basebackupid, '0/1000000', 123456789, + '/var/lib/pgaf-archiver/backups/base_20260804'); + +SELECT basebackupid, status, startlsn, endlsn, sizebytes + FROM pgautofailover.basebackup; + +SELECT basebackupid, formationid, groupid, status + FROM pgautofailover.get_latest_basebackup('archiving_test', 0); + +-- nothing to prune yet: the captured segment's LSN isn't older than this +-- backup's own startlsn +SELECT pgautofailover.prune_archiver_wal('archiving_test', 0); + +-- report_basebackup_deleted() marks status='deleted' (never a real DELETE) +-- and prunes -- with no 'complete' backup left for this group, there's no +-- anchor point to replay forward from, so nothing prunes either +SELECT pgautofailover.report_basebackup_deleted(:basebackupid); +SELECT basebackupid, status, deletedat IS NOT NULL AS was_deleted + FROM pgautofailover.basebackup; + +-- ── rclone_config + archiver_storage ───────────────────────────────────── + +SELECT pgautofailover.create_rclone_config( + 'minio-test', '[minio]' || chr(10) || 'type = s3') + AS rcloneconfigid \gset + +SELECT pgautofailover.archiver_add_storage(:archiverid, 'minio-test') + AS archiverstorageid \gset + +SELECT archiverstorageid, storagemethod, rcloneconfigid + FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid + ORDER BY archiverstorageid; + +-- the mandatory local target cannot be removed +SELECT archiverstorageid AS local_storageid FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid AND storagemethod = 'local' \gset + +SELECT pgautofailover.archiver_remove_storage(:local_storageid); + +-- the non-local target can be +SELECT pgautofailover.archiver_remove_storage(:archiverstorageid); +SELECT count(*) AS remaining_storage_targets FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid; + +-- ── warm-standby archiver_node + maxresidentreplay cap ────────────────────── + +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby', + NULL, NULL, 'archiving_test', 0, 'continuous') + AS archivernodeid1 \gset + +-- default maxresidentreplay is 1: a second resident warm-standby on the +-- same archiver must be refused +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby2', + NULL, NULL, 'archiving_test', 0, 'continuous'); + +-- ── PITR lifecycle ─────────────────────────────────────────────────────── + +SELECT pgautofailover.create_archiver_node( + :archiverid, 'pitr', '/var/lib/pgaf-archiver/pitr-recovery', + NULL, NULL, NULL, NULL, NULL, NULL, 'restoring') + AS pitrnodeid \gset + +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'create', + '{"restore_target_time": "2026-08-04 00:00:00+00"}'::jsonb, + NULL, NULL, 'not paused'); + +SELECT pgautofailover.set_archiver_node_pitr_status(:pitrnodeid, 'paused'); + +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'status', NULL, '0/900000'::pg_lsn, '2026-08-04 00:00:05+00', 'paused'); + +SELECT archivernodeid, archiverid, pitrstatus, lastoperation, + observedlsn, observedpausestate + FROM pgautofailover.pitr_node_status; + +-- ── PITR command queue: pops and clears exactly once ──────────────────────── + +SELECT pgautofailover.pitr_queue_command(:pitrnodeid, 'promote', NULL); +SELECT pgautofailover.pitr_next_command(:pitrnodeid); +SELECT pgautofailover.pitr_next_command(:pitrnodeid); + +-- ── archiver_remove_formation cleans up the ARCHIVING node row ────────────── + +SELECT pgautofailover.archiver_remove_formation(:archiverid, 'archiving_test'); + +SELECT count(*) AS should_be_zero FROM pgautofailover.node + WHERE haspgdata = false AND nodeid = :nodeid; + +SELECT count(*) AS should_also_be_zero FROM pgautofailover.archiver_node + WHERE archiverid = :archiverid AND kind = 'wal-receiver'; diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 276738592..d24d7cfb5 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -67,7 +67,12 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- summary row right before its own detail rows, as a header. -- -- Expected result: empty. Every MonitorFSM[] rule currently has a matching --- KeeperFSM[] row for every current_state it can assign a transition from. +-- KeeperFSM[] row for every current_state it can assign a transition from +-- -- including the pos 367/396/397/398 archiver-related edges (Archiving & +-- Disaster Recovery design, milestone 2): KeeperFSM[]'s own +-- WAIT_STANDBY_STATE/ARCHIVING_STATE/REPORT_LSN_STATE rows +-- (fsm_init_archiver/fsm_archiver_report_lsn/fsm_archiver_follow_new_primary, +-- fsm.c/fsm_transition.c) close this milestone's own gap. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos diff --git a/tests/tap/schedule b/tests/tap/schedule index 3b29df16a..dcf765a74 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -31,6 +31,10 @@ fast_forward demote_timeout_wait_primary_deadlock wait_primary_draining_deadlock timeline_fork_report_lsn_deadlock +archiver_wal_capture +archiver_basebackup_generation +archiver_basebackup_policy +archiver_bootstrap_and_fast_forward keeper_fsm_gap_209_wait_maintenance keeper_fsm_gap_211_wait_maintenance keeper_fsm_gap_209_wait_standby diff --git a/tests/tap/schedules/archiver-multi.sch b/tests/tap/schedules/archiver-multi.sch new file mode 100644 index 000000000..64342d1d6 --- /dev/null +++ b/tests/tap/schedules/archiver-multi.sch @@ -0,0 +1,16 @@ +# Archiving & Disaster Recovery: dynamic multi-formation attach and +# geo-redundant region coverage. Kept out of archiver.sch (WAL capture, +# base backups, rebuild-from-archiver): these specs exercise the +# reconciler's own membership-diffing and the region column's SQL/CLI +# round-trip -- monitor-side and CLI logic, not pg_walsender's wire +# protocol -- so PG17-only matches node-fsm-gaps.sch's own rationale +# ("this is FSM/logic coverage, not version-specific code paths") rather +# than archiver.sch's all-versions one. Also keeps this schedule light: +# archiver_multi_formation.pgaf alone has a mandatory 50s sleep (one +# reconciler tick, ARCHIVER_RECONCILER_INTERVAL_SECONDS) plus several +# more, and archiver.sch already learned the hard way (this same PR, +# CI run 84233594160) what happens when a schedule's own runtime creeps +# past the 20-minute step timeout. +archiver_multi_formation +archiver_budget_architecture_regions +archiver_two_regions diff --git a/tests/tap/schedules/archiver.sch b/tests/tap/schedules/archiver.sch new file mode 100644 index 000000000..26d9d229a --- /dev/null +++ b/tests/tap/schedules/archiver.sch @@ -0,0 +1,18 @@ +# Archiving & Disaster Recovery: WAL capture, base backups, and rebuild- +# from-archiver. Split out of node.sch: adding these 4 specs pushed every +# PG version of that already-tight schedule over the CI step's 20-minute +# timeout (CI run 84233594160: PG16/PG18 timed out at 20 minutes, PG14/ +# PG15/PG19 hit real failures before even getting there -- a cumulative +# time-budget overrun on top of real bugs, same pattern node-fsm-gaps.sch +# was split out for). +# +# Unlike node-fsm-gaps.sch, this schedule runs on every PG version rather +# than PG17 only: pg_walsender speaks the real Postgres replication wire +# protocol to real pg_basebackup/pg_receivewal clients, so its correctness +# is genuinely version-sensitive (this exact split was prompted by a +# version-specific bug: a hardcoded server_version made every non-PG16 +# build fail "incompatible server version" against real pg_basebackup). +archiver_wal_capture +archiver_basebackup_generation +archiver_basebackup_policy +archiver_bootstrap_and_fast_forward diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index 1c9ed4fef..8149d11cd 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -4,6 +4,9 @@ # edge-gap specs that used to live here were split out to node-fsm-gaps.sch # (PG17-only) once this schedule's own combined runtime started timing out # the CI step on every PG version -- see that file's own header comment. +# The archiver specs that briefly lived here too were split out to +# archiver.sch (all PG versions) for the same reason -- see that file's +# own header comment. create_standby_with_pgdata launch_deferred_set_metadata fsm_step_report_advance diff --git a/tests/tap/specs/archiver_basebackup_generation.pgaf b/tests/tap/specs/archiver_basebackup_generation.pgaf new file mode 100644 index 000000000..c4d8fdc51 --- /dev/null +++ b/tests/tap/specs/archiver_basebackup_generation.pgaf @@ -0,0 +1,88 @@ +# Archiving & Disaster Recovery, Milestone 5: base backup generation, +# `live` source then `replay`/`volatile`. +# +# Covers service_archiver_maybe_generate_basebackup() (service_archiver_ +# basebackup.c): a group with no base backups yet gets one immediately, +# sourced live (pg_basebackup run directly against a real node); once that +# lands, the very next tick exercises replay/volatile once -- extract that +# live backup into a throwaway staging instance, replay this archiver's own +# already-captured WAL forward until it promotes, pg_basebackup it over +# loopback, then discard the staging instance. Both are real, +# monitor-tracked pgautofailover.basebackup rows by the end. +# +# The bootstrap backup (a group's very first one) is always sourced live, +# regardless of policy -- a replay needs an existing backup to replay from +# (service_archiver_maybe_generate_basebackup()'s own "bootstrap is always +# live" rule, service_archiver_basebackup.c). Everything *after* bootstrap +# is scheduled and sourced by whichever base-backup policy applies to the +# group -- the schema's own built-in 'default' policy (frequency 24h, +# source 'replay') would make the second backup real, but not within any +# sane test window, so this spec attaches its own short-frequency, +# source=replay policy as soon as archiver1 is up, via the real +# `pg_autoctl create basebackup-policy` CLI + set_archiver_policy() -- the +# same path archiver_basebackup_policy.pgaf (scheduling/retention coverage, +# source=live there) exercises, just with source=replay here so this +# spec's own remaining job -- proving the replay/volatile generation +# pipeline itself actually works (extract the live backup into a staging +# instance, replay this archiver's own captured WAL forward until it +# promotes, pg_basebackup it over loopback, discard the staging instance) +# -- still gets exercised for real. Attached during setup, before test_001's +# own sleep starts, so the fast frequency is already in effect for whichever +# tick first notices the bootstrap backup has landed and a new one is due. +# +# Predecessor: archiver_wal_capture.pgaf (M4). + +cluster { + monitor + formation { + node1 + archiver1 archiver + } +} + +setup { + # A lone node plus an archiver is a genuine single-node formation -- + # the archiver never counts as a real Postgres secondary (see group_ + # state_machine.c's BuildForPrimaryNodeNodeActiveContext, hasPgData- + # gated), so node1's own correct, stable terminal state here is + # "single", not "primary" (there is no other node to ever promote it + # past that). + wait until node1 state is single timeout 60s + wait until archiver1 state is archiving timeout 60s + + exec archiver1 bash -c 'printf "%s" "{\"source\": \"replay\", \"replaymode\": \"volatile\", \"frequency\": \"10 seconds\", \"maxcount\": 3, \"maxage\": \"10 minutes\", \"onpromotion\": false}" > /tmp/replay-policy.json' + exec archiver1 pg_autoctl create basebackup-policy --monitor postgresql://autoctl_node@monitor/pg_auto_failover --name replay-fast --config /tmp/replay-policy.json + sql monitor { + SELECT pgautofailover.set_archiver_policy( + 'default', NULL, 1, + (SELECT basebackuppolicyid + FROM pgautofailover.get_basebackup_policy('replay-fast')), + false); + } +} + +teardown { + compose down +} + +# +# test_001: the bootstrap live backup and (once the attached fast policy's +# first frequency interval has elapsed) a real replay/volatile +# backup both land on their own; check the final state. +# +# Polls rather than sleeping a fixed guess: generating the +# replay/volatile backup is several real Postgres-instance +# lifecycles (extract the live backup into a staging instance, +# replay this archiver's own captured WAL forward, poll for +# promotion, pg_basebackup it over loopback, discard the staging +# instance), not a single fast pg_basebackup call like archiver_ +# basebackup_policy.pgaf's own live-backup cycles, so its wall- +# clock cost varies with host load (CI run 84233594160 hit this +# directly: a fixed 60s sleep wasn't always enough). +# + +step test_001_replay_backup_lands { + wait until basebackup source is replay in default/0 timeout 150s + wait until basebackup replaymode is volatile in default/0 timeout 5s + wait until basebackup status is complete in default/0 timeout 30s +} diff --git a/tests/tap/specs/archiver_basebackup_policy.pgaf b/tests/tap/specs/archiver_basebackup_policy.pgaf new file mode 100644 index 000000000..7a6851163 --- /dev/null +++ b/tests/tap/specs/archiver_basebackup_policy.pgaf @@ -0,0 +1,103 @@ +# Archiving & Disaster Recovery, Milestone 5 (appended): base-backup +# production/retention policy -- frequency-driven scheduling and +# maxcount/maxage pruning, appended to M5 rather than left as a follow-up, +# so the archiver's own base-backup production is a real, bounded resource +# before Milestones 6/7/8 (warm standby, PITR, cloud push) start building +# on top of it. +# +# `frequency: 6 seconds` here is illustrative-fast, not the "1 backup a +# minute" example a real deployment might reasonably use (the design doc's +# own "nightly-cloud" example uses 6h; the schema's own default is 24h) -- +# this spec exists to prove the *mechanism* (does a new backup actually +# fire once frequency has elapsed, does retention actually prune once +# maxcount is exceeded), and a multi-minute real interval would make this +# test unnecessarily slow without adding any real coverage. +# +# The policy is created via the real `pg_autoctl create basebackup-policy` +# CLI (exercising its own --config file-reading path end to end), then +# attached to the formation directly via SQL (set_archiver_policy() -- +# the same function `pg_autoctl create archiver --basebackup-policy` +# calls, just without needing archiver1 declared `create and launch +# deferred` only to re-create it by hand the way the disaster-recovery +# spec does for node2). +# +# `source: "live"` rather than "replay": a live pg_basebackup against a +# near-empty test database completes in a couple of seconds, letting +# several full cycles land inside a practical test runtime -- the replay/ +# volatile pipeline itself is already covered by archiver_basebackup_ +# generation.pgaf, this spec's own job is scheduling/retention, not +# re-proving the replay mechanism. +# +# Predecessor: archiver_basebackup_generation.pgaf (M5, base backup +# generation itself). + +cluster { + monitor + formation { + node1 + archiver1 archiver + } +} + +setup { + wait until node1 state is single timeout 60s + wait until archiver1 state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: create a fast-cycling, maxcount=3 policy via the real CLI, +# attach it to the formation, then let enough cycles pass that +# retention has real pruning to do. Reaching *exactly* maxcount +# after several times frequency has elapsed is strong evidence +# both halves work together: if scheduling never fired past the +# bootstrap backup, the count would be stuck at 1, not 3; if +# retention never pruned, the count would keep growing past 3. +# + +step test_001_policy_scheduling_and_retention { + exec archiver1 bash -c 'printf "%s" "{\"source\": \"live\", \"frequency\": \"6 seconds\", \"maxcount\": 3, \"maxage\": \"10 minutes\", \"onpromotion\": false, \"cache\": \"local\"}" > /tmp/basebackup-policy.json' + exec archiver1 pg_autoctl create basebackup-policy --monitor postgresql://autoctl_node@monitor/pg_auto_failover --name fast-policy --config /tmp/basebackup-policy.json + + sql monitor { + SELECT pgautofailover.set_archiver_policy( + 'default', NULL, 1, + (SELECT basebackuppolicyid + FROM pgautofailover.get_basebackup_policy('fast-policy')), + false); + } + + # ~70s at 6s/cycle covers roughly 11 possible cycles -- comfortably + # past maxcount=3 even accounting for each live backup itself taking a + # few seconds, so retention has settled into a stable state by the + # time this checks. + sleep 70s + + sql monitor { + SELECT count(*) FROM pgautofailover.list_basebackups('default', 0); + } + expect { 3 } + + # the newest retained backup should be recent -- confirms retention + # kept the *newest* maxcount backups (this file's own ORDER BY ... + # DESC), not an arbitrary set. + sql monitor { + SELECT (max(startedat_epoch) > + extract(epoch FROM now() - interval '20 seconds')::bigint) + FROM pgautofailover.list_basebackups('default', 0); + } + expect { t } +} + +# +# test_002: `show basebackup-policy` reads back exactly what test_001's +# own `create basebackup-policy` wrote, through the real CLI on +# both ends. +# + +step test_002_show_basebackup_policy { + exec archiver1 pg_autoctl show basebackup-policy --monitor postgresql://autoctl_node@monitor/pg_auto_failover --name fast-policy --json +} diff --git a/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf b/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf new file mode 100644 index 000000000..4e8d75c7b --- /dev/null +++ b/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf @@ -0,0 +1,161 @@ +# Archiving & Disaster Recovery: `pg_autoctl create postgres --from-archiver` +# (bootstrap a brand new standby from the archiver's own base backup + WAL +# cache instead of the group's live primary), followed by a FAST_FORWARD +# election where the archiver is the only node with the WAL the winning +# candidate is missing. +# +# node2 is declared `create and launch deferred` (compose_gen.c's normal +# per-node command is `pg_autoctl node run `, which just spin-polls +# the ini forever while deferred) so its container starts but never runs +# the normal, ini-driven `create postgres` -- that path has no hook for +# custom flags like --from-archiver (see nodespec.c: NodeSpec has no +# fromArchiver field, only KeeperConfig does, populated exclusively by +# cli_create_node.c's own direct CLI parsing). test_001 instead `exec`s +# into node2's own container and runs `pg_autoctl create postgres +# --from-archiver` by hand, then backgrounds `pg_autoctl run` the same way +# debug_citus_worker_switchover.pgaf backgrounds a long-lived command +# (`bash -c "nohup ... &"` -- `docker compose exec -T` blocks until its +# argv exits, so a foreground `pg_autoctl run` would hang the test step +# forever without this). +# +# test_002/003/004 engineer an actual WAL gap rather than relying on race +# timing: node2 is stopped (so it can't stream anything further from +# node1), more WAL is generated and given time to land in the archiver's +# walcache, *then* node1 is killed and node2 brought back -- at that +# point node2 is the only live standby-kind candidate, the archiver is +# strictly ahead of it, and get_most_advanced_standby() (already proven +# in this milestone's own manual testing to resolve an ARCHIVING row's +# port == 0 sentinel to the archiver's real serve port, keeper.c) selects +# the archiver as fast_forward's WAL source. test_004's final row-count +# check on node2 (post-promotion) confirms real WAL bytes were actually +# fetched and applied, not just that the FSM passed through the right +# state label. +# +# Predecessor: archiver_basebackup_generation.pgaf (M5, base backup +# generation this spec's test_001 depends on already being ready). + +cluster { + monitor + ssl off + formation { + node1 + archiver1 archiver + node2 create and launch deferred + } +} + +setup { + wait until node1 state is single timeout 60s + wait until archiver1 state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: wait for the archiver's live base backup (get_latest_basebackup's +# preferred_source = 'live' overload -- see pgautofailover.sql -- +# is what service_archiver_serve.c's own routes refresh asks for; +# a plain 'live' check here is the same thing this spec can +# observe from the monitor side), then bootstrap node2 from it. +# +# The monitor's own "complete" status and pg_walsender's actual +# ability to serve that backup used to be two different things: +# the archiver only re-read its routes file (what cmd_base_ +# backup.c actually checks -- route->basebackupDir) once every +# ARCHIVER_SERVE_ROUTES_REFRESH_TICKS ticks (service_archiver_ +# serve.c, 30 x the 1s tick). service_archiver_basebackup.c now +# signals archiver-serve (SIGUSR1) the moment a backup finishes +# generating and gets reported complete, prompting an immediate +# refresh instead of waiting for the next tick -- no bridging +# sleep needed here anymore. +# + +step test_001_bootstrap_secondary_from_archiver { + # get_latest_basebackup's 3-arg preferred_source overload -- not the + # plain 2-arg form the "wait until basebackup ..." verb generates -- + # so this stays the generic sql-polling form. + wait until sql monitor { + SELECT source::text FROM pgautofailover.get_latest_basebackup('default', 0, 'live') + } is { live } timeout 30s + wait until sql monitor { + SELECT status::text FROM pgautofailover.get_latest_basebackup('default', 0, 'live') + } is { complete } timeout 10s + exec node2 pg_autoctl create postgres --pgdata /var/lib/postgres/pgaf --monitor postgresql://autoctl_node@monitor/pg_auto_failover --auth trust --no-ssl --name node2 --hostname node2 --from-archiver + exec node2 bash -c "nohup pg_autoctl run --pgdata /var/lib/postgres/pgaf > /tmp/node2-run.log 2>&1 & echo backgrounded pid $!" + wait until node2 state is secondary timeout 90s +} + +# +# test_002: stop node2 so it can no longer stream from node1, then generate +# more WAL on the primary and give the archiver (still capturing +# independently via pg_receivewal) time to land it. node2 now +# knows nothing about this WAL; the archiver does. +# + +step test_002_stop_secondary_and_advance_primary { + compose stop node2 + wait until node2 stopped timeout 60s + sql node1 { CREATE TABLE archiver_ff_probe(a int); } + sql node1 { + INSERT INTO archiver_ff_probe SELECT generate_series(1, 1000); + } + sql node1 { SELECT pg_switch_wal(); } + sql node1 { SELECT pg_switch_wal(); } + sleep 15s +} + +# +# test_003: kill the primary. node2 is still stopped, so at this instant the +# archiver is the only node in the group with any of test_002's +# WAL -- node1 is gone, node2 never received it. node1 has +# already self-demoted from "primary" back to "wait_primary" by +# this point (test_002's own compose stop node2 took away its +# only sync-quorum-satisfying standby), so there's no single +# fixed intermediate assigned-state to assert on here -- just +# kill it and let the monitor's own health check notice. +# + +step test_003_kill_primary_leaving_archiver_only { + compose kill node1 + sleep 45s +} + +# +# test_004: bring node2 back. compose stop/start recreates its container +# from scratch, so test_001's own manually-backgrounded +# `pg_autoctl run` (started via exec, not through the container's +# normal deferred-polling entrypoint) doesn't survive it and +# needs restarting by hand again, the same way test_001 started +# it the first time. Once it does, node2 reports in behind the +# archiver, the monitor assigns fast_forward with the archiver as +# WAL source, node2 fetches the missing WAL from it (standby_ +# fetch_missing_wal, already proven against a real archiver +# during this milestone's own development), and promotes. The +# row count on the other side confirms the fetched WAL was real +# and got applied, not just that the FSM label passed through +# fast_forward. +# +# The terminal assigned state is wait_primary, not primary: this +# is real, correct pg_auto_failover semantics (group_state_ +# machine.c's pos 401-421 PRIMARY_NODE section), not an archiver- +# specific gap -- WAIT_PRIMARY -> PRIMARY requires some other node +# to reach *reported* SECONDARY state first, and neither ever will +# here: node1 is dead (stuck reporting "demoted") and archiver1 is +# a different node kind entirely (reports "archiving", never +# "secondary"). A plain two-node cluster whose original primary +# never rejoins behaves identically -- the surviving node stays in +# wait_primary indefinitely. wait_primary is still a fully active, +# write-serving primary (it's only the synchronous-replication +# guarantee that's unmet), which is exactly what the row-count +# query right below exercises. +# + +step test_004_bring_back_secondary_and_fast_forward { + compose start node2 + exec node2 bash -c "nohup pg_autoctl run --pgdata /var/lib/postgres/pgaf > /tmp/node2-run.log 2>&1 & echo backgrounded pid $!" + wait until node2 state is wait_primary timeout 180s + sql node2 { SELECT count(*) FROM archiver_ff_probe; } + expect { 1000 } +} diff --git a/tests/tap/specs/archiver_budget_architecture_regions.pgaf b/tests/tap/specs/archiver_budget_architecture_regions.pgaf new file mode 100644 index 000000000..0440f277d --- /dev/null +++ b/tests/tap/specs/archiver_budget_architecture_regions.pgaf @@ -0,0 +1,86 @@ +# Archiving & Disaster Recovery: the "budget architecture" (see +# docs/architecture.rst's own "Service Availability" section) -- an +# ordinary two-node primary/secondary group plus an archiver added on top +# for Disaster Recovery, rather than a third live standby. This spec's own +# focus is the --region label pg_autoctl create postgres/create archiver +# both accept: node1 in "dc1", node2 in "dc2", archiver1 (which the +# monitor logically sits alongside, for this topology) in "dc3" -- a +# realistic geo-distributed budget deployment, and the first pgaftest spec +# to actually exercise region end to end for either node kind. +# +# region is purely informational (pg_autoctl watch's own display, get_ +# archivers()'s own output column for an archiver) -- it never affects +# placement, quorum, or failover decisions on its own. This spec proves +# the label round-trips correctly (set at create time, readable back from +# the monitor afterwards) and that the archiver itself still does real +# work regardless of which region it's labelled with. +# +# archiver1 uses pgaftest's top-level "archiver { }" syntax (see +# TestArchiverNode's own comment in test_spec.h), folded into "default"'s +# own node list and launched immediately (the normal default for this +# syntax) via the ordinary ini-driven "pg_autoctl node run" path -- no +# deferred launch needed: group 0 of "default" already exists as soon as +# node1 (which archiver1 depends on being healthy) has registered, +# regardless of whether promotion has completed yet. +# +# Predecessor: basic_operation.pgaf (the same two-node group, without the +# archiver or region labels); archiver_wal_capture.pgaf (the WAL-capture +# proof this borrows its pattern from). + +cluster { + monitor + formation { + node1 region dc1 + node2 region dc2 + } + archiver archiver1 { + formation default + region dc3 + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 + wait until archiver1 state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: region labels round-trip correctly for both ordinary nodes and +# the archiver -- set via --region at create time, readable back +# from the monitor afterwards. pgautofailover.node is covered by +# the blanket "GRANT SELECT ON ALL TABLES" near the top of +# pgautofailover.sql, so a direct SELECT works for node1/node2; +# pgautofailover.archiver itself is not (granted much later in +# the same file), so archiver1's region goes through get_ +# archivers('default'), the same function pg_autoctl watch uses. +# + +step test_001_region_labels_round_trip { + sql monitor { SELECT region FROM pgautofailover.node WHERE nodename = 'node1'; } + expect { dc1 } + sql monitor { SELECT region FROM pgautofailover.node WHERE nodename = 'node2'; } + expect { dc2 } + sql monitor { SELECT region FROM pgautofailover.get_archivers('default') WHERE archiver_name = 'archiver1'; } + expect { dc3 } +} + +# +# test_002: the archiver itself still does real work regardless of its own +# region label -- same idiom as archiver_wal_capture.pgaf's own +# test_001 (segment 3 is this image's own observed slot-creation +# floor; see that spec's header comment for the full reasoning). +# + +step test_002_archiver_captures_wal { + sql node1 { CREATE TABLE t1(a int); INSERT INTO t1 VALUES (1), (2); } + sql node1 { SELECT pg_switch_wal(); } + sql node1 { INSERT INTO t1 VALUES (3); } + sql node1 { SELECT pg_switch_wal(); } + wait until wal segment "000000010000000000000003" archived in default/0 timeout 30s + wait until wal segment "000000010000000000000004" archived in default/0 timeout 30s +} diff --git a/tests/tap/specs/archiver_multi_formation.pgaf b/tests/tap/specs/archiver_multi_formation.pgaf new file mode 100644 index 000000000..a27c7aeb8 --- /dev/null +++ b/tests/tap/specs/archiver_multi_formation.pgaf @@ -0,0 +1,190 @@ +# Archiving & Disaster Recovery, Milestone 5: dynamic multi-formation +# membership for a single archiver identity. +# +# Covers service_archiver_reconciler.c: one archiver process (archiver1) +# is brought up ARCHIVING only "default" (a plain two-node formation), then +# -- while it is already running and capturing that formation's WAL -- is +# attached to a *second*, independent formation ("formation2") purely +# through a monitor-side RPC (pgautofailover.archiver_add_formation()), +# with no restart of the archiver process and no CLI/config change on its +# side at all. archiver_reconciler_tick() only re-lists this archiver's +# memberships (pgautofailover.list_archiver_memberships()) once every +# ARCHIVER_RECONCILER_INTERVAL_SECONDS (30s, service_archiver_reconciler.c) +# -- so the second membership's WAL capture can only start once that +# periodic tick actually notices it, not immediately. That discovery path +# is the thing this spec exists to prove; everything else here (bringing +# up two independent 2-node formations, forcing WAL switches, checking +# wal_archived()) is the same idiom as archiver_wal_capture.pgaf. +# +# Two independent formations coming up concurrently is why this spec uses +# the explicit per-node "wait until state is " forms throughout +# rather than the aggregate "wait until primary, secondary" form: the +# aggregate form is not formation-scoped (test_runner.c's +# wait_for_states()/monitor pg_autoctl inspect monitor formation-states +# path just checks "does *any* node report primary and *any* node report +# secondary" cluster-wide), which is ambiguous once two formations are +# each independently electing their own primary/secondary at once. +# +# archiver1 ends up with two rows in pgautofailover.node once attached to +# both formations (one per (formation, group) membership, both groupid = 0 +# since each formation here has a single, plain-Postgres group). Each row +# is named by archiver_add_formation() itself as 'archiver-- +# ' -- never the plain --name -- so the generic "wait until +# archiver1 state is archiving" form (SELECT reportedstate, goalstate FROM +# pgautofailover.node WHERE nodename = $1 LIMIT 1, no ORDER BY -- see +# test_runner.c's monitor_get_node_state()) wouldn't match either row to +# begin with, and would be ambiguous between the two even if it did. Every +# check on archiver1's per-membership state after test_003 attaches the +# second membership uses "wait until archiver state is ... in " +# instead, which matches nodename LIKE 'archiver-%' AND formationid under +# the hood (safe: this spec has exactly one archiver). +# +# The autoctl_node role has no direct SELECT on pgautofailover.archiver +# (granted much later in pgautofailover.sql than the blanket "GRANT SELECT +# ON ALL TABLES IN SCHEMA pgautofailover" near its own top) -- archiver1's +# archiverid is resolved via pgautofailover.get_archivers('default'), which +# *is* granted and returns archiver_id as its first output column. +# +# archiver1 is declared with pgaftest's top-level "archiver { }" syntax +# (sibling to monitor/formation, not nested inside either formation block) +# -- an archiver attaches to formations by name, it isn't a member of any +# one of them (pgautofailover.archiver has no formationid column at all; +# see TestArchiverNode's own comment in test_spec.h). Internally it's +# folded into "default"'s own node list (the one formation it declares), +# so it launches immediately -- the normal default for this syntax, same +# as any ordinary node -- via the ordinary ini-driven "pg_autoctl node +# run" path: no exec, no deferred launch needed here, since group 0 of +# "default" already exists as soon as node1 (which archiver1 depends on +# being healthy, same as node2 does) has registered, regardless of +# whether promotion has completed yet. +# +# Predecessor: none -- first pgaftest spec covering dynamic multi-formation +# archiver membership; see archiver_wal_capture.pgaf for the base single- +# formation WAL-capture mechanism this builds on. + +cluster { + monitor + formation { + node1 + node2 + } + formation formation2 { + node3 + node4 + } + archiver archiver1 { + formation default + } +} + +setup { + wait until node1 state is primary timeout 90s + wait until node2 state is secondary timeout 90s + promote node1 + wait until archiver1 state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: baseline -- confirm formation "default"'s WAL is captured +# before formation2 even exists, exactly as in +# archiver_wal_capture.pgaf. Segment numbering here follows that +# spec's own observed floor (segment 3 is the archiver's slot +# restart_lsn floor on this image -- see archiver_wal_capture. +# pgaf's header comment for the full reasoning); each switch +# below has a real INSERT immediately before it. +# + +step test_001_capture_formation1_wal { + sql node1 { CREATE TABLE t1(a int); INSERT INTO t1 VALUES (1), (2); } + sql node1 { SELECT pg_switch_wal(); } + sql node1 { INSERT INTO t1 VALUES (3); } + sql node1 { SELECT pg_switch_wal(); } + wait until wal segment "000000010000000000000003" archived in default/0 timeout 30s + wait until wal segment "000000010000000000000004" archived in default/0 timeout 30s +} + +# +# test_002: bring up formation2 independently -- registration against a +# non-default formation can lag behind the default formation's +# own (compose_gen.c: data nodes retry registration until their +# formation exists), so this is generous on timeout. +# + +step test_002_bring_up_formation2 { + wait until node3 state is primary timeout 120s + wait until node4 state is secondary timeout 120s + promote node3 +} + +# +# test_003: the actual behavior under test. Attach the already-running +# archiver1 to formation2 purely via the monitor RPC (not the +# CLI's --formation flag, which only matters at `create archiver` +# time) and wait for the reconciler's own periodic tick (30s, +# ARCHIVER_RECONCILER_INTERVAL_SECONDS) to notice the new +# membership and start a second WAL-capture child for it. +# archiver_add_formation() is idempotent and, since formation2 +# has exactly one group (group 0, plain nodes, no Citus), attaches +# exactly one new ARCHIVING node row. +# + +step test_003_dynamic_attach_to_formation2 { + sql monitor { + SELECT pgautofailover.archiver_add_formation( + (SELECT archiver_id FROM pgautofailover.get_archivers('default') LIMIT 1), + 'formation2'); + } + # polls through the reconciler's own periodic tick (30s, + # ARCHIVER_RECONCILER_INTERVAL_SECONDS) noticing the new membership, + # forking the new capture child, and that child registering/reporting + # far enough to reach ARCHIVING_STATE. + wait until archiver state is archiving in formation2 timeout 90s +} + +# +# test_004: confirm formation2's WAL is actually being captured by the +# newly-started capture child, not just that the FSM state looks +# right. formation2 is a fresh formation/group, but its archiver +# slot's restart_lsn floor still lands on segment 3 -- the same +# bootstrap-consumption behavior archiver_wal_capture.pgaf's own +# header comment documents for "default": a freshly-created +# replication slot's restart_lsn already reflects whatever WAL +# bootstrap/registration itself generated before the slot existed, +# independent of which formation it belongs to. Confirmed live +# (segment 1/2 never actually get archived -- only 3 onward do). +# + +step test_004_capture_formation2_wal { + sql node3 { CREATE TABLE t2(a int); INSERT INTO t2 VALUES (1), (2); } + sql node3 { SELECT pg_switch_wal(); } + sql node3 { INSERT INTO t2 VALUES (3); } + sql node3 { SELECT pg_switch_wal(); } + wait until wal segment "000000010000000000000003" archived in formation2/0 timeout 30s + wait until wal segment "000000010000000000000004" archived in formation2/0 timeout 30s +} + +# +# test_005: formation "default"'s own capture must still be uninterrupted +# -- proves the reconciler's dynamic add of formation2 did not +# restart the archiver process or disrupt the pre-existing +# membership's already-running pg_receivewal. One more switch on +# top of test_001's own two. +# + +step test_005_formation1_still_healthy { + sql node1 { INSERT INTO t1 VALUES (4); } + sql node1 { SELECT pg_switch_wal(); } + wait until wal segment "000000010000000000000005" archived in default/0 timeout 30s + wait until archiver state is archiving in default timeout 30s +} + +sequence + test_001_capture_formation1_wal + test_002_bring_up_formation2 + test_003_dynamic_attach_to_formation2 + test_004_capture_formation2_wal + test_005_formation1_still_healthy diff --git a/tests/tap/specs/archiver_two_regions.pgaf b/tests/tap/specs/archiver_two_regions.pgaf new file mode 100644 index 000000000..3b65e983b --- /dev/null +++ b/tests/tap/specs/archiver_two_regions.pgaf @@ -0,0 +1,110 @@ +# Archiving & Disaster Recovery: two independent archivers, in two +# different regions, both attached to the very same formation -- the +# geographically-redundant DR coverage pattern archiver.region exists for +# (see that column's own comment, pgautofailover.sql, and +# pg_autoctl_create_archiver.rst's own --region section). +# +# archiver_add_formation() names each ARCHIVING node row +# 'archiver--', so two different archivers attaching +# to the same (formation, group) get distinct rows and fully independent +# WAL streams/replication slots against the same primary -- confirmed +# already at the SQL-regression level (src/monitor/sql/archiving_schema. +# sql's own "a second archiver serving the same formation/group" case); +# this spec is the first to prove it end to end, with two real archiver +# processes. +# +# Rather than trying to peek at which specific archiver reported a given +# WAL segment (pgautofailover.wal_archived() aggregates across every +# archiver attached to the group, it doesn't expose a per-archiver +# breakdown to autoctl_node), test_002 proves both are independently +# capturing by raising archiverQuorum from its default of 1 to 2 *after* +# a segment is already confirmed archived at quorum 1, then re-checking +# the very same segment: if only one of the two archivers had actually +# captured and reported it, raising the quorum would make wal_archived() +# flip back to false for that segment. It doesn't -- proving both +# archivers, not just one, independently streamed and reported it. +# +# Both archivers use pgaftest's top-level "archiver { }" syntax (see +# TestArchiverNode's own comment in test_spec.h), each folded into +# "default"'s own node list and launched immediately (the normal default +# for this syntax) via the ordinary ini-driven "pg_autoctl node run" path +# -- no deferred launch needed: group 0 of "default" already exists as +# soon as node1 has registered, regardless of promotion. +# +# Predecessor: archiver_wal_capture.pgaf (single-archiver WAL-capture +# proof this borrows its segment-numbering reasoning from); +# archiver_budget_architecture_regions.pgaf (the first spec to exercise +# --region at all, for a single archiver). + +cluster { + monitor + formation { + node1 + node2 + } + archiver archiver-eu { + formation default + region eu-west + } + archiver archiver-us { + formation default + region us-east + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 + wait until archiver-eu state is archiving timeout 60s + wait until archiver-us state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: both archivers attached to the same formation, each correctly +# reporting its own region -- get_archivers('default') returns +# one row per archiver here (both groupid 0, the only group in +# this plain-Postgres formation), distinguished by archiver_name. +# + +step test_001_both_archivers_attached_and_labelled { + sql monitor { SELECT reported_state::text FROM pgautofailover.get_archivers('default') WHERE archiver_name = 'archiver-eu'; } + expect { archiving } + sql monitor { SELECT region FROM pgautofailover.get_archivers('default') WHERE archiver_name = 'archiver-eu'; } + expect { eu-west } + sql monitor { SELECT reported_state::text FROM pgautofailover.get_archivers('default') WHERE archiver_name = 'archiver-us'; } + expect { archiving } + sql monitor { SELECT region FROM pgautofailover.get_archivers('default') WHERE archiver_name = 'archiver-us'; } + expect { us-east } +} + +# +# test_002: both archivers are independently, actually capturing WAL -- +# not just one of them with the other silently idle -- proven by +# raising archiverQuorum after the fact, see this file's own +# header comment for the full reasoning. Segment 3 is archiver_ +# wal_capture.pgaf's own observed floor for a single archiver on +# this image (see that spec's header comment for the general +# reasoning: a few segments get consumed by node1+node2's own +# bootstrap before any archiver's replication slot exists, +# observed at exactly segment 3, repeatably). Two archivers here +# instead of one shouldn't move that floor -- both slots get +# created around the same bootstrap point and segment 3 hasn't +# been recycled yet by either -- but this hasn't been separately +# confirmed against a real two-archiver run the way the single- +# archiver floor has; if this segment number turns out wrong +# against the real image, this is the line to adjust. +# + +step test_002_both_archivers_capture_independently { + sql node1 { CREATE TABLE t1(a int); INSERT INTO t1 VALUES (1), (2); } + sql node1 { SELECT pg_switch_wal(); } + sql node1 { INSERT INTO t1 VALUES (3); } + sql node1 { SELECT pg_switch_wal(); } + wait until wal segment "000000010000000000000003" archived in default/0 timeout 30s + sql monitor { SELECT pgautofailover.set_archiver_policy('default', NULL, 2, NULL, NULL); } + wait until wal segment "000000010000000000000003" archived in default/0 timeout 30s +} diff --git a/tests/tap/specs/archiver_wal_capture.pgaf b/tests/tap/specs/archiver_wal_capture.pgaf new file mode 100644 index 000000000..a0302f24c --- /dev/null +++ b/tests/tap/specs/archiver_wal_capture.pgaf @@ -0,0 +1,159 @@ +# Archiving & Disaster Recovery, Milestone 4: WAL-capture correctness and +# failover continuity for an ARCHIVING node. +# +# Covers the mechanism added in service_archiver.c's +# service_archiver_report_captured_wal(): once pg_receivewal completes a WAL +# segment (no longer ".partial"), the archiver reports it to the monitor via +# pgautofailover.report_wal_received(), which is what actually populates +# pgautofailover.archiver_wal and makes wal_archived() -- the archive_command +# confirmation check -- return true. Before this milestone nothing in the +# codebase ever called that SQL function, so archiver_wal stayed permanently +# empty no matter how much WAL an archiver captured. +# +# Segment filenames are deterministic within a single run of this spec: each +# pg_switch_wal() call that has real content to flush advances exactly one +# segment (a bare pg_switch_wal() with nothing written since the previous +# one is *not* reliably a no-op in practice -- it still writes the SWITCH +# record itself -- but forcing a long run of them back-to-back with no +# other activity in between measurably slows down how fast pg_receivewal +# can stream and flush all of it under host load, which is exactly the +# wrong kind of margin to add: it turns a segment-numbering assumption into +# a throughput race instead, observed firsthand while developing this +# fix). What is NOT segment 000000010000000000000001, despite an earlier +# version of this spec assuming so: by the time archiver1 finishes +# registering and gets its own replication slot (keeper_create_and_drop_ +# replication_slots(), the same eager per-tick mechanism used for an +# ordinary standby -- see pg_autoctl's service_archiver.c and keeper.c), +# node1 + node2's own cluster/extension bootstrap has typically already +# consumed a few WAL segments -- observed at exactly segment 3 (the +# archiver's slot restart_lsn landing on 0/3000000) across repeated runs of +# this exact Docker image, and segment 3 itself *is* fully retained and +# capturable (it's the archiver's own slot floor, not one before it). A +# replication slot only protects WAL *from its own creation time onward*; +# it can't retroactively un-recycle segments the primary already dropped +# before the slot existed -- confirmed by node2's own real-secondary slot +# *also* not reaching back to segment 1 in the same runs. pgaftest's DSL +# has no way to capture a query result for use in a later query, so there's +# no way to compute "whatever segment we're actually on" dynamically; every +# check below instead uses the fixed, repeatedly-observed floor of segment +# 3 directly, with the minimum number of pg_switch_wal() calls needed +# (each with real content immediately before it) rather than any extra +# margin -- if a future Postgres/build change shifts the real floor, this +# spec will fail fast and clearly (wrong segment name -> wal_archived() +# returns false), not hang or silently pass. +# +# The autoctl_node role has no direct SELECT on archiver_wal (see +# report_wal_received()'s own SECURITY DEFINER indirection in +# pgautofailover.sql) -- wal_archived() is the one function it can call to +# observe that table's contents, so every check here goes through it rather +# than a raw SELECT. +# +# Predecessor: none -- this is the first archiver-kind pgaftest spec. + +cluster { + monitor + formation { + node1 + node2 + archiver1 archiver + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 + wait until archiver1 state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: force two WAL segment switches on the primary and confirm both +# land durably in archiver_wal (archiver_quorum defaults to 1, and +# there is exactly one archiver here, so wal_archived() flips to +# true as soon as service_archiver_report_captured_wal()'s next +# tick reports the segment). Segment 3 is the observed floor (see +# this spec's own header comment) and is itself fully capturable, +# so no throwaway switches are needed before it -- each switch +# below has a real INSERT immediately before it, so it reliably +# produces a genuinely new segment rather than racing pg_ +# receivewal's own throughput under host load. +# + +step test_001_capture_wal { + sql node1 { CREATE TABLE t1(a int); INSERT INTO t1 VALUES (1), (2); } + sql node1 { SELECT pg_switch_wal(); } + sql node1 { INSERT INTO t1 VALUES (3); } + sql node1 { SELECT pg_switch_wal(); } + wait until wal segment "000000010000000000000003" archived in default/0 timeout 30s + wait until wal segment "000000010000000000000004" archived in default/0 timeout 30s +} + +# +# test_002: kill and restart the archiver process while it is already +# ARCHIVING (persisted state, no FSM transition on restart). This +# exercises service_archiver_loop()'s own liveness check: without +# it, pg_receivewal never comes back up after a restart, because +# it is otherwise only (re)started from fsm_init_archiver / +# fsm_archiver_follow_new_primary -- the transition *into* +# ARCHIVING_STATE, which does not run again once current_role and +# assigned_role already agree. +# + +step test_002_archiver_restart_liveness { + compose stop archiver1 + wait until archiver1 stopped timeout 60s + compose start archiver1 + wait until archiver1 state is archiving timeout 60s + # one more switch on top of test_001's own two -- completes segment + # 000000010000000000000005 (see this spec's own header comment on why + # these are fixed numbers rather than derived from segment 1). + sql node1 { INSERT INTO t1 VALUES (4); } + sql node1 { SELECT pg_switch_wal(); } + wait until wal segment "000000010000000000000005" archived in default/0 timeout 30s +} + +# +# test_003: fail node1 (primary) over to node2. Every ARCHIVING row in the +# group is expected to pass through REPORT_LSN_STATE during the +# election (fsm_archiver_report_lsn stops pg_receivewal against the +# now-untrustworthy old primary) and back to ARCHIVING_STATE once +# node2 is confirmed primary (fsm_archiver_follow_new_primary +# re-points pg_receivewal at it) -- the same election phases every +# other node kind goes through, applied to an archiver for the +# first time here. Segments captured before the failover must stay +# recorded: report_wal_received() never deletes archiver_wal rows. +# + +step test_003_failover_continuity { + compose stop node1 + wait until node1 stopped timeout 60s + # wait_primary, not primary, is the correct terminal state here: node1 + # is only stopped (still a registered group member, never rejoining + # within this spec) and archiver1 is a different node kind entirely + # (reports "archiving", never "secondary") -- group_state_machine.c's + # PRIMARY_NODE section only ever promotes WAIT_PRIMARY -> PRIMARY once + # some other node reaches *reported* SECONDARY state, which neither + # ever will. Same real, correct pg_auto_failover semantics as a plain + # two-node cluster whose original primary never rejoins (see this + # project's own archiver_bootstrap_and_fast_forward.pgaf, test_004, + # for the identical reasoning). + wait until node2 state is wait_primary timeout 120s + # plain terminal-state check, no "passing through report_lsn": that + # clause tracks ASSIGNED-state transitions via LISTEN/NOTIFY, and can + # miss one that already happened before this wait started listening -- + # now that node2 reaches wait_primary much faster (no longer stuck + # waiting on a secondary-quorum condition that could never be + # satisfied), archiver1's own report_lsn -> archiving cycle can + # complete before this wait even starts observing it. + wait until archiver1 state is archiving timeout 120s + # re-check the earliest and latest segments already confirmed by test_001 + # and test_002 -- proves they survived the failover, not that they were + # ever really "segment 1" (see this spec's own header comment). + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } + expect { t } + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000005'); } + expect { t } +} diff --git a/tests/tap/specs/citus_basic_operation.pgaf b/tests/tap/specs/citus_basic_operation.pgaf index 47a0cd571..16e26ddac 100644 --- a/tests/tap/specs/citus_basic_operation.pgaf +++ b/tests/tap/specs/citus_basic_operation.pgaf @@ -1,5 +1,10 @@ # Test basic Citus cluster operations: coordinator HA, worker HA with two # worker groups, distributed table writes/reads, and failover at each level. +# test_011/test_012 additionally cover Archiving & Disaster Recovery for a +# Citus formation: a single archiver attached once ends up with one +# membership per group -- the coordinator's own group 0, plus worker1's +# group 1 and worker2's group 2 -- each independently WAL-capturing, proving +# a Citus formation's archiver coverage isn't limited to the coordinator. # # Ported from tests/test_basic_citus_operation.py # Predecessor: tests/test_basic_citus_operation.py @@ -14,6 +19,10 @@ cluster { worker2a worker group 2 worker2b worker group 2 } + archiver archiver1 { + formation default + create and launch deferred + } } setup { @@ -132,3 +141,65 @@ step test_010_perform_failover_coordinator { and coordinator1b state is primary timeout 90s } + +# +# test_011: bring up an archiver attached to this Citus formation. +# Declared with pgaftest's top-level "archiver { }" syntax (see +# the cluster{} block above) with "create and launch deferred": +# its container still runs the ordinary "pg_autoctl node run +# " command, but the ini's own [launch] section makes that +# poll and wait rather than actually registering, until this +# step's `pg_autoctl node start` un-defers it -- exactly the +# same idiom as any other deferred node in this DSL (see +# basic_operation.pgaf's own node3/test_017). Triggered only +# here, well after test_002_init_workers confirms every group +# (0, 1, 2) already exists on the monitor: +# pgautofailover.archiver_add_formation() attaches one ARCHIVING +# row per group already present in the formation at the moment +# it's called, so creating the archiver this late (rather than +# at cluster boot, concurrently with the worker groups still +# registering) guarantees a single `pg_autoctl create archiver +# --formation default` call here covers all three groups, not +# just whichever happened to exist first. +# +# archiver1 ends up with three rows in pgautofailover.node (one +# per group), each named by archiver_add_formation() itself as +# 'archiver--' (never the plain --name) -- +# the generic "wait until archiver1 state is archiving" form is +# ambiguous once more than one such row exists (it matches on +# nodename = $1, no ORDER BY) and wouldn't match this synthesized +# name anyway, so every check below uses "wait until archiver +# state is ... in default/" instead (matches nodename +# LIKE 'archiver-%' AND formationid/groupid under the hood). +# + +step test_011_bring_up_archiver { + exec archiver1 pg_autoctl node start + wait until archiver state is archiving in default/0 timeout 60s + wait until archiver state is archiving in default/1 timeout 60s + wait until archiver state is archiving in default/2 timeout 60s +} + +# +# test_012: prove the archiver is actually capturing real WAL for every +# group, not just reporting a state label. Writes go to each +# group's *current* primary -- coordinator1b (failed over in +# test_010), worker1a (never failed over in this spec), worker2b +# (failed over in test_009) -- and each membership's own +# reportedlsn is checked against '0/0': service_archiver_update_ +# current_lsn() (service_archiver.c) never reports anything until +# a real segment has actually been captured, so a value beyond +# '0/0' can only mean WAL genuinely landed for that group. This +# avoids hardcoding an exact segment name/number, which this +# spec's own substantial prior WAL traffic (distributed table +# creation, several failovers) would make fragile to predict. +# + +step test_012_archiver_captures_every_group { + sql coordinator1b { CREATE TABLE archiver_probe_coord(a int); INSERT INTO archiver_probe_coord SELECT generate_series(1, 100); SELECT pg_switch_wal(); } + sql worker1a { CREATE TABLE archiver_probe_w1(a int); INSERT INTO archiver_probe_w1 SELECT generate_series(1, 100); SELECT pg_switch_wal(); } + sql worker2b { CREATE TABLE archiver_probe_w2(a int); INSERT INTO archiver_probe_w2 SELECT generate_series(1, 100); SELECT pg_switch_wal(); } + wait until sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 0 } is { t } timeout 30s + wait until sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 1 } is { t } timeout 30s + wait until sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 2 } is { t } timeout 30s +}