Archiving & Disaster Recovery: schema through base-backup generation (M1-M5)

.github/workflows/ci.yml

@@ -323,6 +323,16 @@ jobs:
           - { PGVERSION: 17, schedule: node }
           - { PGVERSION: 18, schedule: node }
           - { PGVERSION: 19, schedule: node }
+          # archiver: WAL capture, base backups, rebuild-from-archiver — all
+          #           PG versions, since pg_walsender's wire-protocol
+          #           correctness is version-sensitive (see
+          #           tests/tap/schedules/archiver.sch's own header comment)
+          - { PGVERSION: 14, schedule: archiver }
+          - { PGVERSION: 15, schedule: archiver }
+          - { PGVERSION: 16, schedule: archiver }
+          - { PGVERSION: 17, schedule: archiver }
+          - { PGVERSION: 18, schedule: archiver }
+          - { PGVERSION: 19, schedule: archiver }
           # ssl: enable_ssl, ssl_self_signed, ssl_cert
           - { PGVERSION: 14, schedule: ssl }
           - { PGVERSION: 15, schedule: ssl }
@@ -335,6 +345,7 @@ jobs:
           - { PGVERSION: 17, schedule: multi-misc }
           - { PGVERSION: 17, schedule: multi-async }
           - { PGVERSION: 17, schedule: node-fsm-gaps }
+          - { PGVERSION: 17, schedule: archiver-multi }
           - { PGVERSION: 17, schedule: citus-1 }
           - { PGVERSION: 17, schedule: citus-2 }
           # citus on PG18 (supported); allow failure until officially validated

.gitignore

@@ -56,5 +56,6 @@ valgrind/
 src/bin/pgaftest/test_spec_parse.tab.*
 src/bin/pgaftest/test_spec_parse.output
 src/bin/pgaftest/pgaftest
+src/bin/pg_walsender/pg_walsender
 run-test.sh
 tests/tablespaces/__pycache__/

Dockerfile

@@ -120,6 +120,12 @@ COPY --from=build /usr/lib/postgresql/${PGVERSION}/lib/pgautofailover.so \
 COPY --from=build /usr/share/postgresql/${PGVERSION}/extension/pgautofailover* \
                   /usr/share/postgresql/${PGVERSION}/extension/
 COPY --from=build /usr/local/bin/pg_autoctl /usr/local/bin/
+# Bracket-glob makes this an optional copy: BuildKit treats [r] as a glob,
+# and an empty glob match is not an error for COPY (unlike a literal missing
+# path). This lets tests/upgrade build the "current" Dockerfile against an
+# old release's source tree, which predates pg_walsender and has no binary
+# to copy. Stopgap only -- revisit after the release with a cleaner fix.
+COPY --from=build /usr/local/bin/pg_walsende[r] /usr/local/bin/
 
 RUN mkdir -p /var/lib/postgres \
  && chown -R docker /var/lib/postgres

docs/_static/css/zoom.css

@@ -0,0 +1,77 @@
+/* Click-to-zoom overlay for docs figures -- see js/zoom.js */
+
+figure img.pgaf-zoomable-img {
+  cursor: zoom-in;
+}
+
+html.pgaf-zoom-locked,
+html.pgaf-zoom-locked body {
+  overflow: hidden;
+}
+
+.pgaf-zoom-overlay {
+  display: none;
+  position: fixed;
+  inset: 0;
+  z-index: 10000;
+  background: rgba(20, 20, 20, 0.92);
+}
+
+.pgaf-zoom-overlay.pgaf-zoom-open {
+  display: flex;
+  flex-direction: column;
+}
+
+.pgaf-zoom-viewport {
+  flex: 1 1 auto;
+  overflow: hidden;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  cursor: grab;
+}
+
+.pgaf-zoom-viewport.pgaf-zoom-dragging {
+  cursor: grabbing;
+}
+
+.pgaf-zoom-img {
+  max-width: 92vw;
+  max-height: 82vh;
+  will-change: transform;
+  transition: transform 0.05s linear;
+  user-select: none;
+  -webkit-user-drag: none;
+  background: #fff;
+  border-radius: 4px;
+  box-shadow: 0 10px 40px rgba(0, 0, 0, 0.5);
+}
+
+.pgaf-zoom-close {
+  position: absolute;
+  top: 1.25rem;
+  right: 1.5rem;
+  width: 2.5rem;
+  height: 2.5rem;
+  border-radius: 50%;
+  border: none;
+  background: rgba(255, 255, 255, 0.12);
+  color: #fff;
+  font-size: 1.75rem;
+  line-height: 1;
+  cursor: pointer;
+  z-index: 1;
+}
+
+.pgaf-zoom-close:hover,
+.pgaf-zoom-close:focus {
+  background: rgba(255, 255, 255, 0.25);
+}
+
+.pgaf-zoom-hint {
+  flex: 0 0 auto;
+  text-align: center;
+  color: rgba(255, 255, 255, 0.7);
+  font-size: 0.85rem;
+  padding: 0.5rem 1rem 1.25rem;
+}

docs/_static/js/zoom.js

@@ -0,0 +1,174 @@
+/*
+ * Click-to-zoom for the docs' own figures (architecture/sequence/FSM
+ * diagrams rendered from tikz), generalizing the pan/scroll-to-zoom
+ * already available on Mermaid diagrams (mermaid_d3_zoom, conf.py) to
+ * every other <img> the docs embed via `.. figure::`.
+ *
+ * Mermaid diagrams render as inline <svg>, not <img>, and already ship
+ * their own in-place zoom -- this script only ever wires plain <img>
+ * elements, so the two never compete for the same wheel/drag events.
+ */
+(function () {
+  "use strict";
+
+  function buildOverlay() {
+    var overlay = document.createElement("div");
+    overlay.className = "pgaf-zoom-overlay";
+    overlay.setAttribute("role", "dialog");
+    overlay.setAttribute("aria-modal", "true");
+    overlay.innerHTML =
+      '<button type="button" class="pgaf-zoom-close" aria-label="Close">×</button>' +
+      '<div class="pgaf-zoom-viewport">' +
+      '<img class="pgaf-zoom-img" alt="" draggable="false" />' +
+      "</div>" +
+      '<div class="pgaf-zoom-hint">scroll to zoom &middot; drag to pan &middot; ' +
+      "double-click to reset &middot; Esc to close</div>";
+    document.body.appendChild(overlay);
+
+    var viewport = overlay.querySelector(".pgaf-zoom-viewport");
+    var img = overlay.querySelector(".pgaf-zoom-img");
+    var closeBtn = overlay.querySelector(".pgaf-zoom-close");
+
+    var scale = 1;
+    var panX = 0;
+    var panY = 0;
+    var dragging = false;
+    var startX = 0;
+    var startY = 0;
+    var startPanX = 0;
+    var startPanY = 0;
+    var lastFocused = null;
+
+    function applyTransform() {
+      img.style.transform =
+        "translate(" + panX + "px, " + panY + "px) scale(" + scale + ")";
+    }
+
+    function reset() {
+      scale = 1;
+      panX = 0;
+      panY = 0;
+      applyTransform();
+    }
+
+    function isOpen() {
+      return overlay.classList.contains("pgaf-zoom-open");
+    }
+
+    function open(src, alt) {
+      lastFocused = document.activeElement;
+      img.src = src;
+      img.alt = alt || "";
+      reset();
+      overlay.classList.add("pgaf-zoom-open");
+      document.documentElement.classList.add("pgaf-zoom-locked");
+      closeBtn.focus();
+    }
+
+    function close() {
+      overlay.classList.remove("pgaf-zoom-open");
+      document.documentElement.classList.remove("pgaf-zoom-locked");
+      img.removeAttribute("src");
+      if (lastFocused && typeof lastFocused.focus === "function") {
+        lastFocused.focus();
+      }
+    }
+
+    closeBtn.addEventListener("click", close);
+
+    overlay.addEventListener("click", function (event) {
+      if (event.target === overlay) {
+        close();
+      }
+    });
+
+    document.addEventListener("keydown", function (event) {
+      if (event.key === "Escape" && isOpen()) {
+        close();
+      }
+    });
+
+    viewport.addEventListener(
+      "wheel",
+      function (event) {
+        if (!isOpen()) {
+          return;
+        }
+        event.preventDefault();
+        var factor = event.deltaY < 0 ? 1.15 : 1 / 1.15;
+        scale = Math.min(8, Math.max(0.5, scale * factor));
+        applyTransform();
+      },
+      { passive: false }
+    );
+
+    viewport.addEventListener("dblclick", function (event) {
+      event.preventDefault();
+      reset();
+    });
+
+    viewport.addEventListener("mousedown", function (event) {
+      dragging = true;
+      startX = event.clientX;
+      startY = event.clientY;
+      startPanX = panX;
+      startPanY = panY;
+      viewport.classList.add("pgaf-zoom-dragging");
+      event.preventDefault();
+    });
+
+    window.addEventListener("mousemove", function (event) {
+      if (!dragging) {
+        return;
+      }
+      panX = startPanX + (event.clientX - startX);
+      panY = startPanY + (event.clientY - startY);
+      applyTransform();
+    });
+
+    window.addEventListener("mouseup", function () {
+      dragging = false;
+      viewport.classList.remove("pgaf-zoom-dragging");
+    });
+
+    return { open: open, close: close };
+  }
+
+  function isZoomable(img) {
+    if (img.closest(".pgaf-zoom-overlay")) {
+      return false;
+    }
+    if (img.classList.contains("no-zoom")) {
+      return false;
+    }
+    return !!img.closest("figure");
+  }
+
+  function wireImages(zoom) {
+    var images = document.querySelectorAll("figure img");
+    images.forEach(function (img) {
+      if (!isZoomable(img) || img.dataset.pgafZoomWired) {
+        return;
+      }
+      img.dataset.pgafZoomWired = "1";
+      img.classList.add("pgaf-zoomable-img");
+      img.tabIndex = 0;
+      img.setAttribute("role", "button");
+      img.setAttribute("aria-label", "Click to zoom: " + (img.alt || "image"));
+      img.addEventListener("click", function () {
+        zoom.open(img.currentSrc || img.src, img.alt);
+      });
+      img.addEventListener("keydown", function (event) {
+        if (event.key === "Enter" || event.key === " ") {
+          event.preventDefault();
+          zoom.open(img.currentSrc || img.src, img.alt);
+        }
+      });
+    });
+  }
+
+  document.addEventListener("DOMContentLoaded", function () {
+    var zoom = buildOverlay();
+    wireImages(zoom);
+  });
+})();

docs/architecture.rst

@@ -31,6 +31,69 @@ PostgreSQL service to accept writes when there's a single server available,
 and opens the service for potential data loss if the primary server were
 also to fail.
 
+High Availability
+------------------
+
+pg_auto_failover treats "High Availability" as two related but distinct
+guarantees, rather than one. Most of what follows on this page -- the
+Monitor, the keeper, synchronous replication, node recovery -- is in service
+of the first of the two; :ref:`archiving_architecture` and the pages it
+links to are in service of the second:
+
+- **Service Availability**: the Postgres *service* itself stays reachable
+  and able to accept reads and writes, with as little downtime as
+  possible when a node is lost.
+- **Disaster Recovery**: the *data* survives even in scenarios Service
+  Availability alone can't help with -- an operator mistake, a bad
+  deployment, or every node that ever held the data being lost at once.
+
+Most Postgres setups reach for two separate, independently-operated
+products for these -- an HA tool for the first, a backup tool for the
+second. pg_auto_failover treats them as one system instead; see
+:ref:`ha_dr_backups` for the full comparison.
+
+Service Availability (failover)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This is what the rest of this page, and :ref:`failover_state_machine` /
+:ref:`fault_tolerance` in detail, describe: a primary and one or more
+secondary nodes, a Monitor orchestrating automated failover when the
+primary is lost, and synchronous replication (`Synchronous vs.
+asynchronous replication`_ below) guaranteeing no committed write is lost
+in the process. This is the guarantee that answers "the primary just
+died -- who serves the next query?".
+
+Service Availability can be setup to obtain Business Continuity in the face
+of production incidents with a basic setup of two Postgres nodes, and
+Postgres High Availability starting with a setup of three Postgres nodes.
+
+Given integrated archiving support, a trade-off or *budget* architecture can
+be easily deployed with two Postgres nodes and an archiver to obtain an HA
+setup that complies with many production needs.
+
+Disaster Recovery
+^^^^^^^^^^^^^^^^^^
+
+Service Availability only makes sense for a database system when there is a
+compliant setup for durability, or data safety. When using PostgreSQL, that
+means a proper archiving implementation that allows *Point in Time Recovery*
+operations.
+
+With PITR it's possible to mitigate data loss operations such as a missing
+WHERE clause in a DELETE or a DROP TABLE done in production instead of the
+development environment, also known as human errors.
+
+Disaster Recovery is handled by a physically distinct kind of node, the
+**archiver**, added on top of any of the architectures on this page: it
+continuously captures WAL from the group's current primary and periodically
+produces base backups.
+
+See :ref:`archiving_and_disaster_recovery` for where an archiver fits
+alongside the architectures below, :ref:`archiving_architecture` for exactly
+how WAL capture and base-backup generation work, and
+:ref:`archiving_fault_tolerance` for how this changes what a total loss of
+the rest of the formation actually means.
+
 The pg_auto_failover Monitor
 ----------------------------
 
@@ -152,6 +215,25 @@ As a result, refrain from naming your nodes with the role you intend for them.
 Their roles can change. If they didn't, your system wouldn't need
 pg_auto_failover!
 
+Archiver
+^^^^^^^^
+
+An archiver is a server (virtual or physical) that runs PostgreSQL archiving
+storage for one or many formations. The archiver hosts any number of
+*archiving nodes* and schedules *base backups* in order to be able to
+implement Postgres `Point in Time Recovery`__ which is the foundations for
+implementing Disaster Recovery.
+
+__ https://www.postgresql.org/docs/current/continuous-archiving.html
+
+Archiving Node
+^^^^^^^^^^^^^^
+
+A process managed in an archiver instance that reports to the monitor as a
+node in a group and that runs ``pg_receivewal``. An archiving node as no
+PGDATA, it can participate in the replication quorum but can not be a
+failover candidate: its ``candidate_priority`` is always zero.
+
 State
 ^^^^^