Switch /etc to use confext in mutable mode

build_library/build_image_util.sh

@@ -748,8 +748,30 @@ EOF
   if [[ $(sudo find "${root_fs_dir}/usr/share/flatcar/etc" -size +0 ! -type d 2>/dev/null | wc -l) -gt 0 ]]; then
     die "Unexpected non-empty files in ${root_fs_dir}/usr/share/flatcar/etc"
   fi
+  # Some backwards-compat symlinks still use this folder as target,
+  # we can't remove it yet
   sudo rm -rf "${root_fs_dir}/usr/share/flatcar/etc"
   sudo cp -a "${root_fs_dir}/etc" "${root_fs_dir}/usr/share/flatcar/etc"
+  # Now set up a default confext and enable it.
+  # It's important to use dm-verity not only for stricter image policies
+  # but also because it allows us the refresh to identify this image and
+  # skip setting it up again in the final boot, which not only saves us
+  # a daemon-reload during boot but also from /etc contents shortly
+  # disappearing until systemd-sysext uses mount beneath for an atomic
+  # remount. Instead of a temporary directory we first prepare it as
+  # folder and then convert it to a DDI and remove the folder.
+  sudo rm -rf "${root_fs_dir}/usr/lib/confexts/00-flatcar-default"
+  sudo mkdir -p "${root_fs_dir}/usr/lib/confexts/00-flatcar-default"
+  sudo cp -a "${root_fs_dir}/etc" "${root_fs_dir}/usr/lib/confexts/00-flatcar-default/etc"
+  sudo mkdir -p "${root_fs_dir}/usr/lib/confexts/00-flatcar-default/etc/extension-release.d/"
+  echo ID=_any | sudo tee "${root_fs_dir}/usr/lib/confexts/00-flatcar-default/etc/extension-release.d/extension-release.00-flatcar-default" > /dev/null
+  sudo systemd-repart \
+    --private-key="${SYSEXT_SIGNING_KEY_DIR}/sysexts.key" \
+    --certificate="${SYSEXT_SIGNING_KEY_DIR}/sysexts.crt" \
+    --make-ddi=confext \
+    --copy-source="${root_fs_dir}/usr/lib/confexts/00-flatcar-default" \
+    "${root_fs_dir}/usr/lib/confexts/00-flatcar-default.raw"
+  sudo rm -rf "${root_fs_dir}/usr/lib/confexts/00-flatcar-default"
 
   # Remove the rootfs state as it should be recreated through the
   # tmpfiles and may not be present on updating machines. This

build_library/prod_image_util.sh

@@ -170,6 +170,10 @@ EOF
   # Remove source locale data, only need to ship the compiled archive.
   sudo rm -rf ${root_fs_dir}/usr/share/i18n/
 
+  # Inject ephemeral sysext signing certificate
+  sudo mkdir -p "${root_fs_dir}/usr/lib/verity.d"
+  sudo cp "${SYSEXT_SIGNING_KEY_DIR}/sysexts.crt" "${root_fs_dir}/usr/lib/verity.d"
+
   # Finish image will move files from /etc to /usr/share/flatcar/etc.
   # Note that image filesystem contents generated by finish_image will not
   # include sysext contents (only the sysext squashfs files themselves).

build_library/sysext_prod_builder

@@ -63,11 +63,15 @@ create_prod_sysext() {
   # The --install_root_basename="${name}-base-sysext-rootfs" flag is
   # important - it sets the name of a rootfs directory, which is used
   # to determine the package target in coreos/base/profile.bashrc
-  sudo "FLATCAR_BUILD_ID=$FLATCAR_BUILD_ID" "${SCRIPTS_DIR}/build_sysext" \
+  #
+  # Built-in sysexts are stored in the compressed /usr partition, so we
+  # disable compression to avoid double-compression.
+  sudo -E "FLATCAR_BUILD_ID=$FLATCAR_BUILD_ID" "${SCRIPTS_DIR}/build_sysext" \
             --board="${BOARD}" \
             --image_builddir="${workdir}/sysext-build" \
             --squashfs_base="${base_sysext}" \
             --generate_pkginfo \
+            --compression=none \
             --install_root_basename="${name}-base-sysext-rootfs" \
             "${build_sysext_opts[@]}" \
             "${name}" "${grp_pkg[@]}"
@@ -99,6 +103,14 @@ sysext_mountdir="${BUILD_DIR}/prod-sysext-work/mounts"
 sysext_base="${sysext_workdir}/base-os.squashfs"
 
 function cleanup() {
+  IFS=':' read -r -a mounted_sysexts <<< "$sysext_lowerdirs"
+  # skip the rootfs
+  mounted_sysexts=("${mounted_sysexts[@]:1}")
+
+  for sysext in "${mounted_sysexts[@]}"; do
+    sudo systemd-dissect --umount --rmdir "$sysext"
+  done
+
   sudo umount "${sysext_mountdir}"/* || true
   rm -rf "${sysext_workdir}" || true
 }
@@ -116,6 +128,7 @@ sudo mksquashfs "${root_fs_dir}" "${sysext_base}" -noappend -xattrs-exclude '^bt
 # for combined overlay later.
 prev_pkginfo=""
 sysext_lowerdirs="${sysext_mountdir}/rootfs-lower"
+mkdir -p "${sysext_mountdir}"
 for sysext in ${sysexts_list//,/ }; do
   # format is "<name>:<group>/<package>"
   name="${sysext%|*}"
@@ -129,12 +142,21 @@ for sysext in ${sysexts_list//,/ }; do
                      "${grp_pkg}" \
                      "${prev_pkginfo}"
 
-  mkdir -p "${sysext_mountdir}/${name}" \
-           "${sysext_mountdir}/${name}_pkginfo"
-  sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}.raw" \
-                                    "${sysext_mountdir}/${name}"
-  sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}_pkginfo.raw" \
-                                    "${sysext_mountdir}/${name}_pkginfo"
+  sudo systemd-dissect \
+    --read-only \
+    --mount \
+    --mkdir \
+    --image-policy='root=encrypted+unprotected+absent:usr=encrypted+unprotected+absent' \
+    "${sysext_output_dir}/${name}.raw" \
+    "${sysext_mountdir}/${name}"
+
+  sudo systemd-dissect \
+    --read-only \
+    --mount \
+    --mkdir \
+    --image-policy='root=encrypted+unprotected+absent:usr=encrypted+unprotected+absent' \
+    "${sysext_output_dir}/${name}_pkginfo.raw" \
+    "${sysext_mountdir}/${name}_pkginfo"
 
   sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}"
   sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}_pkginfo"

build_library/vm_image_util.sh

@@ -585,11 +585,15 @@ install_oem_sysext() {
     # important - it sets the name of a rootfs directory, which is
     # used to determine the package target in
     # coreos/base/profile.bashrc
+    #
+    # OEM sysexts are stored in the compressed partition, so we disable
+    # compression to avoid double-compression.
     local build_sysext_flags=(
         --board="${BOARD}"
         --squashfs_base="${VM_SRC_SYSEXT_IMG}"
         --image_builddir="${built_sysext_dir}"
         --metapkgs="${metapkg}"
+        --compression=none
         --install_root_basename="${VM_IMG_TYPE}-oem-sysext-rootfs"
     )
     local overlay_path mangle_fs
@@ -602,7 +606,7 @@ install_oem_sysext() {
     fi
 
     mkdir -p "${built_sysext_dir}"
-    sudo "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}"
+    sudo -E "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}"
 
     local installed_sysext_oem_dir='/oem/sysext'
     local installed_sysext_file_prefix="${oem_sysext}-${version}"

build_sysext

@@ -35,10 +35,10 @@ DEFINE_boolean generate_pkginfo "${FLAGS_FALSE}" \
   "Generate an additional squashfs '<sysext_name>_pkginfo.raw' with portage package meta-information (/var/db ...). Useful for creating sysext dependencies; see 'base_pkginfo' below."
 DEFINE_string base_pkginfo "" \
   "Colon-separated list of pkginfo squashfs paths / files generated via 'generate_pkginfo' to base this sysext on. The corresponding base sysexts are expected to be merged with the sysext generated."
-DEFINE_string compression "zstd" \
-  "Compression to use for sysext squashfs. One of 'gzip', 'lzo', 'lz4', 'xz', or 'zstd'. Must be supported by the Flatcar squashfs kernel module in order for the sysext to work."
-DEFINE_string mksquashfs_opts "" \
-  "Additional command line options to pass to mksquashfs. See 'man 1 mksquashfs'. If <compression> is 'zstd' (the default), this option defaults to '-Xcompression-level 22 -b 512K'. Otherwise the default is empty."
+DEFINE_string compression "lz4hc" \
+  "Compression to use for sysext EROFS image. Options: 'lz4', 'lz4hc', 'zstd', or 'none'. Default is 'lz4hc'."
+DEFINE_string mkerofs_opts "" \
+  "Additional mkfs.erofs options to pass via SYSTEMD_REPART_MKFS_OPTIONS_EROFS. If not specified, defaults are used based on compression type."
 DEFINE_boolean ignore_version_mismatch "${FLAGS_FALSE}" \
   "Ignore version mismatch between SDK board packages and base squashfs. DANGEROUS."
 DEFINE_string install_root_basename "${default_install_root_basename}" \
@@ -112,10 +112,6 @@ fi
 BUILD_DIR=$(realpath "${FLAGS_image_builddir}")
 mkdir -p "${BUILD_DIR}"
 
-if [[ "${FLAGS_compression}" = "zstd" && -z "${FLAGS_mksquashfs_opts}" ]] ; then
-    FLAGS_mksquashfs_opts="-Xcompression-level 22 -b 512k"
-fi
-
 source "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
 source "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
 source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1
@@ -248,7 +244,7 @@ if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then
   mkdir -p "${BUILD_DIR}/img-pkginfo/var/db"
   cp -R "${BUILD_DIR}/${FLAGS_install_root_basename}/var/db/pkg" "${BUILD_DIR}/img-pkginfo/var/db/"
   mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" \
-              -noappend -xattrs-exclude '^btrfs.' -comp "${FLAGS_compression}" ${FLAGS_mksquashfs_opts}
+              -noappend -xattrs-exclude '^btrfs.' -comp zstd -Xcompression-level 22 -b 512k
 fi
 
 info "Writing ${SYSEXTNAME}_packages.txt"
@@ -292,6 +288,7 @@ all_fields=(
   'ID=flatcar'
   "${version_field}"
   "ARCHITECTURE=${ARCH}"
+  "EXTENSION_RELOAD_MANAGER=1"
 )
 printf '%s\n' "${all_fields[@]}" >"${BUILD_DIR}/${FLAGS_install_root_basename}/usr/lib/extension-release.d/extension-release.${SYSEXTNAME}"
 
@@ -304,14 +301,44 @@ if [[ -n "${invalid_files}" ]]; then
   die "Invalid file ownership: ${invalid_files}"
 fi
 
-mksquashfs "${BUILD_DIR}/${FLAGS_install_root_basename}" "${BUILD_DIR}/${SYSEXTNAME}.raw" \
-               -noappend -xattrs-exclude '^btrfs.' -comp "${FLAGS_compression}" ${FLAGS_mksquashfs_opts}
+# Set up EROFS compression options based on compression type
+if [[ "${FLAGS_compression}" != "none" ]]; then
+  export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="-z${FLAGS_compression}"
+
+  if [[ -n "${FLAGS_mkerofs_opts}" ]]; then
+    # User provided custom options
+    export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="${SYSTEMD_REPART_MKFS_OPTIONS_EROFS} ${FLAGS_mkerofs_opts}"
+  elif [[ "${FLAGS_compression}" = "lz4hc" ]]; then
+    # Default options for lz4hc
+    export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="${SYSTEMD_REPART_MKFS_OPTIONS_EROFS},12 -C65536 -Efragments,ztailpacking"
+  elif [[ "${FLAGS_compression}" = "zstd" ]]; then
+    # Default options for zstd
+    export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="${SYSTEMD_REPART_MKFS_OPTIONS_EROFS},level=22 -C524288 -Efragments,ztailpacking"
+  fi
+  info "Building sysext with ${FLAGS_compression} compression"
+else
+  info "Building sysext without compression (built-in sysexts)"
+fi
+
+systemd-repart \
+  --private-key="${SYSEXT_SIGNING_KEY_DIR}/sysexts.key" \
+  --certificate="${SYSEXT_SIGNING_KEY_DIR}/sysexts.crt" \
+  --make-ddi=sysext \
+  --copy-source="${BUILD_DIR}/${FLAGS_install_root_basename}" \
+  "${BUILD_DIR}/${SYSEXTNAME}.raw"
+
 rm -rf "${BUILD_DIR}"/{fs-root,"${FLAGS_install_root_basename}",workdir}
 
 # Generate reports
 mkdir "${BUILD_DIR}/img-rootfs"
-mount -rt squashfs -o loop,nodev "${BUILD_DIR}/${SYSEXTNAME}.raw" "${BUILD_DIR}/img-rootfs"
+systemd-dissect --read-only \
+  --mount \
+  --mkdir \
+  --image-policy='root=encrypted+unprotected+absent:usr=encrypted+unprotected+absent' \
+  "${BUILD_DIR}/${SYSEXTNAME}.raw" \
+  "${BUILD_DIR}/img-rootfs"
+
 write_contents "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents.txt"
 write_contents_with_technical_details "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents_wtd.txt"
 write_disk_space_usage_in_paths "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_disk_usage.txt"
-umount "${BUILD_DIR}/img-rootfs"
+systemd-dissect --umount --rmdir "${BUILD_DIR}/img-rootfs"

changelog/changes/2025-11-05-signed-os-dependent-sysexts.md

@@ -0,0 +1 @@
+- OS-dependent sysexts (e.g., docker-flatcar, containerd-flatcar) are now cryptographically signed using dm-verity roothash signatures. This enables stricter sysext policies via systemd-sysext and provides a foundation for verifying user-provided extensions in future releases. The format changed from squashfs to erofs-based Discoverable Disk Images (DDI). ([scripts#3162](https://github.com/flatcar/scripts/pull/3162))

changelog/changes/2025-12-12-default-systemd-confext.md

@@ -0,0 +1,2 @@
+- Switched `/etc/` from a custom overlayfs for A/B updates to using a systemd-confext extension providing the default contents by using systemd-confext in the mutable mode where `/etc/` gets used as upperdir [scripts#3555](https://github.com/flatcar/scripts/pull/3555)
+- Moved systemd-sysext image mounting into the initrd, so that system extensions can better define the behavior of the final system at boot without workarounds to apply settings late at boot. This means `.wants` symlinks for systemd units work as expected now and, therefore, we dropped the `ensure-sysext.service` workaround. We still recommend extensions to keep their workarounds, e.g., using `.upholds` instead of `.wants`, to better support live reloading. A skipping logic prevents an extension refresh late at boot but only if no changes were found. For extensions that are not stored on a custom filesystem, such as a separate `/var` partition, the new extension mounting from the initrd won't be able to load them early but they will be picked up late at boot through the extension refresh. This is another case where it's good if extensions keep workarounds for late loading.

changelog/updates/2025-12-08-update-systemd.md

@@ -0,0 +1 @@
+- systemd (258.2)

sdk_container/.repo/manifests/mantle-container

@@ -1 +1 @@
-ghcr.io/flatcar/mantle:git-bed79eb716792cbd6f79301f515bafcdb59ee93d
+ghcr.io/flatcar/mantle:pr-720

sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild

@@ -8,7 +8,8 @@ EGIT_REPO_URI="https://github.com/flatcar/init.git"
 if [[ "${PV}" == 9999 ]]; then
 	KEYWORDS="~amd64 ~arm ~arm64 ~x86"
 else
-	EGIT_COMMIT="8bd8a82fb22bc46ea2cf7da94d58655e102ca26d" # flatcar-master
+	#EGIT_COMMIT="8bd8a82fb22bc46ea2cf7da94d58655e102ca26d" # flatcar-master
+	EGIT_BRANCH="kai/default-confext"
 	KEYWORDS="amd64 arm arm64 x86"
 fi
 

sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-apps/systemd

@@ -160,19 +160,6 @@ EOF
         -e '/^C!* \/etc\/pam\.d/d' \
         -e '/^C!* \/etc\/issue/d' || die
 
-    (
-        # Some OEMs prefer chronyd, so allow them to replace
-        # systemd-timesyncd with it.
-        insinto "$(systemd_get_systemunitdir)/systemd-timesyncd.service.d"
-        newins - flatcar.conf <<'EOF'
-# Allow sysexts to ship timesyncd replacements which can have
-# a Conflicts=systemd-timesyncd directive that would result
-# in systemd-timesyncd not being started.
-[Unit]
-After=ensure-sysext.service
-EOF
-    )
-
     (
         # Allow @mount syscalls for systemd-udevd.service
         insinto "$(systemd_get_systemunitdir)/systemd-udevd.service.d"

sdk_container/src/third_party/coreos-overlay/coreos/user-patches/sys-apps/systemd/0001-vpick-Don-t-use-openat-directly-but-resolve-symlinks.patch

@@ -0,0 +1,33 @@
+From 6f4b065b626edd8a06ff0c8028173e060b5e444b Mon Sep 17 00:00:00 2001
+From: Kai Lueke <[email protected]>
+Date: Thu, 20 Nov 2025 23:43:55 +0900
+Subject: [PATCH 03/10] vpick: Don't use openat directly but resolve symlinks
+ in given root
+
+With systemd-sysext --root= all symlinks should be followed relative to
+the given root and direct openat usage doesn't work.
+Change the openat call to use the chase helper function to resolve the
+symlink in the given root.
+---
+ src/shared/vpick.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/shared/vpick.c b/src/shared/vpick.c
+index b1b2d93054..dfe58cafa5 100644
+--- a/src/shared/vpick.c
++++ b/src/shared/vpick.c
+@@ -471,9 +471,9 @@ static int make_choice(
+         if (!p)
+                 return log_oom_debug();
+
+-        object_fd = openat(dir_fd, best_filename, O_CLOEXEC|O_PATH);
++        object_fd = chase_and_openat(toplevel_fd, p, CHASE_AT_RESOLVE_IN_ROOT, O_PATH|O_CLOEXEC, NULL);
+         if (object_fd < 0)
+-                return log_debug_errno(errno, "Failed to open '%s/%s': %m",
++                return log_debug_errno(object_fd, "Failed to open '%s/%s': %m",
+                                        empty_to_root(toplevel_path), skip_leading_slash(inode_path));
+
+         return pin_choice(
+-- 
+2.52.0
+

sdk_container/src/third_party/coreos-overlay/coreos/user-patches/sys-apps/systemd/0001-wait-online-set-any-by-default.patch

@@ -1,7 +1,7 @@
-From 83043596b6cc74b6f049999fa660afd983dc493a Mon Sep 17 00:00:00 2001
+From 61ae07bbf1d7032eef32137b1fe299647602e3de Mon Sep 17 00:00:00 2001
 From: David Michael <[email protected]>
 Date: Tue, 16 Apr 2019 02:44:51 +0000
-Subject: [PATCH 1/8] wait-online: set --any by default
+Subject: [PATCH] wait-online: set --any by default
 
 The systemd-networkd-wait-online command would normally continue
 waiting after a network interface is usable if other interfaces are
@@ -11,22 +11,22 @@ Preserve previous Container Linux behavior for compatibility by
 setting the --any flag by default.  See patches from v241 (or
 earlier) for the original implementation.
 ---
- src/network/wait-online/wait-online.c | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
+ src/network/wait-online/wait-online.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
 
 diff --git a/src/network/wait-online/wait-online.c b/src/network/wait-online/wait-online.c
-index 6f5aef903a..0acb3e76b9 100644
+index b1d0b9cde2..e07c11d807 100644
 --- a/src/network/wait-online/wait-online.c
 +++ b/src/network/wait-online/wait-online.c
-@@ -21,7 +21,7 @@ static Hashmap *arg_interfaces = NULL;
+@@ -24,7 +24,7 @@ static Hashmap *arg_interfaces = NULL;
  static char **arg_ignore = NULL;
  static LinkOperationalStateRange arg_required_operstate = LINK_OPERSTATE_RANGE_INVALID;
  static AddressFamily arg_required_family = ADDRESS_FAMILY_NO;
 -static bool arg_any = false;
 +static bool arg_any = true;
+ static bool arg_requires_dns = false;
 
- STATIC_DESTRUCTOR_REGISTER(arg_interfaces, hashmap_free_free_freep);
- STATIC_DESTRUCTOR_REGISTER(arg_ignore, strv_freep);
+ STATIC_DESTRUCTOR_REGISTER(arg_interfaces, hashmap_freep);
 -- 
 2.51.0