From d78cc8f8f4ba3ac66fcaef8416034dea8c115f87 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Fri, 5 Jul 2024 14:19:11 +0100 Subject: [PATCH 1/7] Stop trying to create lib->lib64 symlink We stopped using profiles with a lib->lib64 symlink a while ago, so there is no point in checking for this any more. We weren't checking against the target SDK architecture anyway. Signed-off-by: James Le Cuirot --- bootstrap_sdk | 11 +---------- build_library/set_lsb_release | 1 + build_library/toolchain_util.sh | 4 ---- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/bootstrap_sdk b/bootstrap_sdk index 834506ee6d2..0c636beda2b 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -65,16 +65,7 @@ ROOT_OVERLAY=${TEMPDIR}/stage4_overlay if [[ "$STAGES" =~ stage4 ]]; then info "Setting release to ${FLATCAR_VERSION}" rm -rf "${ROOT_OVERLAY}" - # need to setup the lib->lib64 symlink correctly - libdir=$(get_sdk_libdir) - mkdir -p "${ROOT_OVERLAY}/usr/${libdir}" - if [[ "${libdir}" != lib ]]; then - if [[ "$(get_sdk_symlink_lib)" == "yes" ]]; then - ln -s "${libdir}" "${ROOT_OVERLAY}/usr/lib" - else - mkdir -p "${ROOT_OVERLAY}/usr/lib" - fi - fi + mkdir -p "${ROOT_OVERLAY}" "${BUILD_LIBRARY_DIR}/set_lsb_release" \ --root "${ROOT_OVERLAY}" fi diff --git a/build_library/set_lsb_release b/build_library/set_lsb_release index f86c968909f..4681cb5eda6 100755 --- a/build_library/set_lsb_release +++ b/build_library/set_lsb_release @@ -45,6 +45,7 @@ sudo ln -sf "../usr/share/flatcar/lsb-release" "${ROOT_FS_DIR}/etc/lsb-release" # And the new standard, os-release # https://www.freedesktop.org/software/systemd/man/os-release.html +sudo mkdir -p "${ROOT_FS_DIR}/usr/lib" sudo_clobber "${ROOT_FS_DIR}/usr/lib/os-release" < Date: Fri, 5 Jul 2024 15:05:53 +0100 Subject: [PATCH 2/7] Stop creating /usr/lib64/os-release compatibility symlink We currently put an os-release symlink in lib64, but we shouldn't assume that the architecture will even have a lib64 directory. I doubt this compatibility symlink was needed anyway. Gentoo doesn't have one, and applications are supposed to check /etc/os-release. I can find almost no reference to /usr/lib64/os-release anywhere, let alone in Flatcar. Signed-off-by: James Le Cuirot --- build_library/set_lsb_release | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build_library/set_lsb_release b/build_library/set_lsb_release index 4681cb5eda6..514bba7c2de 100755 --- a/build_library/set_lsb_release +++ b/build_library/set_lsb_release @@ -63,10 +63,6 @@ CPE_NAME="cpe:2.3:o:${OS_ID}-linux:${OS_ID}_linux:${FLATCAR_VERSION}:*:*:*:*:*:* EOF sudo ln -sf "../usr/lib/os-release" "${ROOT_FS_DIR}/etc/os-release" sudo ln -sf "../../lib/os-release" "${ROOT_FS_DIR}/usr/share/flatcar/os-release" -# Compat for split of lib64 into lib and lib64 -if [ ! -e "${ROOT_FS_DIR}/usr/lib64/os-release" ]; then - sudo ln -sf "../lib/os-release" "${ROOT_FS_DIR}/usr/lib64/os-release" -fi # Create the defaults for the coreos configuration files in the usr directory sudo_clobber "${ROOT_FS_DIR}/usr/share/flatcar/release" < Date: Mon, 15 Jul 2024 12:03:01 +0100 Subject: [PATCH 3/7] Support the SDK on arm64 Catalyst 4 adds support for building with QEMU, so I initially leveraged this, but it turned out to be very much slower than emulating the amd64 SDK on arm64, where an arm64 build could then be mostly run without emulation. I have kept the code for the slower approach anyway since it is small and may be useful to somebody. There were several places where we assumed that amd64 was native and arm64 required emulation via QEMU. The scripts are now more architecture-agnostic, paving the way for riscv support later. We no longer set QEMU_LD_PREFIX because it prevents the SDK itself from being emulated. It also assumes there is only one non-native target, which may not always be the case. bubblewrap does a better job of running binaries under QEMU. Signed-off-by: James Le Cuirot --- build_library/build_image_util.sh | 29 +++----------- build_library/catalyst.sh | 15 ++++++++ build_library/portage/env/releng/qemu | 1 + build_library/portage/package.env/qemu | 1 + build_library/prod_image_util.sh | 1 + common.sh | 38 ------------------- .../coreos/config/env/dev-lang/rust | 3 +- .../profiles/coreos/amd64/sdk/package.use | 3 ++ .../profiles/coreos/arm64/sdk/package.use | 3 ++ .../profiles/coreos/targets/sdk/make.defaults | 6 --- .../profiles/coreos/targets/sdk/package.use | 2 +- setup_board | 26 ++++++++++--- 12 files changed, 52 insertions(+), 76 deletions(-) create mode 100644 build_library/portage/env/releng/qemu create mode 100644 build_library/portage/package.env/qemu create mode 100644 sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/package.use diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 221095d45cd..4ff46dcb03c 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -102,32 +102,13 @@ zip_update_tools() { --arch "$(get_sdk_arch)" --output-dir "${BUILD_DIR}" --zip-name "${update_zip}" } -# ldconfig cannot generate caches for non-native arches. -# Use qemu & the native ldconfig to work around that. -# http://code.google.com/p/chromium/issues/detail?id=378377 run_ldconfig() { - local root_fs_dir=$1 - case ${ARCH} in - arm64) - sudo qemu-aarch64 "${root_fs_dir}"/usr/sbin/ldconfig -r "${root_fs_dir}";; - x86|amd64) - sudo ldconfig -r "${root_fs_dir}";; - *) - die "Unable to run ldconfig for ARCH ${ARCH}" - esac + # This wrapper is created by setup_board. + sudo "ldconfig-${BOARD}" -r "$1" } run_localedef() { - local root_fs_dir="$1" loader=() - case ${ARCH} in - arm64) - loader=( qemu-aarch64 -L "${root_fs_dir}" );; - amd64) - loader=( "${root_fs_dir}/usr/lib64/ld-linux-x86-64.so.2" \ - --library-path "${root_fs_dir}/usr/lib64" );; - *) - die "Unable to run localedef for ARCH ${ARCH}";; - esac + local root_fs_dir="$1" info "Generating C.UTF-8 locale..." local i18n="${root_fs_dir}/usr/share/i18n" # localedef will silently fall back to /usr/share/i18n if missing so @@ -135,8 +116,8 @@ run_localedef() { [[ -f "${i18n}/charmaps/UTF-8.gz" ]] || die [[ -f "${i18n}/locales/C" ]] || die sudo mkdir -p "${root_fs_dir}/usr/lib/locale" - sudo I18NPATH="${i18n}" "${loader[@]}" "${root_fs_dir}/usr/bin/localedef" \ - --prefix="${root_fs_dir}" --charmap=UTF-8 --inputfile=C C.UTF-8 + sudo I18NPATH="${i18n}" "bwrap-${BOARD}" "${root_fs_dir}" /usr/bin/localedef \ + --charmap=UTF-8 --inputfile=C C.UTF-8 } # Basic command to emerge binary packages into the target image. diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index 3e792417bed..c4055cee63a 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -25,6 +25,7 @@ BINPKGS= DISTDIR= TEMPDIR= STAGES= +unset QEMU DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \ "Path to directory for all catalyst images and other files." @@ -97,6 +98,7 @@ cflags: -O2 -pipe cxxflags: -O2 -pipe ldflags: -Wl,-O2 -Wl,--as-needed source_subpath: ${SEED} +${QEMU+interpreter: $(type -P "${QEMU}")} EOF } @@ -207,6 +209,16 @@ catalyst_init() { SEED="seed/${FLAGS_seed_tarball##*/}" SEED="${SEED%.tar.*}" fi + + # Emulate the build, if needed. Note the SDK itself may already be emulated, + # so check the requested arch against the kernel's real arch, not uname -m. + if [[ ${ARCH} != $(get_portage_arch "$(< /proc/sys/kernel/arch)") ]]; then + case "${ARCH}" in + amd64) QEMU=qemu-x86_64 ;; + arm64) QEMU=qemu-aarch64 ;; + riscv) QEMU=qemu-riscv64 ;; + esac + fi } write_configs() { @@ -226,6 +238,9 @@ write_configs() { ln -sfT '/mnt/host/source/src/third_party/coreos-overlay/coreos/user-patches' \ "${TEMPDIR}"/portage/patches + + [[ -n ${QEMU} ]] || + rm "${TEMPDIR}"/portage/package.env/qemu } build_stage() { diff --git a/build_library/portage/env/releng/qemu b/build_library/portage/env/releng/qemu new file mode 100644 index 00000000000..de86517db4d --- /dev/null +++ b/build_library/portage/env/releng/qemu @@ -0,0 +1 @@ +FEATURES="-pid-sandbox -network-sandbox -ipc-sandbox" diff --git a/build_library/portage/package.env/qemu b/build_library/portage/package.env/qemu new file mode 100644 index 00000000000..60c290a8ba7 --- /dev/null +++ b/build_library/portage/package.env/qemu @@ -0,0 +1 @@ +*/* releng/qemu diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index b6d760fba2a..d45936f06da 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -139,6 +139,7 @@ create_prod_image() { sudo rm -rf "${BUILD_DIR}/root_fs_dir2" # clean-ups of things we do not need + sudo find ${root_fs_dir}/usr/bin -empty -delete # Bind mounts created by bwrap sudo rm ${root_fs_dir}/etc/csh.env sudo rm -rf ${root_fs_dir}/etc/env.d sudo rm -rf ${root_fs_dir}/usr/include diff --git a/common.sh b/common.sh index 3dbb8040dc3..0cff99ed563 100644 --- a/common.sh +++ b/common.sh @@ -51,9 +51,6 @@ fi # Turn on bash debug support if available for backtraces. shopt -s extdebug 2>/dev/null -# Source qemu library path -. /etc/profile.d/qemu-aarch64.sh 2> /dev/null || true - # Output a backtrace all the way back to the raw invocation, suppressing # only the _dump_trace frame itself. _dump_trace() { @@ -992,38 +989,3 @@ BOAT echo -e "${V_VIDOFF}" die "$* failed" } - -# The binfmt_misc support in the kernel is required. -# The aarch64 binaries should be executed through -# "/usr/bin/qemu-aarch64-static" -setup_qemu_static() { - local root_fs_dir="$1" - case "${BOARD}" in - amd64-usr) return 0;; - arm64-usr) - if [[ -f "${root_fs_dir}/sbin/ldconfig" ]]; then - sudo cp /usr/bin/qemu-aarch64 "${root_fs_dir}"/usr/bin/qemu-aarch64-static - echo export QEMU_LD_PREFIX=\"/build/arm64-usr/\" | sudo tee /etc/profile.d/qemu-aarch64.sh - . /etc/profile.d/qemu-aarch64.sh - else - die "Missing basic layout in target rootfs" - fi - ;; - *) die "Unsupported arch" ;; - esac -} - -clean_qemu_static() { - local root_fs_dir="$1" - case "${BOARD}" in - amd64-usr) return 0;; - arm64-usr) - if [[ -f "${root_fs_dir}/usr/bin/qemu-aarch64-static" ]]; then - sudo rm "${root_fs_dir}"/usr/bin/qemu-aarch64-static - else - die "File not found" - fi - ;; - *) die "Unsupported arch" ;; - esac -} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-lang/rust b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-lang/rust index c467aedeada..d72efdb3d5e 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-lang/rust +++ b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-lang/rust @@ -2,5 +2,6 @@ INSTALL_MASK+=" *rustdoc*" I_KNOW_WHAT_I_AM_DOING_CROSS=1 RUST_CROSS_TARGETS=( - $(aarch64-cros-linux-gnu-gcc --version >/dev/null && echo "AArch64:aarch64-unknown-linux-gnu:aarch64-cros-linux-gnu") + $(use arm64 || { aarch64-cros-linux-gnu-gcc --version &>/dev/null && echo "AArch64:aarch64-unknown-linux-gnu:aarch64-cros-linux-gnu"; }) + $(use amd64 || { x86_64-cros-linux-gnu-gcc --version &>/dev/null && echo "X86:x86_64-unknown-linux-gnu:x86_64-cros-linux-gnu" ; }) ) diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/amd64/sdk/package.use b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/amd64/sdk/package.use index e69de29bb2d..21b5b3ee045 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/amd64/sdk/package.use +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/amd64/sdk/package.use @@ -0,0 +1,3 @@ +# Don't build the user space emulator for this arch. It's not needed and gets in +# the way when using Catalyst with QEMU. +app-emulation/qemu -qemu_user_targets_x86_64 diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/package.use b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/package.use new file mode 100644 index 00000000000..c3e3f1eaed4 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/package.use @@ -0,0 +1,3 @@ +# Don't build the user space emulator for this arch. It's not needed and gets in +# the way when using Catalyst with QEMU. +app-emulation/qemu -qemu_user_targets_aarch64 diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/make.defaults b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/make.defaults index 3fb2f4e9a72..5a6fa0111d9 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/make.defaults +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/make.defaults @@ -5,12 +5,6 @@ USE="cros_host expat man -pam" # Enable CPU architectures needed by Rust builds LLVM_TARGETS="X86 AArch64" -# Both x86_64 and i386 targets are required for grub testing -QEMU_SOFTMMU_TARGETS="x86_64 i386 aarch64" - -# For cross build support. -QEMU_USER_TARGETS="aarch64" - # add cros_host to bootstrapping USE flags so SDK / toolchains bootstrapping # will use vim's vimrc instead of baselayouts', BOOTSTRAP_USE="$BOOTSTRAP_USE cros_host" diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use index 641b433bda1..2108e23b8b9 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use @@ -12,7 +12,7 @@ app-crypt/gnupg smartcard usb # for qemu app-arch/bzip2 static-libs -app-emulation/qemu -doc -jpeg ncurses python static-user virtfs qemu_softmmu_targets_x86_64 qemu_softmmu_targets_aarch64 +app-emulation/qemu -doc -jpeg ncurses python static-user virtfs qemu_softmmu_targets_aarch64 qemu_softmmu_targets_x86_64 qemu_user_targets_aarch64 qemu_user_targets_x86_64 dev-libs/glib static-libs dev-libs/libaio static-libs dev-libs/libpcre2 static-libs diff --git a/setup_board b/setup_board index c1acdc1f25d..0b97cc7c5b0 100755 --- a/setup_board +++ b/setup_board @@ -92,6 +92,13 @@ generate_all_wrappers() { # the board arch matches the SDK arch and therefore emulation is unnecessary. qemu=$(type -P "qemu-${BOARD_CHOST%%-*}") || unset qemu + # If emulation is necessary, then we need to create a placeholder to bind + # mount QEMU onto now. This avoids needing root to do it later. + if [[ -n ${qemu-} ]]; then + sudo mkdir -p "${BOARD_ROOT}${qemu%/*}" + sudo touch "${BOARD_ROOT}${qemu}" + fi + info "Generating wrapper scripts" for wrapper in emerge ebuild eclean equery portageq \ @@ -113,8 +120,20 @@ exec ${BOARD_CHOST}-gdb -iex 'set sysroot ${BOARD_ROOT}' "\$@" EOF wrappers+=( "${wrapper}" ) + # A general purpose wrapper for effectively chrooting using bubblewrap, + # together with emulation by QEMU if necessary. + wrapper="/usr/local/bin/bwrap-${BOARD_VARIANT}" + sudo_clobber "${wrapper}" < "$GCLIENT_ROOT/src/scripts/.default_board" fi From 734e2b7fb17668d5a89715a655946429d73fda51 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Thu, 22 May 2025 15:20:28 +0100 Subject: [PATCH 4/7] profiles: We don't use wrappers in the board sysroot anymore Signed-off-by: James Le Cuirot --- .../coreos-overlay/profiles/coreos/base/profile.bashrc | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc index 6165f45d053..b8dfb442218 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc @@ -1,8 +1,6 @@ # Dumping ground for build-time helpers to utilize since SYSROOT/tmp/ # can be nuked at any time. CROS_BUILD_BOARD_TREE="${SYSROOT}/build" -CROS_BUILD_BOARD_BIN="${CROS_BUILD_BOARD_TREE}/bin" - CROS_ADDONS_TREE="/mnt/host/source/src/third_party/coreos-overlay/coreos" # Are we merging for the board sysroot, or for the SDK, or for @@ -118,12 +116,6 @@ cros_setup_hooks() { } cros_setup_hooks -# Since we're storing the wrappers in a board sysroot, make sure that -# is actually in our PATH. -cros_pre_pkg_setup_sysroot_build_bin_dir() { - PATH+=":${CROS_BUILD_BOARD_BIN}" -} - # Avoid modifications of the preexisting users - these are provided by # our baselayout and usermod can't change anything there anyway (it # complains that the user is not in /etc/passwd). From d05c42ddba9707efa5116e8a0c7f5b0b5dec3092 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Thu, 24 Apr 2025 15:46:00 +0100 Subject: [PATCH 5/7] profiles: Update stale dev-lang/yasm keywords modifier Signed-off-by: James Le Cuirot --- .../coreos-overlay/profiles/coreos/base/package.accept_keywords | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.accept_keywords b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.accept_keywords index 0f5d73bca2c..d8ca71314da 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.accept_keywords +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.accept_keywords @@ -48,7 +48,7 @@ dev-cpp/azure-security-keyvault-certificates dev-cpp/azure-security-keyvault-keys # Keep versions on both arches in sync. -=dev-lang/yasm-1.3.0-r1 ~arm64 +=dev-lang/yasm-1.3.0-r2 ~arm64 =dev-libs/cowsql-1.15.9 ~arm64 =dev-libs/ding-libs-0.6.2-r1 ~arm64 From 2fe1ded81f019492740c8b0da499bc811247415b Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Fri, 27 Jun 2025 21:24:25 +0100 Subject: [PATCH 6/7] profiles: Adjust package.accept_keywords for a cross-compile fix Signed-off-by: James Le Cuirot --- .../profiles/coreos/base/package.accept_keywords | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.accept_keywords b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.accept_keywords index d8ca71314da..339a81944bc 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.accept_keywords +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.accept_keywords @@ -8,10 +8,12 @@ # (the following packages are "unstable" upstream; we're stabilising these) # Keep versions on both arches in sync. -=app-containers/cri-tools-1.32.0 ~arm64 =app-containers/incus-6.0.4-r1 ~arm64 =app-containers/lxc-6.0.4-r1 ~arm64 +# Needed for a cross-compile fix. +=app-containers/cri-tools-1.33.0 ~amd64 ~arm64 + # CVE-2025-6032 =app-containers/podman-5.5.2 ~amd64 ~arm64 From 89150d0680bc277abee1a535f7267003027ab8e7 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Fri, 30 Jan 2026 17:45:09 +0000 Subject: [PATCH 7/7] fixup! Support the SDK on arm64 --- .../coreos-overlay/profiles/coreos/arm64/sdk/transition/parent | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/transition/parent diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/transition/parent b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/transition/parent new file mode 100644 index 00000000000..627544f8c19 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/transition/parent @@ -0,0 +1,2 @@ +.. +:coreos/targets/sdk/transition