Building on Arch Linux fails

[lahwaacz]

First, I had to add cmake and clang to make dependencies of my package. But then it still fails:

warning: unused import: `Set`
 --> crates/tako/src/internal/worker/resources/pool.rs:8:18
  |
8 | use crate::{Map, Set};
  |                  ^^^
  |
  = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default

warning: method `all_states` is never used
   --> crates/tako/src/internal/worker/resources/concise.rs:202:12
    |
177 | impl ConciseFreeResources {
    | ------------------------- method in this implementation
...
202 |     pub fn all_states(&self) -> &[ConciseResourceState] {
    |            ^^^^^^^^^^
    |
    = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default

   Compiling hyperqueue v0.25.0 (/build/hyperqueue-git/src/hyperqueue/crates/hyperqueue)
warning: `tako` (lib) generated 2 warnings (run `cargo fix --lib -p tako` to apply 1 suggestion)
error: linking with `x86_64-linux-gnu-gcc` failed: exit status: 1
  |
  = note:  "x86_64-linux-gnu-gcc" "-m64" "/tmp/rustc2ml6H2/symbols.o" "<17 object files omitted>" "-Wl,--as-needed" "-Wl,-Bstatic" "<sysroot>/lib/rustlib/x86_64-unknown-linux-gnu/lib/libpanic_abort-*.rlib" "/build/hyperqueue-git/src/hyperqueue/target/release/deps/{libjemallocator-bb852798b6e4a0ba,libjemalloc_sys-3cd94648848b49d1,libhyperqueue-26e09751be425393,libcore_affinity-72c41a44869bf602,libhex-2e240bf36bb6a735,libgethostname-11cf65661a5d3394,libdirs-fbc462828efb3bae,libdirs_sys-59d49483d51331c5,liboption_ext-2b2a96d15553bb1b,libtextwrap-71b1bd8d5913ba3a,libunicode_linebreak-1ed828470ef9f8c4,libsmawk-21c644533b2460c2,libwhich-f5d1aaa5c091e5b7,libenv_home-8d68758a195daebf,libshlex-a2de75d25f7b5395,libtoml-4b5d02961582538b,libtoml_writer-f2c7ba4b8149cee4,libtoml_parser-e0579e05ee134599,libwinnow-9fb4caf2d5afe9e2,libserde_spanned-fc3b06a8f39a5ea5,libtoml_datetime-28d4362e5fcbe016,libopen-8a8201499dd4b330,libpathdiff-38478d0fa7f3be36,libis_wsl-03049fea5acf3fdb,libis_docker-333534457f664bb5,libconst_format-e1d36804f23733b8,liblru-2be6c6ee22ce72c1,libratatui-b2c494ec5cc6e1e1,libunicode_truncate-63f11089e0018652,libunicode_width-b366798ccea5688a,liblru-e8801b2a42a69858,libitertools-27621f83b540598a,libcassowary-a3747e33c5146227,libcompact_str-3377ae7644b9c9ea,libstatic_assertions-655dc6c195d8c3bf,libcastaway-b94568b938812d00,libunicode_segmentation-e4762e94140e08e7,libstrum-268664697510f2d1,libcrossterm-2a232f382b210e1a,librustix-135d0c3fc9b90c22,liblinux_raw_sys-34c4fe5a26e8a297,libcrossterm-da5c5a1acc26f5ed,libsignal_hook_mio-884e056bc1d039a7,libsignal_hook-0a11e00a49823db0,libenv_logger-5a30e8304f1932a8,libjiff-117fc4ed01b3b6a6,libenv_filter-da67c6a9c6c49422,libregex-b0251a862aa191d5,libnom_supreme-a1c9b92abe0c7195,libbrownstone-69f5ab5320504d18,libarrayvec-49b7c0ec507aea5b,libjoinery-ff76a71d69d2f49a,libindent_write-195177d1f0bcece6,libnom-eafa0833bea82e48,libchumsky-af34fd085bbecc82,libstacker-3059599449ecdac1,libpsm-407822c7b05ae968,libhashbrown-0a51db62aec1536e,libahash-cc4ba6131a88988a,libitertools-67e75ae4e563c854,libeither-1b30d79a4da7f7e9,libchrono-f2279791768e034f,libiana_time_zone-57377a1a67e78c93,libnum_traits-dac2f37caefab465,libtako-b6dac1973bc934c3,libanyhow-ac17f181addaf6ef,libserde_bytes-28130de96f50f089,libtracing-99b63e45ce7533ae,libtracing_core-7991f30382fae7f9,libnix-99e03e3e1b67ec17,libtempfile-c8870ca2b102464c,libfastrand-8b2fc838bddf6b68,librustix-73be696df52e5d01,libbitflags-7137199fd66891bb,liblinux_raw_sys-b0f3aca072ebe879,libderive_builder-bf34f5dbd776dc42,libhashbrown-e5749690aa5fbba7,libfoldhash-d72ff8e5e51c1a4d,libbstr-883107db222a73d5,libregex_automata-6a33b56243d7bf72,libaho_corasick-2efb4b42a14ce168,libregex_syntax-c4615027a71c7d4e,libhighs-7c45eb24ec91bfb3,libhighs_sys-444fbd8c8ef4eed3,librand-ccbacc8e14e5d948,librand_chacha-9396e85c5f84c86f,libppv_lite86-512c258c71ea1a5f,libzerocopy-66c9941baa2e292e,librand_core-9654de2c4cfb9dfa,libpsutil-be3f6c2b2abb977d,libnum_cpus-063d8fb59995ccd5,libonce_cell-c0250d9d082cc5b5,libunescape-5abc6176523a2468,libthiserror-2ab758ad0c5c3c31,libnix-3253520310cb0260,libbitflags-f1d33465186f7fdb,libplatforms-29cf86c5eca4d0f8,libglob-83b3341055be8b50,libbincode-daf9dde03018a592,libthin_vec-046cbf469b295713,libtokio_util-6d548054871fea57,libfutures-ef36cc7c1eb7f235,libfutures_executor-bc506a03cb6ccb77,libfutures_util-c45eb120368ff218,libfutures_io-5bf5b3c30c4353c0,libslab-7f93663fa5f69778,libfutures_channel-d6d5371dc8b9aa82,libfutures_sink-ae7233624952089f,libfutures_task-fe1411096f537a67,libpin_utils-9423b90006295bd0,libfutures_core-8f5d95de40c3087c,libserde_json-ef26ba09b71fa493,libmemchr-37c23256ca1527fe,libitoa-ed5bf2a3f13a4c31,libryu-db290e9f513b16df,liborion-4e3a531afd0f1a83,libgetrandom-eb3f6d94e0ea75f7,libct_codecs-a77dc5df5171865e,libfiat_crypto-dc38172672752d12,libzeroize-e7f0c8f2c6b244a1,libsubtle-228a8f4d1487dd43,libpriority_queue-e459bd6b85ac5b92,libindexmap-217f711a857cc818,libhashbrown-fa3203db94a9d5d7,libfoldhash-16622714fce4fce6,libequivalent-0bf727b6aa6716f2,liballocator_api2-db950b9eb7c62d57,libtokio-640c1723d7f14368,libsignal_hook_registry-712a3023e1d52ab3,libsocket2-915f828899f3d89d,libbytes-1015deb8df3efd52,libmio-f9e4af1e79dd58a3,liblog-90c0fd9ab93eaf95,libparking_lot-16161de451b5ec8a,libparking_lot_core-4b3d004ad7ab40b2,liblibc-123fba201cd667a3,libcfg_if-3faf74730df5d5fe,liblock_api-9a236106dd40cc38,libscopeguard-d7a0e7b27e3e1763,libpin_project_lite-c90acb2a2ef1d1f1,libderive_more-1c6b4d38842f722c,libsmallvec-9ccd9b7cb3d8b421,libthiserror-bdd521f61705dcb2,libserde-f53ac03f3833a3ef,libserde_core-53d9d438b69419f4,libfxhash-886d2a303101babf,libbyteorder-c10bfcd49a9f20af,libhumantime-fe6c40e6aa949757,libcolored-e1ce7aad6cd16fc2,libcli_table-6eba5913e03df4d1,libunicode_width-a00a948852022b66,libtermcolor-424c22fc19c78101,libclap_complete-8d6c5fa40fb1e645,libclap-1de2f657fffb3897,libclap_builder-2f8062801a9d9e7e,libstrsim-9e7f07b085d4b2bf,libanstream-d8241ccc4a1338dd,libanstyle_query-0b75fcdeae05cc54,libis_terminal_polyfill-86dcde434d5a0545,libcolorchoice-8b3dded07d7feee5,libanstyle_parse-04317cd3ab01e2b5,libutf8parse-60c1a7c8d7a0cb31,libclap_lex-e38ab752ddd43479,libanstyle-f9e387a024308131}.rlib" "<sysroot>/lib/rustlib/x86_64-unknown-linux-gnu/lib/{libstd-*,libobject-*,libmemchr-*,libaddr2line-*,libgimli-*,libcfg_if-*,librustc_demangle-*,libstd_detect-*,libhashbrown-*,librustc_std_workspace_alloc-*,libminiz_oxide-*,libadler2-*,libunwind-*,liblibc-*,librustc_std_workspace_core-*,liballoc-*,libcore-*,libcompiler_builtins-*}.rlib" "-Wl,-Bdynamic" "-lstdc++" "-lgcc_s" "-lutil" "-lrt" "-lpthread" "-lm" "-ldl" "-lc" "-L" "/tmp/rustc2ml6H2/raw-dylibs" "-fuse-ld=lld" "-Wl,--eh-frame-hdr" "-Wl,-z,noexecstack" "-L" "/build/hyperqueue-git/src/hyperqueue/target/release/build/psm-1f04386c5d8861b8/out" "-L" "/build/hyperqueue-git/src/hyperqueue/target/release/build/jemalloc-sys-e86ee70fe48eba28/out/build/lib" "-L" "/build/hyperqueue-git/src/hyperqueue/target/release/build/highs-sys-008352654bae9efb/out/lib" "-L" "/build/hyperqueue-git/src/hyperqueue/target/release/build/highs-sys-008352654bae9efb/out/lib64" "-L" "<sysroot>/lib/rustlib/x86_64-unknown-linux-gnu/lib" "-o" "/build/hyperqueue-git/src/hyperqueue/target/release/deps/hq-c9afa9cab0745255" "-Wl,--gc-sections" "-pie" "-Wl,-z,relro,-z,now" "-Wl,-O1" "-Wl,--strip-debug" "-nodefaultlibs"
  = note: some arguments are omitted. use `--verbose` to show all linker arguments
  = note: ld.lld: error: undefined symbol: Highs_destroy
          >>> referenced by lib.rs:547 (src/lib.rs:547)
          >>>               highs-7c45eb24ec91bfb3.highs.271cad7b7562350b-cgu.0.rcgu.o:(_$LT$highs..HighsPtr$u20$as$u20$core..ops..drop..Drop$GT$::drop::h352a6a8ec755edd5) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libhighs-7c45eb24ec91bfb3.rlib
          >>> referenced by lib.rs:547 (src/lib.rs:547)
          >>>               highs-7c45eb24ec91bfb3.highs.271cad7b7562350b-cgu.0.rcgu.o:(highs::Model::try_solve::h858f107f054e5842) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libhighs-7c45eb24ec91bfb3.rlib

          ld.lld: error: undefined symbol: Highs_create
          >>> referenced by lib.rs:553 (src/lib.rs:553)
          >>>               highs-7c45eb24ec91bfb3.highs.271cad7b7562350b-cgu.0.rcgu.o:(_$LT$highs..HighsPtr$u20$as$u20$core..default..Default$GT$::default::h58fd77a7abdda95b) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libhighs-7c45eb24ec91bfb3.rlib

          ld.lld: error: undefined symbol: Highs_getNumCols
          >>> referenced by lib.rs:592 (src/lib.rs:592)
          >>>               highs-7c45eb24ec91bfb3.highs.271cad7b7562350b-cgu.0.rcgu.o:(highs::SolvedModel::get_solution::h5185dbb350fd9e1f) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libhighs-7c45eb24ec91bfb3.rlib

          ld.lld: error: undefined symbol: Highs_getNumRows
          >>> referenced by lib.rs:598 (src/lib.rs:598)
          >>>               highs-7c45eb24ec91bfb3.highs.271cad7b7562350b-cgu.0.rcgu.o:(highs::SolvedModel::get_solution::h5185dbb350fd9e1f) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libhighs-7c45eb24ec91bfb3.rlib

          ld.lld: error: undefined symbol: Highs_getSolution
          >>> referenced by lib.rs:650 (src/lib.rs:650)
          >>>               highs-7c45eb24ec91bfb3.highs.271cad7b7562350b-cgu.0.rcgu.o:(highs::SolvedModel::get_solution::h5185dbb350fd9e1f) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libhighs-7c45eb24ec91bfb3.rlib

          ld.lld: error: undefined symbol: Highs_getObjectiveValue
          >>> referenced by lib.rs:618 (src/lib.rs:618)
          >>>               highs-7c45eb24ec91bfb3.highs.271cad7b7562350b-cgu.0.rcgu.o:(highs::SolvedModel::objective_value::h99bc2e4aacda156a) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libhighs-7c45eb24ec91bfb3.rlib

          ld.lld: error: undefined symbol: Highs_getModelStatus
          >>> referenced by lib.rs:623 (src/lib.rs:623)
          >>>               highs-7c45eb24ec91bfb3.highs.271cad7b7562350b-cgu.0.rcgu.o:(highs::SolvedModel::status::h95a5ad1fd4859036) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libhighs-7c45eb24ec91bfb3.rlib

          ld.lld: error: undefined symbol: Highs_changeObjectiveSense
          >>> referenced by lib.rs:275 (src/lib.rs:275)
          >>>               highs-7c45eb24ec91bfb3.highs.271cad7b7562350b-cgu.0.rcgu.o:(highs::Model::set_sense::h842ad7d6c7556bed) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libhighs-7c45eb24ec91bfb3.rlib

          ld.lld: error: undefined symbol: Highs_run
          >>> referenced by lib.rs:234 (src/lib.rs:234)
          >>>               highs-7c45eb24ec91bfb3.highs.271cad7b7562350b-cgu.0.rcgu.o:(highs::Model::try_solve::h858f107f054e5842) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libhighs-7c45eb24ec91bfb3.rlib

          ld.lld: error: undefined symbol: Highs_setBoolOptionValue
          >>> referenced by options.rs:10 (src/options.rs:10)
          >>>               highs-7c45eb24ec91bfb3.highs.271cad7b7562350b-cgu.0.rcgu.o:(highs::HighsPtr::set_option::haf65ae2c8f1a813a) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libhighs-7c45eb24ec91bfb3.rlib

          ld.lld: error: undefined symbol: Highs_passLp
          >>> referenced by lib.rs:234 (/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/highs-1.12.0/src/lib.rs:234)
          >>>               tako-b6dac1973bc934c3.tako.e3f2505efcbc6016-cgu.08.rcgu.o:(_$LT$tako..internal..worker..resources..solver_highs..HighsSolver$u20$as$u20$tako..internal..worker..resources..solver..LpSolver$GT$::solve::h4ebd6389cdaf6a26) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libtako-b6dac1973bc934c3.rlib

          ld.lld: error: undefined symbol: Highs_passMip
          >>> referenced by lib.rs:234 (/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/highs-1.12.0/src/lib.rs:234)
          >>>               tako-b6dac1973bc934c3.tako.e3f2505efcbc6016-cgu.08.rcgu.o:(_$LT$tako..internal..worker..resources..solver_highs..HighsSolver$u20$as$u20$tako..internal..worker..resources..solver..LpSolver$GT$::solve::h4ebd6389cdaf6a26) in archive /build/hyperqueue-git/src/hyperqueue/target/release/deps/libtako-b6dac1973bc934c3.rlib
          collect2: error: ld returned 1 exit status


error: could not compile `hyperqueue` (bin "hq") due to 1 previous error

What is the C/C++ code used here? Can we build it separately as a shared library and link it dynamically?

[lahwaacz]

Note that Arch Linux uses these flags

CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fno-plt -fexceptions \
        -Wp,-D_FORTIFY_SOURCE=3 -Wformat -Werror=format-security \
        -fstack-clash-protection -fcf-protection \
        -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"
CXXFLAGS="$CFLAGS -Wp,-D_GLIBCXX_ASSERTIONS"
LDFLAGS="-Wl,-O1 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now \
         -Wl,-z,pack-relative-relocs"

and we have this override for hyperqueue:

# fix building with LTO
CFLAGS+=' -ffat-lto-objects'

[Kobzol]

You can build with --no-default-features to avoid building Highs. It's an annoying C++ dependency that is not always trivial to build. We haven't found an alternative in Rust with similar performance, but it's not strictly needed to use it with HQ.

[lahwaacz]

Hmm as it is in default features, I'd rather build that 🤷 I expect it provides some functionality

[spirali]

You can enable feature microlp that replace Highs with microlp, that is pure Rust package. Both are MILP solvers, with the same functionality. The problem is that Highs is much more performant than microlp. So far, a MILP solver is used only for NUMA mapping, so if you do not have complex NUMA settings, microlp is ok.

On the other hand, I am building a new global scheduler on MILP, so it may have nontrivial performence implication in the future, so we would like to know how it is actually difficult to build Highs in the wild.

[spirali]

Can we build it separately as a shared library and link it dynamically?

AFAIK Rust binding of Highs links only statically. But there are other solvers and some of them are dynamically linked.

[lahwaacz]

Arch Linux has a highs package ready. It is annoying that we can't link it dynamically and not even compile highs using cargo.

[lahwaacz]

It seems that highs-sys has a discover feature: https://github.com/rust-or/highs-sys?tab=readme-ov-file#using-a-pre-installed-version-of-highs

How can I select it when building hyperqueue?

[Kobzol]

You can't at the moment, we would have to expose it in HQ.

[spirali]

Are we able to expose this? We are not using highs-sys directly, we are using highs crate and it does not seems to have any features.

[Kobzol]

It's a bit of a hack, but we can depend on highs-sys explicitly and then do highs = ["dep:highs", "dep:highs-sys", "highs-sys/discover"] in tako.

I'd just suggest using microlp though.

[spirali]

I will try to benchmark cbc, it should have good performance and it is by default dynamically linked, so no C compiler is needed. Since it adds a dynamic dependency, I have chosen highs as default but it is quite easy to support additional solver.

[lahwaacz]

Is highs the only thing that gets disabled by --no-default-features?

[lahwaacz]

Oh well, it seems that highs-sys gets built even with --no-default-features 🤷

$ cargo build --frozen --release --no-default-features
[...]
error: failed to run custom build command for `highs-sys v1.11.0`

Caused by:
  process didn't exit successfully: `/build/hyperqueue/src/hyperqueue-0.25.0/target/release/build/highs-sys-9fc6312a381b43dc/build-script-build` (exit status: 101)
  --- stdout
  CMAKE_TOOLCHAIN_FILE_x86_64-unknown-linux-gnu = None
  CMAKE_TOOLCHAIN_FILE_x86_64_unknown_linux_gnu = None
  HOST_CMAKE_TOOLCHAIN_FILE = None
  CMAKE_TOOLCHAIN_FILE = None
  CMAKE_GENERATOR_x86_64-unknown-linux-gnu = None
  CMAKE_GENERATOR_x86_64_unknown_linux_gnu = None
  HOST_CMAKE_GENERATOR = None
  CMAKE_GENERATOR = None
  CMAKE_PREFIX_PATH_x86_64-unknown-linux-gnu = None
  CMAKE_PREFIX_PATH_x86_64_unknown_linux_gnu = None
  HOST_CMAKE_PREFIX_PATH = None
  CMAKE_PREFIX_PATH = None
  CMAKE_x86_64-unknown-linux-gnu = None
  CMAKE_x86_64_unknown_linux_gnu = None
  HOST_CMAKE = None
  CMAKE = None
  running: cd "/build/hyperqueue/src/hyperqueue-0.25.0/target/release/build/highs-sys-c9895b6e070a41fb/out/build" && CMAKE_PREFIX_PATH="" LC_ALL="C" "cmake" "/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/highs-sys-1.11.0/HiGHS" "-DFAST_BUILD=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDLL" "-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=FALSE" "-DZLIB=OFF" "-DCMAKE_INSTALL_PREFIX=/build/hyperqueue/src/hyperqueue-0.25.0/target/release/build/highs-sys-c9895b6e070a41fb/out" "-DCMAKE_C_FLAGS= -ffunction-sections -fdata-sections -fPIC -m64 -fno-omit-frame-pointer -march=x86-64 -mtune=generic -pipe -fno-plt -fexceptions -Wp,-D_FORTIFY_SOURCE=3 -Wformat -Werror=format-security -fstack-clash-protection -fcf-protection -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -ffile-prefix-map=/build/hyperqueue/src=/usr/src/debug/hyperqueue -flto=auto -ffat-lto-objects" "-DCMAKE_C_COMPILER=/usr/bin/cc" "-DCMAKE_CXX_FLAGS= -ffunction-sections -fdata-sections -fPIC -m64 -fno-omit-frame-pointer -march=x86-64 -mtune=generic -pipe -fno-plt -fexceptions -Wp,-D_FORTIFY_SOURCE=3 -Wformat -Werror=format-security -fstack-clash-protection -fcf-protection -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -Wp,-D_GLIBCXX_ASSERTIONS -ffile-prefix-map=/build/hyperqueue/src=/usr/src/debug/hyperqueue -flto=auto" "-DCMAKE_CXX_COMPILER=/usr/bin/c++" "-DCMAKE_ASM_FLAGS= -ffunction-sections -fdata-sections -fPIC -m64 -fno-omit-frame-pointer -march=x86-64 -mtune=generic -pipe -fno-plt -fexceptions -Wp,-D_FORTIFY_SOURCE=3 -Wformat -Werror=format-security -fstack-clash-protection -fcf-protection -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -ffile-prefix-map=/build/hyperqueue/src=/usr/src/debug/hyperqueue -flto=auto -ffat-lto-objects" "-DCMAKE_ASM_COMPILER=/usr/bin/cc" "-DCMAKE_BUILD_TYPE=Release"

  --- stderr

  thread 'main' (6203) panicked at /build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/cmake-0.1.54/src/lib.rs:1119:5:

  failed to execute command: No such file or directory (os error 2)
  is `cmake` not installed?

  build script failed, must exit now
  note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
warning: build failed, waiting for other jobs to finish...

[Kobzol]

Woops, that's a bug. Fixed in #1069.

To answer your question, --no-default-features also disables the dashboard and the Jemalloc allocator. You can do e.g. cargo build --features dashboard,jemalloc,microlp.