From e3f828d96dcc90ca7cd5a188bcb7ce870f4c0b9d Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sun, 23 Nov 2025 17:44:52 +0000 Subject: [PATCH 01/11] chore: syntax and structure cleanup This commit does two things to clean up the formatting of the main default.nix file. First, it factors the optimizedBuild function out, mostly for consistency. This does not actually impact the meaning of the expressions. Second, it removes a (now) pointless split of the libyang build into a -dynamic, -static, and a combined build. The -dynamic build is no longer useful now that our build of FRR is almost completely static, so this split was just pointless complexity. Signed-off-by: Daniel Noland --- default.nix | 327 ++++++++++++++++++++++++++-------------------------- 1 file changed, 166 insertions(+), 161 deletions(-) diff --git a/default.nix b/default.nix index 2141b24..2bff2ad 100644 --- a/default.nix +++ b/default.nix @@ -48,10 +48,12 @@ rec { cargo-nextest = fancy.rustOverrides super.cargo-nextest; csview = fancy.rustOverrides super.csview; just = fancy.rustOverrides super.just; - kopium = fancy.rustOverrides (self.callPackage ./nix/kopium { - rev = "c4931d13043994813c9949d90bf3e0fbac949148"; # 0.22.5 - hash = "sha256-zYmb+HxwEKEnzdqAzvki5M+NA2fGP174pRkU6B4WmZI="; - }); + kopium = fancy.rustOverrides ( + self.callPackage ./nix/kopium { + rev = "c4931d13043994813c9949d90bf3e0fbac949148"; # 0.22.5 + hash = "sha256-zYmb+HxwEKEnzdqAzvki5M+NA2fGP174pRkU6B4WmZI="; + } + ); frr-agent = fancy.rustOverrides ( self.callPackage ./nix/frr-agent { rev = versions.frr-agent.rev; @@ -108,24 +110,28 @@ rec { buildWithMyFlags = pkg: (buildWithFlags build-flags pkg); optimizedBuild = pkg: - (buildWithMyFlags (pkg.override { stdenv = fancy.stdenv; })).overrideAttrs (orig: { - nativeBuildInputs = (orig.nativeBuildInputs or [ ]) ++ [ super.fancy.llvmPackages.bintools ]; - LD = "lld"; - withDoc = false; - doCheck = false; - }); - fancy.libmd = (optimizedBuild super.libmd).overrideAttrs (orig: { - configureFlags = orig.configureFlags ++ [ - "--enable-static" - "--disable-shared" - ]; - postFixup = (orig.postFixup or "") + '' - rm $out/lib/*.la - ''; - }); + buildWithMyFlags ( + (pkg.override { stdenv = fancy.stdenv; }).overrideAttrs (orig: { + nativeBuildInputs = (orig.nativeBuildInputs or [ ]) ++ [ super.fancy.llvmPackages.bintools ]; + LD = "lld"; + withDoc = false; + doCheck = false; + }) + ); + fancy.libmd = optimizedBuild ( + super.libmd.overrideAttrs (orig: { + configureFlags = orig.configureFlags ++ [ + "--enable-static" + "--disable-shared" + ]; + postFixup = (orig.postFixup or "") + '' + rm $out/lib/*.la + ''; + }) + ); - fancy.libbsd = - ((optimizedBuild super.libbsd).override { + fancy.libbsd = optimizedBuild ( + (super.libbsd.override { libmd = fancy.libmd; }).overrideAttrs (orig: { @@ -137,7 +143,8 @@ rec { postFixup = (orig.postFixup or "") + '' rm $out/lib/*.la ''; - }); + }) + ); at-spi2-atk = null; # no users in container at-spi2-core = null; # no users in container bluez = null; @@ -173,18 +180,22 @@ rec { ethtool = null; iproute2 = null; fancy.iproute2 = optimizedBuild super.iproute2; - libnl = (optimizedBuild super.libnl).overrideAttrs (orig: { - configureFlags = orig.configureFlags ++ [ - "--enable-static" - "--disable-shared" - ]; - postFixup = (orig.postFixup or "") + '' - rm $out/lib/*.la - ''; - }); - jansson = (optimizedBuild super.jansson).overrideAttrs (orig: { - cmakeFlags = [ "-DJANSSON_BUILD_SHARED_LIBS=OFF" ]; - }); + libnl = optimizedBuild ( + super.libnl.overrideAttrs (orig: { + configureFlags = orig.configureFlags ++ [ + "--enable-static" + "--disable-shared" + ]; + postFixup = (orig.postFixup or "") + '' + rm $out/lib/*.la + ''; + }) + ); + jansson = optimizedBuild ( + super.jansson.overrideAttrs (orig: { + cmakeFlags = [ "-DJANSSON_BUILD_SHARED_LIBS=OFF" ]; + }) + ); libmnl = optimizedBuild super.libmnl; libnetfilter_conntrack = optimizedBuild super.libnetfilter_conntrack; libnftnl = optimizedBuild super.libnftnl; @@ -197,30 +208,30 @@ rec { outputChecks.lib.disallowedRequisites = [ ]; }) ); - numactl = (optimizedBuild super.numactl).overrideAttrs (orig: { - outputs = super.lib.lists.remove "man" orig.outputs; - configurePhase = '' - set -euxo pipefail; - ./configure \ - --prefix=$out \ - --libdir=$out/lib \ - --includedir=$out/include \ - --enable-static \ - --enable-shared; - ''; - buildPhase = '' - set -euxo pipefail; - make; - rm ./.libs/*.la; - ''; - }); - dpdk = ( - optimizedBuild ( - self.callPackage ./nix/dpdk { - libbsd = fancy.libbsd; - libmd = fancy.libmd; - } - ) + numactl = optimizedBuild ( + super.numactl.overrideAttrs (orig: { + outputs = super.lib.lists.remove "man" orig.outputs; + configurePhase = '' + set -euxo pipefail; + ./configure \ + --prefix=$out \ + --libdir=$out/lib \ + --includedir=$out/include \ + --enable-static \ + --enable-shared; + ''; + buildPhase = '' + set -euxo pipefail; + make; + rm ./.libs/*.la; + ''; + }) + ); + dpdk = optimizedBuild ( + self.callPackage ./nix/dpdk { + libbsd = fancy.libbsd; + libmd = fancy.libmd; + } ); dpdk-wrapper = ( optimizedBuild ( @@ -232,34 +243,17 @@ rec { ) ); fancy.xxHash = optimizedBuild super.xxHash; - libyang-dynamic = - ((optimizedBuild super.libyang).override { - pcre2 = self.fancy.pcre2; - xxHash = self.fancy.xxHash; - }).overrideAttrs - (orig: { - cmakeFlags = (orig.cmakeFlags or [ ]) ++ [ "-DBUILD_SHARED_LIBS=ON" ]; - }); - libyang-static = - ((optimizedBuild super.libyang).override { + libyang = optimizedBuild ( + (super.libyang.override { pcre2 = self.fancy.pcre2; xxHash = self.fancy.xxHash; }).overrideAttrs (orig: { cmakeFlags = (orig.cmakeFlags or [ ]) ++ [ "-DBUILD_SHARED_LIBS=OFF" ]; - }); - libyang = self.fancy.stdenv.mkDerivation { - name = "libyang"; - src = null; - dontUnpack = true; - installPhase = '' - mkdir -p $out/lib; - cp -r ${self.libyang-static}/lib/libyang.a $out/lib/ - cp -r ${self.libyang-dynamic}/* $out - ''; - }; - fancy.libcap = - ((optimizedBuild super.libcap).override { + }) + ); + fancy.libcap = optimizedBuild ( + (super.libcap.override { stdenv = fancy.stdenv; usePam = false; }).overrideAttrs @@ -278,47 +272,55 @@ rec { # extant postInstall removes .a files for no reason cp ./libcap/*.a $lib/lib; ''; - }); - fancy.json_c = (optimizedBuild super.json_c).overrideAttrs (orig: { - cmakeFlags = (orig.cmakeFlags or [ ]) ++ [ "-DENABLE_STATIC=1" ]; - postInstall = (orig.postInstall or "") + '' - mkdir -p $dev/lib - $RANLIB libjson-c.a; - cp libjson-c.a $dev/lib; - ''; - }); - rtrlib = (optimizedBuild super.rtrlib).overrideAttrs (orig: { - cmakeFlags = (orig.cmakeFlags or [ ]) ++ [ "-DENABLE_STATIC=1" ]; - }); - abseil-cpp = (optimizedBuild super.abseil-cpp); - protobuf = - (optimizedBuild (super.protobuf.override { enableShared = false; })).overrideAttrs - (orig: { - cmakeFlags = (orig.cmakeFlags or [ ]) ++ [ "-Dprotobuf_BUILD_SHARED_LIBS=OFF" ]; - }); - fancy.zlib = (optimizedBuild super.zlib).override { - static = true; - shared = false; - }; - protobufc = - (optimizedBuild ( - self.callPackage ./nix/protobufc { - stdenv = fancy.stdenv; - zlib = fancy.zlib; - } - )).overrideAttrs + }) + ); + fancy.json_c = optimizedBuild ( + super.json_c.overrideAttrs (orig: { + cmakeFlags = (orig.cmakeFlags or [ ]) ++ [ "-DENABLE_STATIC=1" ]; + postInstall = (orig.postInstall or "") + '' + mkdir -p $dev/lib + $RANLIB libjson-c.a; + cp libjson-c.a $dev/lib; + ''; + }) + ); + rtrlib = optimizedBuild ( + super.rtrlib.overrideAttrs (orig: { + cmakeFlags = (orig.cmakeFlags or [ ]) ++ [ "-DENABLE_STATIC=1" ]; + }) + ); + abseil-cpp = optimizedBuild super.abseil-cpp; + protobuf = optimizedBuild ( + (super.protobuf.override { enableShared = false; }).overrideAttrs (orig: { + cmakeFlags = (orig.cmakeFlags or [ ]) ++ [ "-Dprotobuf_BUILD_SHARED_LIBS=OFF" ]; + }) + ); + fancy.zlib = optimizedBuild ( + super.zlib.override { + static = true; + shared = false; + } + ); + protobufc = optimizedBuild ( + (self.callPackage ./nix/protobufc { + stdenv = fancy.stdenv; + zlib = fancy.zlib; + }).overrideAttrs (orig: { configureFlags = (orig.configureFlags or [ ]) ++ [ "--enable-static" "--disable-shared" ]; - }); - fancy.pcre2 = (optimizedBuild super.pcre2).overrideAttrs (orig: { - configureFlags = (orig.configureFlags or [ ]) ++ [ - "--enable-static" - "--disable-shared" - ]; - }); + }) + ); + fancy.pcre2 = optimizedBuild ( + super.pcre2.overrideAttrs (orig: { + configureFlags = (orig.configureFlags or [ ]) ++ [ + "--enable-static" + "--disable-shared" + ]; + }) + ); fancy.ncurses = optimizedBuild (super.ncurses.override { enableStatic = true; }); fancy.readline = optimizedBuild (super.readline.override { ncurses = fancy.ncurses; }); fancy.libxcrypt = optimizedBuild super.libxcrypt; @@ -337,22 +339,21 @@ rec { rev = versions.frr.rev; hash = versions.frr.hash; - c-ares = fancy.c-ares; - json_c = fancy.json_c.dev; - libcap = fancy.libcap; - libgccjit = fancy.libgccjit; - libxcrypt = fancy.libxcrypt; - libyang = self.libyang-static; - pcre2 = fancy.pcre2; - readline = fancy.readline; - stdenv = fancy.stdenv; - } - )).overrideAttrs + c-ares = fancy.c-ares; + json_c = fancy.json_c.dev; + libcap = fancy.libcap; + libgccjit = fancy.libgccjit; + libxcrypt = fancy.libxcrypt; + libyang = self.libyang; + pcre2 = fancy.pcre2; + readline = fancy.readline; + stdenv = fancy.stdenv; + }).overrideAttrs (orig: { LDFLAGS = (orig.LDFLAGS or "") + " -L${self.fancy.c-ares}/lib -lcares " - + " -L${self.libyang-static}/lib -lyang " + + " -L${self.libyang}/lib -lyang " + " -L${fancy.xxHash}/lib -lxxhash " + " -L${fancy.libxcrypt}/lib -lcrypt " + " -L${protobufc}/lib -lprotobuf-c " @@ -363,7 +364,8 @@ rec { "--enable-static" "--enable-static-bin" ]; - }); + }) + ); dplane-rpc = optimizedBuild ( self.callPackage ./nix/dplane-rpc { rev = versions.dplane-rpc.rev; @@ -376,11 +378,11 @@ rec { hash = versions.dplane-plugin.hash; commit_date = versions.dplane-plugin.commit_date; stdenv = fancy.stdenv; - libyang = libyang-static; + inherit libyang; pcre2 = fancy.pcre2; } ); - frr-config = (optimizedBuild (self.callPackage ./nix/frr-config { })); + frr-config = optimizedBuild (self.callPackage ./nix/frr-config { }); frr-with-dplane-plugin = self.symlinkJoin { name = "frr-with-dplane-plugin"; paths = [ @@ -395,40 +397,42 @@ rec { fancy.xz = optimizedBuild super.xz; fancy.libxml2 = optimizedBuild super.libxml2; fancy.busybox = super.busybox.override { enableStatic = true; }; - fancy.boost = (optimizedBuild super.boost).override { - enableShared = false; - enableStatic = true; - }; + fancy.boost = optimizedBuild ( + super.boost.override { + enableShared = false; + enableStatic = true; + } + ); fancy.expat = optimizedBuild super.expat; fancy.openssl = optimizedBuild ( (super.openssl.override { static = true; }).overrideAttrs (final: { doCheck = false; }) ); - fancy.curl = (optimizedBuild super.curlMinimal).override { zlib = fancy.zlib; }; + fancy.curl = optimizedBuild (super.curlMinimal.override { zlib = fancy.zlib; }); hwdata = optimizedBuild super.hwdata; hwloc = optimizedBuild super.hwloc; - pciutils = (optimizedBuild super.pciutils).override { - zlib = fancy.zlib; - static = true; - }; + pciutils = optimizedBuild ( + super.pciutils.override { + zlib = fancy.zlib; + static = true; + } + ); - mstflint = - (optimizedBuild ( - super.mstflint.override { - openssl = self.fancy.openssl; - zlib = self.fancy.zlib; - xz = self.fancy.xz; - expat = self.fancy.expat; - boost = self.fancy.boost; - curl = self.fancy.curl; - libxml2 = self.fancy.libxml2; - busybox = self.fancy.busybox; - onlyFirmwareUpdater = false; - enableDPA = false; - python3 = self.python3Minimal; - } - )).overrideAttrs + mstflint = optimizedBuild ( + (super.mstflint.override { + openssl = self.fancy.openssl; + zlib = self.fancy.zlib; + xz = self.fancy.xz; + expat = self.fancy.expat; + boost = self.fancy.boost; + curl = self.fancy.curl; + libxml2 = self.fancy.libxml2; + busybox = self.fancy.busybox; + onlyFirmwareUpdater = false; + enableDPA = false; + python3 = self.python3Minimal; + }).overrideAttrs (orig: { configureFlags = [ "--datarootdir=${placeholder "out"}/share" @@ -477,13 +481,14 @@ rec { -e ${stdenv.cc.libc} \ '{}' +; ''); - }); + }) + ); - perftest = ( - optimizedBuild (self.callPackage ./nix/perftest { inherit (versions.perftest) rev hash; }) + perftest = optimizedBuild ( + self.callPackage ./nix/perftest { inherit (versions.perftest) rev hash; } ); - base-image = self.callPackage ./nix/base-image {}; + base-image = optimizedBuild (self.callPackage ./nix/base-image { }); }; pkgs.debug = From 19a381cbd5aac1f9f1cd30c94aa45fba9cd0fa1d Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sun, 23 Nov 2025 17:48:45 +0000 Subject: [PATCH 02/11] fixup! chore: syntax and structure cleanup --- default.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/default.nix b/default.nix index 2bff2ad..a6d201b 100644 --- a/default.nix +++ b/default.nix @@ -333,11 +333,10 @@ rec { ]; }) ); - frr = - (optimizedBuild ( - self.callPackage ./nix/frr { - rev = versions.frr.rev; - hash = versions.frr.hash; + frr = optimizedBuild ( + (self.callPackage ./nix/frr { + rev = versions.frr.rev; + hash = versions.frr.hash; c-ares = fancy.c-ares; json_c = fancy.json_c.dev; From aa5ec75cc2dbf7a9a10a17402f5ae1d07714131e Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sun, 23 Nov 2025 17:49:45 +0000 Subject: [PATCH 03/11] fixup! fixup! chore: syntax and structure cleanup --- default.nix | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/default.nix b/default.nix index a6d201b..ab05f19 100644 --- a/default.nix +++ b/default.nix @@ -155,27 +155,28 @@ rec { tinysparql = null; pandoc = null; util-linux = super.util-linux.override { systemdSupport = false; }; - rdma-core = (optimizedBuild super.rdma-core).overrideAttrs (orig: { - version = "60.0"; - src = self.fetchFromGitHub { - owner = "githedgehog"; - repo = "rdma-core"; - rev = "fix-lto-60.0"; - hash = "sha256-JAf7r0I+MuppH/cbBd0ZrXVAjPPeWo/oFcDFpJ7TTbY="; - }; - outputs = [ - "out" - "dev" - ]; - perl = null; - cmakeFlags = orig.cmakeFlags ++ [ - "-DENABLE_STATIC=1" - "-DNO_PYVERBS=1" - "-DNO_MAN_PAGES=1" - "-DIOCTL_MODE=write" - "-DNO_COMPAT_SYMS=1" - ]; - }); + rdma-core = optimizedBuild ( + super.rdma-core.overrideAttrs (orig: { + version = "60.0"; + src = self.fetchFromGitHub { + owner = "githedgehog"; + repo = "rdma-core"; + rev = "fix-lto-60.0"; + hash = "sha256-JAf7r0I+MuppH/cbBd0ZrXVAjPPeWo/oFcDFpJ7TTbY="; + }; + outputs = [ + "out" + "dev" + ]; + perl = null; + cmakeFlags = orig.cmakeFlags ++ [ + "-DENABLE_STATIC=1" + "-DNO_PYVERBS=1" + "-DNO_MAN_PAGES=1" + "-DNO_COMPAT_SYMS=1" + ]; + }) + ); iptables = null; ethtool = null; iproute2 = null; From 5919f5fe3e939f733ad7f7d6757ccfc61d399af7 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sun, 23 Nov 2025 17:52:46 +0000 Subject: [PATCH 04/11] feat: add thread profile Signed-off-by: Daniel Noland --- default.nix | 30 +++++++++++++++++++++++++++--- nix/dpdk/default.nix | 1 - nix/flags.nix | 33 +++++++++++++++++++++++---------- 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/default.nix b/default.nix index ab05f19..18b41cd 100644 --- a/default.nix +++ b/default.nix @@ -529,6 +529,24 @@ rec { ]; }).pkgsCross; + pkgs.fuzz = + (import toolchainPkgs.path { + overlays = [ + (self: prev: { + pkgsCross.gnu64 = import prev.path { + overlays = [ + llvm-overlay + helpersOverlay + fenix-overlay + rust-overlay + (crossOverlay { + build-flags = build-flags.fuzz; + }) + ]; + }; + }) + ]; + }).pkgsCross; sysrootPackageListFn = pkgs: with pkgs; [ dpdk @@ -604,6 +622,10 @@ rec { name = "${project-name}-env-release-sysroot-gnu64"; paths = sysrootPackageListFn pkgs.release.gnu64; }; + sysroot.gnu64.fuzz = toolchainPkgs.symlinkJoin { + name = "${project-name}-env-release-sysroot-gnu64"; + paths = sysrootPackageListFn pkgs.fuzz.gnu64; + }; compile = toolchainPkgs.symlinkJoin { name = "${project-name}-env-compile"; paths = compileEnvPackageList; @@ -632,9 +654,6 @@ rec { rsync -rLhP \ "${env.sysroot.gnu64.${profile}}/include/" \ "$out/sysroot/x86_64-unknown-linux-gnu/${profile}/include/" - if [ ${profile} = "release" ]; then - ln -rs $out/sysroot/x86_64-unknown-linux-gnu/release $out/sysroot/x86_64-unknown-linux-gnu/fuzz - fi ''; postFixup = # libm.a file contains a GROUP instruction which contains absolute paths to /nix @@ -659,10 +678,12 @@ rec { sysroot.gnu64.debug = sysrootFn "debug"; sysroot.gnu64.release = sysrootFn "release"; + sysroot.gnu64.fuzz = sysrootFn "fuzz"; sysroots = with sysroot; [ gnu64.debug gnu64.release + gnu64.fuzz ]; compile-env = toolchainPkgs.symlinkJoin { @@ -671,10 +692,13 @@ rec { env.compile pkgs.debug.gnu64.glibc.static pkgs.release.gnu64.glibc.static + pkgs.fuzz.gnu64.glibc.static pkgs.debug.gnu64.glibc.dev pkgs.release.gnu64.glibc.dev + pkgs.fuzz.gnu64.glibc.dev pkgs.debug.gnu64.glibc.out pkgs.release.gnu64.glibc.out + pkgs.fuzz.gnu64.glibc.out ] ++ sysroots; }; diff --git a/nix/dpdk/default.nix b/nix/dpdk/default.nix index d5b25db..4e76855 100644 --- a/nix/dpdk/default.nix +++ b/nix/dpdk/default.nix @@ -263,7 +263,6 @@ stdenv.mkDerivation rec { "-Db_pch=true" "-Db_pgo=off" "-Db_pie=true" - "-Db_sanitize=none" "-Dbackend=ninja" "-Ddefault_library=static" "-Denable_docs=false" diff --git a/nix/flags.nix b/nix/flags.nix index 8094389..ca06301 100644 --- a/nix/flags.nix +++ b/nix/flags.nix @@ -6,26 +6,38 @@ let debug = "-ggdb3 -gdwarf-5 -gembed-source"; security = "-fstack-protector-strong"; errors = "-Werror=odr -Werror=strict-aliasing"; - profile = { - debug = "-Og -fno-inline -fno-omit-frame-pointer"; - release = "-O3 -flto=thin"; - }; + profile = + let + release = "-O3 -flto=thin"; + profile = "-fprofile-instr-generate -fcoverage-mapping -fno-omit-frame-pointer -fno-sanitize-merge"; + in + { + debug = "-Og -fno-inline -fno-omit-frame-pointer"; + inherit release; + fuzz = "${release} ${profile} -fsanitize=address,leak,undefined,local-bounds"; + }; end = "-Qunused-arguments"; }; - link = { - linker = "-fuse-ld=lld"; - profile = { - debug = ""; + link = + let release = "-flto=thin -Wl,-O3 -Wl,-z,relro,-z,now"; + in + { + linker = "-fuse-ld=lld"; + profile = { + debug = ""; + inherit release; + fuzz = "${release} -shared-libasan -fsanitize=address,leak,undefined,local-bounds"; + }; + end = "-Qunused-arguments"; }; - end = "-Qunused-arguments"; - }; }; cflags = type: with flags.compile; "${machine} ${debug} ${security} ${errors} ${profile.${type}} ${end}"; cxxflags = type: cflags type; ldflags = type: with flags.link; "${linker} ${profile.${type}} ${end}"; configuration = type: { + profile = type; CFLAGS = cflags type; CXXFLAGS = cxxflags type; LDFLAGS = ldflags type; @@ -34,4 +46,5 @@ in { release = configuration "release"; debug = configuration "debug"; + fuzz = configuration "fuzz"; } From c9fc89ce66e06433a37bc88a55507fda2d198cb6 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sun, 23 Nov 2025 17:53:55 +0000 Subject: [PATCH 05/11] feat: add symbolizer for sanitize builds Signed-off-by: Daniel Noland --- default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/default.nix b/default.nix index 18b41cd..f895819 100644 --- a/default.nix +++ b/default.nix @@ -590,6 +590,7 @@ rec { fancy.llvmPackages.clang fancy.llvmPackages.compiler-rt fancy.llvmPackages.libclang.lib + fancy.llvmPackages.libllvm fancy.llvmPackages.lld getent glibc.dev From 1dc5decab33f08929df8a194447eecbbe8e53707 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sun, 23 Nov 2025 17:54:59 +0000 Subject: [PATCH 06/11] feat: add thread sanitizer fuzz profile Signed-off-by: Daniel Noland --- default.nix | 33 +++++++++++++++++++++++++++++++++ nix/dpdk/default.nix | 2 +- nix/flags.nix | 3 +++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index f895819..4f2c6d2 100644 --- a/default.nix +++ b/default.nix @@ -170,6 +170,10 @@ rec { ]; perl = null; cmakeFlags = orig.cmakeFlags ++ [ + # this allows thread sanitizer to build (thread sanitizer does not like -Wl,-z,defs or -Wl,--no-undefined) + # Normally I would say that disabling -Wl,--no-undefined is a bad idea, but we throw away all the shared + # libs and executables from this build anwyway, so it is quite harmless. + "-DSUPPORTS_NO_UNDEFINED=0" # todo: find a way to enable this for only fuzz_thread builds. "-DENABLE_STATIC=1" "-DNO_PYVERBS=1" "-DNO_MAN_PAGES=1" @@ -547,6 +551,26 @@ rec { }) ]; }).pkgsCross; + + pkgs.fuzz_thread = + (import toolchainPkgs.path { + overlays = [ + (self: prev: { + pkgsCross.gnu64 = import prev.path { + overlays = [ + llvm-overlay + helpersOverlay + fenix-overlay + rust-overlay + (crossOverlay { + build-flags = build-flags.fuzz_thread; + }) + ]; + }; + }) + ]; + }).pkgsCross; + sysrootPackageListFn = pkgs: with pkgs; [ dpdk @@ -627,6 +651,10 @@ rec { name = "${project-name}-env-release-sysroot-gnu64"; paths = sysrootPackageListFn pkgs.fuzz.gnu64; }; + sysroot.gnu64.fuzz_thread = toolchainPkgs.symlinkJoin { + name = "${project-name}-env-release-sysroot-gnu64"; + paths = sysrootPackageListFn pkgs.fuzz_thread.gnu64; + }; compile = toolchainPkgs.symlinkJoin { name = "${project-name}-env-compile"; paths = compileEnvPackageList; @@ -680,11 +708,13 @@ rec { sysroot.gnu64.debug = sysrootFn "debug"; sysroot.gnu64.release = sysrootFn "release"; sysroot.gnu64.fuzz = sysrootFn "fuzz"; + sysroot.gnu64.fuzz_thread = sysrootFn "fuzz_thread"; sysroots = with sysroot; [ gnu64.debug gnu64.release gnu64.fuzz + gnu64.fuzz_thread ]; compile-env = toolchainPkgs.symlinkJoin { @@ -694,12 +724,15 @@ rec { pkgs.debug.gnu64.glibc.static pkgs.release.gnu64.glibc.static pkgs.fuzz.gnu64.glibc.static + pkgs.fuzz_thread.gnu64.glibc.static pkgs.debug.gnu64.glibc.dev pkgs.release.gnu64.glibc.dev pkgs.fuzz.gnu64.glibc.dev + pkgs.fuzz_thread.gnu64.glibc.dev pkgs.debug.gnu64.glibc.out pkgs.release.gnu64.glibc.out pkgs.fuzz.gnu64.glibc.out + pkgs.fuzz_thread.gnu64.glibc.out ] ++ sysroots; }; diff --git a/nix/dpdk/default.nix b/nix/dpdk/default.nix index 4e76855..f74e189 100644 --- a/nix/dpdk/default.nix +++ b/nix/dpdk/default.nix @@ -259,7 +259,7 @@ stdenv.mkDerivation rec { "-Db_colorout=never" "-Db_coverage=false" "-Db_lto=true" - "-Db_lundef=true" + "-Db_lundef=false" "-Db_pch=true" "-Db_pgo=off" "-Db_pie=true" diff --git a/nix/flags.nix b/nix/flags.nix index ca06301..5ba36ad 100644 --- a/nix/flags.nix +++ b/nix/flags.nix @@ -15,6 +15,7 @@ let debug = "-Og -fno-inline -fno-omit-frame-pointer"; inherit release; fuzz = "${release} ${profile} -fsanitize=address,leak,undefined,local-bounds"; + fuzz_thread = "${release} ${profile} -fsanitize=thread"; }; end = "-Qunused-arguments"; }; @@ -28,6 +29,7 @@ let debug = ""; inherit release; fuzz = "${release} -shared-libasan -fsanitize=address,leak,undefined,local-bounds"; + fuzz_thread = "${release} -fsanitize=thread -Wl,--allow-shlib-undefined"; }; end = "-Qunused-arguments"; }; @@ -47,4 +49,5 @@ in release = configuration "release"; debug = configuration "debug"; fuzz = configuration "fuzz"; + fuzz_thread = configuration "fuzz_thread"; } From 7090648024a4321dfeb43a0acad1ffd48f47da1b Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sun, 23 Nov 2025 17:55:40 +0000 Subject: [PATCH 07/11] feat(dpdk): increase max numa nodes Signed-off-by: Daniel Noland --- nix/dpdk/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/dpdk/default.nix b/nix/dpdk/default.nix index f74e189..15f81fd 100644 --- a/nix/dpdk/default.nix +++ b/nix/dpdk/default.nix @@ -268,7 +268,7 @@ stdenv.mkDerivation rec { "-Denable_docs=false" "-Denable_driver_sdk=false" "-Dibverbs_link=static" - "-Dmax_numa_nodes=4" + "-Dmax_numa_nodes=8" "-Dstrip=false" # We should strip binaries in a separate step to preserve detached debug info "-Dtests=false" # Running DPDK tests in CI is usually silly "-Duse_hpet=false" From 5f9c20c3cabf2898b4b325ae14976b6dbbda345f Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sun, 23 Nov 2025 17:56:32 +0000 Subject: [PATCH 08/11] chore(base-image): format Signed-off-by: Daniel Noland --- nix/base-image/default.nix | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/nix/base-image/default.nix b/nix/base-image/default.nix index 1a6a0c5..b8e2ccc 100644 --- a/nix/base-image/default.nix +++ b/nix/base-image/default.nix @@ -1,19 +1,20 @@ { - stdenv, - glibc, - libgcc, -}: stdenv.mkDerivation rec { - pname = "base-image"; - version = "25.07"; + stdenv, + glibc, + libgcc, +}: +stdenv.mkDerivation { + pname = "base-image"; + version = "25.07"; - src = ./root; + src = ./root; - installPhase = '' - cp -r $src $out - ''; - dontUnpack = true; - buildInputs = [ - glibc.out - libgcc.libgcc - ]; + installPhase = '' + cp -r $src $out + ''; + dontUnpack = true; + buildInputs = [ + glibc.out + libgcc.libgcc + ]; } From fd1f9ecd2d672e0d6505979e918a873235bc3469 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sun, 23 Nov 2025 17:56:55 +0000 Subject: [PATCH 09/11] docs: remove obsolete design docs These are very very out of date. Signed-off-by: Daniel Noland --- design-docs/book.toml | 6 -- design-docs/src/SUMMARY.md | 4 - design-docs/src/principals/index.md | 105 ------------------------- design-docs/src/principals/no-shell.md | 50 ------------ 4 files changed, 165 deletions(-) delete mode 100644 design-docs/book.toml delete mode 100644 design-docs/src/SUMMARY.md delete mode 100644 design-docs/src/principals/index.md delete mode 100644 design-docs/src/principals/no-shell.md diff --git a/design-docs/book.toml b/design-docs/book.toml deleted file mode 100644 index baaa4cf..0000000 --- a/design-docs/book.toml +++ /dev/null @@ -1,6 +0,0 @@ -[book] -authors = ["Daniel Noland"] -language = "en" -multilingual = false -src = "src" -title = "dpdk-sys design docs" diff --git a/design-docs/src/SUMMARY.md b/design-docs/src/SUMMARY.md deleted file mode 100644 index 8fde30f..0000000 --- a/design-docs/src/SUMMARY.md +++ /dev/null @@ -1,4 +0,0 @@ -# Summary - -- [Design Principals](./principals/index.md) - - [Why no shell in prod?](./principals/no-shell.md) diff --git a/design-docs/src/principals/index.md b/design-docs/src/principals/index.md deleted file mode 100644 index dca741a..0000000 --- a/design-docs/src/principals/index.md +++ /dev/null @@ -1,105 +0,0 @@ -# Design Principals - -> _Or how I stopped worrying and learned to tolerate the build._ - -## Trial Balloons - -Our product should have a good answer to the following situations. -Any product we ship should be able to roll with any of these punches. - -1. Bug or vulnerability on a production system -2. Bug or vulnerability on a deployed dependency. - -## Possible Responses - -1. **Bad Idea:** Ask to get a shell on the production system and start debugging. - - **Analysis**: ☢️⚠️☣️☠️ _**[NEVER DO THIS!]**_ ☠️☣️⚠️☢️ - -2. **Bad Idea:** Get on a call with the customer and ask them to describe the problem in detail. - - **Analysis**: ¯\\\_(ツ)\_/¯ - This is likely a necessary first step, but it is not going to cut it in most cases. - At minimum, we need to understand that - - 1. The customer may not be able to provide enough details to respond in a useful way. - 2. What details the customer can or will provide may not be accurate or complete. Maybe they don't know how to get the info we need or are legally prohibited from sharing it. - 3. Debugging is a time-consuming process. The customer has limits to their patience and availability. This is especially true when their internet gateway is down or their network is under attack. - -3. **Bad Idea:** Use a typical development break/fix cycle - - **Analysis:** Avoid this plan. - - For starters, break/fix doesn't efficiently help us find the cause of the problem. - We might be able to make it work, but it is very inefficient to push a commit, wait for CI, and pray you are even on the right track. - - More, break/fix loops are a very frustrating experience for the customer at the best of times, but they are even worse in our case. - - The whole premise of this project is to provide an internet gateway for a data center. - Every time our attempted fix doesn't work, we have disrupted the customer's entire network. - Public internet BGP timers make this far worse in our case. - Public internet BGP timers are tuned for flap mitigation. - This means that we will be down for the close order of 30 minutes every time we try a fix in production. - - Now imagine it takes 19 attempts to isolate and fix the problem. - That's 9.5 hours of downtime in the best case scenario. - **This method isn't going to work.** - -4. **Bad Idea:** Shrug and ask the customer to upgrade to "the latest" software. - - **Analysis:** This is a non-starter for us. - - The customer is running a production system in this trial balloon. - This is asking them to (outside of a maintenance window) - - 1. drop their BGP sessions, and then - 2. upgrade to software they haven't qualified internally, - 3. to address a problem that we don't even know if we have solved in an update, - 4. because we didn't have the ability to test the fix in a controlled environment. - - Yeah. Don't do this if at all possible. - -5. **Good Idea:** - - * Duplicate the customer's production environment in the lab. - * Reproduce the bug. - * Fix the bug. - * Test the fix. - * Roll out the fix in a controlled manner. - - **Analysis:** This is the best of the bad options we have. - - By duplicating the customer's environment, we can test our fixes in a controlled environment. - The break fix cycle is faster, the job is less stressful, and we can be more confident in our fixes. - - This plan has two fundamental requirements: - - 1. That you can semi-accurately replicate the hardware environment. - 2. That you can accurately replicate the software environment. - - The first requirement is a challenge which we may not be able to meet. - Who knows what kind of hardware the customer has our product plugged into? - But that doesn't mean that this plan is useless, it just means that it will work better in some cases than others. - More, if we can't actually replicate the hardware environment, we can (and should) still use this plan to test our fixes in a controlled environment. - - The second requirement is a challenge we _really_ need to meet. - It sounds easy enough on the surface: "just" install the same software the customer is running on a lab machine. - Problems with that: - - 1. Do you know the exact version of every piece of software the customer is running? - The answer to this question should be yes! - If the answer is not yes then we have a failure in our operational procedures. - 2. Do you know the exact configuration of every piece of software the customer is running? - The answer to this question should be _kinda_. - You **won't** get the exact configs (encryption keys and such), but you can and should get a high-level description of the workload it is running and enough telemetry to understand what it is doing roughly. - 3. **<>**: Assuming you know the versions and the config they are using, can you - 1. Find the source code which produced that software? - 2. Build that source code into the **exact same** software they are running? - 3. Debug the issue? - 4. Compose a fix and apply the patch **and no other patches** to the flawed software? - 5. Test the patch in a controlled environment? - 6. Roll out the patch to the customer's production environment with reasonable confidence that it will work? - - -[NEVER DO THIS!]: ./no-shell.md "Why getting shell is a bad idea" - diff --git a/design-docs/src/principals/no-shell.md b/design-docs/src/principals/no-shell.md deleted file mode 100644 index 69819ef..0000000 --- a/design-docs/src/principals/no-shell.md +++ /dev/null @@ -1,50 +0,0 @@ -# Why "getting shell" is a bad idea for production systems - -## The customer's security team should not allow this - -Let's game out this conversation with their security team. - -> **Us:** Hey, can I get a shell on your production system to investigate that bug you reported? - -> **Them:** Oh that system that is processing _our_ customer's traffic and adjudicating their TLS keys? -> -> No. 🖕️ - -## The customer's legal team should not allow this. - -Let's game out this conversation with their legal team - -> **Us:** Hey, can I get a shell on your production system to investigate that bug you reported? - -> **Them:** No. ❤️ -> -> (decides this was a billable conversation) -> -> That will be $18,372.29. -> Check or credit card? - -They don't want the liability. ¯\\\_(ツ)\_/¯ - -## Our engineering policy should not allow this - -The entire concept of getting a shell on a production system fundamentally violates the scientific method and best practices for debugging. - -Getting shell is an inherently "impure" proposition. -If you get shell on the production system, then we can't really "trust" that system anymore. -The simple act of mutating the system this way is problematic. -Even if you fix whatever problem you set out to fix, we will no longer be sure of the system's state. -How do we test this fix? -How do we roll out the fix to other systems? - -Even if you are confident you are doing read-only operations, you are still exposing yourself to massive risk. -You are one typo away from a massive security incident _for which Hedgehog will be liable_. - -_We should never put a Hedgehog engineer in this position._ It is the perfect blend of useless and risky. - - -## Our legal team should not allow this. - -**Never** put a Hedgehog **employee** in a position where they can be plausibly accused of unauthorized exfiltration of customer data. -Especially cryptographic keys. - -**Never** put a Hedgehog **customer** in a position where they need to explain to _their_ customers why they allowed us to poke around on a system responsible for their security. From 8e1b9f965a81188f9d34f7111865a0aba9a8b10a Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sun, 23 Nov 2025 17:57:31 +0000 Subject: [PATCH 10/11] docs(dpdk): remove misleading comments Signed-off-by: Daniel Noland --- nix/dpdk/default.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nix/dpdk/default.nix b/nix/dpdk/default.nix index 15f81fd..d0d34d8 100644 --- a/nix/dpdk/default.nix +++ b/nix/dpdk/default.nix @@ -105,11 +105,11 @@ stdenv.mkDerivation rec { "cryptodev" # required for vhost "dmadev" # required by vhost "ethdev" - "eventdev" # needed for i40e + "eventdev" "pci" - "security" # needed for i40e - "timer" # needed for i40e - "vhost" # enabled to facilitate testing with vm runner + "security" + "timer" + "vhost" ]; disabledDrivers = [ From cb21dd1d1398f4b81aa4eade7707485dbe78dd18 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sun, 23 Nov 2025 17:57:51 +0000 Subject: [PATCH 11/11] feat: add profile name to build flags Signed-off-by: Daniel Noland --- nix/flags.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/flags.nix b/nix/flags.nix index 5ba36ad..89fe8ba 100644 --- a/nix/flags.nix +++ b/nix/flags.nix @@ -39,7 +39,6 @@ let cxxflags = type: cflags type; ldflags = type: with flags.link; "${linker} ${profile.${type}} ${end}"; configuration = type: { - profile = type; CFLAGS = cflags type; CXXFLAGS = cxxflags type; LDFLAGS = ldflags type;