diff --git a/README.md b/README.md index 0da145d0fe29a..8e3f67e91d869 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,263 @@ -NetBSD -====== +NetBSD-perf +=========== -NetBSD is a free, fast, secure, and highly portable Unix-like Open -Source operating system. It is available for a [wide range of -platforms](https://wiki.NetBSD.org/ports/), from large-scale servers -and powerful desktop systems to handheld and embedded devices. +This is my attempt at replicating iMil results that were reported on https://mail-index.netbsd.org/tech-kern/2024/01/23/msg029450.html and https://old.reddit.com/r/BSD/comments/197vfmq by using the perf branch he published on https://github.com/NetBSDfr/NetBSD-src -Building --------- +Currently, I'm still about 5x slower as I boot in 200 ms, but this is a good beginning: -You can cross-build NetBSD from most UNIX-like operating systems. -To build for amd64 (x86_64), in the src directory: +``` +# sh test-netbsd.sh +[ 1.0000000] cpu_rng: rdrand/rdseed +[ 1.0000000] entropy: ready +[ 1.0000000] NetBSD 10.99.10 (MICROVM) Notice: this software is protected by copyright +[ 1.0000000] Copyright (c) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, +[ 1.0000000] 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, +[ 1.0000000] 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, +[ 1.0000000] 2024 +[ 1.0000000] The NetBSD Foundation, Inc. All rights reserved. +[ 1.0000000] Copyright (c) 1982, 1986, 1989, 1991, 1993 +[ 1.0000000] The Regents of the University of California. All rights reserved. - ./build.sh -U -u -j4 -m amd64 -O ~/obj release +[ 1.0000000] NetBSD 10.99.10 (MICROVM) #8: Wed Jan 31 09:01:58 CST 2024 +[ 1.0000000] charlotte@x1ng2:/home/charlotte/obj/sys/arch/amd64/compile/MICROVM +[ 1.0000000] total memory = 127 MB +[ 1.0000000] avail memory = 77184 KB +[ 1.0000000] timecounter: Timecounters tick every 10.000 msec +[ 1.0000000] timecounter: Timecounter "i8254" frequency 1193182 Hz quality 100 +[ 1.0000030] Hypervisor: KVM +[ 1.0000030] VMM: Generic PVH +[ 1.0000030] mainbus0 (root) +[ 1.0000030] mainbus0: Intel MP Specification (Version 1.4) (QBOOT 000000000000) +[ 1.0000030] cpu0 at mainbus0 apid 0 +[ 1.0000030] cpu0: Use lfence to serialize rdtsc +[ 1.0000030] got tsc from vmware compatible cpuid +[ 1.0000030] cpu0: TSC freq CPUID 2496000000 Hz +[ 1.0000030] cpu0: 12th Gen Intel(R) Core(TM) i7-1270P, id 0x906a2 +[ 1.0000030] cpu0: node 0, package 0, core 0, smt 0 +[ 1.0000030] cpu1 at mainbus0 apid 1 +[ 1.0000030] cpu1: 12th Gen Intel(R) Core(TM) i7-1270P, id 0x906a2 +[ 1.0000030] cpu1: node 0, package 0, core 1, smt 0 +[ 1.0000030] mpbios: bus 0 is type ISA +[ 1.0000030] ioapic0 at mainbus0 apid 3: pa 0xfec00000, version 0x20, 24 pins +[ 1.0000030] isa0 at mainbus0 +[ 1.0000030] com0 at isa0 port 0x3f8-0x3ff irq 4: ns16550a, 16-byte FIFO +[ 1.0000030] com0: console +[ 1.0000030] allocated pic ioapic0 type edge pin 4 level 8 to cpu0 slot 0 idt entry 129 +[ 1.0000030] pv0 at mainbus0 +[ 1.0000030] virtio0 at pv0 +[ 1.0000030] kernel parameters: root=ld0c rw console=com -z -v virtio_mmio.device=512@0xfeb00e00:12 virtio_mmio.device=512@0xfeb00c00:11 +[ 1.0000030] viommio: 512@0xfeb00e00:12 +[ 1.0000030] virtio0: VirtIO-MMIO-v2 +[ 1.0000030] virtio0: entropy device (id 4, rev. 0x01) +[ 1.0000030] viornd0 at virtio0: features: 0x110000000 +[ 1.0000030] virtio0: allocated 32768 byte for virtqueue 0 for Entropy request, size 1024 +[ 1.0000030] allocated pic ioapic0 type level pin 12 level 6 to cpu0 slot 1 idt entry 96 +[ 1.0000030] virtio0: interrupting on -1 +[ 1.0000030] virtio1 at pv0 +[ 1.0000030] viommio: 512@0xfeb00c00:11 +[ 1.0000030] virtio1: VirtIO-MMIO-v2 +[ 1.0000030] virtio1: block device (id 2, rev. 0x01) +[ 1.0000030] ld0 at virtio1: features: 0x110000a54 +[ 1.0000030] virtio1: allocated 4227072 byte for virtqueue 0 for I/O request, size 1024 +[ 1.0000030] virtio1: using 4194304 byte (262144 entries) indirect descriptors +[ 1.0000030] allocated pic ioapic0 type level pin 11 level 6 to cpu0 slot 2 idt entry 97 +[ 1.0000030] virtio1: interrupting on -1 +[ 1.0000030] ld0: 512 MB, 1040 cyl, 16 head, 63 sec, 512 bytes/sect x 1048576 sectors +[ 1.0000030] pvclock0 at pv0 +[ 1.0000030] timecounter: Timecounter "pvclock0" frequency 1000000000 Hz quality 1500 +[ 1.0001371] timecounter: Timecounter "lapic" frequency 999983000 Hz quality -100 +[ 1.0001371] timecounter: Timecounter "clockinterrupt" frequency 100 Hz quality 0 +[ 1.0001371] timecounter: Timecounter "TSC" frequency 2496000000 Hz quality 3000 +[ 1.0575299] boot device: ld0 +[ 1.0575299] root on ld0c dumps on ld0b +[ 1.0575299] root file system type: ffs +[ 1.0575299] kern.module.path=/stand/amd64/10.99.10/modules +[ 1.0611265] boot: 198ms (entry tsc: 251543752) +[ 1.0611265] exec /sbin/init: error 8 +[ 1.0611265] init: trying /sbin/oinit +bslinit v7 starting on NetBSD, will handle 12 signals, reaping zombies every 5 s + arg 0: oinit +oinit: Failed to mount procfs on /proc: No such device +oinit: Failed to mount kernfs on /kern: Invalid argument +``` -Additional build information available in the [BUILDING](BUILDING) file. +For reference, using the exact same filesystem but with the original kernel: -Binaries --------- +``` +[ 1.0000000] cpu_rng: rdrand/rdseed +[ 1.0000000] entropy: ready +[ 1.0000000] NetBSD 10.99.10 (MICROVM) Notice: this software is protected by copyright +[ 1.0000000] Copyright (c) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, +[ 1.0000000] 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, +[ 1.0000000] 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, +[ 1.0000000] 2024 +[ 1.0000000] The NetBSD Foundation, Inc. All rights reserved. +[ 1.0000000] Copyright (c) 1982, 1986, 1989, 1991, 1993 +[ 1.0000000] The Regents of the University of California. All rights reserved. -- [Daily builds](https://nycdn.NetBSD.org/pub/NetBSD-daily/HEAD/latest/) -- [Releases](https://cdn.NetBSD.org/pub/NetBSD/) +[ 1.0000000] NetBSD 10.99.10 (MICROVM) #1891: Tue Jan 23 06:43:58 CET 2024 +[ 1.0000000] imil@tatooine:/home/imil/src/github.com/NetBSD-src/sys/arch/amd64/compile/obj/MICROVM +[ 1.0000000] total memory = 127 MB +[ 1.0000000] avail memory = 77184 KB +[ 1.0000000] timecounter: Timecounters tick every 10.000 msec +[ 1.0000000] timecounter: Timecounter "i8254" frequency 1193182 Hz quality 100 +[ 1.0000030] Hypervisor: KVM +[ 1.0000030] VMM: Generic PVH +[ 1.0000030] mainbus0 (root) +[ 1.0000030] mainbus0: Intel MP Specification (Version 1.4) (QBOOT 000000000000) +[ 1.0000030] cpu0 at mainbus0 apid 0 +[ 1.0000030] cpu0: Use lfence to serialize rdtsc +[ 1.0000030] got tsc from vmware compatible cpuid +[ 1.0000030] cpu0: TSC freq CPUID 2496000000 Hz +[ 1.0000030] cpu0: 12th Gen Intel(R) Core(TM) i7-1270P, id 0x906a2 +[ 1.0000030] cpu0: node 0, package 0, core 0, smt 0 +[ 1.0000030] mpbios: bus 0 is type ISA +[ 1.0000030] ioapic0 at mainbus0 apid 2: pa 0xfec00000, version 0x20, 24 pins +[ 1.0000030] isa0 at mainbus0 +[ 1.0000030] com0 at isa0 port 0x3f8-0x3ff irq 4: ns16550a, 16-byte FIFO +[ 1.0000030] com0: console +[ 1.0000030] allocated pic ioapic0 type edge pin 4 level 8 to cpu0 slot 0 idt entry 129 +[ 1.0000030] pv0 at mainbus0 +[ 1.0000030] virtio0 at pv0 +[ 1.0000030] kernel parameters: root=ld0c console=com rw -z -v virtio_mmio.device=512@0xfeb00e00:12 +[ 1.0000030] viommio: 512@0xfeb00e00:12 +[ 1.0000030] virtio0: VirtIO-MMIO-v2 +[ 1.0000030] virtio0: block device (id 2, rev. 0x01) +[ 1.0000030] ld0 at virtio0: features: 0x110000a54 +[ 1.0000030] virtio0: allocated 4227072 byte for virtqueue 0 for I/O request, size 1024 +[ 1.0000030] virtio0: using 4194304 byte (262144 entries) indirect descriptors +[ 1.0000030] allocated pic ioapic0 type level pin 12 level 6 to cpu0 slot 1 idt entry 96 +[ 1.0000030] virtio0: interrupting on -1 +[ 1.0000030] ld0: 512 MB, 1040 cyl, 16 head, 63 sec, 512 bytes/sect x 1048576 sectors +[ 1.0000030] pvclock0 at pv0 +[ 1.0000030] timecounter: Timecounter "pvclock0" frequency 1000000000 Hz quality 1500 +[ 1.0070044] allocated pic ioapic0 type level pin 2 level 7 to cpu0 slot 2 idt entry 112 +[ 1.0232075] timecounter: Timecounter "clockinterrupt" frequency 100 Hz quality 0 +[ 1.0232075] timecounter: Timecounter "TSC" frequency 2496000000 Hz quality 3000 +[ 1.0244222] boot device: ld0 +[ 1.0244222] root on ld0c dumps on ld0b +[ 1.0244222] root file system type: ffs +[ 1.0244222] kern.module.path=/stand/amd64/10.99.10/modules +[ 1.0285561] boot: 72ms (entry tsc: 168962022) +[ 1.0294073] exec /sbin/init: error 8 +[ 1.0294073] init: trying /sbin/oinit +bslinit v7 starting on NetBSD, will handle 12 signals, reaping zombies every 30 s + arg 0: oinit +oinit: mount /proc failed: No such device +``` -Testing -------- +Note also that the MMIO in qemu append doesn't seem to survive a reboot: -On a running NetBSD system: +``` +# init: can't add utmpx record for `system boot': No such file or directory +init: can't add utmpx record for `system down': No such file or directory +qemu-system-x86_64: terminating on signal 2 +[ 1.0000000] cpu_rng: rdrand/rdseed +[ 1.0000000] entropy: ready +[ 1.0000000] NetBSD 10.99.10 (MICROVM) Notice: this software is protected by copyright +[ 1.0000000] Copyright (c) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, +[ 1.0000000] 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, +[ 1.0000000] 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, +[ 1.0000000] 2024 +[ 1.0000000] The NetBSD Foundation, Inc. All rights reserved. +[ 1.0000000] Copyright (c) 1982, 1986, 1989, 1991, 1993 +[ 1.0000000] The Regents of the University of California. All rights reserved. +(...) +[ 1.0155438] root on ld0c dumps on ld0b +[ 1.0155438] vfs_mountroot: can't open root device +[ 1.0155438] cannot mount root, error = 6 +[ 1.0155438] root device (default ld0c): qemu-system-x86_64: terminating on signal 2 +``` - cd /usr/tests; atf-run | atf-report +Also the performance is not matching yet: -Troubleshooting ---------------- +``` +# sysctl -b debug.tslog > /tmp/tslog +0x0 275799932 ENTER main +0x2 295914148 THREAD idle/0 +0x3 295927199 THREAD softnet/0 +0x4 295946662 THREAD softbio/0 +0x5 295962333 THREAD softclk/0 +0x6 296029811 THREAD softser/0 +0x7 296101498 THREAD xcall/0 +0x8 296160256 THREAD modunload +0x9 296242469 THREAD pooldisp +0xa 297478103 THREAD iflnkst +0xb 297483183 THREAD ifwdog +0xc 297495935 THREAD sopendfree +0xd 297511476 THREAD pmfevent +0xe 297516835 THREAD pmfsuspend +0x0 300470812 ENTER config_attach_internal mainbus +0x0 302535470 ENTER config_attach_internal cpu +0x0 334981541 EXIT config_attach_internal +0x0 334995085 ENTER config_attach_internal cpu +0xf 335760098 THREAD idle/1 +0x10 335798245 THREAD softnet/1 +0x11 335825899 THREAD softbio/1 +0x12 335850895 THREAD softclk/1 +0x13 335875037 THREAD softser/1 +0x14 335928985 THREAD xcall/1 +0x0 390242684 EXIT config_attach_internal +0x0 391082852 ENTER config_attach_internal ioapic +0x0 569050116 EXIT config_attach_internal +0x0 569084610 ENTER config_attach_internal isa +0x0 569827191 ENTER config_attach_internal com +0x0 589518701 EXIT config_attach_internal +0x0 589525847 EXIT config_attach_internal +0x0 589529639 ENTER config_attach_internal pv +0x0 590079490 ENTER config_attach_internal virtio +0x0 595773519 ENTER config_attach_internal viornd +0x0 683067128 EXIT config_attach_internal +0x0 683078608 ENTER config_attach_internal virtio +0x0 696013228 ENTER config_attach_internal ld +0x0 828659203 EXIT config_attach_internal +0x0 828662014 EXIT config_attach_internal +0x0 828663123 EXIT config_attach_internal +0x0 828673724 ENTER config_attach_internal pvclock +0x0 831876908 EXIT config_attach_internal +0x0 831878663 EXIT config_attach_internal +0x0 831884395 EXIT config_attach_internal +0x15 833672855 THREAD entbutler +0x16 964636813 THREAD configintr +0x19 964651639 THREAD configintr +0x18 965754770 THREAD configintr +0x19 965772930 THREAD configintr +0x6f 965776243 THREAD configintr +0x6d 965782151 THREAD configintr +0x6a 965785487 THREAD configintr +0x61 965789332 THREAD configintr +0x1e 965860007 THREAD vmem_rehash +0x1f 965973444 THREAD rt_timer +0x36 965988880 THREAD icmp_wqinput/0 +0x60 966064940 THREAD icmp_wqinput/1 +0x61 966403212 THREAD nd6_timer +0x62 966673756 THREAD icmp6_wqinput/0 +0x63 966678725 THREAD icmp6_wqinput/1 +0x64 966715553 THREAD unpgc +0x65 966741324 THREAD rt_free +0x6b 976822639 THREAD configroot +0x67 976832285 THREAD configroot +0x68 977580970 THREAD pgdaemon +0x69 977592989 THREAD ioflush +0x6a 977695961 THREAD pooldrain +0x0 977726594 EXIT main +``` -- Send bugs and patches [via web form](https://www.NetBSD.org/cgi-bin/sendpr.cgi?gndb=netbsd). -- Subscribe to the [mailing lists](https://www.NetBSD.org/mailinglists/). - The [netbsd-users](https://www.NetBSD.org/mailinglists/#netbsd-users) list is a good choice for many problems; watch [current-users](https://www.NetBSD.org/mailinglists/#current-users) if you follow the bleeding edge of NetBSD-current. -- Join the community IRC channel [#netbsd @ libera.chat](https://web.libera.chat/#netbsd). +Building +-------- -Latest sources --------------- +The all-in-one compile.sh will build a kernel for you in ~/obj/sys/arch/amd64/compile/MICROVM/netbsd -To fetch the main CVS repository: + sh ./compile.sh - cvs -d anoncvs@anoncvs.NetBSD.org:/cvsroot checkout -P src +Binaries +-------- -To work in the Git mirror, which is updated every few hours from CVS: +When it'll be ready, you'll see the releases on the right handside - git clone https://github.com/NetBSD/src.git +TODO +---- -Additional Links ----------------- +Adding flamecharts to understand why it's so slow -- [The NetBSD Guide](https://www.NetBSD.org/docs/guide/en/) -- [NetBSD manual pages](https://man.NetBSD.org/) -- [NetBSD Cross-Reference](https://nxr.NetBSD.org/) +Adding a basic rootfs with simple tools such as bslinit, the tslog script to create flamecharts, and a shell diff --git a/compile.sh b/compile.sh new file mode 100755 index 0000000000000..96b338c7bbbbc --- /dev/null +++ b/compile.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# Replication of iMil result reported on https://mail-index.netbsd.org/tech-kern/2024/01/23/msg029450.html +m="amd64" +a="x86_64" +CPUS=4 +PATHSKC="sys/arch/${m}/conf/" +TOOLCHAIN="${HOME}/obj" + +[ -z ${1} ] \ + && printf "# Defaulting to MICROVM\n" \ + && KERNELCONFIG="MICROVM" \ + || KERNELCONFIG="${1}" + +[ ! -f "${PATHSKC}/${1}" ] \ + && echo "Missing ${KERNELCONFIG} in ${PATHSKC}" \ + && exit 1 + +[ -f "${PATHSKC}/${1}" ] \ + && printf "# Using kernel config ${1}" \ + && KERNELCONFIG="${1}" \ + && stat -c "%y %A %n size: %s" ${PATHSKC}/${KERNELCONFIG} \ + || echo "Missing ${KERNELCONFIG} in ${PATHSKC}" + +# Get the sources +#git clone https://github.com/NetBSDfr/NetBSD-src +git clone https://github.com/csdvrx/NetBSD-fr-src + +# Show the branches available +git branch -a + +# Change to result-replication, based on the perf branch +git branch| grep result-replication2 || git checkout result-replication2 || exit 2 + +#cf https://www.netbsd.org/docs/guide/en/chap-build.html#chap-boot-cross-compiling-kernel +echo "# Building the cross compilation toolchain into ${TOOLCHAIN}": + +stat -c "%y %A %n size: %s" ${TOOLCHAIN}/tooldir*x86_64/bin/nbmake \ + || sh ./build.sh -U -O ${TOOLCHAIN} -j ${CPUS} -m ${m} -a ${a} tools \ + || exit 3 + +echo "# Using the kernel configuration ${KERNELCONFIG}": +stat -c "%y %A %n size: %s" sys/arch/amd64/conf/${KERNELCONFIG} \ + || exit 4 + +#echo "# Checking pvclock is disabled as it requires the missing " +#grep -v ^pvclock sys/arch/amd64/conf/${KERNELCONFIG} \ +# || exit 5 + +echo "# Refreshing the kernel build with -u" \ + && sh ./build.sh -u -U -O ${TOOLCHAIN} -j ${CPUS} -m ${m} kernel=${KERNELCONFIG} \ + && stat -c "%y %A %n size: %s" ${TOOLCHAIN}/sys/arch/${m}/compile/${KERNELCONFIG}/netbsd \ + || exit 6 + +## Alternatives: +# Use nbconfig and nbmake directly like sys/arch/amd64/conf/kern-compile.sh: +#NBCONFIG=$( ls ~/obj/tooldir*${a}/bin/nbconfig | tail -n1 ) +#NBMAKE=$( ls ~/obj/tooldir*${a}/bin/nbmake-${m} | tail -n1 ) +#printf "\tnbconfig:\t${NBCONFIG}\n\tnbmake-xxx:\t${NBMAKE}\n\n" +#${NBCONFIG} ${KERNELCONFIG} | grep "Build directory is" \ +# && cd ${TOOLCHAIN}/sys/arch/${m}/compile \ +# && ${NBMAKE} depend + +# Don't just update the kernel: remove the '-u' that's for update only + +# Compile from scratch the release and the kernel +#sh ./build.sh -U -O ${TOOLCHAIN} -j ${CPUS} -m amd64 -a x86_64 release kernel=${KERNELCONFIG} + +# TODO: add qemu scripts, which should have console=com etc +# TODO: add a basic init_args parsing arguments like sys/arch/x86/x86/x86_autoconf.c diff --git a/sys/arch/amd64/conf/MICROVM b/sys/arch/amd64/conf/MICROVM index bc84c3ed6cfc8..38e98de9756d8 100644 --- a/sys/arch/amd64/conf/MICROVM +++ b/sys/arch/amd64/conf/MICROVM @@ -185,6 +185,7 @@ makeoptions COPTS="-O2 -fno-omit-frame-pointer" file-system FFS file-system EXT2FS file-system KERNFS +file-system PROCFS file-system MFS # File system options diff --git a/sys/arch/x86/x86/cpu.c b/sys/arch/x86/x86/cpu.c index 0271fb261965b..bd49033792ccd 100644 --- a/sys/arch/x86/x86/cpu.c +++ b/sys/arch/x86/x86/cpu.c @@ -74,7 +74,7 @@ __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.209 2023/07/16 19:55:43 riastradh Exp $"); #include "ioapic.h" #include "acpica.h" #include "hpet.h" -#include "pvclock.h" +//#include "pvclock.h" #include #include #include diff --git a/sys/arch/x86/x86/lapic.c b/sys/arch/x86/x86/lapic.c index 7f8cc59a0980f..e5307f7382338 100644 --- a/sys/arch/x86/x86/lapic.c +++ b/sys/arch/x86/x86/lapic.c @@ -42,7 +42,7 @@ __KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.89 2022/09/07 00:40:19 knakahara Exp $") #include "opt_multiprocessor.h" #include "opt_ntp.h" #include "opt_xen.h" -#include "pvclock.h" +//#include "pvclock.h" #include #include #include diff --git a/sys/kern/kern_tslog.c b/sys/kern/kern_tslog.c index 15819cf6188a9..e3e050342d8da 100644 --- a/sys/kern/kern_tslog.c +++ b/sys/kern/kern_tslog.c @@ -64,7 +64,7 @@ #include #include -#include +#include #include #ifndef TSLOGSIZE