From a1709c76db4ba70de526d7eec18c6b0637ebf7b0 Mon Sep 17 00:00:00 2001 From: Jean-Francois Roy Date: Mon, 23 Sep 2024 09:19:51 -0700 Subject: [PATCH] feat: enable module unloading and memory hotplug (for NVIDIA UVM) This patch enables module unloading. This will be needed by machined to unload kernel modules when the machine config is changed. Module unloading is also required by NVIDIA driver containers. These containers are managed by the NVIDIA GPU Operator and will load and unload the nvidia kernel modules as part of their lifecycle. This patch enables memory hotplug and a few other settings (e.g. `DEVICE_MEMORY`) to support NVIDIA unified virtual memory (UVM) and the NVIDIA GPU Operator. Memory hotplug is required to provide `/sys/devices/system/memory`. In particular, driver containers use `/sys/devices/system/memory/auto_online_blocks` to manage NVIDIA UVM. Signed-off-by: Jean-Francois Roy --- kernel/build/config-amd64 | 30 +++++++++++++++++++++++++++--- kernel/build/config-arm64 | 24 ++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/kernel/build/config-amd64 b/kernel/build/config-amd64 index 42d5c314f..ac1e2ba9b 100644 --- a/kernel/build/config-amd64 +++ b/kernel/build/config-amd64 @@ -447,6 +447,7 @@ CONFIG_X86_64_ACPI_NUMA=y CONFIG_NODES_SHIFT=6 CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_ARCH_SPARSEMEM_DEFAULT=y +# CONFIG_ARCH_MEMORY_PROBE is not set CONFIG_ILLEGAL_POINTER_VALUE=0xdead000000000000 # CONFIG_X86_PMEM_LEGACY is not set CONFIG_X86_CHECK_BIOS_CORRUPTION=y @@ -494,7 +495,7 @@ CONFIG_X86_NEED_RELOCS=y CONFIG_PHYSICAL_ALIGN=0x200000 CONFIG_DYNAMIC_MEMORY_LAYOUT=y CONFIG_RANDOMIZE_MEMORY=y -CONFIG_RANDOMIZE_MEMORY_PHYSICAL_PADDING=0x0 +CONFIG_RANDOMIZE_MEMORY_PHYSICAL_PADDING=0x1 CONFIG_ADDRESS_MASKING=y CONFIG_HOTPLUG_CPU=y # CONFIG_COMPAT_VDSO is not set @@ -587,6 +588,7 @@ CONFIG_ACPI_TABLE_UPGRADE=y # CONFIG_ACPI_DEBUG is not set # CONFIG_ACPI_PCI_SLOT is not set CONFIG_ACPI_CONTAINER=y +CONFIG_ACPI_HOTPLUG_MEMORY=y CONFIG_ACPI_HOTPLUG_IOAPIC=y # CONFIG_ACPI_SBS is not set # CONFIG_ACPI_HED is not set @@ -887,7 +889,9 @@ CONFIG_MODULE_SIG_FORMAT=y CONFIG_MODULES=y # CONFIG_MODULE_DEBUG is not set # CONFIG_MODULE_FORCE_LOAD is not set -# CONFIG_MODULE_UNLOAD is not set +CONFIG_MODULE_UNLOAD=y +# CONFIG_MODULE_FORCE_UNLOAD is not set +# CONFIG_MODULE_UNLOAD_TAINT_TRACKING is not set CONFIG_MODVERSIONS=y CONFIG_ASM_MODVERSIONS=y CONFIG_MODULE_SRCVERSION_ALL=y @@ -1030,9 +1034,16 @@ CONFIG_SPARSEMEM_VMEMMAP=y CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP=y CONFIG_ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP=y CONFIG_HAVE_FAST_GUP=y +CONFIG_NUMA_KEEP_MEMINFO=y +CONFIG_MEMORY_ISOLATION=y CONFIG_EXCLUSIVE_SYSTEM_RAM=y +CONFIG_HAVE_BOOTMEM_INFO_NODE=y CONFIG_ARCH_ENABLE_MEMORY_HOTPLUG=y -# CONFIG_MEMORY_HOTPLUG is not set +CONFIG_ARCH_ENABLE_MEMORY_HOTREMOVE=y +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE=y +CONFIG_MEMORY_HOTREMOVE=y +CONFIG_MHP_MEMMAP_ON_MEMORY=y CONFIG_ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE=y CONFIG_SPLIT_PTLOCK_CPUS=4 CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK=y @@ -1042,8 +1053,10 @@ CONFIG_COMPACTION=y CONFIG_COMPACT_UNEVICTABLE_DEFAULT=1 CONFIG_PAGE_REPORTING=y CONFIG_MIGRATION=y +CONFIG_DEVICE_MIGRATION=y CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION=y CONFIG_ARCH_ENABLE_THP_MIGRATION=y +CONFIG_CONTIG_ALLOC=y CONFIG_PCP_BATCH_SCALE_MAX=5 CONFIG_PHYS_ADDR_T_64BIT=y CONFIG_MMU_NOTIFIER=y @@ -1072,7 +1085,10 @@ CONFIG_ARCH_HAS_PTE_DEVMAP=y CONFIG_ARCH_HAS_ZONE_DMA_SET=y CONFIG_ZONE_DMA=y CONFIG_ZONE_DMA32=y +CONFIG_ZONE_DEVICE=y CONFIG_HMM_MIRROR=y +CONFIG_GET_FREE_REGION=y +CONFIG_DEVICE_PRIVATE=y CONFIG_VMAP_PFN=y CONFIG_ARCH_USES_HIGH_VMA_FLAGS=y CONFIG_ARCH_HAS_PKEYS=y @@ -1860,6 +1876,7 @@ CONFIG_PCI_LOCKLESS_CONFIG=y CONFIG_PCI_IOV=y CONFIG_PCI_PRI=y CONFIG_PCI_PASID=y +# CONFIG_PCI_P2PDMA is not set CONFIG_PCI_LABEL=y CONFIG_PCI_HYPERV=y # CONFIG_PCIE_BUS_TUNE_OFF is not set @@ -4185,6 +4202,7 @@ CONFIG_DRM_AMD_DC_FP=y # end of Display Engine Configuration CONFIG_HSA_AMD=y +CONFIG_HSA_AMD_SVM=y # CONFIG_DRM_NOUVEAU is not set CONFIG_DRM_I915=m CONFIG_DRM_I915_FORCE_PROBE="" @@ -5061,6 +5079,7 @@ CONFIG_VIRTIO_MENU=y CONFIG_VIRTIO_PCI=m CONFIG_VIRTIO_PCI_LEGACY=y CONFIG_VIRTIO_BALLOON=m +# CONFIG_VIRTIO_MEM is not set CONFIG_VIRTIO_INPUT=m CONFIG_VIRTIO_MMIO=m CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y @@ -5088,6 +5107,8 @@ CONFIG_HYPERV_BALLOON=y # Xen driver support # CONFIG_XEN_BALLOON=y +CONFIG_XEN_BALLOON_MEMORY_HOTPLUG=y +CONFIG_XEN_MEMORY_HOTPLUG_LIMIT=512 CONFIG_XEN_SCRUB_PAGES_DEFAULT=y CONFIG_XEN_DEV_EVTCHN=y # CONFIG_XEN_BACKEND is not set @@ -5106,6 +5127,7 @@ CONFIG_XEN_EFI=y CONFIG_XEN_AUTO_XLATE=y CONFIG_XEN_ACPI=y CONFIG_XEN_HAVE_VPMU=y +CONFIG_XEN_UNPOPULATED_ALLOC=y # CONFIG_XEN_VIRTIO is not set # end of Xen driver support @@ -5450,6 +5472,7 @@ CONFIG_BTRFS_FS_POSIX_ACL=y # CONFIG_BTRFS_FS_REF_VERIFY is not set # CONFIG_NILFS2_FS is not set # CONFIG_F2FS_FS is not set +# CONFIG_FS_DAX is not set CONFIG_FS_POSIX_ACL=y CONFIG_EXPORTFS=y # CONFIG_EXPORTFS_BLOCK_OPS is not set @@ -6601,6 +6624,7 @@ CONFIG_RUNTIME_TESTING_MENU=y # CONFIG_TEST_MEMCAT_P is not set # CONFIG_TEST_OBJAGG is not set # CONFIG_TEST_MEMINIT is not set +# CONFIG_TEST_HMM is not set # CONFIG_TEST_FREE_PAGES is not set # CONFIG_TEST_FPU is not set # CONFIG_TEST_CLOCKSOURCE_WATCHDOG is not set diff --git a/kernel/build/config-arm64 b/kernel/build/config-arm64 index 47ef820c7..e26e16338 100644 --- a/kernel/build/config-arm64 +++ b/kernel/build/config-arm64 @@ -687,6 +687,7 @@ CONFIG_ACPI_TABLE_UPGRADE=y # CONFIG_ACPI_DEBUG is not set CONFIG_ACPI_PCI_SLOT=y CONFIG_ACPI_CONTAINER=y +CONFIG_ACPI_HOTPLUG_MEMORY=y CONFIG_ACPI_HED=y # CONFIG_ACPI_CUSTOM_METHOD is not set CONFIG_ACPI_BGRT=y @@ -870,7 +871,9 @@ CONFIG_MODULE_SIG_FORMAT=y CONFIG_MODULES=y # CONFIG_MODULE_DEBUG is not set # CONFIG_MODULE_FORCE_LOAD is not set -# CONFIG_MODULE_UNLOAD is not set +CONFIG_MODULE_UNLOAD=y +# CONFIG_MODULE_FORCE_UNLOAD is not set +# CONFIG_MODULE_UNLOAD_TAINT_TRACKING is not set CONFIG_MODVERSIONS=y CONFIG_ASM_MODVERSIONS=y CONFIG_MODULE_SRCVERSION_ALL=y @@ -1062,11 +1065,15 @@ CONFIG_SPARSEMEM_VMEMMAP_ENABLE=y CONFIG_SPARSEMEM_VMEMMAP=y CONFIG_HAVE_FAST_GUP=y CONFIG_ARCH_KEEP_MEMBLOCK=y +CONFIG_NUMA_KEEP_MEMINFO=y CONFIG_MEMORY_ISOLATION=y CONFIG_EXCLUSIVE_SYSTEM_RAM=y CONFIG_ARCH_ENABLE_MEMORY_HOTPLUG=y CONFIG_ARCH_ENABLE_MEMORY_HOTREMOVE=y -# CONFIG_MEMORY_HOTPLUG is not set +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE=y +CONFIG_MEMORY_HOTREMOVE=y +CONFIG_MHP_MEMMAP_ON_MEMORY=y CONFIG_ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE=y CONFIG_SPLIT_PTLOCK_CPUS=4 CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK=y @@ -1076,6 +1083,7 @@ CONFIG_COMPACTION=y CONFIG_COMPACT_UNEVICTABLE_DEFAULT=1 CONFIG_PAGE_REPORTING=y CONFIG_MIGRATION=y +CONFIG_DEVICE_MIGRATION=y CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION=y CONFIG_ARCH_ENABLE_THP_MIGRATION=y CONFIG_CONTIG_ALLOC=y @@ -1110,6 +1118,9 @@ CONFIG_ARCH_HAS_PTE_DEVMAP=y CONFIG_ARCH_HAS_ZONE_DMA_SET=y CONFIG_ZONE_DMA=y CONFIG_ZONE_DMA32=y +CONFIG_ZONE_DEVICE=y +CONFIG_GET_FREE_REGION=y +CONFIG_DEVICE_PRIVATE=y CONFIG_ARCH_USES_HIGH_VMA_FLAGS=y CONFIG_ARCH_USES_PG_ARCH_X=y CONFIG_VM_EVENT_COUNTERS=y @@ -1879,6 +1890,7 @@ CONFIG_PCI_BRIDGE_EMUL=y CONFIG_PCI_IOV=y CONFIG_PCI_PRI=y CONFIG_PCI_PASID=y +# CONFIG_PCI_P2PDMA is not set CONFIG_PCI_LABEL=y CONFIG_PCI_HYPERV=y # CONFIG_PCI_DYNAMIC_OF_NODES is not set @@ -6462,6 +6474,7 @@ CONFIG_VIRTIO_PCI=y CONFIG_VIRTIO_PCI_LEGACY=y CONFIG_VIRTIO_PMEM=y CONFIG_VIRTIO_BALLOON=y +# CONFIG_VIRTIO_MEM is not set CONFIG_VIRTIO_INPUT=y CONFIG_VIRTIO_MMIO=y CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y @@ -6487,6 +6500,7 @@ CONFIG_HYPERV_BALLOON=y # Xen driver support # CONFIG_XEN_BALLOON=y +CONFIG_XEN_BALLOON_MEMORY_HOTPLUG=y CONFIG_XEN_SCRUB_PAGES_DEFAULT=y CONFIG_XEN_DEV_EVTCHN=y # CONFIG_XEN_BACKEND is not set @@ -6502,6 +6516,7 @@ CONFIG_SWIOTLB_XEN=y CONFIG_XEN_PRIVCMD=y CONFIG_XEN_EFI=y CONFIG_XEN_AUTO_XLATE=y +CONFIG_XEN_UNPOPULATED_ALLOC=y # CONFIG_XEN_VIRTIO is not set # end of Xen driver support @@ -7988,6 +8003,9 @@ CONFIG_BLK_DEV_PMEM=y CONFIG_ND_CLAIM=y CONFIG_ND_BTT=y CONFIG_BTT=y +CONFIG_ND_PFN=y +CONFIG_NVDIMM_PFN=y +CONFIG_NVDIMM_DAX=y CONFIG_OF_PMEM=y CONFIG_DAX=y # CONFIG_DEV_DAX is not set @@ -8086,6 +8104,7 @@ CONFIG_BTRFS_FS_POSIX_ACL=y # CONFIG_BTRFS_FS_REF_VERIFY is not set # CONFIG_NILFS2_FS is not set # CONFIG_F2FS_FS is not set +# CONFIG_FS_DAX is not set CONFIG_FS_POSIX_ACL=y CONFIG_EXPORTFS=y # CONFIG_EXPORTFS_BLOCK_OPS is not set @@ -9279,6 +9298,7 @@ CONFIG_RUNTIME_TESTING_MENU=y # CONFIG_TEST_MEMCAT_P is not set # CONFIG_TEST_OBJAGG is not set # CONFIG_TEST_MEMINIT is not set +# CONFIG_TEST_HMM is not set # CONFIG_TEST_FREE_PAGES is not set CONFIG_ARCH_USE_MEMTEST=y # CONFIG_MEMTEST is not set