From 4869e4d4ad6a41d6f829e52f1c36e51b031cd44a Mon Sep 17 00:00:00 2001 From: Koichi Date: Thu, 28 Aug 2025 01:04:57 +0900 Subject: [PATCH 1/8] add DmaMap Signed-off-by: Koichi --- awkernel_lib/src/dma_map.rs | 240 ++++++++++++++++++++++++++++++++++++ awkernel_lib/src/lib.rs | 3 + 2 files changed, 243 insertions(+) create mode 100644 awkernel_lib/src/dma_map.rs diff --git a/awkernel_lib/src/dma_map.rs b/awkernel_lib/src/dma_map.rs new file mode 100644 index 000000000..d7ebdf13b --- /dev/null +++ b/awkernel_lib/src/dma_map.rs @@ -0,0 +1,240 @@ +//! DMA Mapping API +//! +//! This DMA mapping layer expects that transfers passed to it are already +//! limited to the device's Maximum Data Transfer Size (MDTS, stored in +//! DmaTag::maxsize). If a transfer exceeds maxsize, this layer will return +//! DmaError::SizeTooLarge rather than attempting to split it. +//! +//! In OpenBSD, the physio() layer calls minphys() to limit transfer sizes +//! before they reach device drivers. AWKernel currently lacks an equivalent +//! physio layer, so upper layers (storage, filesystem) are responsible for +//! ensuring transfers do not exceed device limits. +//! +//! For transfers within MDTS but requiring too many segments (exceeding +//! DmaTag::nsegments), this layer WILL automatically use bounce buffers +//! to consolidate segments. + +use crate::{ + addr::{phy_addr::PhyAddr, virt_addr::VirtAddr, Addr}, + dma_pool::DMAPool, + paging::{self, PAGESIZE}, +}; +use alloc::vec::Vec; + +/// DMA constraints for a device +#[derive(Debug, Clone, Copy)] +pub struct DmaTag { + pub boundary: u64, + pub maxsegsz: usize, + pub nsegments: usize, + pub maxsize: usize, + pub alignment: usize, +} + +impl Default for DmaTag { + /// Create a default DMA tag for 64-bit devices + fn default() -> Self { + Self { + boundary: 0, + maxsegsz: PAGESIZE, + nsegments: 1, + maxsize: usize::MAX, + alignment: 1, + } + } +} + +/// DMA segment descriptor +/// +/// NOTE: OpenBSD tracks _ds_va and _ds_bounce_va per segment for copying data +/// between original and bounce buffers. AWKernel doesn't need these because: +/// 1. We store virtual addresses at the map level (orig_vaddr, owned_memory) +/// 2. Bounce buffer is a single contiguous segment - we copy the entire range at once +#[derive(Debug, Clone, Copy)] +pub struct DmaSegment { + pub ds_addr: PhyAddr, + pub ds_len: usize, +} + +/// DMA map structure +pub struct DmaMap { + tag: DmaTag, + segments: Vec, + /// Memory owned by this map (either bounce buffer or pre-allocated DMA pool) + owned_memory: Option>, + /// Original virtual address (for sync operations) + orig_vaddr: Option, + mapsize: usize, + numa_id: usize, +} + +/// DMA synchronization operations +#[derive(Debug, Clone, Copy)] +pub enum DmaSyncOp { + PreRead, + PostRead, + PreWrite, + PostWrite, +} + +/// Errors that can occur during DMA operations +#[derive(Debug)] +pub enum DmaError { + AddressTooHigh, + SizeTooLarge, + TooManySegments, + BadAlignment, + OutOfMemory, + NotLoaded, + InvalidAddress, +} + +impl DmaMap { + /// Create a new DMA map + pub fn new(tag: DmaTag, numa_id: usize) -> Result { + Ok(Self { + tag, + segments: Vec::new(), + owned_memory: None, + orig_vaddr: None, + mapsize: 0, + numa_id, + }) + } + + pub fn mapsize(&self) -> usize { + self.mapsize + } + + /// Load a buffer into the DMA map + /// Based on OpenBSD's _bus_dmamap_load_buffer (sys/arch/amd64/amd64/bus_dma.c:722-823) + pub fn load(&mut self, vaddr: VirtAddr, size: usize) -> Result<(), DmaError> { + if size > self.tag.maxsize { + return Err(DmaError::SizeTooLarge); + } + + if vaddr.as_usize() & (self.tag.alignment - 1) != 0 { + return Err(DmaError::BadAlignment); + } + + self.segments.clear(); + + let mut lastaddr = PhyAddr::new(0); + let bmask = if self.tag.boundary > 0 { + !(self.tag.boundary - 1) + } else { + 0 // No masking needed when boundary = 0 + }; + + let mut buflen = size; + let mut vaddr_current = vaddr.as_usize(); + let mut seg = 0usize; + let mut first = true; + + while buflen > 0 { + // Get the physical address for this segment + let curaddr = match paging::vm_to_phy(VirtAddr::new(vaddr_current)) { + Some(paddr) => paddr, + None => { + return Err(DmaError::InvalidAddress); + } + }; + + // TODO: Skipping check for PCIeDevice with 32bit addressing. + + // NOTE: OpenBSD has a check here for pre-allocated bounce + // buffers in SEV guest mode. AWKernel skips this because: + // 1. No SEV support yet + // 2. We use dynamic bounce buffer allocation instead + + // Compute the segment size, and adjust counts. + let mut sgsize = PAGESIZE - (vaddr_current & (PAGESIZE - 1)); + if buflen < sgsize { + sgsize = buflen; + } + + // Make sure we don't cross any boundaries + if self.tag.boundary > 0 { + let baddr = (curaddr.as_usize() as u64 + self.tag.boundary) & bmask; + let max_size = baddr - curaddr.as_usize() as u64; + if sgsize > max_size as usize { + sgsize = max_size as usize; + } + } + + // Insert chunk into a segment, coalescing with + // previous segment if possible + if first { + self.segments.push(DmaSegment { + ds_addr: curaddr, + ds_len: sgsize, + }); + first = false; + } else { + let can_coalesce = curaddr.as_usize() == lastaddr.as_usize() + && (self.segments[seg].ds_len + sgsize) <= self.tag.maxsegsz + && (self.tag.boundary == 0 + || (self.segments[seg].ds_addr.as_usize() as u64 & bmask) + == (curaddr.as_usize() as u64 & bmask)); + + if can_coalesce { + self.segments[seg].ds_len += sgsize; + } else { + seg += 1; + if seg >= self.tag.nsegments { + return self.load_with_bounce(vaddr, size); + } + self.segments.push(DmaSegment { + ds_addr: curaddr, + ds_len: sgsize, + }); + } + } + + lastaddr = PhyAddr::new(curaddr.as_usize() + sgsize); + vaddr_current += sgsize; + buflen -= sgsize; + } + + self.orig_vaddr = Some(vaddr); + self.mapsize = size; + Ok(()) + } + + /// Load buffer using bounce buffer + fn load_with_bounce(&mut self, vaddr: VirtAddr, size: usize) -> Result<(), DmaError> { + let pages = size.div_ceil(PAGESIZE); + log::warn!( + "Allocating bounce buffer: size={} bytes, pages={}", + size, + pages + ); + + let bounce = DMAPool::::new(self.numa_id, pages).ok_or(DmaError::OutOfMemory)?; + + let bounce_paddr = bounce.get_phy_addr(); + + // NOTE: Data copying happens in sync(), not here! + // This matches OpenBSD's design where load only sets up mappings + + self.segments.clear(); + self.segments.push(DmaSegment { + ds_addr: bounce_paddr, + ds_len: size, + }); + + self.owned_memory = Some(bounce); + self.orig_vaddr = Some(vaddr); + self.mapsize = size; + + Ok(()) + } + + /// Unload the DMA map + pub fn unload(&mut self) { + self.segments.clear(); + self.owned_memory = None; + self.orig_vaddr = None; + self.mapsize = 0; + } +} diff --git a/awkernel_lib/src/lib.rs b/awkernel_lib/src/lib.rs index 37be10fb9..25aae2bed 100644 --- a/awkernel_lib/src/lib.rs +++ b/awkernel_lib/src/lib.rs @@ -43,6 +43,9 @@ pub mod heap; #[cfg(not(feature = "std"))] pub mod dma_pool; +#[cfg(not(feature = "std"))] +pub mod dma_map; + #[cfg(not(feature = "std"))] pub mod context; From 87a6837c508676dd97d8e04e6d0542267201bf45 Mon Sep 17 00:00:00 2001 From: Koichi Date: Thu, 28 Aug 2025 10:35:12 +0900 Subject: [PATCH 2/8] fix clippy Signed-off-by: Koichi --- awkernel_lib/src/dma_map.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/awkernel_lib/src/dma_map.rs b/awkernel_lib/src/dma_map.rs index d7ebdf13b..270e52e70 100644 --- a/awkernel_lib/src/dma_map.rs +++ b/awkernel_lib/src/dma_map.rs @@ -204,11 +204,7 @@ impl DmaMap { /// Load buffer using bounce buffer fn load_with_bounce(&mut self, vaddr: VirtAddr, size: usize) -> Result<(), DmaError> { let pages = size.div_ceil(PAGESIZE); - log::warn!( - "Allocating bounce buffer: size={} bytes, pages={}", - size, - pages - ); + log::warn!("Allocating bounce buffer: size={size} bytes, pages={pages}"); let bounce = DMAPool::::new(self.numa_id, pages).ok_or(DmaError::OutOfMemory)?; From 82c2ff88bc6d06b4639130b9db23605f04589146 Mon Sep 17 00:00:00 2001 From: Koichi Date: Thu, 28 Aug 2025 11:12:11 +0900 Subject: [PATCH 3/8] delete comments and unnecessary changes Signed-off-by: Koichi --- awkernel_lib/src/dma_map.rs | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/awkernel_lib/src/dma_map.rs b/awkernel_lib/src/dma_map.rs index 270e52e70..98971dcee 100644 --- a/awkernel_lib/src/dma_map.rs +++ b/awkernel_lib/src/dma_map.rs @@ -3,16 +3,9 @@ //! This DMA mapping layer expects that transfers passed to it are already //! limited to the device's Maximum Data Transfer Size (MDTS, stored in //! DmaTag::maxsize). If a transfer exceeds maxsize, this layer will return -//! DmaError::SizeTooLarge rather than attempting to split it. -//! -//! In OpenBSD, the physio() layer calls minphys() to limit transfer sizes -//! before they reach device drivers. AWKernel currently lacks an equivalent -//! physio layer, so upper layers (storage, filesystem) are responsible for -//! ensuring transfers do not exceed device limits. -//! -//! For transfers within MDTS but requiring too many segments (exceeding -//! DmaTag::nsegments), this layer WILL automatically use bounce buffers -//! to consolidate segments. +//! DmaError::SizeTooLarge rather than attempting to split it. For transfers +//! within MDTS but requiring too many segments (exceeding DmaTag::nsegments), +//! this layer WILL automatically use bounce buffers to consolidate segments. use crate::{ addr::{phy_addr::PhyAddr, virt_addr::VirtAddr, Addr}, @@ -45,11 +38,6 @@ impl Default for DmaTag { } /// DMA segment descriptor -/// -/// NOTE: OpenBSD tracks _ds_va and _ds_bounce_va per segment for copying data -/// between original and bounce buffers. AWKernel doesn't need these because: -/// 1. We store virtual addresses at the map level (orig_vaddr, owned_memory) -/// 2. Bounce buffer is a single contiguous segment - we copy the entire range at once #[derive(Debug, Clone, Copy)] pub struct DmaSegment { pub ds_addr: PhyAddr, @@ -68,15 +56,6 @@ pub struct DmaMap { numa_id: usize, } -/// DMA synchronization operations -#[derive(Debug, Clone, Copy)] -pub enum DmaSyncOp { - PreRead, - PostRead, - PreWrite, - PostWrite, -} - /// Errors that can occur during DMA operations #[derive(Debug)] pub enum DmaError { @@ -107,7 +86,6 @@ impl DmaMap { } /// Load a buffer into the DMA map - /// Based on OpenBSD's _bus_dmamap_load_buffer (sys/arch/amd64/amd64/bus_dma.c:722-823) pub fn load(&mut self, vaddr: VirtAddr, size: usize) -> Result<(), DmaError> { if size > self.tag.maxsize { return Err(DmaError::SizeTooLarge); From e917e5892115572d6fbe6e6e3d7f979c1d6dd332 Mon Sep 17 00:00:00 2001 From: Koichi Date: Thu, 28 Aug 2025 13:13:14 +0900 Subject: [PATCH 4/8] rename to DmaConstraints Signed-off-by: Koichi --- awkernel_lib/src/dma_map.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/awkernel_lib/src/dma_map.rs b/awkernel_lib/src/dma_map.rs index 98971dcee..4c57a3bff 100644 --- a/awkernel_lib/src/dma_map.rs +++ b/awkernel_lib/src/dma_map.rs @@ -2,9 +2,9 @@ //! //! This DMA mapping layer expects that transfers passed to it are already //! limited to the device's Maximum Data Transfer Size (MDTS, stored in -//! DmaTag::maxsize). If a transfer exceeds maxsize, this layer will return +//! DmaConstraints::maxsize). If a transfer exceeds maxsize, this layer will return //! DmaError::SizeTooLarge rather than attempting to split it. For transfers -//! within MDTS but requiring too many segments (exceeding DmaTag::nsegments), +//! within MDTS but requiring too many segments (exceeding DmaConstraints::nsegments), //! this layer WILL automatically use bounce buffers to consolidate segments. use crate::{ @@ -16,7 +16,7 @@ use alloc::vec::Vec; /// DMA constraints for a device #[derive(Debug, Clone, Copy)] -pub struct DmaTag { +pub struct DmaConstraints { pub boundary: u64, pub maxsegsz: usize, pub nsegments: usize, @@ -24,8 +24,8 @@ pub struct DmaTag { pub alignment: usize, } -impl Default for DmaTag { - /// Create a default DMA tag for 64-bit devices +impl Default for DmaConstraints { + /// Create default DMA constraints for 64-bit devices fn default() -> Self { Self { boundary: 0, @@ -46,7 +46,7 @@ pub struct DmaSegment { /// DMA map structure pub struct DmaMap { - tag: DmaTag, + tag: DmaConstraints, segments: Vec, /// Memory owned by this map (either bounce buffer or pre-allocated DMA pool) owned_memory: Option>, @@ -70,7 +70,7 @@ pub enum DmaError { impl DmaMap { /// Create a new DMA map - pub fn new(tag: DmaTag, numa_id: usize) -> Result { + pub fn new(tag: DmaConstraints, numa_id: usize) -> Result { Ok(Self { tag, segments: Vec::new(), From b6e8dc32dd8600835823939bcbc834a3c6350eb2 Mon Sep 17 00:00:00 2001 From: Koichi Date: Fri, 29 Aug 2025 10:06:08 +0900 Subject: [PATCH 5/8] delete unnecessary comments Signed-off-by: Koichi --- awkernel_lib/src/dma_map.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/awkernel_lib/src/dma_map.rs b/awkernel_lib/src/dma_map.rs index 4c57a3bff..008114b09 100644 --- a/awkernel_lib/src/dma_map.rs +++ b/awkernel_lib/src/dma_map.rs @@ -14,7 +14,6 @@ use crate::{ }; use alloc::vec::Vec; -/// DMA constraints for a device #[derive(Debug, Clone, Copy)] pub struct DmaConstraints { pub boundary: u64, @@ -25,7 +24,6 @@ pub struct DmaConstraints { } impl Default for DmaConstraints { - /// Create default DMA constraints for 64-bit devices fn default() -> Self { Self { boundary: 0, @@ -37,14 +35,12 @@ impl Default for DmaConstraints { } } -/// DMA segment descriptor #[derive(Debug, Clone, Copy)] pub struct DmaSegment { pub ds_addr: PhyAddr, pub ds_len: usize, } -/// DMA map structure pub struct DmaMap { tag: DmaConstraints, segments: Vec, @@ -56,7 +52,6 @@ pub struct DmaMap { numa_id: usize, } -/// Errors that can occur during DMA operations #[derive(Debug)] pub enum DmaError { AddressTooHigh, @@ -69,7 +64,6 @@ pub enum DmaError { } impl DmaMap { - /// Create a new DMA map pub fn new(tag: DmaConstraints, numa_id: usize) -> Result { Ok(Self { tag, @@ -85,7 +79,6 @@ impl DmaMap { self.mapsize } - /// Load a buffer into the DMA map pub fn load(&mut self, vaddr: VirtAddr, size: usize) -> Result<(), DmaError> { if size > self.tag.maxsize { return Err(DmaError::SizeTooLarge); @@ -179,7 +172,6 @@ impl DmaMap { Ok(()) } - /// Load buffer using bounce buffer fn load_with_bounce(&mut self, vaddr: VirtAddr, size: usize) -> Result<(), DmaError> { let pages = size.div_ceil(PAGESIZE); log::warn!("Allocating bounce buffer: size={size} bytes, pages={pages}"); @@ -204,7 +196,6 @@ impl DmaMap { Ok(()) } - /// Unload the DMA map pub fn unload(&mut self) { self.segments.clear(); self.owned_memory = None; From ec9719fa4bd6b2af8593e66eb7eec62f0ac4a30f Mon Sep 17 00:00:00 2001 From: Koichi Date: Fri, 29 Aug 2025 13:21:44 +0900 Subject: [PATCH 6/8] add sync method Signed-off-by: Koichi --- awkernel_lib/src/dma_map.rs | 93 +++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/awkernel_lib/src/dma_map.rs b/awkernel_lib/src/dma_map.rs index 008114b09..b0950dbe1 100644 --- a/awkernel_lib/src/dma_map.rs +++ b/awkernel_lib/src/dma_map.rs @@ -202,4 +202,97 @@ impl DmaMap { self.orig_vaddr = None; self.mapsize = 0; } + + pub fn sync(&self, offset: usize, len: usize, op: DmaSyncOp) -> Result<(), DmaError> { + if self.segments.is_empty() { + return Err(DmaError::NotLoaded); + } + + // Validate offset and length + if offset + len > self.mapsize { + return Err(DmaError::SizeTooLarge); + } + + if let (Some(ref bounce), Some(orig_vaddr)) = (&self.owned_memory, self.orig_vaddr) { + match op { + // READ: device -> memory + DmaSyncOp::PostRead => unsafe { + core::ptr::copy_nonoverlapping( + bounce.get_virt_addr().as_ptr::().add(offset), + orig_vaddr.as_mut_ptr::().add(offset), + len, + ); + }, + // WRITE: memory -> device + DmaSyncOp::PreWrite => unsafe { + core::ptr::copy_nonoverlapping( + orig_vaddr.as_ptr::().add(offset), + bounce.get_virt_addr().as_mut_ptr::().add(offset), + len, + ); + }, + // PREREAD and POSTWRITE are no-ops. + _ => (), + } + } + + // TODO: Implement cache operations for non-coherent devices. + // Currently only memory barriers are implemented, which is insufficient + // for non-coherent devices but works for cache-coherent systems. + // + // Note: The barrier and cache invalidation are not atomic, which could + // theoretically allow speculative loads between them. This is mitigated + // in practice by running in interrupt context and exclusive buffer access. + + #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] + if matches!(op, DmaSyncOp::PostRead) { + crate::barrier::membar_sync(); + // TODO: cpu_dcache_inv_range(paddr, len) + } + + #[cfg(target_arch = "aarch64")] + if matches!(op, DmaSyncOp::PostRead) { + crate::barrier::membar_sync(); + // TODO: cpu_dcache_inv_range(paddr, len) + } + + Ok(()) + } + + pub fn get_segment(&self) -> Option<&DmaSegment> { + self.segments.first() + } + + pub fn get_segments(&self) -> &[DmaSegment] { + &self.segments + } + + pub fn is_bounced(&self) -> bool { + self.owned_memory.is_some() + } +} + +impl DmaMap { + pub fn from_dma_pool(pool: DMAPool, tag: DmaConstraints) -> Result { + let paddr = pool.get_phy_addr(); + let size = pool.get_size(); + let vaddr = pool.get_virt_addr(); + let numa_id = pool.get_numa_id(); + + if size > tag.maxsize { + return Err(DmaError::SizeTooLarge); + } + + Ok(Self { + tag, + segments: alloc::vec![DmaSegment { + ds_addr: paddr, + ds_len: size, + }], + owned_memory: Some(pool.into_bytes()), + orig_vaddr: Some(vaddr), + mapsize: size, + numa_id, + }) + } } From 9e9ed69c169e0a3781eb4fac148730986cd48cd2 Mon Sep 17 00:00:00 2001 From: Koichi Date: Fri, 29 Aug 2025 13:24:32 +0900 Subject: [PATCH 7/8] delete get_segment Signed-off-by: Koichi --- awkernel_lib/src/dma_map.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/awkernel_lib/src/dma_map.rs b/awkernel_lib/src/dma_map.rs index b0950dbe1..30321d01d 100644 --- a/awkernel_lib/src/dma_map.rs +++ b/awkernel_lib/src/dma_map.rs @@ -259,10 +259,6 @@ impl DmaMap { Ok(()) } - pub fn get_segment(&self) -> Option<&DmaSegment> { - self.segments.first() - } - pub fn get_segments(&self) -> &[DmaSegment] { &self.segments } From e37683e60759990430eafc25baa05f0bd73ad2b0 Mon Sep 17 00:00:00 2001 From: Koichi Date: Tue, 23 Sep 2025 11:01:09 +0900 Subject: [PATCH 8/8] fix Signed-off-by: Koichi --- awkernel_lib/src/dma_map.rs | 8 ++++++++ awkernel_lib/src/dma_pool.rs | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/awkernel_lib/src/dma_map.rs b/awkernel_lib/src/dma_map.rs index 30321d01d..6e83146fd 100644 --- a/awkernel_lib/src/dma_map.rs +++ b/awkernel_lib/src/dma_map.rs @@ -52,6 +52,14 @@ pub struct DmaMap { numa_id: usize, } +#[derive(Debug, Clone, Copy)] +pub enum DmaSyncOp { + PreRead, + PostRead, + PreWrite, + PostWrite, +} + #[derive(Debug)] pub enum DmaError { AddressTooHigh, diff --git a/awkernel_lib/src/dma_pool.rs b/awkernel_lib/src/dma_pool.rs index aae52405b..5b9ecadd4 100644 --- a/awkernel_lib/src/dma_pool.rs +++ b/awkernel_lib/src/dma_pool.rs @@ -89,6 +89,24 @@ impl DMAPool { ptr } + /// Convert DMAPool to DMAPool + pub fn into_bytes(self) -> DMAPool { + let virt_addr = self.virt_addr; + let phy_addr = self.phy_addr; + let size = self.size; + let numa_id = self.numa_id; + + core::mem::forget(self); + + DMAPool { + virt_addr, + phy_addr, + size, + numa_id, + ptr: unsafe { NonNull::new_unchecked(virt_addr.as_mut_ptr()) }, + } + } + #[inline(always)] pub fn get_virt_addr(&self) -> VirtAddr { self.virt_addr