From 0cfd74b4066facb4fd3ed259606edd36e32bc32f Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Fri, 27 Dec 2024 17:33:31 +0900 Subject: [PATCH 01/31] feat:sched_preempt Signed-off-by: kobayu858 --- .../tests/test_sched_preempt/src/lib.rs | 51 ++++++++++++++++++- awkernel_async_lib/src/scheduler.rs | 26 ++++++++-- awkernel_async_lib/src/scheduler/fifo.rs | 4 +- awkernel_async_lib/src/scheduler/gedf.rs | 42 ++------------- .../src/scheduler/prioritized_fifo.rs | 3 ++ .../src/scheduler/priority_based_rr.rs | 5 +- awkernel_async_lib/src/scheduler/rr.rs | 5 +- awkernel_async_lib/src/task.rs | 4 ++ 8 files changed, 94 insertions(+), 46 deletions(-) diff --git a/applications/tests/test_sched_preempt/src/lib.rs b/applications/tests/test_sched_preempt/src/lib.rs index 5b0c89cb5..ed4fa96a6 100644 --- a/applications/tests/test_sched_preempt/src/lib.rs +++ b/applications/tests/test_sched_preempt/src/lib.rs @@ -9,6 +9,7 @@ use awkernel_lib::{ #[allow(dead_code)] enum TestType { GetlowestTask, + SchedPreempt, } /// Tests related to preemption between schedulers @@ -21,9 +22,10 @@ pub async fn run() { cpu_id() ); - let test_type = TestType::GetlowestTask; + let test_type = TestType::SchedPreempt; match test_type { TestType::GetlowestTask => check_lowest_task().await, + TestType::SchedPreempt => check_sched_preempt().await, } log::info!( @@ -69,3 +71,50 @@ async fn check_lowest_task() { log::debug!("Task ID: {}, cpu_id: {}", task_id, cpu_id,); } } + +async fn check_sched_preempt() { + log::info!("[{}] GEDF_H1 spawn", uptime()); + spawn( + "GEDF_H1".into(), + async move { + log::info!("[{}] GEDF_H1 is start at cpu_id: {}", uptime(), cpu_id()); + wait_microsec(10000000); + log::info!("[{}] GEDF_H1 is end at cpu_id: {}", uptime(), cpu_id()); + }, + SchedulerType::GEDF(99000000), + ) + .await; + log::info!("[{}] FIFO_L1 spawn", uptime()); + spawn( + "FIFO_M1".into(), + async move { + log::info!("[{}] FIFO_M1 is start at cpu_id: {}", uptime(), cpu_id()); + wait_microsec(10000000); + log::info!("[{}] FIFO_M1 is end at cpu_id: {}", uptime(), cpu_id()); + }, + SchedulerType::FIFO, + ) + .await; + log::info!("[{}] RR_L1 spawn", uptime()); + spawn( + "RR_L1".into(), + async move { + log::info!("[{}] RR_L1 is start at cpu_id: {}", uptime(), cpu_id()); + wait_microsec(10000000); + log::info!("[{}] RR_L1 is end at cpu_id: {}", uptime(), cpu_id()); + }, + SchedulerType::RR, + ) + .await; + log::info!("[{}] GEDF_H2 spawn", uptime()); + spawn( + "GEDF_H2".into(), + async move { + log::info!("[{}] GEDF_H2 is start at cpu_id: {}", uptime(), cpu_id()); + wait_microsec(10000000); + log::info!("[{}] GEDF_H2 is end at cpu_id: {}", uptime(), cpu_id()); + }, + SchedulerType::GEDF(98000000), + ) + .await; +} \ No newline at end of file diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 4e384a6db..7244e604d 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -1,7 +1,10 @@ //! Define types and trait for the Autoware Kernel scheduler. //! This module contains `SleepingTasks` for sleeping. -use crate::task::{get_current_task, get_scheduler_type_by_task_id}; +use crate::task::{ + get_current_task, get_lowest_priority_task_info, get_scheduler_type_by_task_id, + set_need_preemption, PriorityInfo, +}; use crate::{delay::uptime, task::Task}; use alloc::sync::Arc; use awkernel_async_lib_verified::delta_list::DeltaList; @@ -89,8 +92,23 @@ pub(crate) trait Scheduler { /// Get the scheduler name. fn scheduler_name(&self) -> SchedulerType; - #[allow(dead_code)] // TODO: to be removed + /// Get the priority of the scheduler. fn priority(&self) -> u8; + + /// Invoke preemption. + fn invoke_preemption(&self, wake_task_priority: PriorityInfo) { + let (task_id, cpu_id, lowest_priority_info) = match get_lowest_priority_task_info() { + Some(info) => info, + None => return, + }; + + if wake_task_priority < lowest_priority_info { + let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); + set_need_preemption(task_id); + awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); + } + return; + } } /// Get the next executable task. @@ -190,9 +208,9 @@ pub fn wake_task() { for cpu_id in 1..num_cpu() { if let Some(task_id) = get_current_task(cpu_id) { match get_scheduler_type_by_task_id(task_id) { - Some(SchedulerType::RR) => rr::SCHEDULER.invoke_preemption(cpu_id, task_id), + Some(SchedulerType::RR) => rr::SCHEDULER.invoke_rr_preemption(cpu_id, task_id), Some(SchedulerType::PriorityBasedRR(_)) => { - priority_based_rr::SCHEDULER.invoke_preemption(cpu_id, task_id) + priority_based_rr::SCHEDULER.invoke_rr_preemption(cpu_id, task_id) } _ => (), } diff --git a/awkernel_async_lib/src/scheduler/fifo.rs b/awkernel_async_lib/src/scheduler/fifo.rs index f1fad34ad..bfd489390 100644 --- a/awkernel_async_lib/src/scheduler/fifo.rs +++ b/awkernel_async_lib/src/scheduler/fifo.rs @@ -14,7 +14,7 @@ impl Scheduler for FIFOScheduler { fn wake_task(&self, task: Arc) { let mut node = MCSNode::new(); let mut queue = self.queue.lock(&mut node); - + let wake_task_priority = task.priority.clone(); if let Some(queue) = queue.as_mut() { queue.push_back(task); } else { @@ -22,6 +22,8 @@ impl Scheduler for FIFOScheduler { q.push_back(task); *queue = Some(q); } + + self.invoke_preemption(wake_task_priority); } fn get_next(&self) -> Option> { diff --git a/awkernel_async_lib/src/scheduler/gedf.rs b/awkernel_async_lib/src/scheduler/gedf.rs index 2881f86a5..b75d75adb 100644 --- a/awkernel_async_lib/src/scheduler/gedf.rs +++ b/awkernel_async_lib/src/scheduler/gedf.rs @@ -1,15 +1,9 @@ //! A GEDF scheduler. use super::{Scheduler, SchedulerType, Task}; -use crate::{ - scheduler::get_priority, - task::{get_absolute_deadline_by_task_id, get_tasks_running, set_need_preemption, State}, -}; +use crate::{scheduler::get_priority, task::State}; use alloc::{collections::BinaryHeap, sync::Arc}; -use awkernel_lib::{ - cpu::num_cpu, - sync::mutex::{MCSNode, Mutex}, -}; +use awkernel_lib::sync::mutex::{MCSNode, Mutex}; pub struct GEDFScheduler { data: Mutex>, // Run queue. @@ -83,7 +77,8 @@ impl Scheduler for GEDFScheduler { wake_time, }); - self.invoke_preemption(absolute_deadline); + let wake_task_priority = task.priority.clone(); + self.invoke_preemption(wake_task_priority); } fn get_next(&self) -> Option> { @@ -129,32 +124,3 @@ pub static SCHEDULER: GEDFScheduler = GEDFScheduler { data: Mutex::new(None), priority: get_priority(SchedulerType::GEDF(0)), }; - -impl GEDFScheduler { - fn invoke_preemption(&self, absolute_deadline: u64) { - // Get running tasks and filter out tasks with task_id == 0. - let mut tasks = get_tasks_running(); - tasks.retain(|task| task.task_id != 0); - - // If the number of running tasks is less than the number of non-primary CPUs, preempt is not required. - let num_non_primary_cpus = num_cpu() - 1; - if tasks.len() < num_non_primary_cpus { - return; - } - - let task_with_max_deadline = tasks - .iter() - .filter_map(|task| { - get_absolute_deadline_by_task_id(task.task_id).map(|deadline| (task, deadline)) - }) - .max_by_key(|&(_, deadline)| deadline); - - if let Some((task, max_absolute_deadline)) = task_with_max_deadline { - if max_absolute_deadline > absolute_deadline { - let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); - set_need_preemption(task.task_id); - awkernel_lib::interrupt::send_ipi(preempt_irq, task.cpu_id as u32); - } - } - } -} diff --git a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs index 72fd3e938..9a107d55a 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs @@ -64,6 +64,9 @@ impl Scheduler for PrioritizedFIFOScheduler { ); *data = Some(prioritized_fifo_data); } + + let wake_task_priority = task.priority.clone(); + self.invoke_preemption(wake_task_priority); } fn get_next(&self) -> Option> { diff --git a/awkernel_async_lib/src/scheduler/priority_based_rr.rs b/awkernel_async_lib/src/scheduler/priority_based_rr.rs index b5e7a9933..71ca95149 100644 --- a/awkernel_async_lib/src/scheduler/priority_based_rr.rs +++ b/awkernel_async_lib/src/scheduler/priority_based_rr.rs @@ -49,6 +49,9 @@ impl Scheduler for PriorityBasedRRScheduler { let mut guard = self.data.lock(&mut node); let data = guard.get_or_insert_with(PriorityBasedRRData::new); data.queue.push(priority as u32, new_task); + + let wake_task_priority = task.priority.clone(); + self.invoke_preemption(wake_task_priority); } fn get_next(&self) -> Option> { @@ -97,7 +100,7 @@ pub static SCHEDULER: PriorityBasedRRScheduler = PriorityBasedRRScheduler { impl PriorityBasedRRScheduler { // Invoke a preemption if the task exceeds the time quantum - pub fn invoke_preemption(&self, cpu_id: usize, task_id: u32) { + pub fn invoke_rr_preemption(&self, cpu_id: usize, task_id: u32) { let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); if let Some(last_executed) = get_last_executed_by_task_id(task_id) { let elapsed = awkernel_lib::delay::uptime() - last_executed; diff --git a/awkernel_async_lib/src/scheduler/rr.rs b/awkernel_async_lib/src/scheduler/rr.rs index 7a67a39ca..34b87968f 100644 --- a/awkernel_async_lib/src/scheduler/rr.rs +++ b/awkernel_async_lib/src/scheduler/rr.rs @@ -19,7 +19,10 @@ impl Scheduler for RRScheduler { fn wake_task(&self, task: Arc) { let mut node = MCSNode::new(); let mut guard = self.queue.lock(&mut node); + let wake_task_priority = task.priority.clone(); guard.get_or_insert_with(VecDeque::new).push_back(task); + + self.invoke_preemption(wake_task_priority); } fn get_next(&self) -> Option> { @@ -68,7 +71,7 @@ pub static SCHEDULER: RRScheduler = RRScheduler { impl RRScheduler { // Invoke a preemption if the task exceeds the time quantum - pub fn invoke_preemption(&self, cpu_id: usize, task_id: u32) { + pub fn invoke_rr_preemption(&self, cpu_id: usize, task_id: u32) { let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); if let Some(last_executed) = get_last_executed_by_task_id(task_id) { let elapsed = awkernel_lib::delay::uptime() - last_executed; diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 7696e24be..c152dd1bf 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1086,6 +1086,10 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { .filter(|task| task.task_id != 0) .collect(); + if running_tasks.len() != awkernel_lib::cpu::num_cpu().saturating_sub(1) { + return None; + } + let priority_infos: Vec<(u32, usize, PriorityInfo)> = { let mut node = MCSNode::new(); let tasks = TASKS.lock(&mut node); From c73e53a8de0bf189fe9eeccf1bfa9468d0c12f8e Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Tue, 7 Jan 2025 16:56:11 +0900 Subject: [PATCH 02/31] feat:check num ipi Signed-off-by: kobayu858 --- awkernel_async_lib/src/scheduler.rs | 31 +++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 7244e604d..3cf5d6419 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -12,6 +12,9 @@ use awkernel_lib::{ cpu::num_cpu, sync::mutex::{MCSNode, Mutex}, }; +use core::{ + sync::atomic::{AtomicU32, Ordering}, +}; #[cfg(not(feature = "std"))] use alloc::boxed::Box; @@ -24,6 +27,7 @@ mod priority_based_rr; mod rr; static SLEEPING: Mutex = Mutex::new(SleepingTasks::new()); +static NUM_SEND_IPI: AtomicU32 = AtomicU32::new(0); // The number of send IPI. /// Type of scheduler. /// `u8` is the priority of priority based schedulers. @@ -97,17 +101,24 @@ pub(crate) trait Scheduler { /// Invoke preemption. fn invoke_preemption(&self, wake_task_priority: PriorityInfo) { - let (task_id, cpu_id, lowest_priority_info) = match get_lowest_priority_task_info() { - Some(info) => info, - None => return, - }; - - if wake_task_priority < lowest_priority_info { - let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); - set_need_preemption(task_id); - awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); + loop { + let (task_id, cpu_id, lowest_priority_info) = match get_lowest_priority_task_info() { + Some(info) => info, + None => return, + }; + + if wake_task_priority < lowest_priority_info { + let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); + if NUM_SEND_IPI.load(Ordering::Relaxed) != 0 { + continue; + } + NUM_SEND_IPI.fetch_add(1, Ordering::Relaxed); + set_need_preemption(task_id); + awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); + NUM_SEND_IPI.fetch_sub(1, Ordering::Relaxed); + } + return; } - return; } } From d9baee02c45f2d04e79292e3be33bbd32dbe2bfa Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Tue, 7 Jan 2025 19:46:23 +0900 Subject: [PATCH 03/31] feat:check preempt done Signed-off-by: kobayu858 --- awkernel_async_lib/src/scheduler.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 3cf5d6419..2330934cf 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -13,7 +13,7 @@ use awkernel_lib::{ sync::mutex::{MCSNode, Mutex}, }; use core::{ - sync::atomic::{AtomicU32, Ordering}, + sync::atomic::{AtomicUsize, Ordering}, }; #[cfg(not(feature = "std"))] @@ -27,7 +27,7 @@ mod priority_based_rr; mod rr; static SLEEPING: Mutex = Mutex::new(SleepingTasks::new()); -static NUM_SEND_IPI: AtomicU32 = AtomicU32::new(0); // The number of send IPI. +static NUM_SEND_IPI: AtomicUsize = AtomicUsize::new(0); // The number of send IPI. /// Type of scheduler. /// `u8` is the priority of priority based schedulers. @@ -101,17 +101,20 @@ pub(crate) trait Scheduler { /// Invoke preemption. fn invoke_preemption(&self, wake_task_priority: PriorityInfo) { - loop { - let (task_id, cpu_id, lowest_priority_info) = match get_lowest_priority_task_info() { - Some(info) => info, - None => return, - }; + let mut last_preempt_task_id = None; + + while let Some((task_id, cpu_id, lowest_priority_info)) = get_lowest_priority_task_info() { + if Some(task_id) == last_preempt_task_id { + continue; + } if wake_task_priority < lowest_priority_info { - let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); if NUM_SEND_IPI.load(Ordering::Relaxed) != 0 { + last_preempt_task_id = Some(task_id); continue; } + + let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); NUM_SEND_IPI.fetch_add(1, Ordering::Relaxed); set_need_preemption(task_id); awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); From e1accb0fb3467c9256bd2f690bf10ef01e1fdde6 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Tue, 7 Jan 2025 19:56:55 +0900 Subject: [PATCH 04/31] fix: cargo fmt Signed-off-by: kobayu858 --- applications/tests/test_sched_preempt/src/lib.rs | 2 +- awkernel_async_lib/src/scheduler.rs | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/applications/tests/test_sched_preempt/src/lib.rs b/applications/tests/test_sched_preempt/src/lib.rs index ed4fa96a6..5ef5f010e 100644 --- a/applications/tests/test_sched_preempt/src/lib.rs +++ b/applications/tests/test_sched_preempt/src/lib.rs @@ -117,4 +117,4 @@ async fn check_sched_preempt() { SchedulerType::GEDF(98000000), ) .await; -} \ No newline at end of file +} diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 2330934cf..fb068b2d9 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -12,9 +12,7 @@ use awkernel_lib::{ cpu::num_cpu, sync::mutex::{MCSNode, Mutex}, }; -use core::{ - sync::atomic::{AtomicUsize, Ordering}, -}; +use core::sync::atomic::{AtomicUsize, Ordering}; #[cfg(not(feature = "std"))] use alloc::boxed::Box; @@ -109,18 +107,17 @@ pub(crate) trait Scheduler { } if wake_task_priority < lowest_priority_info { + let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); if NUM_SEND_IPI.load(Ordering::Relaxed) != 0 { last_preempt_task_id = Some(task_id); continue; } - - let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); NUM_SEND_IPI.fetch_add(1, Ordering::Relaxed); set_need_preemption(task_id); awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); NUM_SEND_IPI.fetch_sub(1, Ordering::Relaxed); } - return; + break; } } } From 8340bedc7309792cbfc75b3616a25f7767d10c99 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 14:29:07 +0900 Subject: [PATCH 05/31] feat: wait until all task statuses are ready to load Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 26 ++++++++++++++++++++++++-- kernel/src/main.rs | 3 ++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index c152dd1bf..cc700da5d 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -25,7 +25,7 @@ use awkernel_lib::{ unwind::catch_unwind, }; use core::{ - sync::atomic::{AtomicU32, AtomicU64, Ordering}, + sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, task::{Context, Poll}, }; use futures::{ @@ -48,6 +48,8 @@ pub type TaskResult = Result<(), Cow<'static, str>>; static TASKS: Mutex = Mutex::new(Tasks::new()); // Set of tasks. static RUNNING: [AtomicU32; NUM_MAX_CPU] = array![_ => AtomicU32::new(0); NUM_MAX_CPU]; // IDs of running tasks. +pub static IS_LOAD_RUNNING: [AtomicBool; NUM_MAX_CPU] = + array![_ => AtomicBool::new(false); NUM_MAX_CPU]; // Whether or not RUNNING can be loaded. static MAX_TASK_PRIORITY: u64 = (1 << 56) - 1; // Maximum task priority. /// Task has ID, future, information, and a reference to a scheduler. @@ -676,6 +678,7 @@ pub fn run_main() { perf::add_kernel_time_start(awkernel_lib::cpu::cpu_id(), cpu_counter()); if let Some(task) = get_next_task() { + IS_LOAD_RUNNING[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); #[cfg(not(feature = "no_preempt"))] { // If the next task is a preempted task, then the current task will yield to the thread holding the next task. @@ -750,6 +753,7 @@ pub fn run_main() { }; RUNNING[cpu_id].store(task.id, Ordering::Relaxed); + IS_LOAD_RUNNING[cpu_id].store(true, Ordering::Relaxed); // Invoke a task. catch_unwind(|| { @@ -797,6 +801,7 @@ pub fn run_main() { awkernel_lib::heap::TALLOC.use_primary_then_backup_cpu_id(cpu_id) }; + IS_LOAD_RUNNING[cpu_id].store(false, Ordering::Relaxed); let running_id = RUNNING[cpu_id].swap(0, Ordering::Relaxed); assert_eq!(running_id, task.id); @@ -855,6 +860,7 @@ pub fn run_main() { } } } else { + IS_LOAD_RUNNING[awkernel_lib::cpu::cpu_id()].store(true, Ordering::Relaxed); #[cfg(feature = "perf")] perf::add_idle_time_start(awkernel_lib::cpu::cpu_id(), cpu_counter()); @@ -1078,15 +1084,31 @@ impl Ord for PriorityInfo { } } +pub fn wait_until_true_count(target_count: usize) { + loop { + let true_count = IS_LOAD_RUNNING + .iter() + .filter(|flag| flag.load(Ordering::Relaxed)) + .count(); + + if true_count >= target_count { + break; + } + } +} + pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { let mut lowest_task: Option<(u32, usize, PriorityInfo)> = None; // (task_id, cpu_id, priority_info) + let non_primary_cpus = awkernel_lib::cpu::num_cpu().saturating_sub(1); + + wait_until_true_count(non_primary_cpus); let running_tasks: Vec = get_tasks_running() .into_iter() .filter(|task| task.task_id != 0) .collect(); - if running_tasks.len() != awkernel_lib::cpu::num_cpu().saturating_sub(1) { + if running_tasks.len() != non_primary_cpus { return None; } diff --git a/kernel/src/main.rs b/kernel/src/main.rs index ff80bf2ec..dcfa36cc6 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -14,6 +14,7 @@ extern crate alloc; use awkernel_async_lib::{ scheduler::{wake_task, SchedulerType}, task, + task::IS_LOAD_RUNNING, }; use core::{ fmt::Debug, @@ -42,6 +43,7 @@ static NUM_READY_WORKER: AtomicU16 = AtomicU16::new(0); /// `Info` of `KernelInfo` represents architecture specific information. fn main(kernel_info: KernelInfo) { log::info!("CPU#{} is starting.", kernel_info.cpu_id); + IS_LOAD_RUNNING[usize::from(kernel_info.cpu_id)].store(true, Ordering::Relaxed); unsafe { awkernel_lib::cpu::increment_num_cpu() }; @@ -93,7 +95,6 @@ fn main(kernel_info: KernelInfo) { loop { awkernel_lib::interrupt::disable(); - wake_task(); // Wake executable tasks periodically. awkernel_lib::net::poll(); // Poll network devices. From 980f87a5f818cacffea09ec37365b2b8117f5f9d Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 14:29:11 +0900 Subject: [PATCH 06/31] feat: wait until all task statuses are ready to load Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index cc700da5d..13c024d88 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1084,7 +1084,11 @@ impl Ord for PriorityInfo { } } -pub fn wait_until_true_count(target_count: usize) { +pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { + let mut lowest_task: Option<(u32, usize, PriorityInfo)> = None; // (task_id, cpu_id, priority_info) + let non_primary_cpus = awkernel_lib::cpu::num_cpu().saturating_sub(1); + + // Wait until all task statuses are ready to load. loop { let true_count = IS_LOAD_RUNNING .iter() @@ -1095,13 +1099,6 @@ pub fn wait_until_true_count(target_count: usize) { break; } } -} - -pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { - let mut lowest_task: Option<(u32, usize, PriorityInfo)> = None; // (task_id, cpu_id, priority_info) - let non_primary_cpus = awkernel_lib::cpu::num_cpu().saturating_sub(1); - - wait_until_true_count(non_primary_cpus); let running_tasks: Vec = get_tasks_running() .into_iter() From 444a9831623be10225ca6d54777ae6f7131418c0 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 14:50:37 +0900 Subject: [PATCH 07/31] refactor: variable name Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 13c024d88..2f8321c16 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1095,7 +1095,7 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { .filter(|flag| flag.load(Ordering::Relaxed)) .count(); - if true_count >= target_count { + if true_count >= non_primary_cpus { break; } } From 9425d6ef855874f395da3b7435947bbfa5ac03c4 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 15:37:48 +0900 Subject: [PATCH 08/31] feat: support for Interruptions Signed-off-by: kobayu858 --- awkernel_async_lib/src/scheduler.rs | 7 ------- awkernel_async_lib/src/task/preempt.rs | 3 ++- kernel/src/arch/aarch64/exception.rs | 3 +++ kernel/src/arch/x86_64/interrupt_handler.rs | 2 ++ 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index fb068b2d9..738607f42 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -99,17 +99,10 @@ pub(crate) trait Scheduler { /// Invoke preemption. fn invoke_preemption(&self, wake_task_priority: PriorityInfo) { - let mut last_preempt_task_id = None; - while let Some((task_id, cpu_id, lowest_priority_info)) = get_lowest_priority_task_info() { - if Some(task_id) == last_preempt_task_id { - continue; - } - if wake_task_priority < lowest_priority_info { let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); if NUM_SEND_IPI.load(Ordering::Relaxed) != 0 { - last_preempt_task_id = Some(task_id); continue; } NUM_SEND_IPI.fetch_add(1, Ordering::Relaxed); diff --git a/awkernel_async_lib/src/task/preempt.rs b/awkernel_async_lib/src/task/preempt.rs index 04b0deaa8..aef2aadaa 100644 --- a/awkernel_async_lib/src/task/preempt.rs +++ b/awkernel_async_lib/src/task/preempt.rs @@ -1,4 +1,4 @@ -use crate::task::{get_current_task, Task}; +use crate::task::{get_current_task, Task, IS_LOAD_RUNNING}; use alloc::{collections::VecDeque, sync::Arc}; use array_macro::array; use awkernel_lib::{ @@ -176,6 +176,7 @@ impl Drop for RunningTaskGuard { } super::RUNNING[cpu_id].store(self.0, Ordering::Relaxed); + IS_LOAD_RUNNING[usize::from(cpu_id)].store(true, Ordering::Relaxed); } } diff --git a/kernel/src/arch/aarch64/exception.rs b/kernel/src/arch/aarch64/exception.rs index f21a23c14..428348390 100644 --- a/kernel/src/arch/aarch64/exception.rs +++ b/kernel/src/arch/aarch64/exception.rs @@ -1,3 +1,4 @@ +use awkernel_async_lib::task::IS_LOAD_RUNNING; #[cfg(feature = "perf")] use awkernel_async_lib::{ cpu_counter, @@ -11,6 +12,7 @@ use awkernel_lib::{ interrupt, }; use core::str::from_utf8_unchecked; +use core::sync::atomic::Ordering; const _ESR_EL1_EC_MASK: u64 = 0b111111 << 26; const _ESR_EL1_EC_UNKNOWN: u64 = 0b000000 << 26; @@ -224,6 +226,7 @@ ESR = 0x{:x} #[no_mangle] pub extern "C" fn curr_el_spx_irq_el1(_ctx: *mut Context, _sp: usize, _esr: usize) { + IS_LOAD_RUNNING[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); #[cfg(feature = "perf")] { add_task_end(awkernel_lib::cpu::cpu_id(), cpu_counter()); diff --git a/kernel/src/arch/x86_64/interrupt_handler.rs b/kernel/src/arch/x86_64/interrupt_handler.rs index be1bd73f8..cbfbdd49e 100644 --- a/kernel/src/arch/x86_64/interrupt_handler.rs +++ b/kernel/src/arch/x86_64/interrupt_handler.rs @@ -7,6 +7,7 @@ use awkernel_async_lib::{ }, }; +use awkernel_async_lib::task::IS_LOAD_RUNNING; use awkernel_lib::delay::wait_forever; use x86_64::structures::idt::{InterruptDescriptorTable, InterruptStackFrame, PageFaultErrorCode}; @@ -518,6 +519,7 @@ irq_handler!(irq253, 253); irq_handler!(irq254, 254); extern "x86-interrupt" fn preemption(_stack_frame: InterruptStackFrame) { + IS_LOAD_RUNNING[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); #[cfg(feature = "perf")] { add_task_end(awkernel_lib::cpu::cpu_id(), cpu_counter()); From beb0c0e42f6cac0d27bcd995920242e128fd2e1a Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 15:40:39 +0900 Subject: [PATCH 09/31] fix: revert white line Signed-off-by: kobayu858 --- kernel/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/src/main.rs b/kernel/src/main.rs index dcfa36cc6..8c498b243 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -95,6 +95,7 @@ fn main(kernel_info: KernelInfo) { loop { awkernel_lib::interrupt::disable(); + wake_task(); // Wake executable tasks periodically. awkernel_lib::net::poll(); // Poll network devices. From a2a651d8aa372643881bfe3ff22b172e9b22bdb9 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 16:05:33 +0900 Subject: [PATCH 10/31] fix: add import Signed-off-by: kobayu858 --- kernel/src/arch/x86_64/interrupt_handler.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/src/arch/x86_64/interrupt_handler.rs b/kernel/src/arch/x86_64/interrupt_handler.rs index cbfbdd49e..034b5dc7c 100644 --- a/kernel/src/arch/x86_64/interrupt_handler.rs +++ b/kernel/src/arch/x86_64/interrupt_handler.rs @@ -9,6 +9,7 @@ use awkernel_async_lib::{ use awkernel_async_lib::task::IS_LOAD_RUNNING; use awkernel_lib::delay::wait_forever; +use core::sync::atomic::Ordering; use x86_64::structures::idt::{InterruptDescriptorTable, InterruptStackFrame, PageFaultErrorCode}; static mut IDT: InterruptDescriptorTable = InterruptDescriptorTable::new(); From ad0822eb44b539eeb245aca9dbc30a43e1468234 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 16:10:05 +0900 Subject: [PATCH 11/31] fix: remove from Signed-off-by: kobayu858 --- awkernel_async_lib/src/task/preempt.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awkernel_async_lib/src/task/preempt.rs b/awkernel_async_lib/src/task/preempt.rs index aef2aadaa..152d128c1 100644 --- a/awkernel_async_lib/src/task/preempt.rs +++ b/awkernel_async_lib/src/task/preempt.rs @@ -176,7 +176,7 @@ impl Drop for RunningTaskGuard { } super::RUNNING[cpu_id].store(self.0, Ordering::Relaxed); - IS_LOAD_RUNNING[usize::from(cpu_id)].store(true, Ordering::Relaxed); + IS_LOAD_RUNNING[cpu_id].store(true, Ordering::Relaxed); } } From f4907b9c236736a598169a376c31cde2b28c262e Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 16:17:43 +0900 Subject: [PATCH 12/31] fix: remove from Signed-off-by: kobayu858 --- kernel/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 8c498b243..b40187666 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -43,7 +43,7 @@ static NUM_READY_WORKER: AtomicU16 = AtomicU16::new(0); /// `Info` of `KernelInfo` represents architecture specific information. fn main(kernel_info: KernelInfo) { log::info!("CPU#{} is starting.", kernel_info.cpu_id); - IS_LOAD_RUNNING[usize::from(kernel_info.cpu_id)].store(true, Ordering::Relaxed); + IS_LOAD_RUNNING[kernel_info.cpu_id].store(true, Ordering::Relaxed); unsafe { awkernel_lib::cpu::increment_num_cpu() }; From 1a3af6f04cd51748f68d6f2be125844c2be9a59d Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 16:48:19 +0900 Subject: [PATCH 13/31] refactor: IS_SEND_IPI Signed-off-by: kobayu858 --- awkernel_async_lib/src/scheduler.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 738607f42..b6663868b 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -12,7 +12,7 @@ use awkernel_lib::{ cpu::num_cpu, sync::mutex::{MCSNode, Mutex}, }; -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicBool, Ordering}; #[cfg(not(feature = "std"))] use alloc::boxed::Box; @@ -25,7 +25,7 @@ mod priority_based_rr; mod rr; static SLEEPING: Mutex = Mutex::new(SleepingTasks::new()); -static NUM_SEND_IPI: AtomicUsize = AtomicUsize::new(0); // The number of send IPI. +static IS_SEND_IPI: AtomicBool = AtomicBool::new(false); // The number of send IPI. /// Type of scheduler. /// `u8` is the priority of priority based schedulers. @@ -102,13 +102,13 @@ pub(crate) trait Scheduler { while let Some((task_id, cpu_id, lowest_priority_info)) = get_lowest_priority_task_info() { if wake_task_priority < lowest_priority_info { let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); - if NUM_SEND_IPI.load(Ordering::Relaxed) != 0 { + if IS_SEND_IPI.load(Ordering::Relaxed) { continue; } - NUM_SEND_IPI.fetch_add(1, Ordering::Relaxed); + IS_SEND_IPI.store(true, Ordering::Relaxed); set_need_preemption(task_id); awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); - NUM_SEND_IPI.fetch_sub(1, Ordering::Relaxed); + IS_SEND_IPI.store(false, Ordering::Relaxed); } break; } From 6ac8ee488f2f6a336e553dec3d02ee2dbbc139f5 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 17:03:45 +0900 Subject: [PATCH 14/31] docs: fix comment Signed-off-by: kobayu858 --- awkernel_async_lib/src/scheduler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index b6663868b..610a82c16 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -25,7 +25,7 @@ mod priority_based_rr; mod rr; static SLEEPING: Mutex = Mutex::new(SleepingTasks::new()); -static IS_SEND_IPI: AtomicBool = AtomicBool::new(false); // The number of send IPI. +static IS_SEND_IPI: AtomicBool = AtomicBool::new(false); // Whether IPI was sent or not. /// Type of scheduler. /// `u8` is the priority of priority based schedulers. From 91466363c563ef0cacd0d0b7a0c70090f3a4f14c Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 18:00:02 +0900 Subject: [PATCH 15/31] fix: IS_LOAD_RUNNING Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 1 - awkernel_async_lib/src/task/preempt.rs | 2 ++ kernel/src/main.rs | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 2f8321c16..c1b114df9 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -801,7 +801,6 @@ pub fn run_main() { awkernel_lib::heap::TALLOC.use_primary_then_backup_cpu_id(cpu_id) }; - IS_LOAD_RUNNING[cpu_id].store(false, Ordering::Relaxed); let running_id = RUNNING[cpu_id].swap(0, Ordering::Relaxed); assert_eq!(running_id, task.id); diff --git a/awkernel_async_lib/src/task/preempt.rs b/awkernel_async_lib/src/task/preempt.rs index 152d128c1..15a651649 100644 --- a/awkernel_async_lib/src/task/preempt.rs +++ b/awkernel_async_lib/src/task/preempt.rs @@ -115,6 +115,8 @@ fn re_schedule() { let mut node = MCSNode::new(); let mut tasks = PREEMPTED_TASKS[cpu_id].lock(&mut node); + IS_LOAD_RUNNING[awkernel_lib::cpu::cpu_id()].store(true, Ordering::Relaxed); + while let Some(task) = tasks.pop_front() { task.scheduler.wake_task(task); } diff --git a/kernel/src/main.rs b/kernel/src/main.rs index b40187666..58e3f4817 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -49,6 +49,7 @@ fn main(kernel_info: KernelInfo) { if kernel_info.cpu_id == 0 { // Primary CPU. + IS_LOAD_RUNNING[0].store(false, Ordering::Relaxed); let _ = draw_splash(); From e3f05e318793e1cfcb2b044170b0d07f3c3c4f01 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 18:44:20 +0900 Subject: [PATCH 16/31] fix: get_lowest_priority_task_info Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 77 ++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index c1b114df9..e3c434325 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1084,50 +1084,57 @@ impl Ord for PriorityInfo { } pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { - let mut lowest_task: Option<(u32, usize, PriorityInfo)> = None; // (task_id, cpu_id, priority_info) let non_primary_cpus = awkernel_lib::cpu::num_cpu().saturating_sub(1); + let mut lowest_task: Option<(u32, usize, PriorityInfo)> = None; // (task_id, cpu_id, priority_info) - // Wait until all task statuses are ready to load. loop { - let true_count = IS_LOAD_RUNNING - .iter() - .filter(|flag| flag.load(Ordering::Relaxed)) - .count(); + // Wait until all task statuses are ready to load. + loop { + let true_count = IS_LOAD_RUNNING + .iter() + .filter(|flag| flag.load(Ordering::Relaxed)) + .count(); - if true_count >= non_primary_cpus { - break; + if true_count >= non_primary_cpus { + break; + } } - } - let running_tasks: Vec = get_tasks_running() - .into_iter() - .filter(|task| task.task_id != 0) - .collect(); + let running_tasks: Vec = get_tasks_running() + .into_iter() + .filter(|task| task.task_id != 0) + .collect(); - if running_tasks.len() != non_primary_cpus { - return None; - } + if running_tasks.len() != non_primary_cpus { + return None; + } - let priority_infos: Vec<(u32, usize, PriorityInfo)> = { - let mut node = MCSNode::new(); - let tasks = TASKS.lock(&mut node); - running_tasks - .into_iter() - .filter_map(|task| { - tasks - .id_to_task - .get(&task.task_id) - .map(|task_data| (task.task_id, task.cpu_id, task_data.priority.clone())) - }) - .collect() - }; + let priority_infos: Vec<(u32, usize, PriorityInfo)> = { + let mut node = MCSNode::new(); + let tasks = TASKS.lock(&mut node); + running_tasks + .into_iter() + .filter_map(|task| { + tasks + .id_to_task + .get(&task.task_id) + .map(|task_data| (task.task_id, task.cpu_id, task_data.priority.clone())) + }) + .collect() + }; - for (task_id, cpu_id, priority_info) in priority_infos { - if lowest_task - .as_ref() - .is_none_or(|(_, _, lowest_priority_info)| priority_info > *lowest_priority_info) - { - lowest_task = Some((task_id, cpu_id, priority_info)); + for (task_id, cpu_id, priority_info) in priority_infos { + if lowest_task + .as_ref() + .is_none_or(|(_, _, lowest_priority_info)| priority_info > *lowest_priority_info) + { + lowest_task = Some((task_id, cpu_id, priority_info)); + } + } + + // Check to confirm that the information has not changed while getting priority_info. + if RUNNING[cpu_id] == task_id { + break; } } From 70c015e52586df82fee7324476d14e74caee6866 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 18:52:09 +0900 Subject: [PATCH 17/31] fix: get_lowest_priority_task_info Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index e3c434325..a4a22c1ce 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1133,8 +1133,10 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { } // Check to confirm that the information has not changed while getting priority_info. - if RUNNING[cpu_id] == task_id { - break; + if let Some((task_id, cpu_id, _)) = lowest_task { + if RUNNING[cpu_id].load(Ordering::Relaxed) == task_id { + break; + } } } From 52ee1659dc3db36d29e3fc16a14e7351a3036635 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 18:58:35 +0900 Subject: [PATCH 18/31] refactor: where to get preempt_irq Signed-off-by: kobayu858 --- awkernel_async_lib/src/scheduler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 610a82c16..cce1c77fd 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -101,11 +101,11 @@ pub(crate) trait Scheduler { fn invoke_preemption(&self, wake_task_priority: PriorityInfo) { while let Some((task_id, cpu_id, lowest_priority_info)) = get_lowest_priority_task_info() { if wake_task_priority < lowest_priority_info { - let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); if IS_SEND_IPI.load(Ordering::Relaxed) { continue; } IS_SEND_IPI.store(true, Ordering::Relaxed); + let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); set_need_preemption(task_id); awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); IS_SEND_IPI.store(false, Ordering::Relaxed); From 25d703b43bacef4c53060cbaa5ef75ff65b197bc Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 19:38:15 +0900 Subject: [PATCH 19/31] refactor:rename NOT_IN_TRANSITION Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 10 +++++----- awkernel_async_lib/src/task/preempt.rs | 6 +++--- kernel/src/arch/aarch64/exception.rs | 4 ++-- kernel/src/arch/x86_64/interrupt_handler.rs | 4 ++-- kernel/src/main.rs | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index a4a22c1ce..28f925e09 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -48,7 +48,7 @@ pub type TaskResult = Result<(), Cow<'static, str>>; static TASKS: Mutex = Mutex::new(Tasks::new()); // Set of tasks. static RUNNING: [AtomicU32; NUM_MAX_CPU] = array![_ => AtomicU32::new(0); NUM_MAX_CPU]; // IDs of running tasks. -pub static IS_LOAD_RUNNING: [AtomicBool; NUM_MAX_CPU] = +pub static NOT_IN_TRANSITION: [AtomicBool; NUM_MAX_CPU] = array![_ => AtomicBool::new(false); NUM_MAX_CPU]; // Whether or not RUNNING can be loaded. static MAX_TASK_PRIORITY: u64 = (1 << 56) - 1; // Maximum task priority. @@ -678,7 +678,7 @@ pub fn run_main() { perf::add_kernel_time_start(awkernel_lib::cpu::cpu_id(), cpu_counter()); if let Some(task) = get_next_task() { - IS_LOAD_RUNNING[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); + NOT_IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); #[cfg(not(feature = "no_preempt"))] { // If the next task is a preempted task, then the current task will yield to the thread holding the next task. @@ -753,7 +753,7 @@ pub fn run_main() { }; RUNNING[cpu_id].store(task.id, Ordering::Relaxed); - IS_LOAD_RUNNING[cpu_id].store(true, Ordering::Relaxed); + NOT_IN_TRANSITION[cpu_id].store(true, Ordering::Relaxed); // Invoke a task. catch_unwind(|| { @@ -859,7 +859,7 @@ pub fn run_main() { } } } else { - IS_LOAD_RUNNING[awkernel_lib::cpu::cpu_id()].store(true, Ordering::Relaxed); + NOT_IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(true, Ordering::Relaxed); #[cfg(feature = "perf")] perf::add_idle_time_start(awkernel_lib::cpu::cpu_id(), cpu_counter()); @@ -1090,7 +1090,7 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { loop { // Wait until all task statuses are ready to load. loop { - let true_count = IS_LOAD_RUNNING + let true_count = NOT_IN_TRANSITION .iter() .filter(|flag| flag.load(Ordering::Relaxed)) .count(); diff --git a/awkernel_async_lib/src/task/preempt.rs b/awkernel_async_lib/src/task/preempt.rs index 15a651649..e69a16a66 100644 --- a/awkernel_async_lib/src/task/preempt.rs +++ b/awkernel_async_lib/src/task/preempt.rs @@ -1,4 +1,4 @@ -use crate::task::{get_current_task, Task, IS_LOAD_RUNNING}; +use crate::task::{get_current_task, Task, NOT_IN_TRANSITION}; use alloc::{collections::VecDeque, sync::Arc}; use array_macro::array; use awkernel_lib::{ @@ -115,7 +115,7 @@ fn re_schedule() { let mut node = MCSNode::new(); let mut tasks = PREEMPTED_TASKS[cpu_id].lock(&mut node); - IS_LOAD_RUNNING[awkernel_lib::cpu::cpu_id()].store(true, Ordering::Relaxed); + NOT_IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(true, Ordering::Relaxed); while let Some(task) = tasks.pop_front() { task.scheduler.wake_task(task); @@ -178,7 +178,7 @@ impl Drop for RunningTaskGuard { } super::RUNNING[cpu_id].store(self.0, Ordering::Relaxed); - IS_LOAD_RUNNING[cpu_id].store(true, Ordering::Relaxed); + NOT_IN_TRANSITION[cpu_id].store(true, Ordering::Relaxed); } } diff --git a/kernel/src/arch/aarch64/exception.rs b/kernel/src/arch/aarch64/exception.rs index 428348390..22c0b16a3 100644 --- a/kernel/src/arch/aarch64/exception.rs +++ b/kernel/src/arch/aarch64/exception.rs @@ -1,4 +1,4 @@ -use awkernel_async_lib::task::IS_LOAD_RUNNING; +use awkernel_async_lib::task::NOT_IN_TRANSITION; #[cfg(feature = "perf")] use awkernel_async_lib::{ cpu_counter, @@ -226,7 +226,7 @@ ESR = 0x{:x} #[no_mangle] pub extern "C" fn curr_el_spx_irq_el1(_ctx: *mut Context, _sp: usize, _esr: usize) { - IS_LOAD_RUNNING[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); + NOT_IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); #[cfg(feature = "perf")] { add_task_end(awkernel_lib::cpu::cpu_id(), cpu_counter()); diff --git a/kernel/src/arch/x86_64/interrupt_handler.rs b/kernel/src/arch/x86_64/interrupt_handler.rs index 034b5dc7c..7e231303a 100644 --- a/kernel/src/arch/x86_64/interrupt_handler.rs +++ b/kernel/src/arch/x86_64/interrupt_handler.rs @@ -7,7 +7,7 @@ use awkernel_async_lib::{ }, }; -use awkernel_async_lib::task::IS_LOAD_RUNNING; +use awkernel_async_lib::task::NOT_IN_TRANSITION; use awkernel_lib::delay::wait_forever; use core::sync::atomic::Ordering; use x86_64::structures::idt::{InterruptDescriptorTable, InterruptStackFrame, PageFaultErrorCode}; @@ -520,7 +520,7 @@ irq_handler!(irq253, 253); irq_handler!(irq254, 254); extern "x86-interrupt" fn preemption(_stack_frame: InterruptStackFrame) { - IS_LOAD_RUNNING[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); + NOT_IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); #[cfg(feature = "perf")] { add_task_end(awkernel_lib::cpu::cpu_id(), cpu_counter()); diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 58e3f4817..ee3affa5e 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -14,7 +14,7 @@ extern crate alloc; use awkernel_async_lib::{ scheduler::{wake_task, SchedulerType}, task, - task::IS_LOAD_RUNNING, + task::NOT_IN_TRANSITION, }; use core::{ fmt::Debug, @@ -43,13 +43,13 @@ static NUM_READY_WORKER: AtomicU16 = AtomicU16::new(0); /// `Info` of `KernelInfo` represents architecture specific information. fn main(kernel_info: KernelInfo) { log::info!("CPU#{} is starting.", kernel_info.cpu_id); - IS_LOAD_RUNNING[kernel_info.cpu_id].store(true, Ordering::Relaxed); + NOT_IN_TRANSITION[kernel_info.cpu_id].store(true, Ordering::Relaxed); unsafe { awkernel_lib::cpu::increment_num_cpu() }; if kernel_info.cpu_id == 0 { // Primary CPU. - IS_LOAD_RUNNING[0].store(false, Ordering::Relaxed); + NOT_IN_TRANSITION[0].store(false, Ordering::Relaxed); let _ = draw_splash(); From cefb7fdb808c5b17b8fc8a5534308e6d4abf79fb Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 21:00:16 +0900 Subject: [PATCH 20/31] refactor: check preemption trigger Signed-off-by: kobayu858 --- awkernel_async_lib/src/scheduler.rs | 10 ++++++++-- awkernel_async_lib/src/task.rs | 7 +++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index cce1c77fd..36e4b2399 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -5,7 +5,10 @@ use crate::task::{ get_current_task, get_lowest_priority_task_info, get_scheduler_type_by_task_id, set_need_preemption, PriorityInfo, }; -use crate::{delay::uptime, task::Task}; +use crate::{ + delay::uptime, + task::{Task, NOT_IN_TRANSITION}, +}; use alloc::sync::Arc; use awkernel_async_lib_verified::delta_list::DeltaList; use awkernel_lib::{ @@ -25,7 +28,7 @@ mod priority_based_rr; mod rr; static SLEEPING: Mutex = Mutex::new(SleepingTasks::new()); -static IS_SEND_IPI: AtomicBool = AtomicBool::new(false); // Whether IPI was sent or not. +pub static IS_SEND_IPI: AtomicBool = AtomicBool::new(false); // Whether IPI was sent or not. /// Type of scheduler. /// `u8` is the priority of priority based schedulers. @@ -104,6 +107,9 @@ pub(crate) trait Scheduler { if IS_SEND_IPI.load(Ordering::Relaxed) { continue; } + if !NOT_IN_TRANSITION[cpu_id].load(Ordering::Relaxed) { + continue; + } IS_SEND_IPI.store(true, Ordering::Relaxed); let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); set_need_preemption(task_id); diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 28f925e09..53778a336 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -12,7 +12,7 @@ mod preempt; #[cfg(feature = "perf")] use crate::cpu_counter; -use crate::scheduler::{self, get_scheduler, Scheduler, SchedulerType}; +use crate::scheduler::{self, get_scheduler, Scheduler, SchedulerType, IS_SEND_IPI}; use alloc::{ borrow::Cow, collections::{btree_map, BTreeMap}, @@ -1134,7 +1134,10 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { // Check to confirm that the information has not changed while getting priority_info. if let Some((task_id, cpu_id, _)) = lowest_task { - if RUNNING[cpu_id].load(Ordering::Relaxed) == task_id { + if !IS_SEND_IPI.load(Ordering::Relaxed) + && NOT_IN_TRANSITION[cpu_id].load(Ordering::Relaxed) + && RUNNING[cpu_id].load(Ordering::Relaxed) == task_id + { break; } } From 3ead65f9b47f158f54f2b257e92f6bfb75f518dd Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 21:03:30 +0900 Subject: [PATCH 21/31] refactor: where to init lowest_task Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 53778a336..399f41e58 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1085,9 +1085,8 @@ impl Ord for PriorityInfo { pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { let non_primary_cpus = awkernel_lib::cpu::num_cpu().saturating_sub(1); - let mut lowest_task: Option<(u32, usize, PriorityInfo)> = None; // (task_id, cpu_id, priority_info) - loop { + let mut lowest_task: Option<(u32, usize, PriorityInfo)> = None; // (task_id, cpu_id, priority_info) // Wait until all task statuses are ready to load. loop { let true_count = NOT_IN_TRANSITION From 85ffac0a9f921d316257760b3b44350dc66a12af Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 21:07:18 +0900 Subject: [PATCH 22/31] fix: cargo fmt Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 399f41e58..7789d2524 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1087,7 +1087,7 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { let non_primary_cpus = awkernel_lib::cpu::num_cpu().saturating_sub(1); loop { let mut lowest_task: Option<(u32, usize, PriorityInfo)> = None; // (task_id, cpu_id, priority_info) - // Wait until all task statuses are ready to load. + // Wait until all task statuses are ready to load. loop { let true_count = NOT_IN_TRANSITION .iter() From 7cf1a43d31326673741ee36183ee9a4f0bdcc392 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 21:14:00 +0900 Subject: [PATCH 23/31] fix: build error Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 7789d2524..c49511300 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1085,9 +1085,10 @@ impl Ord for PriorityInfo { pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { let non_primary_cpus = awkernel_lib::cpu::num_cpu().saturating_sub(1); + let mut lowest_task: Option<(u32, usize, PriorityInfo)> = None; // (task_id, cpu_id, priority_info) + loop { - let mut lowest_task: Option<(u32, usize, PriorityInfo)> = None; // (task_id, cpu_id, priority_info) - // Wait until all task statuses are ready to load. + // Wait until all task statuses are ready to load. loop { let true_count = NOT_IN_TRANSITION .iter() @@ -1139,6 +1140,8 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { { break; } + } else { + lowest_task = None; } } From 793372ffe461d2a38ad36dab9d60ca4ae9928f17 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 22:00:36 +0900 Subject: [PATCH 24/31] fix: make test Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index c49511300..a777f84cb 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1105,7 +1105,7 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { .filter(|task| task.task_id != 0) .collect(); - if running_tasks.len() != non_primary_cpus { + if running_tasks.len() == 0 || running_tasks.len() != non_primary_cpus { return None; } @@ -1146,4 +1146,4 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { } lowest_task -} +} \ No newline at end of file From b2bf3657ee05fa22c2d2a14f0a9d11a793c2c548 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 22:03:07 +0900 Subject: [PATCH 25/31] fix: cargo fmt Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index a777f84cb..9a206fb53 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1146,4 +1146,4 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { } lowest_task -} \ No newline at end of file +} From 70ac8af584edbb37290bbbc2bcf50fb325adfa92 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Wed, 8 Jan 2025 22:07:59 +0900 Subject: [PATCH 26/31] fix: clippy Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 9a206fb53..e96dbafff 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1105,7 +1105,7 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { .filter(|task| task.task_id != 0) .collect(); - if running_tasks.len() == 0 || running_tasks.len() != non_primary_cpus { + if running_tasks.is_empty() || running_tasks.len() != non_primary_cpus { return None; } From 23fa42e8117ed9e94df513c9412fdc9f7f90bb65 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Tue, 14 Jan 2025 19:12:40 +0900 Subject: [PATCH 27/31] fix: preemption Signed-off-by: kobayu858 --- awkernel_async_lib/src/scheduler.rs | 32 +----------- awkernel_async_lib/src/scheduler/fifo.rs | 3 -- awkernel_async_lib/src/scheduler/gedf.rs | 3 -- .../src/scheduler/prioritized_fifo.rs | 3 -- .../src/scheduler/priority_based_rr.rs | 3 -- awkernel_async_lib/src/scheduler/rr.rs | 3 -- awkernel_async_lib/src/task.rs | 50 ++++++++++++++++++- awkernel_async_lib/src/task/preempt.rs | 2 - 8 files changed, 50 insertions(+), 49 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 36e4b2399..a10414f1f 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -1,21 +1,14 @@ //! Define types and trait for the Autoware Kernel scheduler. //! This module contains `SleepingTasks` for sleeping. -use crate::task::{ - get_current_task, get_lowest_priority_task_info, get_scheduler_type_by_task_id, - set_need_preemption, PriorityInfo, -}; -use crate::{ - delay::uptime, - task::{Task, NOT_IN_TRANSITION}, -}; +use crate::task::{get_current_task, get_scheduler_type_by_task_id}; +use crate::{delay::uptime, task::Task}; use alloc::sync::Arc; use awkernel_async_lib_verified::delta_list::DeltaList; use awkernel_lib::{ cpu::num_cpu, sync::mutex::{MCSNode, Mutex}, }; -use core::sync::atomic::{AtomicBool, Ordering}; #[cfg(not(feature = "std"))] use alloc::boxed::Box; @@ -28,7 +21,6 @@ mod priority_based_rr; mod rr; static SLEEPING: Mutex = Mutex::new(SleepingTasks::new()); -pub static IS_SEND_IPI: AtomicBool = AtomicBool::new(false); // Whether IPI was sent or not. /// Type of scheduler. /// `u8` is the priority of priority based schedulers. @@ -99,26 +91,6 @@ pub(crate) trait Scheduler { /// Get the priority of the scheduler. fn priority(&self) -> u8; - - /// Invoke preemption. - fn invoke_preemption(&self, wake_task_priority: PriorityInfo) { - while let Some((task_id, cpu_id, lowest_priority_info)) = get_lowest_priority_task_info() { - if wake_task_priority < lowest_priority_info { - if IS_SEND_IPI.load(Ordering::Relaxed) { - continue; - } - if !NOT_IN_TRANSITION[cpu_id].load(Ordering::Relaxed) { - continue; - } - IS_SEND_IPI.store(true, Ordering::Relaxed); - let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); - set_need_preemption(task_id); - awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); - IS_SEND_IPI.store(false, Ordering::Relaxed); - } - break; - } - } } /// Get the next executable task. diff --git a/awkernel_async_lib/src/scheduler/fifo.rs b/awkernel_async_lib/src/scheduler/fifo.rs index bfd489390..a4dc49fef 100644 --- a/awkernel_async_lib/src/scheduler/fifo.rs +++ b/awkernel_async_lib/src/scheduler/fifo.rs @@ -14,7 +14,6 @@ impl Scheduler for FIFOScheduler { fn wake_task(&self, task: Arc) { let mut node = MCSNode::new(); let mut queue = self.queue.lock(&mut node); - let wake_task_priority = task.priority.clone(); if let Some(queue) = queue.as_mut() { queue.push_back(task); } else { @@ -22,8 +21,6 @@ impl Scheduler for FIFOScheduler { q.push_back(task); *queue = Some(q); } - - self.invoke_preemption(wake_task_priority); } fn get_next(&self) -> Option> { diff --git a/awkernel_async_lib/src/scheduler/gedf.rs b/awkernel_async_lib/src/scheduler/gedf.rs index b75d75adb..a7be9a0a3 100644 --- a/awkernel_async_lib/src/scheduler/gedf.rs +++ b/awkernel_async_lib/src/scheduler/gedf.rs @@ -76,9 +76,6 @@ impl Scheduler for GEDFScheduler { absolute_deadline, wake_time, }); - - let wake_task_priority = task.priority.clone(); - self.invoke_preemption(wake_task_priority); } fn get_next(&self) -> Option> { diff --git a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs index 9a107d55a..72fd3e938 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs @@ -64,9 +64,6 @@ impl Scheduler for PrioritizedFIFOScheduler { ); *data = Some(prioritized_fifo_data); } - - let wake_task_priority = task.priority.clone(); - self.invoke_preemption(wake_task_priority); } fn get_next(&self) -> Option> { diff --git a/awkernel_async_lib/src/scheduler/priority_based_rr.rs b/awkernel_async_lib/src/scheduler/priority_based_rr.rs index 71ca95149..ff04037a7 100644 --- a/awkernel_async_lib/src/scheduler/priority_based_rr.rs +++ b/awkernel_async_lib/src/scheduler/priority_based_rr.rs @@ -49,9 +49,6 @@ impl Scheduler for PriorityBasedRRScheduler { let mut guard = self.data.lock(&mut node); let data = guard.get_or_insert_with(PriorityBasedRRData::new); data.queue.push(priority as u32, new_task); - - let wake_task_priority = task.priority.clone(); - self.invoke_preemption(wake_task_priority); } fn get_next(&self) -> Option> { diff --git a/awkernel_async_lib/src/scheduler/rr.rs b/awkernel_async_lib/src/scheduler/rr.rs index 34b87968f..3274ce01d 100644 --- a/awkernel_async_lib/src/scheduler/rr.rs +++ b/awkernel_async_lib/src/scheduler/rr.rs @@ -19,10 +19,7 @@ impl Scheduler for RRScheduler { fn wake_task(&self, task: Arc) { let mut node = MCSNode::new(); let mut guard = self.queue.lock(&mut node); - let wake_task_priority = task.priority.clone(); guard.get_or_insert_with(VecDeque::new).push_back(task); - - self.invoke_preemption(wake_task_priority); } fn get_next(&self) -> Option> { diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index e96dbafff..966777003 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -12,7 +12,7 @@ mod preempt; #[cfg(feature = "perf")] use crate::cpu_counter; -use crate::scheduler::{self, get_scheduler, Scheduler, SchedulerType, IS_SEND_IPI}; +use crate::scheduler::{self, get_scheduler, Scheduler, SchedulerType}; use alloc::{ borrow::Cow, collections::{btree_map, BTreeMap}, @@ -50,6 +50,9 @@ static TASKS: Mutex = Mutex::new(Tasks::new()); // Set of tasks. static RUNNING: [AtomicU32; NUM_MAX_CPU] = array![_ => AtomicU32::new(0); NUM_MAX_CPU]; // IDs of running tasks. pub static NOT_IN_TRANSITION: [AtomicBool; NUM_MAX_CPU] = array![_ => AtomicBool::new(false); NUM_MAX_CPU]; // Whether or not RUNNING can be loaded. +pub static NO_NEED_PREEMPT: [AtomicBool; NUM_MAX_CPU] = + array![_ => AtomicBool::new(false); NUM_MAX_CPU]; // Whether or not preemption is needed. +static IS_SEND_IPI: Mutex = Mutex::new(false); // Whether or not send IPI to other CPUs. static MAX_TASK_PRIORITY: u64 = (1 << 56) - 1; // Maximum task priority. /// Task has ID, future, information, and a reference to a scheduler. @@ -306,6 +309,14 @@ pub fn spawn( let scheduler = get_scheduler(sched_type); + #[cfg(all( + any(target_arch = "aarch64", target_arch = "x86_64"), + not(feature = "std") + ))] + { + awkernel_lib::interrupt::disable(); + } + let mut node = MCSNode::new(); let mut tasks = TASKS.lock(&mut node); let id = tasks.spawn(name, future.fuse(), scheduler, sched_type); @@ -313,7 +324,17 @@ pub fn spawn( drop(tasks); if let Some(task) = task { + let priority = task.priority.clone(); task.wake(); + invoke_preemption(priority); + } + + #[cfg(all( + any(target_arch = "aarch64", target_arch = "x86_64"), + not(feature = "std") + ))] + { + awkernel_lib::interrupt::enable(); } id @@ -801,6 +822,7 @@ pub fn run_main() { awkernel_lib::heap::TALLOC.use_primary_then_backup_cpu_id(cpu_id) }; + NOT_IN_TRANSITION[cpu_id].store(false, Ordering::Relaxed); let running_id = RUNNING[cpu_id].swap(0, Ordering::Relaxed); assert_eq!(running_id, task.id); @@ -1134,7 +1156,9 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { // Check to confirm that the information has not changed while getting priority_info. if let Some((task_id, cpu_id, _)) = lowest_task { - if !IS_SEND_IPI.load(Ordering::Relaxed) + let mut node = MCSNode::new(); + let is_send_ipi = IS_SEND_IPI.lock(&mut node); + if !*is_send_ipi && NOT_IN_TRANSITION[cpu_id].load(Ordering::Relaxed) && RUNNING[cpu_id].load(Ordering::Relaxed) == task_id { @@ -1147,3 +1171,25 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { lowest_task } + +/// Invoke preemption. +fn invoke_preemption(wake_task_priority: PriorityInfo) { + while let Some((task_id, cpu_id, lowest_priority_info)) = get_lowest_priority_task_info() { + if wake_task_priority < lowest_priority_info { + let mut node = MCSNode::new(); + let mut is_send_ipi = IS_SEND_IPI.lock(&mut node); + if *is_send_ipi { + continue; + } + *is_send_ipi = true; + if !NOT_IN_TRANSITION[cpu_id].load(Ordering::Relaxed) { + continue; + } + let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); + set_need_preemption(task_id); + awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); + *is_send_ipi = false; + } + break; + } +} diff --git a/awkernel_async_lib/src/task/preempt.rs b/awkernel_async_lib/src/task/preempt.rs index e69a16a66..1a6f2c731 100644 --- a/awkernel_async_lib/src/task/preempt.rs +++ b/awkernel_async_lib/src/task/preempt.rs @@ -115,8 +115,6 @@ fn re_schedule() { let mut node = MCSNode::new(); let mut tasks = PREEMPTED_TASKS[cpu_id].lock(&mut node); - NOT_IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(true, Ordering::Relaxed); - while let Some(task) = tasks.pop_front() { task.scheduler.wake_task(task); } From e8e200680364fb2bfe1053ed21563d1970477cfd Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Tue, 14 Jan 2025 19:14:46 +0900 Subject: [PATCH 28/31] fix: revert Signed-off-by: kobayu858 --- awkernel_async_lib/src/scheduler.rs | 4 ++-- awkernel_async_lib/src/scheduler/fifo.rs | 1 + awkernel_async_lib/src/scheduler/priority_based_rr.rs | 2 +- awkernel_async_lib/src/scheduler/rr.rs | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index a10414f1f..ea2472927 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -190,9 +190,9 @@ pub fn wake_task() { for cpu_id in 1..num_cpu() { if let Some(task_id) = get_current_task(cpu_id) { match get_scheduler_type_by_task_id(task_id) { - Some(SchedulerType::RR) => rr::SCHEDULER.invoke_rr_preemption(cpu_id, task_id), + Some(SchedulerType::RR) => rr::SCHEDULER.invoke_preemption(cpu_id, task_id), Some(SchedulerType::PriorityBasedRR(_)) => { - priority_based_rr::SCHEDULER.invoke_rr_preemption(cpu_id, task_id) + priority_based_rr::SCHEDULER.invoke_preemption(cpu_id, task_id) } _ => (), } diff --git a/awkernel_async_lib/src/scheduler/fifo.rs b/awkernel_async_lib/src/scheduler/fifo.rs index a4dc49fef..f1fad34ad 100644 --- a/awkernel_async_lib/src/scheduler/fifo.rs +++ b/awkernel_async_lib/src/scheduler/fifo.rs @@ -14,6 +14,7 @@ impl Scheduler for FIFOScheduler { fn wake_task(&self, task: Arc) { let mut node = MCSNode::new(); let mut queue = self.queue.lock(&mut node); + if let Some(queue) = queue.as_mut() { queue.push_back(task); } else { diff --git a/awkernel_async_lib/src/scheduler/priority_based_rr.rs b/awkernel_async_lib/src/scheduler/priority_based_rr.rs index ff04037a7..b5e7a9933 100644 --- a/awkernel_async_lib/src/scheduler/priority_based_rr.rs +++ b/awkernel_async_lib/src/scheduler/priority_based_rr.rs @@ -97,7 +97,7 @@ pub static SCHEDULER: PriorityBasedRRScheduler = PriorityBasedRRScheduler { impl PriorityBasedRRScheduler { // Invoke a preemption if the task exceeds the time quantum - pub fn invoke_rr_preemption(&self, cpu_id: usize, task_id: u32) { + pub fn invoke_preemption(&self, cpu_id: usize, task_id: u32) { let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); if let Some(last_executed) = get_last_executed_by_task_id(task_id) { let elapsed = awkernel_lib::delay::uptime() - last_executed; diff --git a/awkernel_async_lib/src/scheduler/rr.rs b/awkernel_async_lib/src/scheduler/rr.rs index 3274ce01d..7a67a39ca 100644 --- a/awkernel_async_lib/src/scheduler/rr.rs +++ b/awkernel_async_lib/src/scheduler/rr.rs @@ -68,7 +68,7 @@ pub static SCHEDULER: RRScheduler = RRScheduler { impl RRScheduler { // Invoke a preemption if the task exceeds the time quantum - pub fn invoke_rr_preemption(&self, cpu_id: usize, task_id: u32) { + pub fn invoke_preemption(&self, cpu_id: usize, task_id: u32) { let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); if let Some(last_executed) = get_last_executed_by_task_id(task_id) { let elapsed = awkernel_lib::delay::uptime() - last_executed; From 4496ac8a463e40a0f6a0c024bfcc90cb3091f274 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Tue, 14 Jan 2025 20:00:44 +0900 Subject: [PATCH 29/31] refactor: deleted unuse variable Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 966777003..bc924d014 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -50,8 +50,6 @@ static TASKS: Mutex = Mutex::new(Tasks::new()); // Set of tasks. static RUNNING: [AtomicU32; NUM_MAX_CPU] = array![_ => AtomicU32::new(0); NUM_MAX_CPU]; // IDs of running tasks. pub static NOT_IN_TRANSITION: [AtomicBool; NUM_MAX_CPU] = array![_ => AtomicBool::new(false); NUM_MAX_CPU]; // Whether or not RUNNING can be loaded. -pub static NO_NEED_PREEMPT: [AtomicBool; NUM_MAX_CPU] = - array![_ => AtomicBool::new(false); NUM_MAX_CPU]; // Whether or not preemption is needed. static IS_SEND_IPI: Mutex = Mutex::new(false); // Whether or not send IPI to other CPUs. static MAX_TASK_PRIORITY: u64 = (1 << 56) - 1; // Maximum task priority. From 84a51a02dc1e82caa87f09306d8d5271682cc75c Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Mon, 20 Jan 2025 10:52:06 +0900 Subject: [PATCH 30/31] refactor: minor corrections Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 40 ++++++++++----------- awkernel_async_lib/src/task/preempt.rs | 4 +-- kernel/src/arch/aarch64/exception.rs | 4 +-- kernel/src/arch/x86_64/interrupt_handler.rs | 4 +-- kernel/src/main.rs | 6 ++-- 5 files changed, 28 insertions(+), 30 deletions(-) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index bc924d014..9a65d0115 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -48,9 +48,9 @@ pub type TaskResult = Result<(), Cow<'static, str>>; static TASKS: Mutex = Mutex::new(Tasks::new()); // Set of tasks. static RUNNING: [AtomicU32; NUM_MAX_CPU] = array![_ => AtomicU32::new(0); NUM_MAX_CPU]; // IDs of running tasks. -pub static NOT_IN_TRANSITION: [AtomicBool; NUM_MAX_CPU] = - array![_ => AtomicBool::new(false); NUM_MAX_CPU]; // Whether or not RUNNING can be loaded. -static IS_SEND_IPI: Mutex = Mutex::new(false); // Whether or not send IPI to other CPUs. +pub static IN_TRANSITION: [AtomicBool; NUM_MAX_CPU] = + array![_ => AtomicBool::new(true); NUM_MAX_CPU]; // Whether or not RUNNING can be loaded. +static IS_SEND_IPI: AtomicBool = AtomicBool::new(false); // Whether or not send IPI to other CPUs. static MAX_TASK_PRIORITY: u64 = (1 << 56) - 1; // Maximum task priority. /// Task has ID, future, information, and a reference to a scheduler. @@ -697,7 +697,7 @@ pub fn run_main() { perf::add_kernel_time_start(awkernel_lib::cpu::cpu_id(), cpu_counter()); if let Some(task) = get_next_task() { - NOT_IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); + IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(true, Ordering::Relaxed); #[cfg(not(feature = "no_preempt"))] { // If the next task is a preempted task, then the current task will yield to the thread holding the next task. @@ -772,7 +772,7 @@ pub fn run_main() { }; RUNNING[cpu_id].store(task.id, Ordering::Relaxed); - NOT_IN_TRANSITION[cpu_id].store(true, Ordering::Relaxed); + IN_TRANSITION[cpu_id].store(false, Ordering::Relaxed); // Invoke a task. catch_unwind(|| { @@ -820,7 +820,7 @@ pub fn run_main() { awkernel_lib::heap::TALLOC.use_primary_then_backup_cpu_id(cpu_id) }; - NOT_IN_TRANSITION[cpu_id].store(false, Ordering::Relaxed); + IN_TRANSITION[cpu_id].store(true, Ordering::Relaxed); let running_id = RUNNING[cpu_id].swap(0, Ordering::Relaxed); assert_eq!(running_id, task.id); @@ -879,7 +879,7 @@ pub fn run_main() { } } } else { - NOT_IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(true, Ordering::Relaxed); + IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); #[cfg(feature = "perf")] perf::add_idle_time_start(awkernel_lib::cpu::cpu_id(), cpu_counter()); @@ -1110,12 +1110,12 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { loop { // Wait until all task statuses are ready to load. loop { - let true_count = NOT_IN_TRANSITION + let false_count = IN_TRANSITION .iter() - .filter(|flag| flag.load(Ordering::Relaxed)) + .filter(|flag| !flag.load(Ordering::Relaxed)) .count(); - if true_count >= non_primary_cpus { + if false_count == non_primary_cpus { break; } } @@ -1125,7 +1125,7 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { .filter(|task| task.task_id != 0) .collect(); - if running_tasks.is_empty() || running_tasks.len() != non_primary_cpus { + if running_tasks.is_empty() || running_tasks.len() < non_primary_cpus { return None; } @@ -1154,10 +1154,8 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { // Check to confirm that the information has not changed while getting priority_info. if let Some((task_id, cpu_id, _)) = lowest_task { - let mut node = MCSNode::new(); - let is_send_ipi = IS_SEND_IPI.lock(&mut node); - if !*is_send_ipi - && NOT_IN_TRANSITION[cpu_id].load(Ordering::Relaxed) + if !IS_SEND_IPI.load(Ordering::Relaxed) + && !IN_TRANSITION[cpu_id].load(Ordering::Relaxed) && RUNNING[cpu_id].load(Ordering::Relaxed) == task_id { break; @@ -1174,19 +1172,19 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { fn invoke_preemption(wake_task_priority: PriorityInfo) { while let Some((task_id, cpu_id, lowest_priority_info)) = get_lowest_priority_task_info() { if wake_task_priority < lowest_priority_info { - let mut node = MCSNode::new(); - let mut is_send_ipi = IS_SEND_IPI.lock(&mut node); - if *is_send_ipi { + if IS_SEND_IPI + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { continue; } - *is_send_ipi = true; - if !NOT_IN_TRANSITION[cpu_id].load(Ordering::Relaxed) { + if IN_TRANSITION[cpu_id].load(Ordering::Relaxed) { continue; } let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); set_need_preemption(task_id); awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); - *is_send_ipi = false; + IS_SEND_IPI.store(false, Ordering::Relaxed); } break; } diff --git a/awkernel_async_lib/src/task/preempt.rs b/awkernel_async_lib/src/task/preempt.rs index 1a6f2c731..3623905ec 100644 --- a/awkernel_async_lib/src/task/preempt.rs +++ b/awkernel_async_lib/src/task/preempt.rs @@ -1,4 +1,4 @@ -use crate::task::{get_current_task, Task, NOT_IN_TRANSITION}; +use crate::task::{get_current_task, Task, IN_TRANSITION}; use alloc::{collections::VecDeque, sync::Arc}; use array_macro::array; use awkernel_lib::{ @@ -176,7 +176,7 @@ impl Drop for RunningTaskGuard { } super::RUNNING[cpu_id].store(self.0, Ordering::Relaxed); - NOT_IN_TRANSITION[cpu_id].store(true, Ordering::Relaxed); + IN_TRANSITION[cpu_id].store(false, Ordering::Relaxed); } } diff --git a/kernel/src/arch/aarch64/exception.rs b/kernel/src/arch/aarch64/exception.rs index 22c0b16a3..8b1ec1453 100644 --- a/kernel/src/arch/aarch64/exception.rs +++ b/kernel/src/arch/aarch64/exception.rs @@ -1,4 +1,4 @@ -use awkernel_async_lib::task::NOT_IN_TRANSITION; +use awkernel_async_lib::task::IN_TRANSITION; #[cfg(feature = "perf")] use awkernel_async_lib::{ cpu_counter, @@ -226,7 +226,7 @@ ESR = 0x{:x} #[no_mangle] pub extern "C" fn curr_el_spx_irq_el1(_ctx: *mut Context, _sp: usize, _esr: usize) { - NOT_IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); + IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(true, Ordering::Relaxed); #[cfg(feature = "perf")] { add_task_end(awkernel_lib::cpu::cpu_id(), cpu_counter()); diff --git a/kernel/src/arch/x86_64/interrupt_handler.rs b/kernel/src/arch/x86_64/interrupt_handler.rs index 7e231303a..c4d762c19 100644 --- a/kernel/src/arch/x86_64/interrupt_handler.rs +++ b/kernel/src/arch/x86_64/interrupt_handler.rs @@ -7,7 +7,7 @@ use awkernel_async_lib::{ }, }; -use awkernel_async_lib::task::NOT_IN_TRANSITION; +use awkernel_async_lib::task::IN_TRANSITION; use awkernel_lib::delay::wait_forever; use core::sync::atomic::Ordering; use x86_64::structures::idt::{InterruptDescriptorTable, InterruptStackFrame, PageFaultErrorCode}; @@ -520,7 +520,7 @@ irq_handler!(irq253, 253); irq_handler!(irq254, 254); extern "x86-interrupt" fn preemption(_stack_frame: InterruptStackFrame) { - NOT_IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(false, Ordering::Relaxed); + IN_TRANSITION[awkernel_lib::cpu::cpu_id()].store(true, Ordering::Relaxed); #[cfg(feature = "perf")] { add_task_end(awkernel_lib::cpu::cpu_id(), cpu_counter()); diff --git a/kernel/src/main.rs b/kernel/src/main.rs index ee3affa5e..8328d7e4a 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -14,7 +14,7 @@ extern crate alloc; use awkernel_async_lib::{ scheduler::{wake_task, SchedulerType}, task, - task::NOT_IN_TRANSITION, + task::IN_TRANSITION, }; use core::{ fmt::Debug, @@ -43,13 +43,13 @@ static NUM_READY_WORKER: AtomicU16 = AtomicU16::new(0); /// `Info` of `KernelInfo` represents architecture specific information. fn main(kernel_info: KernelInfo) { log::info!("CPU#{} is starting.", kernel_info.cpu_id); - NOT_IN_TRANSITION[kernel_info.cpu_id].store(true, Ordering::Relaxed); + IN_TRANSITION[kernel_info.cpu_id].store(false, Ordering::Relaxed); unsafe { awkernel_lib::cpu::increment_num_cpu() }; if kernel_info.cpu_id == 0 { // Primary CPU. - NOT_IN_TRANSITION[0].store(false, Ordering::Relaxed); + IN_TRANSITION[0].store(true, Ordering::Relaxed); let _ = draw_splash(); From d5b1cbea2f366a10a5f9d6f7792ed1e65d953a17 Mon Sep 17 00:00:00 2001 From: kobayu858 Date: Mon, 20 Jan 2025 10:56:53 +0900 Subject: [PATCH 31/31] docs: add explain Signed-off-by: kobayu858 --- awkernel_async_lib/src/task.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 9a65d0115..9a3225a2e 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1125,6 +1125,7 @@ pub fn get_lowest_priority_task_info() -> Option<(u32, usize, PriorityInfo)> { .filter(|task| task.task_id != 0) .collect(); + // 'is_empty()' is required to pass 'make test' if running_tasks.is_empty() || running_tasks.len() < non_primary_cpus { return None; }