"fix" segv from #3198, #3199, and #3205

src/machine/heap.rs

@@ -599,7 +599,7 @@ impl Heap {
     pub(crate) fn with_cell_capacity(cap: usize) -> Result<Self, AllocError> {
         let ptr = unsafe {
             let layout = alloc::Layout::from_size_align(
-                cap * size_of::<HeapCellValue>(),
+                heap_index_checked!(cap).ok_or(AllocError)?,
                 size_of::<HeapCellValue>(),
             )
             .unwrap();
@@ -623,7 +623,7 @@ impl Heap {
 
     pub fn reserve(&mut self, num_cells: usize) -> Result<HeapWriter<'_>, AllocError> {
         let section;
-        let len = heap_index!(num_cells);
+        let len = heap_index_checked!(num_cells).ok_or(AllocError)?;
 
         loop {
             unsafe {
@@ -1148,7 +1148,9 @@ pub fn sized_iter_to_heap_list<SrcT: Into<HeapCellValue>>(
 ) -> Result<HeapCellValue, AllocError> {
     if size > 0 {
         let h = heap.cell_len();
-        let mut writer = heap.reserve(1 + 2 * size)?;
+        // not using checked_add for 1 + as the result of multiplying by 2 will be even and the largest representable usize is odd,
+        // so the addition cannot overflow
+        let mut writer = heap.reserve(1 + size.checked_mul(2).ok_or(AllocError)?)?;
 
         writer.write_with(|section| {
             for (idx, value) in values.enumerate() {

src/machine/machine_errors.rs

@@ -76,7 +76,6 @@ impl ValidType {
 
 #[derive(Debug, Clone, Copy)]
 pub(crate) enum ResourceError {
-    FiniteMemory(HeapCellValue),
     OutOfFiles,
 }
 
@@ -330,12 +329,6 @@ impl MachineState {
 
     pub(super) fn resource_error(err: ResourceError) -> MachineError {
         let stub = match err {
-            ResourceError::FiniteMemory(size_requested) => {
-                functor!(
-                    atom!("resource_error"),
-                    [atom_as_cell((atom!("finite_memory"))), cell(size_requested)]
-                )
-            }
             ResourceError::OutOfFiles => {
                 functor!(
                     atom!("resource_error"),
@@ -778,9 +771,16 @@ impl MachineState {
 
     // throw an error pre-allocated in the heap
     pub(super) fn throw_resource_error(&mut self, err: AllocError) {
+        if self.throwing_resource_error {
+            panic!("attempted to throw `error(resource_error(memory), [])` while attempting to throw `error(resource_error(memory), [])`");
+        }
+        self.throwing_resource_error = true;
+
         self.registers[1] = str_loc_as_cell!(err.resource_error_offset(&mut self.heap));
         self.set_ball();
         self.unwind_stack();
+
+        self.throwing_resource_error = false;
     }
 
     pub(super) fn throw_exception(&mut self, err: MachineStub) {

src/machine/machine_state.rs

@@ -74,6 +74,7 @@ pub struct MachineState {
     pub(super) cp: usize,
     pub(super) attr_var_init: AttrVarInitializer,
     pub(super) fail: bool,
+    pub throwing_resource_error: bool,
     pub heap: Heap,
     pub(super) mode: MachineMode,
     pub(crate) stack: Stack,

src/machine/machine_state_impl.rs

@@ -67,6 +67,7 @@ impl MachineState {
             unify_fn: MachineState::unify,
             bind_fn: MachineState::bind,
             run_cleaners_fn: |_| false,
+            throwing_resource_error: false,
         }
     }
 

src/machine/system_calls.rs

@@ -4371,16 +4371,15 @@ impl Machine {
 
     #[inline(always)]
     pub(crate) fn det_length_rundown(&mut self) -> CallResult {
-        let stub_gen = || functor_stub(atom!("length"), 2);
         let len = self.deref_register(2);
 
         let n = match Number::try_from((len, &self.machine_st.arena.f64_tbl)) {
             Ok(Number::Fixnum(n)) => n.get_num() as usize,
             Ok(Number::Integer(n)) => match (&*n).try_into() as Result<usize, _> {
                 Ok(n) => n,
                 Err(_) => {
-                    let err = MachineState::resource_error(ResourceError::FiniteMemory(len));
-                    return Err(self.machine_st.error_form(err, stub_gen()));
+                    self.machine_st.throw_resource_error(AllocError);
+                    return Ok(());
                 }
             },
             _ => {

src/macros.rs

@@ -470,12 +470,27 @@ macro_rules! resource_error_call_result {
     };
 }
 
-macro_rules! heap_index {
+macro_rules! heap_index_checked {
     ($idx:expr) => {
-        ($idx) * std::mem::size_of::<HeapCellValue>()
+        std::mem::size_of::<HeapCellValue>().checked_mul($idx)
     };
 }
 
+pub(crate) use heap_index_checked;
+
+macro_rules! heap_index {
+    ($idx:expr) => {{
+        let idx = $idx;
+        $crate::macros::heap_index_checked!(idx).unwrap_or_else(|| {
+            panic!(
+                "overflow while calculating heap index {idx} * {} > {}",
+                std::mem::size_of::<HeapCellValue>(),
+                usize::MAX,
+            )
+        })
+    }};
+}
+
 macro_rules! cell_index {
     ($idx:expr) => {
         ($idx) / std::mem::size_of::<HeapCellValue>()