diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp
index 5806b9ba70d34..25b96c80e41d7 100644
--- a/sycl/source/detail/global_handler.cpp
+++ b/sycl/source/detail/global_handler.cpp
@@ -330,6 +330,22 @@ void shutdown_early(bool CanJoinThreads = true) {
     GlobalHandler::RTGlobalObjHandler->MHostTaskThreadPool.Inst.reset(nullptr);
   }
 
+  // Reset in-memory cache before releasing default contexts.
+  {
+    // Keeping the default context for platforms in the global cache to avoid
+    // shared_ptr based circular dependency between platform and context classes
+    std::lock_guard<std::mutex> Lock{
+        GlobalHandler::RTGlobalObjHandler
+            ->getPlatformToDefaultContextCacheMutex()};
+
+    auto &PlatformToDefaultContextCache =
+        GlobalHandler::RTGlobalObjHandler->getPlatformToDefaultContextCache();
+
+    for (auto &Pair : PlatformToDefaultContextCache) {
+      Pair.second->getKernelProgramCache().reset();
+    }
+  }
+
   // This releases OUR reference to the default context, but
   // other may yet have refs
   GlobalHandler::RTGlobalObjHandler->releaseDefaultContexts();
diff --git a/sycl/source/detail/kernel_program_cache.hpp b/sycl/source/detail/kernel_program_cache.hpp
index 55919b74a6e3f..04930b6edb55e 100644
--- a/sycl/source/detail/kernel_program_cache.hpp
+++ b/sycl/source/detail/kernel_program_cache.hpp
@@ -573,9 +573,8 @@ class KernelProgramCache {
 
       traceProgram("Program evicted.", CacheKey);
     } else
-      // This should never happen.
-      throw sycl::exception(sycl::make_error_code(sycl::errc::runtime),
-                            "Program not found in the cache.");
+      // This can happen if we try to remove cache entries after early shutdown.
+      return 0;
 
     return MCachedPrograms.ProgramCacheSizeInBytes;
   }
@@ -684,9 +683,9 @@ class KernelProgramCache {
     std::lock_guard<std::mutex> L1(MProgramCacheMutex);
     std::lock_guard<std::mutex> L2(MKernelsPerProgramCacheMutex);
     FastKernelCacheWriteLockT L3(MFastKernelCacheMutex);
-    MCachedPrograms = ProgramCache{};
-    MKernelsPerProgramCache = KernelCacheT{};
-    MFastKernelCache = FastKernelCacheT{};
+    MCachedPrograms.Cache.clear();
+    MKernelsPerProgramCache.clear();
+    MFastKernelCache.clear();
     MProgramToFastKernelCacheKeyMap.clear();
     // Clear the eviction lists and its mutexes.
     MEvictionList.clear();
diff --git a/sycl/test-e2e/Basic/queue/release.cpp b/sycl/test-e2e/Basic/queue/release.cpp
index 5f4dc58c92575..67e8438092877 100644
--- a/sycl/test-e2e/Basic/queue/release.cpp
+++ b/sycl/test-e2e/Basic/queue/release.cpp
@@ -23,7 +23,7 @@ int main() {
 // of these UR objects. So, we currently shutdown without releasing them and
 // windows should handle the memory cleanup.
 
-// CHECK-RELEASE: <--- urContextRelease(
 // CHECK-RELEASE: <--- urKernelRelease(
 // CHECK-RELEASE: <--- urProgramRelease(
+// CHECK-RELEASE: <--- urContextRelease(
 // CHECK-RELEASE: <--- urDeviceRelease(
diff --git a/sycl/test-e2e/KernelAndProgram/disable-caching.cpp b/sycl/test-e2e/KernelAndProgram/disable-caching.cpp
index 7d283372e2ae6..271265b8fe122 100644
--- a/sycl/test-e2e/KernelAndProgram/disable-caching.cpp
+++ b/sycl/test-e2e/KernelAndProgram/disable-caching.cpp
@@ -112,12 +112,13 @@ int main() {
 // of these UR objects. So, we currently shutdown without releasing them and
 // windows should handle the memory cleanup.
 
-// (Program cache releases)
+// (Program cache releases during early shutdown)
+// CHECK-RELEASE: <--- urQueueRelease
+// CHECK-RELEASE: <--- urProgramRelease
+// CHECK-RELEASE: <--- urProgramRelease
 // CHECK-RELEASE: <--- urProgramRelease
-// CHECK-RELEASE: <--- urKernelRelease
 // CHECK-RELEASE: <--- urKernelRelease
 // CHECK-RELEASE: <--- urKernelRelease
 // CHECK-RELEASE: <--- urKernelRelease
 // CHECK-RELEASE: <--- urProgramRelease
-// CHECK-RELEASE: <--- urProgramRelease
-// CHECK-RELEASE: <--- urProgramRelease
+// CHECK-RELEASE: <--- urKernelRelease
diff --git a/sycl/test-e2e/Scheduler/ReleaseResourcesTest.cpp b/sycl/test-e2e/Scheduler/ReleaseResourcesTest.cpp
index fb634c832114d..d691314ff46c6 100644
--- a/sycl/test-e2e/Scheduler/ReleaseResourcesTest.cpp
+++ b/sycl/test-e2e/Scheduler/ReleaseResourcesTest.cpp
@@ -55,6 +55,6 @@ int main() {
 // windows should handle the memory cleanup.
 
 // CHECK-RELEASE: <--- urQueueRelease
-// CHECK-RELEASE: <--- urContextRelease
-// CHECK-RELEASE: <--- urKernelRelease
 // CHECK-RELEASE: <--- urProgramRelease
+// CHECK-RELEASE: <--- urKernelRelease
+// CHECK-RELEASE: <--- urContextRelease