diff --git a/Cargo.lock b/Cargo.lock index 06b44c9..2bf08bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,12 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + [[package]] name = "cfg-if" version = "1.0.4" @@ -36,6 +42,7 @@ version = "0.1.0" dependencies = [ "libc", "tempfile", + "uuid", ] [[package]] @@ -50,6 +57,16 @@ dependencies = [ "wasip2", ] +[[package]] +name = "js-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.180" @@ -68,6 +85,24 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -87,6 +122,23 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.24.0" @@ -100,6 +152,23 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "uuid" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" +dependencies = [ + "getrandom", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -109,6 +178,51 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 9b6aae0..d16c039 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,13 +3,14 @@ name = "first" version = "0.1.0" edition = "2024" description = "Deterministic crash testing framework for storage engines" -license = "MIT" +license = "Apache-2.0" repository = "https://github.com/siphonite/first" keywords = ["crash-testing", "storage", "database", "testing", "fault-injection"] categories = ["development-tools::testing"] [dependencies] libc = "0.2" +uuid = { version = "1.20.0", features = ["v4"] } [dev-dependencies] tempfile = "3" diff --git a/LICENSE b/LICENSE index e69de29..bc6f5a8 100644 --- a/LICENSE +++ b/LICENSE @@ -0,0 +1,204 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Copyright 2026 Aman Kumar diff --git a/README.md b/README.md index d5ea8db..8dcae97 100644 --- a/README.md +++ b/README.md @@ -1,501 +1,157 @@

- FIRST logo + FIRST logo

-# FIRST +

FIRST

-> FIRST is a deterministic crash and recovery testing framework for storage engines and WAL-based systems. - -## The Problem - -Modern databases and storage engines rely on strong crash-consistency guarantees. - -They write data using WALs, manifests, page files, and metadata updates, then recover state after crashes by replaying logs or rebuilding in-memory structures. These recovery paths are some of the most critical — and most fragile — code in the system. - -However, crash behavior is rarely tested systematically. - -Across the ecosystem today: - -- Unit tests validate logic assuming uninterrupted execution -- Integration tests exercise happy paths -- Benchmarks measure performance, not correctness -- Fault injection exists, but is often coarse or non-deterministic -- Crash consistency and power-loss behavior are tested ad-hoc, per project -- Storage correctness is largely enforced by tribal knowledge and past incidents - -Real failures do not happen at function boundaries. - -They happen: -- Between writes -- During system calls -- After data reaches the page cache but before persistence -- After a WAL append but before a commit marker -- After a rename but before directory fsync - -These failure modes are difficult to reason about and easy to miss. - -As a result, many storage systems ship with latent crash-consistency bugs that: -- Pass unit tests -- Pass integration tests -- Pass benchmarks -- Surface only under real crashes, often years later - -Every major database eventually builds internal crash-testing and chaos infrastructure to address this. - -But these solutions are: -- Reinvented independently -- Tightly coupled to internal codebases -- Hard to maintain -- Rarely exhaustive -- Not reusable across projects - -There is no standard framework, shared language, or common methodology for testing crash correctness in storage systems. - -**FIRST exists to fill this gap.** - ---- - -## Why FIRST - -Crash consistency failures are difficult to test because crashes are not normal execution paths. - -Most testing frameworks treat crashes as rare, external events. When failures are injected, they are often: -- Random -- Non-deterministic -- Coarse-grained -- Difficult to reproduce - -This makes failures hard to debug and discourages exhaustive testing. - -**FIRST takes a different approach.** - -FIRST treats crashes as a **first-class part of execution**. - -Instead of asking: -> "Does this code work?" - -FIRST asks: -> "At every possible crash point, does recovery preserve the system's invariants?" - -To answer this, FIRST provides: - -- **Deterministic crash injection** - Crashes are injected at precise, replayable points, making failures reproducible. - -- **Systematic exploration** - Crash points are enumerated deliberately, not discovered by chance. - -- **Recovery-driven testing** - Every crash is followed by a restart and execution of recovery logic. - -- **Invariant-based validation** - Tests assert properties that must always hold, rather than expected outputs. - -By making crashes deterministic, observable, and repeatable, FIRST turns crash consistency from an ad-hoc practice into a testable property. +

+ Deterministic crash testing for storage engines
+ Filesystem Injection and Reliability Stress Test +

-This allows storage systems to reason about correctness under failure with the same rigor applied to normal execution. +

+ Design • + Limitations • + Examples +

--- -## How It Works - -FIRST operates at the **persistence boundary** between your application and the filesystem. +## What is FIRST? -Here's the execution flow: +FIRST tests what happens when your storage system crashes. It injects crashes at precise points in your code, restarts, runs recovery, and validates invariants — deterministically and reproducibly. -1. **Crash Injection** - Mark crash points explicitly with `first::crash_point("label")` or let FIRST automatically intercept persistence syscalls (write, fsync, rename). - -2. **Workload Execution** - Your test workload runs normally until a scheduled crash point is reached, then the process terminates immediately (SIGKILL). - -3. **State Preservation** - Filesystem state is captured exactly as it existed at crash time using copy-on-write snapshots. - -4. **Restart & Recovery** - The process restarts, your storage system's recovery logic runs (WAL replay, index rebuild, etc.), and invariants are validated. - -5. **Systematic Iteration** - The test repeats for each crash point discovered in the workload, exploring all meaningful failure scenarios. +```rust +first::test() + .run(|env| { + let mut wal = Wal::open(&env.path("wal")).unwrap(); + let tx = wal.begin(); + wal.put(tx, "key", "value"); + first::crash_point("before_commit"); + wal.commit(tx); + }) + .verify(|env, _crash| { + let wal = Wal::open(&env.path("wal")).unwrap(); + // Invariant: uncommitted data must not be visible + assert!(wal.get("key").is_none() || wal.get("key") == Some("value")); + }) + .execute(); +``` -**Performance:** Each crash point takes approximately 50ms to test. A typical workload with 100 operations completes in seconds to minutes, making FIRST practical for continuous integration. +## Why FIRST? -**Determinism:** Given the same seed, FIRST produces identical crash sequences. Every failure is reproducible. +Real crashes don't happen at function boundaries. They happen: +- After `write()` but before `fsync()` +- After appending to WAL but before the commit marker +- After `rename()` but before directory sync -See [`DESIGN.md`](DESIGN.md) for detailed architecture and implementation approach. +Traditional tests don't catch these. FIRST does. ---- +| Traditional Testing | FIRST | +|---------------------|-------| +| Tests happy paths | Tests crash paths | +| Random fault injection | Deterministic crash points | +| Failures are flaky | Every failure is reproducible | +| Recovery rarely exercised | Recovery runs after every crash | -## Example +## Quick Start -Here's how you test an append-only log for crash consistency: +```toml +# Cargo.toml +[dev-dependencies] +first = "0.1" +``` ```rust -use first::test; - -/// Test that log entries are atomic - either fully written or not visible -fn test_log_atomicity() { +// tests/my_crash_test.rs +#[test] +fn test_atomicity() { first::test() - // Define the workload .run(|env| { - let log = AppendLog::create(env.path("test.log")); - - log.append("entry1").expect("append failed"); - // Implicit crash point: after write() syscall - - first::crash_point("after_first_append"); - // Explicit crash point: test critical boundary - - log.sync().expect("sync failed"); - // Implicit crash point: after fsync() syscall - - log.append("entry2").expect("append failed"); - // Implicit crash point: after write() syscall - - log.sync().expect("sync failed"); - // Implicit crash point: after fsync() syscall + // Your workload with crash_point() calls }) - - // Define recovery and validation - .verify(|env, crash_info| { - println!("Recovering from crash at: {:?}", crash_info); - - // Recovery: reopen the log (triggers internal recovery logic) - let log = AppendLog::open(env.path("test.log")) - .expect("log should always be recoverable"); - - let entries = log.read_all(); - - // Invariant: Atomicity - entries are all-or-nothing - match entries.len() { - 0 => { - // Crashed before first sync - no data persisted - assert!(entries.is_empty()); - }, - 1 => { - // Crashed after first sync, before second - assert_eq!(entries, vec!["entry1"]); - }, - 2 => { - // Crashed after both syncs - all data persisted - assert_eq!(entries, vec!["entry1", "entry2"]); - }, - _ => { - // Partial write visible - atomicity violated! - panic!("Invalid state after recovery: {:?}", entries); - } - } - - // Additional invariants - assert!(log.verify_checksums(), "Checksums must be valid"); - assert!(log.is_well_formed(), "Log structure must be consistent"); - }); + .verify(|env, crash| { + // Recovery + invariant checks + }) + .execute(); } ``` -**What this test does:** - -- Executes the workload with 5 crash points (2 explicit + 3 implicit from syscalls) -- For each crash point, terminates the process and preserves filesystem state -- Restarts and runs recovery logic -- Validates that invariants hold after recovery -- Reports violations with exact crash point for reproduction +```bash +cargo test +``` -**Example failure report:** +## How It Works ``` -FIRST Test Failure -═══════════════════════════════════════════════ -Test: test_log_atomicity -Crash Point: 3 ("after_first_append") -Seed: 42 - -Invariant Violation: - Partial write detected - entry1 has incomplete data - Expected: valid checksum - Actual: checksum mismatch (0x00000000) - -Filesystem state preserved at: /tmp/first/crash_3/ - -Reproduce: - $ FIRST_SEED=42 FIRST_CRASH_POINT=3 cargo test test_log_atomicity -═══════════════════════════════════════════════ +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ Orchestrator│────▶│ Execution │────▶│ Verify │ +│ (Parent) │ │ (Child) │ │ (Child) │ +└─────────────┘ └─────────────┘ └─────────────┘ + │ │ │ + │ SIGKILL at Recovery + + │ crash_point() Invariants + │ │ │ + └───────────────────┴────────────────────┘ + Repeat for each crash point ``` -This reproducibility transforms intermittent production bugs into debuggable test failures. - ---- - -## Performance - -FIRST is designed for continuous integration and fast feedback: - -| Workload | Crash Points | Test Time (Sequential) | Test Time (8 cores) | -|----------|--------------|------------------------|---------------------| -| Simple log (10 operations) | ~30 | 1.5 seconds | 0.3 seconds | -| Medium DB (100 operations) | ~300 | 15 seconds | 2 seconds | -| Complex LSM (1000 operations) | ~500 | 25 seconds | 4 seconds | +1. **Execution**: Your workload runs until hitting the target crash point → `SIGKILL` +2. **Verify**: Fresh process opens the preserved filesystem state, runs recovery, checks invariants +3. **Iterate**: Repeat for each crash point discovered -**Key characteristics:** +## Example Output -- **Linear scaling**: O(N) where N = number of crash points -- **Per-point overhead**: ~50ms on tmpfs (in-memory filesystem) -- **Parallel execution**: Near-linear speedup on multi-core systems -- **Practical for CI**: Most tests complete in seconds to minutes - -State space exploration is **bounded and systematic**, not exhaustive. FIRST focuses on high-value crash boundaries (fsync, rename, multi-step operations) rather than attempting to test every possible execution interleaving. - -See [`DESIGN.md § State Space and Performance`](DESIGN.md#7-state-space-and-performance) for detailed analysis. - ---- - -## How FIRST Differs from Existing Tools - -FIRST fills a specific gap in the crash testing ecosystem: - -| Tool | **FIRST** | CrashMonkey | Jepsen | dm-flakey | FoundationDB Sim | -|------|-----------|-------------|--------|-----------|------------------| -| **Target** | Storage Engines | Filesystems | Distributed Systems | Block Devices | Full System | -| **Operates At** | Application persistence boundary | Block I/O layer | Network layer | Kernel block device | Internal runtime | -| **Knowledge Level** | App-aware (WAL, manifests) | FS-aware (bio) | Black box | Block-level only | Full system | -| **Deterministic** | [YES] | [MOSTLY] | [NO] | [YES] (scriptable) | [YES] | -| **Setup Required** | `cargo test` | VM + kernel module | Cluster setup | Root + kernel module | Not extractable | -| **Recovery Testing** | [YES] Built-in | [YES] | [NO] | [LIMITED] | [YES] | -| **Open Source** | [YES] (planned) | [YES] | [YES] | [YES] | [NO] Proprietary | - -**Why not use existing tools?** - -- **CrashMonkey** tests filesystem implementations, not storage engines. It operates at the block I/O layer and doesn't understand application-level persistence semantics (e.g., "after WAL commit but before manifest update"). - -- **Jepsen** tests distributed systems by injecting network faults. FIRST tests local persistence and crash recovery—different problem domains. - -- **dm-flakey** injects faults at the block device level, requiring kernel modules and root access. FIRST operates at syscall boundaries and runs as a regular user process. - -- **FoundationDB's simulator** is proprietary and deeply integrated into their codebase. FIRST is designed as a reusable, open-source framework. - -**FIRST's unique position:** - -Unlike filesystem or distributed testing tools, FIRST understands **application-level persistence semantics** and provides **deterministic, reproducible** crash scenarios specifically for storage engine developers. - -It operates where your code meets the filesystem—at the exact boundary where crash bugs happen. - ---- - -## What FIRST Guarantees - -FIRST provides guarantees about **testing behavior**, not about application correctness by itself. - -Specifically, FIRST guarantees that: - -- **Crashes are deterministic and replayable** - Given the same test, crash schedule, and seed, failures can be reproduced exactly. - -- **Crash points are explicit and observable** - Tests run with well-defined crash boundaries rather than implicit assumptions about execution. - -- **Recovery logic is always exercised** - Every injected crash is followed by a restart and execution of user-provided recovery code. - -- **Persistent state after crashes is inspectable** - Filesystem state can be observed, compared, and validated after each crash. - -- **Invariant violations are surfaced early** - Violations appear as minimal, reproducible test failures rather than intermittent production bugs. - -FIRST does **not** guarantee that a system is correct. - -Instead, it guarantees that: -- Crash behavior is systematically explored -- Recovery paths are continuously tested -- Violations are reproducible and debuggable - -**Correctness remains the responsibility of the storage system and the invariants it defines.** - -FIRST provides the testing infrastructure to verify those invariants hold under crash conditions. - ---- - -## Non-Goals - -FIRST is intentionally focused and does not attempt to solve every testing problem. - -Specifically, FIRST is **not**: - -- **A fuzzer or property-based testing framework** - FIRST prioritizes determinism and systematic exploration over randomness. - -- **A performance or benchmarking tool** - FIRST focuses exclusively on correctness under crashes, not throughput or latency. - -- **A database or storage engine** - FIRST provides testing infrastructure, not storage primitives or recovery logic. +``` +[first] crash point 1: OK +[first] crash point 2: OK +[first] crash point 3: FAILED (see /tmp/first/run_3) +[first] crash label: "after_commit_write" +[first] to reproduce: + FIRST_PHASE=VERIFY FIRST_CRASH_TARGET=3 FIRST_WORK_DIR=/tmp/first/run_3 \ + cargo test my_test -- --exact +``` -- **A replacement for unit or integration tests** - FIRST complements existing testing by targeting crash and recovery paths. +## Key Features -- **A distributed systems testing framework** - FIRST targets local persistence and crash recovery, not network partitions or consensus failures. +| Feature | Description | +|---------|-------------| +| **Deterministic** | Same seed/target = same crash = same failure | +| **Reproducible** | Every failure includes exact reproduction command | +| **Zero Setup** | Works with `cargo test`, no kernel modules or VMs | +| **Invariant-Based** | Assert properties, not expected outputs | +| **Artifact Preservation** | Filesystem state preserved on failure for debugging | -These non-goals are deliberate. +## Limitations (v0.1) -By remaining narrowly scoped, FIRST aims to be reliable, predictable, and easy to integrate into existing storage systems. +- Linux only +- Single-threaded execution +- Explicit crash points (no syscall interception yet) +- One `first::test()` per `#[test]` function +- No async test support ---- +See [docs/limitations.md](docs/limitations.md) for details. ## Project Status -FIRST is in early development, with design solidified and implementation in progress. - -### Current State - -**Available Now:** -- ✓ Comprehensive design document ([`DESIGN.md`](DESIGN.md)) -- ✓ Core architecture defined -- ✓ API design finalized -- ✓ Implementation approach validated - -**In Progress:** -- → Rust implementation (core framework) -- → Example storage systems (append-only log, simple LSM) -- → Test suite and validation - -**Not Yet Available:** -- Public repository (coming soon) -- Stable release -- Language bindings (C++, Go planned for later phases) - -### Development Focus - -The current focus is on: -- Defining a clear and minimal core model -- Establishing deterministic crash and recovery semantics -- Validating the design through small, targeted examples - -The API is intentionally unstable and subject to change during this phase. - -This early stage prioritizes: -- **Correctness** over completeness -- **Clarity** over feature count -- **Design rigor** over rapid expansion - -### Feedback Welcome - -We're sharing the design early to gather feedback from storage and database engineers before the first public release. - -Feedback is especially valuable on: -- Invariant modeling patterns -- Crash boundary selection -- Recovery semantics -- API ergonomics -- Integration with existing storage systems - ---- - -## Roadmap - -FIRST is being developed in deliberate phases. - -### Phase 1 — Core Crash Testing (Current Focus) - -- [x] Design document and architecture -- [ ] Deterministic crash scheduler -- [ ] Explicit crash point annotations -- [ ] Process termination and restart (SIGKILL) -- [ ] Filesystem state preservation (CoW snapshots) -- [ ] Persistent state inspection -- [ ] Minimal example storage systems -- [ ] Basic invariant validation framework - -**Target:** Q2 2026 - -### Phase 2 — Reusable Test Harness - -- [ ] Ergonomic test API refinements -- [ ] Syscall interception mode (LD_PRELOAD) -- [ ] Crash point pruning and optimization -- [ ] Parallel crash point exploration -- [ ] Improved diagnostics and failure reproduction -- [ ] Integration with `cargo test` -- [ ] Documentation and examples - -**Target:** Q3 2026 - -### Phase 3 — Language Bindings and Ecosystem - -- [ ] C/C++ bindings -- [ ] Go bindings -- [ ] Java bindings (exploratory) -- [ ] Support for Windows and macOS -- [ ] Integration with existing test frameworks -- [ ] Community examples and case studies - -**Target:** Q4 2026 and beyond - -The roadmap is intentionally conservative. - -Features are added only when they preserve determinism, reproducibility, and clarity. Each phase builds on proven foundations from the previous phase. - ---- - -## Contributing - -FIRST is an early-stage project focused on correctness, determinism, and clear design. +**v0.1** — Core crash testing functionality complete. -Contributions are welcome, especially in the form of: -- Design feedback and architectural suggestions -- Invariant modeling ideas and patterns -- Small, focused examples of storage systems -- Documentation improvements -- Bug reports and edge case identification +- ✅ Deterministic crash injection +- ✅ Orchestrator lifecycle +- ✅ Invariant validation +- ✅ Reference WAL example +- ⬜ Syscall interception (planned v0.2) +- ⬜ Parallel execution (planned v0.2) -### Before Contributing Code +## Documentation -Please open an issue to discuss your approach before submitting code. - -This helps: -- Keep the core design coherent -- Avoid premature complexity -- Ensure contributions align with project goals -- Prevent duplicate or conflicting work - -### Development Principles - -All contributions should maintain FIRST's core principles: - -1. **Determinism first** — Every failure must be reproducible -2. **Simplicity** — Prefer clarity over cleverness -3. **Focused scope** — Stay within defined boundaries -4. **Test what you build** — Demonstrate correctness -5. **Document your decisions** — Explain the "why" - -See `CONTRIBUTING.md` for detailed guidelines (coming soon). - ---- - -## Learn More - -- **[Design Document](DESIGN.md)** — Comprehensive technical design, implementation details, and architectural decisions -- **[Examples](examples/)** — Reference implementations and test patterns (coming soon) -- **[API Documentation](docs/api.md)** — Detailed API reference (coming soon) -- **[FAQ](docs/faq.md)** — Common questions and answers (coming soon) - ---- +- **[Design Document](docs/design.md)** — Architecture and implementation details +- **[Limitations](docs/limitations.md)** — Known constraints and crash model +- **[Reference WAL](examples/reference_wal/)** — Complete example with crash-consistency proof ## License -[To be announced] - ---- - -## Acknowledgments - -FIRST builds on research in crash consistency testing, including: - -- **ALICE** (OSDI '14) — Logical crash consistency checking -- **CrashMonkey** (OSDI '18) — Automated filesystem crash testing -- **AGAMOTTO** (SOSP '21) — Filesystem testing via record/replay - -We're grateful to the storage systems research community for establishing the foundations that make FIRST possible. +Licensed under the [Apache License, Version 2.0](LICENSE). --- diff --git a/assets/image.jpg b/assets/image.jpg new file mode 100644 index 0000000..7b753a1 Binary files /dev/null and b/assets/image.jpg differ diff --git a/assets/logo.svg b/assets/logo.svg index c9790a3..5968cfe 100644 --- a/assets/logo.svg +++ b/assets/logo.svg @@ -1,16 +1,8 @@ - - - - - - - - - - - - - - - - + + + + + + + + \ No newline at end of file diff --git a/docs/architecture/001_structure.md b/docs/architecture/001_structure.md index ee1a06b..8cea9ba 100644 --- a/docs/architecture/001_structure.md +++ b/docs/architecture/001_structure.md @@ -1,115 +1,54 @@ -# Architecture Decision: Crate Structure and Execution Model +# ADR-001: Crate Structure -**Status:** Proposed -**Issue:** #1 +**Status:** Accepted -## 1. High-Level Structure +## Decision -**Decision:** `first` will be a **Library Crate**. - -Users will add `first` to their `dev-dependencies`. -Tests will be written as standard Rust integration tests (in `tests/*.rs`) or unit tests. +FIRST is a **library crate** with no CLI. Users add it to `dev-dependencies` and run tests via `cargo test`. ```toml [dev-dependencies] first = "0.1" ``` -There is no separate `first-cli` binary required to run tests. Standard `cargo test` is the interface. - -## 2. Execution Model: The "Self-Spawning Supervisor" - -To achieve process isolation and deterministic crashes, `first` uses a multiprocess architecture where the test binary spawns *itself*. - -### Three Execution Contexts +## Execution Model -1. **Orchestrator (The Supervisor)** - * **Role:** Manages the test lifecycle, FS snapshots, and scheduling. - * **Trigger:** Default state when `first::test()` is called with no special env vars. - * **Behavior:** - * Does *not* run the user's workload directly. - * Loops through the **Crash Schedule**. - * For each crash point: - 1. Prepares a fresh workspace (`/tmp/first/run_N`). - 2. Spawns **Child: Crash Phase**. - 3. Waits for child to exit (expected signal: SIGKILL). - 4. Snapshots the filesystem (CoW). - 5. Spawns **Child: Verify Phase** against the snapshot. - 6. Reports pass/fail. +FIRST uses a **self-spawning supervisor** pattern with three execution contexts: -2. **Execution Phase (The Victim)** - * **Role:** Runs the actual workload until it crashes. - * **Trigger:** `FIRST_PHASE=EXECUTION` - * **Behavior:** - * `first::test()` sees the env var and immediately executes the `.run()` closure. - * Ignores `.verify()` closure. - * `first::crash_point()` checks the global counter. - * If target reached -> `kill(SIGKILL, self)`. - -3. **Verification Phase (The Inspector)** - * **Role:** Recovers the system and checks invariants. - * **Trigger:** `FIRST_PHASE=VERIFY` - * **Behavior:** - * `first::test()` sees the env var. - * Skips `.run()` closure. - * Executes `.verify()` closure. - * Exits 0 on success, panic on failure. - -### Diagram +| Context | Trigger | Behavior | +|---------|---------|----------| +| **Orchestrator** | Default (no env vars) | Manages lifecycle, spawns children | +| **Execution** | `FIRST_PHASE=EXECUTION` | Runs `.run()`, crashes at target | +| **Verify** | `FIRST_PHASE=VERIFY` | Runs `.verify()`, checks invariants | ```mermaid sequenceDiagram - participant User as cargo test - participant Orch as Orchestrator (Pid 100) - participant Crash as Child: Execution (Pid 101) - participant Verify as Child: Verify (Pid 102) - - User->>Orch: Spawn test_foo - Orch->>Orch: Initialize Env, Plan Schedule - - loop For each Crash Point - Orch->>Crash: Spawn(FIRST_PHASE=EXECUTION) - Crash->>Crash: Run Workload... - Crash->>Crash: Crash Point Reached! (SIGKILL) - Crash--xOrch: Exit(SIGKILL) - - Orch->>Orch: Snapshot Filesystem - + participant Orch as Orchestrator + participant Exec as Execution (Child) + participant Verify as Verify (Child) + + loop For each crash point + Orch->>Exec: Spawn(FIRST_PHASE=EXECUTION) + Exec->>Exec: Run workload → SIGKILL + Exec--xOrch: Exit(137) Orch->>Verify: Spawn(FIRST_PHASE=VERIFY) - Verify->>Verify: Run Recovery & Invariants + Verify->>Verify: Recovery + invariants Verify-->>Orch: Exit(0) end - - Orch-->>User: Test Pass/Fail ``` -## 3. Interface Design - -### Mode Selection -We will use Environment Variables, handled internally by `first`. - -* `FIRST_PHASE`: `EXECUTION` | `VERIFY` (Default: `ORCHESTRATOR`) -* `FIRST_CRASH_TARGET`: Integer ID of the crash point to trigger. -* `FIRST_WORK_DIR`: Absolute path to the isolated directory for this run. - -### Global State -Since crash points can be deep in the call stack, we need global state access. - -* **Location:** `first::rt` (Runtime module) -* **Storage:** `static AtomicUsize` specific to the current process. - * `GLOBAL_CRASH_COUNTER`: Incremented by `crash_point()`. - * `TARGET_CRASH_POINT`: Loaded from env var at startup. - -## 4. Addressing Constraints +## Environment Variables -* **How does `cargo test` work with this?** - * `cargo test` runs the Orchestrator. - * The Orchestrator uses `std::env::current_exe()` to find the test binary. - * It re-runs the binary, filtering for the *specific test name* to avoid recursive bombs. +| Variable | Description | +|----------|-------------| +| `FIRST_PHASE` | `EXECUTION` / `VERIFY` (default: Orchestrator) | +| `FIRST_CRASH_TARGET` | Target crash point ID (1-indexed) | +| `FIRST_WORK_DIR` | Isolated directory for this run | -* **Cleanup?** - * The Orchestrator owns the temp dirs. It cleans them up unless `FIRST_KEEP_ARTIFACTS=1` is set. +## Global State -## 5. Recommendation +Crash points require global state access: -This structure keeps the API simple (`cargo test` just works) while providing the necessary isolation for crash testing. +- **Location:** `first::rt` +- **Storage:** `static AtomicUsize` for crash counter +- **Initialization:** Env vars read once at startup via `OnceLock` diff --git a/docs/architecture/002_crash_point.md b/docs/architecture/002_crash_point.md index c59c21a..0c8bc29 100644 --- a/docs/architecture/002_crash_point.md +++ b/docs/architecture/002_crash_point.md @@ -1,128 +1,51 @@ -# Architecture Decision: `crash_point()` Semantics +# ADR-002: `crash_point()` Semantics -## 1. Overview +**Status:** Accepted -`first::crash_point()` is the core primitive for explicit crash injection. This document defines its exact behavior for v0.1. - ---- - -## 2. Function Signature +## Function Signature ```rust -/// A potential crash location in the execution. -/// -/// When FIRST is active during the "Execution" phase, this function: -/// 1. Increments the global crash counter. -/// 2. If `counter == target`, terminates the process immediately. -/// 3. Otherwise, returns normally. -/// -/// When FIRST is inactive (Orchestrator or Verify phase), this is a no-op. pub fn crash_point(label: &str); ``` -**Labels are required.** This ensures: -- Every crash point is identifiable in logs/reports. -- Developers think intentionally about where crashes matter. -- Future features (e.g., filtering by label) are possible. +**Labels are required** — ensures every crash point is identifiable in logs and failure reports. ---- +## Behavior -## 3. Counter Increment Timing +| Phase | Behavior | +|-------|----------| +| Execution | Increments counter, may terminate process | +| Orchestrator | No-op | +| Verify | No-op | -The counter increments **at the start of `crash_point()`**, before any check. +## Counter Semantics + +Counter increments **before** the target check (1-indexed): ``` -crash_point("A") → Counter: 0 → 1. Check: 1 == target? -crash_point("B") → Counter: 1 → 2. Check: 2 == target? -crash_point("C") → Counter: 2 → 3. Check: 3 == target? +crash_point("A") → Counter: 0 → 1. Check: 1 == target? +crash_point("B") → Counter: 1 → 2. Check: 2 == target? ``` -**Rationale:** Incrementing first means "crash point N" refers to "crash *before* the Nth point completes." This models "power loss during operation N." - ---- +## Crash Metadata -## 4. Crash Metadata +When triggered, emits JSON to stderr before `SIGKILL`: -When a crash is triggered, the following is recorded (written to a file or stderr before `SIGKILL`): - -| Field | Description | Example | -|-------|-------------|---------| -| `point_id` | The numeric crash counter value | `42` | -| `label` | The string passed to `crash_point()` | `"after_wal_write"` | -| `seed` | The test's random seed (if any) | `12345` | -| `work_dir` | Path to the test's isolated directory | `/tmp/first/run_42` | - -**Format (JSON to stderr before kill):** ```json -{"event":"crash","point_id":42,"label":"after_wal_write","seed":12345,"work_dir":"/tmp/first/run_42"} +{"event":"crash","point_id":5,"label":"after_commit","seed":null,"work_dir":"/tmp/first/run_5"} ``` -The Orchestrator parses this from the child's stderr after `SIGKILL`. - ---- - -## 5. Target Not Reached - -If the "Execution" phase completes *without* the target crash point being reached: - -1. The child process exits normally (exit code 0). -2. The Orchestrator detects this. -3. **Interpretation:** The schedule is exhausted. All crash points have been explored. -4. The test passes (assuming all previous iterations passed). - -**Edge Case:** If `target == 0`, we crash immediately at the first `crash_point()` call. - ---- - -## 6. Determinism Guarantees (v0.1) - -For v0.1, we guarantee: - -| Property | Guarantee | -|----------|-----------| -| **Same crash point** | Given the same `target`, the crash occurs at the same logical location. | -| **Same label** | The label is always the one passed to the matching `crash_point()` call. | -| **Same filesystem state** | If the workload is deterministic, the FS state at crash is identical across runs. | - -**NOT Guaranteed (v0.1):** -- Thread interleaving (single-threaded model assumed). -- Timing of background OS operations (page cache flush). - ---- - -## 7. No-Op Behavior - -When `FIRST_PHASE` is not `EXECUTION`, `crash_point()` does nothing: - -```rust -pub fn crash_point(label: &str) { - if !is_execution_phase() { - return; // No-op: Orchestrator or Verify phase - } - // ... increment and check ... -} -``` - -This allows the same test code to run in all phases without conditional compilation. - ---- - -## 8. Summary - -| Question | Answer | -|----------|--------| -| When does counter increment? | At the *start* of `crash_point()`, before the check. | -| Are labels required? | **Yes**, always required. | -| What metadata is recorded? | `point_id`, `label`, `seed`, `work_dir` (JSON to stderr). | -| What if target not reached? | Child exits 0; Orchestrator treats it as "schedule exhausted." | -| Determinism for v0.1? | Same target → same crash location & FS state (single-threaded). | - ---- +## Exit Behavior -## 9. Open Questions (for later) +| Condition | Exit Code | Orchestrator Interpretation | +|-----------|-----------|----------------------------| +| Target reached | 137 (SIGKILL) | Expected crash → run VERIFY | +| Target not reached | 0 | Schedule exhausted → success | -- Should we support `crash_point_if(condition, label)` for conditional crashes? -- Should labels be globally unique, or can they repeat (e.g., in a loop)? -- Should we log *all* crash points encountered, not just the triggered one? +## Determinism Guarantees -These can be deferred to v0.2+. +| Guaranteed | Not Guaranteed | +|------------|----------------| +| Same target → same crash location | Thread interleaving | +| Same label | OS page cache timing | +| Same FS state (if workload is deterministic) | | diff --git a/docs/architecture/004_orchestrator.md b/docs/architecture/004_orchestrator.md index 69547cf..7e261df 100644 --- a/docs/architecture/004_orchestrator.md +++ b/docs/architecture/004_orchestrator.md @@ -1,136 +1,64 @@ -# Architecture Decision: Test Runner and Orchestration Loop +# ADR-004: Orchestrator -## 1. Overview +**Status:** Accepted -This document defines the orchestration engine for FIRST v0.1 — the supervisor that turns one test function into N crash-verify cycles. - ---- - -## 2. Core Loop +## Core Loop ``` target = 1 loop { - 1. Create work_dir: /tmp/first/run_{target} - 2. Spawn EXECUTION child with target + 1. Create /tmp/first/run_{target} + 2. Spawn EXECUTION child 3. Wait for exit: - - SIGKILL (137) → crash occurred, run VERIFY - - Exit 0 → schedule exhausted, DONE - - Other → test failure, STOP - 4. Spawn VERIFY child against same work_dir - 5. If VERIFY fails → STOP with failure + - 137 (SIGKILL) → run VERIFY + - 0 → done (schedule exhausted) + - other → failure + 4. Spawn VERIFY child + 5. If VERIFY fails → stop 6. target += 1 } ``` ---- +## Decisions -## 3. Locked Decisions (v0.1) - -| Decision | Choice | -|----------|--------| +| Aspect | Choice | +|--------|--------| | Discovery | Iterative (no pre-counting) | | Filesystem | Fresh directory per target | -| Phase selection | Environment variables | | Self-spawning | `std::env::current_exe()` | -| Crash detection | Exit code 137 (SIGKILL) | -| Cleanup on success | Delete (unless `FIRST_KEEP_ARTIFACTS=1`) | -| Cleanup on failure | **Always keep** | - ---- - -## 4. Test Name Filtering - -**Constraint:** FIRST v0.1 assumes one FIRST test per Rust test function. - -**Mechanism:** -1. Orchestrator extracts test name from `std::env::args()` or env var -2. Children are spawned with: `cargo test -- --exact` - -**Environment variable:** -``` -FIRST_TEST_NAME= -``` - ---- - -## 5. Environment Variables - -| Variable | Phase | Description | -|----------|-------|-------------| -| `FIRST_PHASE` | All | `EXECUTION` / `VERIFY` (default: Orchestrator) | -| `FIRST_CRASH_TARGET` | Execution | Target crash point ID (1-indexed) | -| `FIRST_WORK_DIR` | All | Isolated directory for this run | -| `FIRST_SEED` | All | Random seed for reproducibility | -| `FIRST_TEST_NAME` | Children | Exact test name to run | -| `FIRST_KEEP_ARTIFACTS` | Orchestrator | Set to `1` to keep dirs on success | +| Crash detection | Exit code 137 | +| Cleanup | Delete on success (keep on failure) | ---- +## Environment Variables -## 6. Exit Code Interpretation +| Variable | Description | +|----------|-------------| +| `FIRST_PHASE` | `EXECUTION` / `VERIFY` | +| `FIRST_CRASH_TARGET` | Target crash point (1-indexed) | +| `FIRST_WORK_DIR` | Isolated directory | +| `FIRST_SEED` | Random seed | +| `FIRST_KEEP_ARTIFACTS` | Set to `1` to preserve dirs | -| Exit Code | Meaning | Orchestrator Action | -|-----------|---------|---------------------| -| 137 | SIGKILL (crash occurred) | Run VERIFY phase | -| 0 | Normal completion | Schedule exhausted (success) | -| 101 | Rust panic | Test failure | -| Other | Unexpected | Test failure | +## Exit Codes ---- +| Code | Meaning | Action | +|------|---------|--------| +| 137 | SIGKILL | Run VERIFY | +| 0 | Normal | Schedule exhausted | +| 101 | Panic | Test failure | -## 7. Progress Output - -Minimal, one line per crash point: - -``` -[first] crash point 1: OK -[first] crash point 2: OK -[first] crash point 3: FAILED (see /tmp/first/run_3) -``` - -No spinners, percentages, or verbosity flags. - ---- - -## 8. API Shape +## API ```rust first::test() - .run(|env| { - // Workload: executed in EXECUTION phase only - }) - .verify(|env, crash_info| { - // Recovery + invariants: executed in VERIFY phase only - }); + .run(|env| { /* workload */ }) + .verify(|env, crash| { /* invariants */ }) + .execute(); ``` -**Behavior by phase:** - -| Phase | `.run()` | `.verify()` | -|-------|----------|-------------| -| Orchestrator | Stored, not called | Stored, not called | -| Execution | **Called** | Ignored | -| Verify | Ignored | **Called** | - ---- - -## 9. Components to Implement - -| Component | Purpose | File | -|-----------|---------|------| -| `TestBuilder` | Stores closures, detects mode | `src/test.rs` | -| `Env` | Provides `path()` to closures | `src/env.rs` | -| `CrashInfo` | Parsed crash metadata | `src/env.rs` | -| `Orchestrator` | The supervisor loop | `src/orchestrator.rs` | -| `spawn_child()` | Re-invokes with env vars | `src/orchestrator.rs` | - ---- - -## 10. Explicitly Deferred (NOT v0.1) +## Deferred (v0.2+) -- Snapshots / CoW filesystem -- Parallel crash runs +- CoW snapshots +- Parallel execution - Crash pruning -- Multiple FIRST tests per function -- Structured output formats (JSON) -- Windows/macOS support +- Windows/macOS diff --git a/docs/invariants.md b/docs/invariants.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/limitations.md b/docs/limitations.md index 4b2c1da..9332548 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -1,187 +1,60 @@ # FIRST Limitations -This document describes known limitations of the FIRST framework. - ---- - ## Crash Model (v0.1) -FIRST v0.1 models process crashes using `SIGKILL`. - -### What v0.1 Accurately Simulates - -- Sudden process termination -- Loss of in-memory state -- Incomplete syscalls (operations interrupted mid-execution) -- Recovery from filesystem state left by prior operations -- Page cache flush ordering non-determinism - -### What v0.1 Does NOT Simulate - -- **Power loss** — Actual power failure affects the block layer and hardware -- **Block-layer rollback** — Storage devices may lose data in their write caches -- **Torn writes** — Partial sector writes that can occur during power loss -- **Filesystem metadata rollback** — Directory entries applied but not journaled - -### Detectable Bug Classes - -| Bug Type | Detectable? | Notes | -|----------|-------------|-------| -| Recovery logic errors | ✅ Yes | SIGKILL accurately tests recovery paths | -| Transaction atomicity bugs | ✅ Yes | Commit-before-durable patterns exposed | -| Missing fsync before commit | ✅ Yes | Page cache flush ordering is undefined | -| Missing parent-directory fsync | ❌ No | Requires filesystem interposition | -| Partial sector writes | ❌ No | Requires block-layer simulation | -| Torn writes | ❌ No | Requires hardware-level simulation | - ---- - -## Reference WAL Bug (Intentional) - -The reference WAL contained one intentional recovery-semantic bug, which was introduced and fixed as part of the Milestone 3 proof. This demonstrates that FIRST can expose real crash-consistency violations. - -### Bug Description - -The WAL writes the transaction `COMMIT` marker before ensuring all transaction records are durable: - -``` -write(record_1) -write(record_2) -write(record_3) -write(COMMIT) -fsync() ← records and commit flushed together, order undefined -``` - -The correct implementation would be: - -``` -write(record_1) -write(record_2) -write(record_3) -fsync() ← ensure records are durable -write(COMMIT) -fsync() ← then write commit -``` - -### Failure Mode - -When a crash occurs at `after_commit_write` (after COMMIT is written but before fsync): - -1. Both records and COMMIT are in the kernel page cache -2. The process is killed with SIGKILL -3. The kernel flushes the page cache in undefined order -4. COMMIT may reach disk before all records -5. Recovery sees a committed transaction with missing records -6. **Atomicity invariant violated** - -### Why This Bug Is Valid Under SIGKILL - -This bug does not require power loss or block-layer simulation. It relies only on: - -- **Page cache behavior**: Writes go to page cache, not directly to disk -- **Flush ordering**: Kernel may flush pages in any order -- **Missing durability barrier**: No fsync between records and COMMIT - -When SIGKILL terminates the process, the kernel continues to manage the page cache. The order in which dirty pages are written to disk is not guaranteed to match the order they were written by the application. +FIRST uses `SIGKILL` to simulate crashes. ---- +### What It Simulates -## Proof of FIRST Effectiveness (v0.1) +✅ Sudden process termination +✅ Loss of in-memory state +✅ Recovery from filesystem state +✅ Page cache flush ordering non-determinism -The following proof chain demonstrates that FIRST v0.1 can expose real crash-consistency bugs: +### What It Does NOT Simulate -1. **FIRST injects a crash** at the `after_commit_write` crash point -2. **The process is killed** with SIGKILL -3. **Filesystem state is preserved** exactly as it existed at crash time -4. **Recovery runs** and opens the WAL -5. **Recovery observes** a `COMMIT` marker for transaction 1 -6. **Recovery attempts** to replay the transaction -7. **Some records are missing** (not yet flushed to disk) -8. **Atomicity invariant fails**: committed transaction has partial visibility -9. **Failure is deterministic** and reproducible with the same crash point +❌ Power loss (block-layer effects) +❌ Torn writes (partial sector writes) +❌ Filesystem metadata rollback -This proof validates: +### Detectable Bugs -- Crash point injection works correctly -- SIGKILL-based process termination preserves filesystem state -- Recovery logic is exercised after each crash -- Invariant checking detects atomicity violations -- Failures are reproducible for debugging +| Bug Type | Detectable | +|----------|------------| +| Recovery logic errors | ✅ | +| Transaction atomicity bugs | ✅ | +| Missing fsync before commit | ✅ | +| Missing parent-directory fsync | ❌ | +| Torn writes | ❌ | --- -## Future Versions - -Future versions of FIRST may add enhanced crash simulation: - -### v0.2 — Filesystem Interposition - -- `LD_PRELOAD` to intercept syscalls -- Simulate incomplete `rename()` or `write()` operations -- Expose directory-fsync bugs - -### v0.3 — FUSE-based Simulation - -- Virtual filesystem with selective data loss -- Full control over what survives a crash - -### v0.4 — Block-layer Simulation +## Integration Constraints -- Device-level interception -- Simulate write cache behavior and torn writes +| Constraint | Status | +|------------|--------| +| One `first::test()` per `#[test]` | Required | +| Single-threaded `.run()` closure | Required | +| `#[tokio::test]` / async | ❌ Not supported | +| `crash_point()` from spawned threads | ❌ Undefined | +| Nested workspaces | ❌ Not supported | --- -## Other Known Limitations - -### Single-Process Only (v0.1) - -FIRST v0.1 runs tests in a single-process model. Multi-process or distributed scenarios are not supported. - -### No Concurrency Testing (v0.1) - -Crash points are serialized. Concurrent operations are not explored for race conditions related to persistence. - -### Linux Only (v0.1) +## Platform Support -The current implementation uses Linux-specific APIs (`SIGKILL`, `/proc`). macOS and Windows support is not yet available. +| Platform | Status | +|----------|--------| +| Linux | ✅ Supported | +| macOS | ❌ Planned | +| Windows | ❌ Planned | --- -## Integration Constraints (v0.1) - -### One `first::test()` Per `#[test]` - -| Pattern | Status | -|---------|--------| -| One `first::test()` per `#[test]` | ✅ Supported | -| Multiple `first::test()` in one `#[test]` | ❌ Undefined | - -**Rationale**: The orchestrator manages process lifecycle. Nested or sequential calls conflict. - -### Async Tests Not Supported - -| Pattern | Status | -|---------|--------| -| `#[test]` (sync) | ✅ Supported | -| `#[tokio::test]` | ❌ Not supported | -| `#[async_std::test]` | ❌ Not supported | - -**Rationale**: FIRST uses `fork()` + `SIGKILL` — async runtimes don't survive this. - -### Not Thread-Safe - -| Pattern | Status | -|---------|--------| -| Single-threaded `.run()` closure | ✅ Supported | -| `crash_point()` from spawned threads | ❌ Undefined | - -**Rationale**: Crash point counter uses atomic state but determinism requires single-threaded execution. - -### No Nested Workspaces - -| Pattern | Status | -|---------|--------| -| User-managed dirs inside `env.path()` | ✅ Supported | -| Expecting FIRST to manage nested isolation | ❌ Not supported | +## Roadmap +| Version | Feature | +|---------|---------| +| v0.2 | `LD_PRELOAD` syscall interception | +| v0.3 | FUSE-based simulation | +| v0.4 | Block-layer simulation | diff --git a/docs/proof/reference_wal.md b/docs/proof/reference_wal.md index 9bd3954..800af82 100644 --- a/docs/proof/reference_wal.md +++ b/docs/proof/reference_wal.md @@ -1,148 +1,54 @@ -# Reference WAL: FIRST Proof of Correctness +# Reference WAL: Proof of Correctness -## Summary - -FIRST deterministically exposed and verified the fix for an atomicity bug in the reference WAL where transactions could be partially visible after crash recovery. - ---- +FIRST deterministically exposed an atomicity bug in the reference WAL. ## The Bug -The WAL wrote the `COMMIT` marker before ensuring all transaction records were durable: - ```rust -// BUGGY: records may not be on disk when COMMIT is written +// BUGGY: COMMIT written before records are durable writeln!(self.file, "COMMIT {}", txid)?; crash_point("after_commit_write"); self.file.sync_all()?; // ← too late ``` -If a crash occurred after writing COMMIT but before `fsync()`, the kernel could flush COMMIT to disk before all PUT records, resulting in a committed transaction with missing records. - ---- +If crash occurs after COMMIT but before `fsync()`, records may not reach disk, but COMMIT does → **atomicity violation**. ## Why It's Subtle -- **Invisible in normal testing**: Without crashes, all writes complete successfully -- **Page cache hides the problem**: Writes appear atomic from the application's perspective -- **Correct-looking code**: A single `fsync()` after COMMIT *seems* sufficient -- **Non-deterministic in production**: Depends on kernel flush timing -- **Common pattern**: Many WAL implementations make this exact mistake - ---- +- Invisible without crashes (all writes succeed) +- Page cache hides the non-atomicity +- Single `fsync()` *looks* correct +- Common mistake in WAL implementations -## Crash Point That Exposed It +## FIRST Output ``` -after_commit_write -``` - -This crash point fires after the COMMIT record is written to the page cache but before any `fsync()` call. - ---- - -## FIRST Output (Failing Run) - -Before the fix, FIRST reported: - -``` -[first] crash point 1: OK -[first] crash point 2: OK -[first] crash point 3: OK -[first] crash point 4: OK [first] crash point 5: FAILED (see /tmp/first/run_5) [first] crash label: "after_commit_write" -[first] reason: verification failed with exit code 101 -[first] to reproduce: - FIRST_PHASE=VERIFY FIRST_CRASH_TARGET=5 FIRST_WORK_DIR=/tmp/first/run_5 \ - FIRST_CRASH_POINT_ID=5 FIRST_CRASH_LABEL="after_commit_write" \ - cargo test transaction_atomicity_under_crash -- --exact -``` -The verification phase panicked with: - -``` -Atomicity violation at crash point 'after_commit_write': -committed transaction has only 1/3 records visible. -key1=Some("value1"), key2=None, key3=None +Atomicity violation: committed transaction has only 1/3 records visible. ``` ---- - ## The Fix -Add `fsync()` before writing COMMIT to ensure records are durable first: - ```diff - pub fn commit(&mut self, txid: TxId) { -+ self.file.sync_all().expect("failed to fsync records"); - writeln!(self.file, "COMMIT {}", txid).expect("failed to write COMMIT"); - crash_point("after_commit_write"); - self.file.sync_all().expect("failed to fsync after COMMIT"); + pub fn commit(&mut self, txid: TxId) { ++ self.file.sync_all()?; // ← ensure records durable FIRST + writeln!(self.file, "COMMIT {}", txid)?; + self.file.sync_all()?; ``` -Now the ordering is: - -1. Write all PUT records -2. `fsync()` ← records durable -3. Write COMMIT -4. `fsync()` ← COMMIT durable - -A crash at any point now guarantees atomicity: -- Before step 2: no records, no COMMIT -- After step 2, before step 4: records durable, no COMMIT visible -- After step 4: records and COMMIT both durable - ---- - ## Verification -After the fix, FIRST reports all crash points passed: +After fix: ``` -[first] crash point 1: OK -[first] crash point 2: OK -[first] crash point 3: OK -[first] crash point 4: OK -[first] crash point 5: OK -[first] crash point 6: OK -[first] crash point 7: OK [first] all 7 crash points passed ``` ---- - -## How to Reproduce the Original Bug - -1. Check out the commit immediately before the fix (see `Fixes #9` in git history) -2. Run the crash consistency test: +## Reproduce ```bash cd examples/reference_wal cargo test transaction_atomicity_under_crash -- --nocapture ``` - -3. Observe the failure at `after_commit_write` - -To re-run just the failing verification: - -```bash -FIRST_PHASE=VERIFY \ -FIRST_CRASH_TARGET=5 \ -FIRST_WORK_DIR=/tmp/first/run_5 \ -FIRST_CRASH_POINT_ID=5 \ -FIRST_CRASH_LABEL="after_commit_write" \ -cargo test transaction_atomicity_under_crash -- --exact --nocapture -``` - ---- - -## Conclusion - -FIRST successfully: - -1. **Found** a real atomicity bug via systematic crash point exploration -2. **Reproduced** the bug deterministically with a single command -3. **Verified** the fix by confirming all crash points pass - -This demonstrates FIRST's effectiveness for crash-consistency testing of storage engines. diff --git a/src/orchestrator.rs b/src/orchestrator.rs index e5a53d8..11f5e0e 100644 --- a/src/orchestrator.rs +++ b/src/orchestrator.rs @@ -4,14 +4,12 @@ use std::fs; use std::io::{BufRead, BufReader}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::{Command, ExitStatus, Stdio}; +use uuid::Uuid; use crate::env::{CrashInfo, Env}; -/// Base directory for FIRST test runs. -const FIRST_BASE_DIR: &str = "/tmp/first"; - /// Exit code for SIGKILL (128 + 9). const SIGKILL_EXIT_CODE: i32 = 137; @@ -42,8 +40,17 @@ where let mut target: usize = 1; + // Generate a unique base directory for this validation run. + // This ensures that parallel `cargo test` invocations don't collide. + let run_uuid = Uuid::new_v4(); + let pid = std::process::id(); + let base_dir = std::env::temp_dir() + .join("first") + .join(format!("first-{}-{}", pid, run_uuid)); + loop { - let work_dir = PathBuf::from(FIRST_BASE_DIR).join(format!("run_{}", target)); + // The orchestrator owns the creation and cleanup of run_N directories. + let work_dir = base_dir.join(format!("run_{}", target)); // Create fresh work directory if let Err(e) = fs::create_dir_all(&work_dir) { @@ -52,13 +59,15 @@ where } // Spawn EXECUTION phase - let exec_result = spawn_child(&exe, &test_name, "EXECUTION", target, &work_dir); + // Note: We pass `base_dir` as FIRST_WORK_DIR, not `work_dir`. + // The child process will derive the leaf `run_N` path itself. + let exec_result = spawn_child(&exe, &test_name, "EXECUTION", target, &base_dir); match exec_result { ChildResult::Crashed(crash_info) => { // Child crashed as expected, now verify let verify_result = - spawn_child_with_crash_info(&exe, &test_name, target, &work_dir, &crash_info); + spawn_child_with_crash_info(&exe, &test_name, target, &base_dir, &crash_info); match verify_result { ChildResult::Success => { @@ -72,6 +81,7 @@ where print_failure_info( target, &work_dir, + &base_dir, // Pass base_dir for reproduction command &crash_info, &test_name, &format!("verification failed with exit code {}", code), @@ -82,6 +92,7 @@ where print_failure_info( target, &work_dir, + &base_dir, // Pass base_dir for reproduction command &crash_info, &test_name, "verify phase crashed unexpectedly", @@ -108,7 +119,7 @@ where eprintln!( " FIRST_PHASE=EXECUTION FIRST_CRASH_TARGET={} FIRST_WORK_DIR={} cargo test{} -- --exact", target, - work_dir.display(), + base_dir.display(), // Use base_dir for reproduction test_name .as_ref() .map(|n| format!(" {}", n)) @@ -125,7 +136,8 @@ where /// Print detailed failure information for debugging. fn print_failure_info( target: usize, - work_dir: &Path, + work_dir: &Path, // The leaf directory (for "see ...") + base_dir: &Path, // The base directory (for env var) crash_info: &CrashInfo, test_name: &Option, reason: &str, @@ -141,7 +153,7 @@ fn print_failure_info( eprintln!( " FIRST_PHASE=VERIFY FIRST_CRASH_TARGET={} FIRST_WORK_DIR={} FIRST_CRASH_POINT_ID={} FIRST_CRASH_LABEL=\"{}\" cargo test{} -- --exact", target, - work_dir.display(), + base_dir.display(), // Use base_dir for reproduction crash_info.point_id, crash_info.label, test_name @@ -344,5 +356,9 @@ fn extract_test_name() -> Option { return Some(arg.clone()); } - None + // Fallback: use thread name (standard in cargo test) + match std::thread::current().name() { + Some(name) if name != "main" => Some(name.to_string()), + _ => None, + } } diff --git a/src/test.rs b/src/test.rs index aeccecf..303b3ff 100644 --- a/src/test.rs +++ b/src/test.rs @@ -105,7 +105,7 @@ where /// | Verify | Test fails | Verification OK | Test fails | pub fn execute(self) { let config = runtime(); - let work_dir = std::env::var("FIRST_WORK_DIR") + let base_dir = std::env::var("FIRST_WORK_DIR") .map(PathBuf::from) .unwrap_or_else(|_| std::env::temp_dir().join("first").join("default")); @@ -116,12 +116,26 @@ where } Phase::Execution => { if let Some(run_fn) = self.run_fn { + // Derive leaf directory from base + target + let target = std::env::var("FIRST_CRASH_TARGET") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(1); + let work_dir = base_dir.join(format!("run_{}", target)); + let env = Env::new(work_dir); run_fn(&env); } } Phase::Verify => { if let Some(verify_fn) = self.verify_fn { + // Derive leaf directory from base + target + let target = std::env::var("FIRST_CRASH_TARGET") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(1); + let work_dir = base_dir.join(format!("run_{}", target)); + let env = Env::new(work_dir); // Parse crash info from env var let crash_info = parse_crash_info(); diff --git a/tests/repro_issue_29.rs b/tests/repro_issue_29.rs new file mode 100644 index 0000000..f62605d --- /dev/null +++ b/tests/repro_issue_29.rs @@ -0,0 +1,65 @@ +//! Reproduction test for Issue #29: Work directory collisions. +//! +//! Two tests running in parallel should collide on `/tmp/first/run_1` +//! if the fix is not applied. + +use std::fs::File; +use std::io::Write; +use std::thread; +use std::time::Duration; + +#[test] +fn collision_test_a() { + first::test() + .run(|env| { + let path = env.path("collision.txt"); + + // Write unique content + let mut file = File::create(&path).unwrap(); + file.write_all(b"TEST_A").unwrap(); + + // Sleep to increase chance of overlap + thread::sleep(Duration::from_millis(100)); + + first::crash_point("point_a"); + }) + .verify(|env, _| { + let path = env.path("collision.txt"); + let contents = std::fs::read_to_string(&path).unwrap_or_default(); + + // If we collided, we might see "TEST_B" or mixed content + assert_eq!( + contents, "TEST_A", + "Content mismatch! Likely collision with concurrent test." + ); + }) + .execute(); +} + +#[test] +fn collision_test_b() { + first::test() + .run(|env| { + let path = env.path("collision.txt"); + + // Write unique content + let mut file = File::create(&path).unwrap(); + file.write_all(b"TEST_B").unwrap(); + + // Sleep to increase chance of overlap + thread::sleep(Duration::from_millis(100)); + + first::crash_point("point_b"); + }) + .verify(|env, _| { + let path = env.path("collision.txt"); + let contents = std::fs::read_to_string(&path).unwrap_or_default(); + + // If we collided, we might see "TEST_A" or mixed content + assert_eq!( + contents, "TEST_B", + "Content mismatch! Likely collision with concurrent test." + ); + }) + .execute(); +}