Compare commits
37
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3e02d6b679 | ||
|
|
a6b8ce4a8f | ||
|
|
fe8de2784b | ||
|
|
b8204c4ed9 | ||
|
|
49be497964 | ||
|
|
b48b3a5a2e | ||
|
|
711cebdd6e | ||
|
|
5bfb284c82 | ||
|
|
c02740798b | ||
|
|
b6ff1f81e6 | ||
|
|
7999996bca | ||
|
|
a75defa5a6 | ||
|
|
f342e33f16 | ||
|
|
4ef7e69cdd | ||
|
|
6f7a58e4c5 | ||
|
|
86787b3a0d | ||
|
|
0137296fb4 | ||
|
|
064c2a15be | ||
|
|
37260f6b3a | ||
|
|
37e4d46bac | ||
|
|
f82205baf4 | ||
|
|
4ad88bcd68 | ||
|
|
1c9034de71 | ||
|
|
8f6472ea22 | ||
|
|
098b2711d8 | ||
|
|
a8e39804d2 | ||
|
|
ee3c9f4417 | ||
|
|
85ccd7e507 | ||
|
|
bacac903b3 | ||
|
|
d7806043e1 | ||
|
|
f4160d2387 | ||
|
|
df39dd80b9 | ||
|
|
8e507d76f3
|
||
|
|
9ad3bd174f
|
||
|
|
979232bd67
|
||
|
|
8ee4092f9a
|
||
|
|
fad97dbf16
|
No files matched your search
Binary file not shown.
@@ -1,142 +0,0 @@
|
||||
# cargo-mutants configuration for the dagx crate
|
||||
#
|
||||
# This configuration optimizes mutation testing by:
|
||||
# - Focusing on correctness-critical code
|
||||
# - Excluding low-value mutations
|
||||
# - Setting appropriate timeouts
|
||||
# - Configuring parallel execution
|
||||
|
||||
# Overall timeout for each mutant test
|
||||
# Increased from default to handle async tests
|
||||
timeout = 300 # 5 minutes per mutant
|
||||
|
||||
# Number of parallel jobs
|
||||
# Set to number of CPU cores for faster execution
|
||||
# jobs = 8 # Uncomment and adjust for your system
|
||||
|
||||
# Test tool to use
|
||||
# nextest is faster than cargo test
|
||||
test_tool = "nextest"
|
||||
|
||||
# Additional arguments to pass to the test command
|
||||
# --no-fail-fast ensures all tests run even if one fails
|
||||
test_args = ["--no-fail-fast"]
|
||||
|
||||
# Exclude patterns for files that shouldn't be mutated
|
||||
# These are low-value targets that produce noise
|
||||
exclude_globs = [
|
||||
# Test files - we test the tests, not mutate them
|
||||
"tests/**",
|
||||
"**/tests.rs",
|
||||
"benches/**",
|
||||
"examples/**",
|
||||
|
||||
# Proc macro crate
|
||||
"dagx-macros/**",
|
||||
|
||||
# Build scripts
|
||||
"build.rs",
|
||||
]
|
||||
|
||||
# Exclude specific functions/methods that shouldn't be mutated
|
||||
# These are low-value mutations that don't test correctness
|
||||
[[exclude_functions]]
|
||||
# Display trait implementations - not correctness-critical
|
||||
regex = ".*::fmt"
|
||||
reason = "Display/Debug implementations don't affect correctness"
|
||||
|
||||
[[exclude_functions]]
|
||||
# Clone implementations - trivial mutations
|
||||
regex = ".*::clone"
|
||||
reason = "Clone implementations are trivial"
|
||||
|
||||
[[exclude_functions]]
|
||||
# Default implementations - trivial
|
||||
regex = ".*::default"
|
||||
reason = "Default implementations are trivial"
|
||||
|
||||
# Specific functions to focus on (include)
|
||||
# These are the correctness-critical paths
|
||||
[[examine_functions]]
|
||||
# Core DAG execution logic
|
||||
regex = "DagRunner::run"
|
||||
reason = "Critical: Core DAG execution orchestration"
|
||||
|
||||
[[examine_functions]]
|
||||
# Task addition and dependency tracking
|
||||
regex = "DagRunner::add_task"
|
||||
reason = "Critical: Task registration and node creation"
|
||||
|
||||
[[examine_functions]]
|
||||
# Dependency management
|
||||
regex = "TaskBuilder::depends_on"
|
||||
reason = "Critical: Dependency edge creation"
|
||||
|
||||
[[examine_functions]]
|
||||
# Output extraction and type checking
|
||||
regex = "DagRunner::get"
|
||||
reason = "Critical: Type-safe output retrieval"
|
||||
|
||||
[[examine_functions]]
|
||||
# Node execution with listeners
|
||||
regex = "ExecutableNode::execute_with_listeners"
|
||||
reason = "Critical: Task execution and value propagation"
|
||||
|
||||
[[examine_functions]]
|
||||
# Listener extraction from announcements
|
||||
regex = "ExtractFromListeners::extract"
|
||||
reason = "Critical: Input extraction for dependent tasks"
|
||||
|
||||
[[examine_functions]]
|
||||
# Dependency tuple conversion
|
||||
regex = "DepsTuple::to_node_ids"
|
||||
reason = "Critical: Dependency tracking correctness"
|
||||
|
||||
# Minimum test time threshold
|
||||
# If tests complete faster than this, warn about insufficient testing
|
||||
# minimum_test_time = 1.0 # Uncomment to enable
|
||||
|
||||
# Output formats
|
||||
# Generate both text and JSON for tracking
|
||||
# output_formats = ["text", "json"] # Uncomment to enable
|
||||
|
||||
# Baseline
|
||||
# Compare mutations against a baseline to detect regressions
|
||||
# baseline = "main" # Uncomment to enable baseline comparison
|
||||
|
||||
# Notes on configuration strategy:
|
||||
#
|
||||
# 1. FOCUS: We explicitly list critical functions to mutate
|
||||
# - DAG execution: run(), execute_with_listeners()
|
||||
# - Dependency tracking: add_task(), depends_on(), to_node_ids()
|
||||
# - Type safety: get() with downcasting
|
||||
# - Value propagation: extract() from listeners
|
||||
#
|
||||
# 2. EXCLUDE: We exclude low-value targets
|
||||
# - Display/Debug impls: String formatting doesn't affect correctness
|
||||
# - Clone/Default impls: Trivial, unlikely to have bugs
|
||||
# - Tests themselves: We test production code, not tests
|
||||
# - Proc macros: Generated code tested via integration
|
||||
#
|
||||
# 3. TIMEOUT: 5 minutes is generous
|
||||
# - Most mutants complete in seconds
|
||||
# - Allows for async tests with real execution
|
||||
# - Catches actual deadlocks without false positives
|
||||
#
|
||||
# 4. NEXTEST: Faster test execution
|
||||
# - Parallel test execution
|
||||
# - Better output formatting
|
||||
# - Improved reliability
|
||||
#
|
||||
# 5. NO-FAIL-FAST: See all failures
|
||||
# - Don't stop at first test failure
|
||||
# - Get complete picture of mutation impact
|
||||
# - Better for debugging
|
||||
#
|
||||
# Expected mutation counts with this configuration:
|
||||
# - Target: ~30-50 mutants for core logic
|
||||
# - Score: 90%+ (high bar for DAG correctness)
|
||||
#
|
||||
# Runtime with this configuration:
|
||||
# - Current: ~5-10 minutes
|
||||
# - With nextest: ~30% faster
|
||||
@@ -2,9 +2,9 @@ name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
branches: ["main"]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
branches: ["main"]
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
@@ -40,11 +40,28 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Extract MSRV from Cargo.toml
|
||||
id: msrv
|
||||
run: |
|
||||
MSRV=$(yq '.package.rust-version' Cargo.toml --unwrapScalar)
|
||||
|
||||
if [ -z "$MSRV" ]; then
|
||||
echo "No MSRV specified in Cargo.toml, using stable"
|
||||
MSRV="stable"
|
||||
else
|
||||
echo "msrv=$MSRV" >> $GITHUB_OUTPUT
|
||||
echo "Using MSRV: $MSRV"
|
||||
fi
|
||||
|
||||
- name: Install Rust stable
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@master
|
||||
with:
|
||||
toolchain: ${{ steps.msrv.outputs.msrv }}
|
||||
|
||||
- name: Install cargo-msrv
|
||||
run: cargo install cargo-msrv
|
||||
uses: taiki-e/install-action@v2
|
||||
with:
|
||||
tool: cargo-msrv
|
||||
|
||||
- name: Verify MSRV
|
||||
run: cargo msrv verify ${{ matrix.features }}
|
||||
@@ -119,10 +136,10 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Install cargo-tarpaulin
|
||||
run: cargo install cargo-tarpaulin
|
||||
- uses: taiki-e/install-action@cargo-tarpaulin
|
||||
|
||||
- name: Generate code coverage
|
||||
run: cargo tarpaulin --lib --all-features --out Xml --out Lcov --output-dir coverage --fail-under 80
|
||||
run: cargo tarpaulin --out Xml --out Lcov --output-dir coverage --fail-under 80
|
||||
|
||||
security:
|
||||
name: Security Audit
|
||||
@@ -134,7 +151,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Install cargo-audit
|
||||
run: cargo install cargo-audit
|
||||
uses: taiki-e/install-action@cargo-audit
|
||||
|
||||
- name: Run cargo audit
|
||||
run: cargo audit
|
||||
@@ -152,4 +169,4 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Build documentation
|
||||
run: cargo doc --no-deps ${{ matrix.features }}
|
||||
run: cargo doc --no-deps ${{ matrix.features }}
|
||||
@@ -2,4 +2,3 @@
|
||||
/.jj
|
||||
|
||||
/target
|
||||
Cargo.lock
|
||||
@@ -5,6 +5,37 @@ All notable changes to dagx will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.3.1] - 2025-10-10
|
||||
|
||||
[View changes](https://github.com/swaits/dagx/compare/v0.3.0...v0.3.1)
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Custom types now work automatically!** The `#[task]` macro now generates inline `extract_and_run()`
|
||||
methods with type-specific extraction logic. This means **ANY type** implementing `Clone + Send + Sync + 'static`
|
||||
works automatically without implementing any special traits.
|
||||
- Previously, only types with `ExtractInput` implementations worked (primitives, String, Vec, HashMap, etc.)
|
||||
- Now, custom user types work seamlessly - just derive `Clone` and you're good to go!
|
||||
- No `ExtractInput` trait implementation needed for custom types
|
||||
- Nested structs, collections of custom types, and complex type hierarchies all work out of the box
|
||||
- New example: `examples/custom_types.rs` demonstrating custom types flowing through a DAG
|
||||
|
||||
### Changed
|
||||
|
||||
- **Documentation improvements**
|
||||
- Updated README.md to highlight custom type support as a key feature
|
||||
- Added "Custom Types" section to Core Concepts with examples
|
||||
- Added `custom_types.rs` to the examples list
|
||||
- Updated feature list to emphasize "Works with ANY type"
|
||||
|
||||
### Internal
|
||||
|
||||
- `ExtractInput` trait is now only used by the internal `task_fn` test helper
|
||||
- The `#[task]` macro generates custom extraction logic per task, bypassing `ExtractInput` entirely
|
||||
- This architectural change enables ANY type to work without requiring trait implementations
|
||||
|
||||
## [0.3.0] - 2025-10-08
|
||||
|
||||
[View changes](https://github.com/swaits/dagx/compare/v0.2.3...v0.3.0)
|
||||
@@ -255,6 +286,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- macOS (compatible)
|
||||
- Windows (compatible)
|
||||
|
||||
[0.3.1]: https://github.com/swaits/dagx/tree/v0.3.1
|
||||
[0.3.0]: https://github.com/swaits/dagx/tree/v0.3.0
|
||||
[0.2.3]: https://github.com/swaits/dagx/tree/v0.2.3
|
||||
[0.2.2]: https://github.com/swaits/dagx/tree/v0.2.2
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
We track work in Beads instead of Markdown. Run \`bd quickstart\` to see how.
|
||||
Generated
+1393
File diff suppressed because it is too large.
Load diff
+20
-11
@@ -1,9 +1,9 @@
|
||||
[workspace]
|
||||
members = ["dagx-macros"]
|
||||
members = ["dagx-macros", "dagx-test"]
|
||||
|
||||
[package]
|
||||
name = "dagx"
|
||||
version = "0.3.0"
|
||||
version = "0.3.1"
|
||||
edition = "2021"
|
||||
rust-version = "1.81"
|
||||
authors = ["Stephen Waits <steve@waits.net>"]
|
||||
@@ -21,23 +21,31 @@ exclude = [".claude/", ".jj/", "dagx-spec.md"]
|
||||
default = []
|
||||
tracing = ["dep:tracing"]
|
||||
|
||||
[lints.rust]
|
||||
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tarpaulin)', 'cfg(tarpaulin_include)'] }
|
||||
[workspace.lints.rust]
|
||||
unexpected_cfgs = { level = "warn", check-cfg = [
|
||||
'cfg(tarpaulin)',
|
||||
'cfg(tarpaulin_include)',
|
||||
] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
announcement = "0.1.0"
|
||||
futures = "0.3"
|
||||
futures-util = { version = "0.3", default-features = false, features = ["std"] }
|
||||
parking_lot = "0.12"
|
||||
dagx-macros = { version = "0.3.0", path = "./dagx-macros" }
|
||||
dagx-macros = { version = "0.3.1", path = "./dagx-macros" }
|
||||
tracing = { version = "0.1", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1", features = ["macros", "rt-multi-thread", "time", "sync"] }
|
||||
async-std = { version = "1", features = ["attributes"] }
|
||||
dagx-test = { path = "./dagx-test" }
|
||||
tokio = { version = "1", features = [
|
||||
"macros",
|
||||
"rt-multi-thread",
|
||||
"time",
|
||||
"sync",
|
||||
] }
|
||||
smol = "2"
|
||||
criterion = "0.7"
|
||||
# Property-based testing
|
||||
proptest = "1"
|
||||
# Quirky async runtimes for edge-case testing
|
||||
async-executor = "1.13"
|
||||
pollster = "0.4"
|
||||
@@ -47,6 +55,7 @@ dagrs = { version = "0.5", default-features = false }
|
||||
async-trait = "0.1"
|
||||
# Tracing support for tests and examples
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
test-case = "3.3.1"
|
||||
|
||||
[[bench]]
|
||||
name = "dagx_bench"
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
# Performance Regression Analysis: ETL Benchmark
|
||||
|
||||
## Summary
|
||||
|
||||
The ETL scaling benchmark regressed by ~17% (3.24ms -> 3.30ms for 10k records) due to expensive data cloning when using the announcement crate.
|
||||
|
||||
## Root Cause
|
||||
|
||||
In both `dagx-macros/src/lib.rs:327` and `src/task.rs:253`:
|
||||
|
||||
```rust
|
||||
let _ = announcer_typed.announce((*output_arc).clone());
|
||||
```
|
||||
|
||||
This dereferences the Arc and clones the actual data, which is extremely expensive for large types.
|
||||
|
||||
### For ETL benchmark with 10k strings:
|
||||
|
||||
**Current (Broken) Flow:**
|
||||
1. Task produces `Vec<String>` (10k items)
|
||||
2. Wrap in `Arc<Vec<String>>`
|
||||
3. **Announce `(*arc).clone()`** → Full Vec clone #1 (EXPENSIVE!)
|
||||
4. Announcement stores cloned Vec in `OnceLock<Vec<String>>`
|
||||
5. Each of 3 listeners calls `.listen()` → Vec clone #2, #3, #4 (EXPENSIVE!)
|
||||
|
||||
**Total**: 4 full Vec clones = 40k string clones
|
||||
|
||||
### Why other benchmarks improved:
|
||||
|
||||
Most benchmarks use primitive types (i32, etc.) where cloning is cheap. The announcement crate's broadcast pattern is more efficient than multiple oneshot channels for small types.
|
||||
|
||||
## The Fix
|
||||
|
||||
Change announcements to use `Arc<T>` instead of `T`:
|
||||
|
||||
**New (Correct) Flow:**
|
||||
1. Task produces `Vec<String>` (10k items)
|
||||
2. Wrap in `Arc<Vec<String>>`
|
||||
3. **Announce `Arc::clone(&arc)`** → Arc clone (CHEAP!)
|
||||
4. Announcement stores Arc in `OnceLock<Arc<Vec<String>>>`
|
||||
5. Each of 3 listeners calls `.listen()` → Arc clone #2, #3, #4 (CHEAP!)
|
||||
|
||||
**Total**: 4 Arc clones = just refcount increments (~40ns total)
|
||||
|
||||
## Required Changes
|
||||
|
||||
### 1. dagx-macros/src/lib.rs
|
||||
|
||||
```rust
|
||||
// Line 327: Change announcement call
|
||||
- let _ = announcer_typed.announce((*output_arc).clone());
|
||||
+ let _ = announcer_typed.announce(Arc::clone(&output_arc));
|
||||
|
||||
// Line 325: Change announcer type
|
||||
- .downcast::<announcement::Announcer<#output_type>>()
|
||||
+ .downcast::<announcement::Announcer<Arc<#output_type>>>()
|
||||
|
||||
// Line 352: Change announcement creation
|
||||
- let (announcer, announcement) = announcement::Announcement::<#output_type>::new();
|
||||
+ let (announcer, announcement) = announcement::Announcement::<Arc<#output_type>>::new();
|
||||
|
||||
// Line 257: Change listener type in parameter extraction
|
||||
- .downcast_ref::<announcement::Listener<#ty>>()
|
||||
+ .downcast_ref::<announcement::Listener<Arc<#ty>>>()
|
||||
|
||||
// Line 361: Change announcement downcast
|
||||
- .downcast_ref::<announcement::Announcement<#output_type>>()
|
||||
+ .downcast_ref::<announcement::Announcement<Arc<#output_type>>>()
|
||||
```
|
||||
|
||||
### 2. src/task.rs
|
||||
|
||||
```rust
|
||||
// Line 253: Change announcement call
|
||||
- let _ = announcer_typed.announce((*output_arc).clone());
|
||||
+ let _ = announcer_typed.announce(Arc::clone(&output_arc));
|
||||
|
||||
// Line 249: Change announcer type
|
||||
- .downcast::<announcement::Announcer<O>>()
|
||||
+ .downcast::<announcement::Announcer<Arc<O>>>()
|
||||
|
||||
// Line 261: Change announcement creation
|
||||
- let (announcer, announcement) = announcement::Announcement::<O>::new();
|
||||
+ let (announcer, announcement) = announcement::Announcement::<Arc<O>>::new();
|
||||
|
||||
// Line 270: Change announcement downcast
|
||||
- .downcast_ref::<announcement::Announcement<O>>()
|
||||
+ .downcast_ref::<announcement::Announcement<Arc<O>>>()
|
||||
|
||||
// Lines 34, 57, 67, 84, 102, 125: Change all listener downcasts from:
|
||||
- .downcast::<announcement::Listener<T>>()
|
||||
+ .downcast::<announcement::Listener<Arc<T>>>()
|
||||
|
||||
// Then unwrap the Arc after listening:
|
||||
let arc_value = listener.listen().await?;
|
||||
let value = (*arc_value).clone(); // Only clone primitives, Arc is already cheap for complex types
|
||||
```
|
||||
|
||||
### 3. Handle Arc unwrapping in ExtractFromListeners
|
||||
|
||||
For each type implementation, extract from `Listener<Arc<T>>` and then dereference:
|
||||
|
||||
```rust
|
||||
impl ExtractFromListeners for Vec<T> {
|
||||
async fn extract(mut listeners: Vec<Box<dyn Any + Send>>) -> Result<Self, String> {
|
||||
let listener = *listeners.pop().unwrap()
|
||||
.downcast::<announcement::Listener<Arc<Vec<T>>>>() // Now Arc<Vec<T>>
|
||||
.map_err(...)?;
|
||||
let arc_vec = listener.listen().await.map_err(...)?;
|
||||
Ok((*arc_vec).clone()) // Clone the Vec out of the Arc
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Actually, a better approach: keep the Arc! For types that are expensive to clone, just return the Arc:
|
||||
|
||||
```rust
|
||||
// For expensive types, use Arc directly in function signatures
|
||||
async fn run(&self, data: &Arc<Vec<String>>) -> Result { ... }
|
||||
```
|
||||
|
||||
Or even simpler: change ExtractFromListeners to extract as the Arc type directly, and modify the macro's parameter extraction to work with Arc types.
|
||||
|
||||
## Expected Performance Impact
|
||||
|
||||
- **ETL 10k benchmark**: Should improve by ~17% back to baseline (or better)
|
||||
- **Other benchmarks**: Should remain the same or slightly improve (Arc clone overhead is minimal)
|
||||
- **Memory efficiency**: Better! Arc refcounting instead of deep copies
|
||||
|
||||
## Why This Wasn't Caught
|
||||
|
||||
- Most tests use small primitive types where clone() is fast
|
||||
- The regression only shows up with large data structures (Vec, HashMap, etc.)
|
||||
- The ETL benchmark with 10k records is the perfect stress test for this issue
|
||||
@@ -38,7 +38,7 @@ See [how it works](docs/CYCLE_PREVENTION.md).
|
||||
|
||||
```rust
|
||||
let sum = dag.add_task(Add).depends_on((&x, &y));
|
||||
dag.run(|fut| tokio::spawn(fut)).await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await?;
|
||||
```
|
||||
|
||||
That's it. No trait boilerplate, no manual channels, no node IDs.
|
||||
@@ -84,13 +84,14 @@ async fn main() {
|
||||
|
||||
// Add source tasks with no dependencies
|
||||
let x = dag.add_task(Value(2));
|
||||
let y = dag.add_task(Value(3));
|
||||
let y: TaskHandle<_> = dag.add_task(Value(3)).into();
|
||||
|
||||
// Add task that depends on both x and y
|
||||
let sum = dag.add_task(Add).depends_on((&x, &y));
|
||||
// Add task that depends on both x and y.
|
||||
// Here, y could be reused as a dependency for other tasks, while x could not.
|
||||
let sum = dag.add_task(Add).depends_on((x, &y));
|
||||
|
||||
// Execute with true parallelism
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
|
||||
// Retrieve results
|
||||
assert_eq!(dag.get(sum).unwrap(), 5);
|
||||
@@ -104,7 +105,6 @@ dagx provides true parallel execution with sub-microsecond overhead per task.
|
||||
**Why is dagx so fast?**
|
||||
|
||||
- **Inline fast-path**: Sequential chains execute inline without spawning (8.9-129x faster)
|
||||
- **Primitives as scheduler**: No custom scheduler — channels coordinate execution
|
||||
- **Adaptive execution**: Inline for sequential work, true parallelism for concurrent work
|
||||
- **Zero-cost abstractions**: Generics and monomorphization eliminate overhead
|
||||
|
||||
@@ -114,6 +114,7 @@ See [design philosophy](docs/DESIGN_PHILOSOPHY.md) for details.
|
||||
|
||||
- **Compile-time cycle prevention**: Type system makes cycles impossible — no runtime checks
|
||||
- **Compile-time type safety**: Dependencies validated at compile time, no runtime type errors
|
||||
- **Works with ANY type**: Custom types work automatically — just `Send + Sync + 'static`, no trait implementations needed
|
||||
- **Runtime-agnostic**: Works with Tokio, async-std, smol, or any async runtime
|
||||
- **True parallelism**: Tasks spawn to multiple threads for genuine parallel execution
|
||||
- **Type-state pattern**: API prevents incorrect wiring through compile-time errors
|
||||
@@ -186,19 +187,51 @@ let task = dag.add_task(my_task).depends_on(&upstream);
|
||||
let task = dag.add_task(my_task).depends_on((&upstream1, &upstream2));
|
||||
```
|
||||
|
||||
### Custom Types
|
||||
|
||||
**dagx works with ANY type automatically!** As long as your type implements `Send + Sync + 'static`, it works seamlessly:
|
||||
|
||||
```rust
|
||||
struct User {
|
||||
name: String,
|
||||
age: u32,
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl CreateUser {
|
||||
async fn run(&self) -> User {
|
||||
User { name: "Alice".to_string(), age: 30 }
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl ProcessUser {
|
||||
async fn run(user: &User) -> String {
|
||||
format!("{} is {} years old", user.name, user.age)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**No trait implementations needed!** The `#[task]` macro generates type-specific extraction logic automatically. This includes nested structs, collections, enums, and any other type you define.
|
||||
|
||||
### Runtime Agnostic
|
||||
|
||||
dagx works with any async runtime. Provide a spawner function to `run()`:
|
||||
|
||||
```rust
|
||||
// With Tokio
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
// The join handle result can be unwrapped because dagx catches panics internally
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
|
||||
// With async-std
|
||||
dag.run(|fut| { async_std::task::spawn(fut); }).await.unwrap();
|
||||
dag.run(|fut| async_std::task::spawn(fut)).await.unwrap();
|
||||
|
||||
// With smol
|
||||
dag.run(|fut| { smol::spawn(fut).detach(); }).await.unwrap();
|
||||
dag.run(|fut| smol::spawn(fut)).await.unwrap();
|
||||
|
||||
// Single-threaded on the invoking runtime
|
||||
// Faster for situations with many computationally light tasks
|
||||
dag.run(|fut| fut).await.unwrap()
|
||||
```
|
||||
|
||||
## Tutorials & Examples
|
||||
@@ -226,18 +259,10 @@ cargo run --example 04_parallel_computation
|
||||
Real-world patterns:
|
||||
|
||||
- [`circuit_breaker.rs`](examples/circuit_breaker.rs) - Circuit breaker pattern for resilient systems
|
||||
- [`complex_dag.rs`](examples/complex_dag.rs) - Multi-layer workflows with complex dependencies
|
||||
- [`conditional_workflow.rs`](examples/conditional_workflow.rs) - Conditional execution with Result types
|
||||
- [`data_pipeline.rs`](examples/data_pipeline.rs) - ETL data processing pipeline
|
||||
- [`debug_tracing.rs`](examples/debug_tracing.rs) - Debug metadata and task naming
|
||||
- [`error_handling.rs`](examples/error_handling.rs) - Error propagation and recovery
|
||||
- [`large_dag.rs`](examples/large_dag.rs) - Performance at 10,000+ nodes
|
||||
- [`parallelism_proof.rs`](examples/parallelism_proof.rs) - Proof of true parallel execution
|
||||
- [`retry_strategies.rs`](examples/retry_strategies.rs) - Retry patterns for transient failures
|
||||
- [`timeout.rs`](examples/timeout.rs) - Managing task timeouts
|
||||
- [`tracing_example.rs`](examples/tracing_example.rs) - Observability with tracing support
|
||||
|
||||
Run any example: `cargo run --example data_pipeline`
|
||||
Run any example: `cargo run --example custom_types`
|
||||
|
||||
## Advanced Topics
|
||||
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
//! Basic DAG execution benchmarks
|
||||
|
||||
use criterion::Criterion;
|
||||
use dagx::{task_fn, DagRunner};
|
||||
|
||||
pub fn bench_dag_execution(c: &mut Criterion) {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
|
||||
c.bench_function("execute_10_independent_tasks", |b| {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
let dag = DagRunner::new();
|
||||
for i in 0..10 {
|
||||
dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
}
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
c.bench_function("execute_100_independent_tasks", |b| {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
let dag = DagRunner::new();
|
||||
for i in 0..100 {
|
||||
dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
}
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
c.bench_function("execute_deep_chain_20", |b| {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Build a manual chain (API doesn't support loop-based chaining due to type system)
|
||||
struct Value(i32);
|
||||
#[dagx::task]
|
||||
impl Value {
|
||||
async fn run(&self) -> i32 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
let t0 = dag.add_task(Value(0));
|
||||
let t1 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(&t0);
|
||||
let t2 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t1);
|
||||
let t3 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t2);
|
||||
let t4 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t3);
|
||||
let t5 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t4);
|
||||
let t6 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t5);
|
||||
let t7 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t6);
|
||||
let t8 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t7);
|
||||
let t9 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t8);
|
||||
let t10 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t9);
|
||||
let t11 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t10);
|
||||
let t12 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t11);
|
||||
let t13 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t12);
|
||||
let t14 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t13);
|
||||
let t15 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t14);
|
||||
let t16 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t15);
|
||||
let t17 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t16);
|
||||
let t18 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t17);
|
||||
let _t19 = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(t18);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
//! Basic DAG benchmarks - creation, execution, scaling
|
||||
|
||||
pub mod creation;
|
||||
pub mod execution;
|
||||
pub mod scaling;
|
||||
@@ -1,7 +1,8 @@
|
||||
//! DAG scaling benchmarks - how performance scales with task count
|
||||
|
||||
use criterion::{BenchmarkId, Criterion};
|
||||
use dagx::{task_fn, DagRunner};
|
||||
use dagx::DagRunner;
|
||||
use dagx_test::task_fn;
|
||||
|
||||
pub fn bench_dag_scaling(c: &mut Criterion) {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
@@ -16,13 +17,11 @@ pub fn bench_dag_scaling(c: &mut Criterion) {
|
||||
rt.block_on(async {
|
||||
let dag = DagRunner::new();
|
||||
for i in 0..size {
|
||||
dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
dag.add_task(task_fn::<(), _, _>(move |_: ()| i));
|
||||
}
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
//! Long sequential chain: 100 tasks in sequence
|
||||
|
||||
use criterion::Criterion;
|
||||
use dagx::TaskHandle;
|
||||
|
||||
pub fn bench_deep_chain(c: &mut Criterion) {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
@@ -12,26 +13,25 @@ pub fn bench_deep_chain(c: &mut Criterion) {
|
||||
group.bench_function("dagx", |b| {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
use dagx::{task_fn, DagRunner};
|
||||
use dagx::DagRunner;
|
||||
use dagx_test::task_fn;
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let first = dag.add_task(task_fn(|_: ()| async { 0 }));
|
||||
let first: TaskHandle<_> = dag.add_task(task_fn::<(), _, _>(|_: ()| 0)).into();
|
||||
let mut prev = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(&first);
|
||||
.add_task(task_fn::<i32, _, _>(|&x: &i32| x + 1))
|
||||
.depends_on(first);
|
||||
|
||||
for i in 2..100 {
|
||||
prev = dag
|
||||
.add_task(task_fn(move |x: i32| async move { x + i }))
|
||||
.add_task(task_fn::<i32, _, _>(move |&x: &i32| x + i))
|
||||
.depends_on(prev);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
dag.get(prev).unwrap()
|
||||
})
|
||||
});
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
//! Classic diamond: A → B,C → D
|
||||
|
||||
use criterion::Criterion;
|
||||
use dagx::TaskHandle;
|
||||
|
||||
pub fn bench_diamond_pattern(c: &mut Criterion) {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
@@ -12,26 +13,25 @@ pub fn bench_diamond_pattern(c: &mut Criterion) {
|
||||
group.bench_function("dagx", |b| {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
use dagx::{task_fn, DagRunner};
|
||||
use dagx::DagRunner;
|
||||
use dagx_test::task_fn;
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let a = dag.add_task(task_fn(|_: ()| async { 10 }));
|
||||
let a: TaskHandle<_> = dag.add_task(task_fn::<(), _, _>(|_: ()| 10)).into();
|
||||
let b = dag
|
||||
.add_task(task_fn(|x: i32| async move { x * 2 }))
|
||||
.depends_on(&a);
|
||||
.add_task(task_fn::<i32, _, _>(|&x: &i32| x * 2))
|
||||
.depends_on(a);
|
||||
let c = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 5 }))
|
||||
.depends_on(&a);
|
||||
.add_task(task_fn::<i32, _, _>(|&x: &i32| x + 5))
|
||||
.depends_on(a);
|
||||
let d = dag
|
||||
.add_task(task_fn(|(x, y): (i32, i32)| async move { x + y }))
|
||||
.add_task(task_fn::<(i32, i32), _, _>(|(x, y): (&i32, &i32)| x + y))
|
||||
.depends_on((&b, &c));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
dag.get(d).unwrap()
|
||||
})
|
||||
});
|
||||
|
||||
@@ -12,39 +12,40 @@ pub fn bench_etl_pipeline(c: &mut Criterion) {
|
||||
group.bench_function("dagx", |b| {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
use dagx::{task_fn, DagRunner};
|
||||
use dagx::DagRunner;
|
||||
use dagx_test::task_fn;
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Extract from 3 sources
|
||||
let source1 = dag.add_task(task_fn(|_: ()| async { vec![1, 2, 3, 4, 5] }));
|
||||
let source2 = dag.add_task(task_fn(|_: ()| async { vec![6, 7, 8, 9, 10] }));
|
||||
let source3 = dag.add_task(task_fn(|_: ()| async { vec![11, 12, 13, 14, 15] }));
|
||||
let source1 = dag.add_task(task_fn::<(), _, _>(|_: ()| vec![1, 2, 3, 4, 5]));
|
||||
let source2 = dag.add_task(task_fn::<(), _, _>(|_: ()| vec![6, 7, 8, 9, 10]));
|
||||
let source3 = dag.add_task(task_fn::<(), _, _>(|_: ()| vec![11, 12, 13, 14, 15]));
|
||||
|
||||
// Transform each source
|
||||
let transform1 = dag
|
||||
.add_task(task_fn(|data: Vec<i32>| async move {
|
||||
data.into_iter().map(|x| x * 2).collect::<Vec<_>>()
|
||||
.add_task(task_fn::<Vec<_>, _, _>(|data: &Vec<i32>| {
|
||||
data.iter().map(|x| x * 2).collect::<Vec<_>>()
|
||||
}))
|
||||
.depends_on(&source1);
|
||||
.depends_on(source1);
|
||||
|
||||
let transform2 = dag
|
||||
.add_task(task_fn(|data: Vec<i32>| async move {
|
||||
data.into_iter().map(|x| x * 2).collect::<Vec<_>>()
|
||||
.add_task(task_fn::<Vec<_>, _, _>(|data: &Vec<i32>| {
|
||||
data.iter().map(|x| x * 2).collect::<Vec<_>>()
|
||||
}))
|
||||
.depends_on(&source2);
|
||||
.depends_on(source2);
|
||||
|
||||
let transform3 = dag
|
||||
.add_task(task_fn(|data: Vec<i32>| async move {
|
||||
data.into_iter().map(|x| x * 2).collect::<Vec<_>>()
|
||||
.add_task(task_fn::<Vec<_>, _, _>(|data: &Vec<i32>| {
|
||||
data.iter().map(|x| x * 2).collect::<Vec<_>>()
|
||||
}))
|
||||
.depends_on(&source3);
|
||||
.depends_on(source3);
|
||||
|
||||
// Merge all transformed data
|
||||
let merge = dag
|
||||
.add_task(task_fn(
|
||||
|(d1, d2, d3): (Vec<i32>, Vec<i32>, Vec<i32>)| async move {
|
||||
let mut result = d1;
|
||||
.add_task(task_fn::<(Vec<_>, Vec<_>, Vec<_>), _, _>(
|
||||
|(d1, d2, d3): (&Vec<i32>, &Vec<i32>, &Vec<i32>)| {
|
||||
let mut result = d1.clone();
|
||||
result.extend(d2);
|
||||
result.extend(d3);
|
||||
result
|
||||
@@ -54,23 +55,24 @@ pub fn bench_etl_pipeline(c: &mut Criterion) {
|
||||
|
||||
// Validate
|
||||
let validate = dag
|
||||
.add_task(task_fn(|data: Vec<i32>| async move {
|
||||
data.into_iter().filter(|&x| x > 0).collect::<Vec<_>>()
|
||||
.add_task(task_fn::<Vec<_>, _, _>(|data: &Vec<i32>| {
|
||||
data.iter()
|
||||
.copied()
|
||||
.filter(|&x| x > 0)
|
||||
.collect::<Vec<i32>>()
|
||||
}))
|
||||
.depends_on(merge);
|
||||
|
||||
// Load (compute sum)
|
||||
let load = dag
|
||||
.add_task(task_fn(|data: Vec<i32>| async move {
|
||||
data.into_iter().sum::<i32>()
|
||||
.add_task(task_fn::<Vec<_>, _, _>(|data: &Vec<i32>| {
|
||||
data.iter().sum::<i32>()
|
||||
}))
|
||||
.depends_on(validate);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
dag.get(load).unwrap()
|
||||
})
|
||||
});
|
||||
|
||||
@@ -14,18 +14,17 @@ pub fn bench_large_scale(c: &mut Criterion) {
|
||||
group.bench_function("dagx", |b| {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
use dagx::{task_fn, DagRunner};
|
||||
use dagx::DagRunner;
|
||||
use dagx_test::task_fn;
|
||||
|
||||
let dag = DagRunner::new();
|
||||
for i in 0..10_000 {
|
||||
dag.add_task(task_fn(move |_: ()| async move { i * 2 }));
|
||||
dag.add_task(task_fn::<(), _, _>(move |_: ()| i * 2));
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,29 +12,28 @@ pub fn bench_linear_pipeline(c: &mut Criterion) {
|
||||
group.bench_function("dagx", |b| {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
use dagx::{task_fn, DagRunner};
|
||||
use dagx::DagRunner;
|
||||
use dagx_test::task_fn;
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let a = dag.add_task(task_fn(|_: ()| async { 1 }));
|
||||
let a = dag.add_task(task_fn::<(), _, _>(|_: ()| 1));
|
||||
let b = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(&a);
|
||||
.add_task(task_fn::<i32, _, _>(|&x: &i32| x + 1))
|
||||
.depends_on(a);
|
||||
let c = dag
|
||||
.add_task(task_fn(|x: i32| async move { x * 2 }))
|
||||
.add_task(task_fn::<i32, _, _>(|&x: &i32| x * 2))
|
||||
.depends_on(b);
|
||||
let d = dag
|
||||
.add_task(task_fn(|x: i32| async move { x - 1 }))
|
||||
.add_task(task_fn::<i32, _, _>(|&x: &i32| x - 1))
|
||||
.depends_on(c);
|
||||
let e = dag
|
||||
.add_task(task_fn(|x: i32| async move { x * 3 }))
|
||||
.add_task(task_fn::<i32, _, _>(|&x: &i32| x * 3))
|
||||
.depends_on(d);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
dag.get(e).unwrap()
|
||||
})
|
||||
});
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
//! One source task feeding 100 dependent tasks
|
||||
|
||||
use criterion::Criterion;
|
||||
use dagx::TaskHandle;
|
||||
|
||||
pub fn bench_wide_fanout(c: &mut Criterion) {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
@@ -12,22 +13,21 @@ pub fn bench_wide_fanout(c: &mut Criterion) {
|
||||
group.bench_function("dagx", |b| {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
use dagx::{task_fn, DagRunner};
|
||||
use dagx::DagRunner;
|
||||
use dagx_test::task_fn;
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let source = dag.add_task(task_fn(|_: ()| async { 42 }));
|
||||
let source: TaskHandle<_> = dag.add_task(task_fn::<(), _, _>(|_: ()| 42)).into();
|
||||
|
||||
// Create 100 tasks that all depend on source
|
||||
for i in 0..100 {
|
||||
dag.add_task(task_fn(move |x: i32| async move { x + i }))
|
||||
.depends_on(&source);
|
||||
dag.add_task(task_fn::<i32, _, _>(move |&x: &i32| x + i))
|
||||
.depends_on(source);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
@@ -28,12 +28,6 @@ criterion_group! {
|
||||
targets = basic::creation::bench_dag_creation
|
||||
}
|
||||
|
||||
criterion_group! {
|
||||
name = execution_benches;
|
||||
config = configure_criterion();
|
||||
targets = basic::execution::bench_dag_execution
|
||||
}
|
||||
|
||||
criterion_group! {
|
||||
name = scaling_benches;
|
||||
config = configure_criterion();
|
||||
@@ -54,7 +48,6 @@ criterion_group! {
|
||||
config = configure_criterion();
|
||||
targets =
|
||||
realistic::etl::bench_etl_pipeline,
|
||||
realistic::config_broadcast::bench_config_broadcast,
|
||||
realistic::memory_efficiency::bench_memory_efficiency,
|
||||
realistic::mixed::bench_mixed_patterns
|
||||
}
|
||||
@@ -73,7 +66,6 @@ criterion_group! {
|
||||
|
||||
criterion_main!(
|
||||
creation_benches,
|
||||
execution_benches,
|
||||
scaling_benches,
|
||||
pattern_benches,
|
||||
realistic_benches,
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
//! 10,000 task breakdown benchmarks - construction vs execution
|
||||
|
||||
use criterion::Criterion;
|
||||
use dagx::{task_fn, DagRunner};
|
||||
use criterion::{BatchSize, Criterion};
|
||||
use dagx::DagRunner;
|
||||
use dagx_test::task_fn;
|
||||
|
||||
pub fn bench_10k_breakdown(c: &mut Criterion) {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
@@ -13,7 +14,7 @@ pub fn bench_10k_breakdown(c: &mut Criterion) {
|
||||
b.iter(|| {
|
||||
let dag = DagRunner::new();
|
||||
for i in 0..10_000 {
|
||||
dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
dag.add_task(task_fn::<(), _, _>(move |_: ()| i));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -24,34 +25,34 @@ pub fn bench_10k_breakdown(c: &mut Criterion) {
|
||||
rt.block_on(async {
|
||||
let dag = DagRunner::new();
|
||||
for i in 0..10_000 {
|
||||
dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
dag.add_task(task_fn::<(), _, _>(move |_: ()| i));
|
||||
}
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
// c) Execution only (pre-built DAG, just run)
|
||||
group.bench_function("execution_only", |b| {
|
||||
// Pre-build the DAG outside the measurement
|
||||
let dag = DagRunner::new();
|
||||
for i in 0..10_000 {
|
||||
dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
}
|
||||
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
b.iter_batched(
|
||||
|| {
|
||||
let dag = DagRunner::new();
|
||||
for i in 0..10_000 {
|
||||
dag.add_task(task_fn::<(), _, _>(move |_: ()| i));
|
||||
}
|
||||
dag
|
||||
},
|
||||
|dag| {
|
||||
rt.block_on(async {
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
},
|
||||
BatchSize::LargeInput,
|
||||
);
|
||||
});
|
||||
|
||||
group.finish();
|
||||
|
||||
+12
-13
@@ -1,8 +1,8 @@
|
||||
//! Diamond pattern benchmarks (fan-out then fan-in)
|
||||
|
||||
use criterion::Criterion;
|
||||
use dagx::{task_fn, DagRunner};
|
||||
|
||||
use dagx::{DagRunner, TaskHandle};
|
||||
use dagx_test::task_fn;
|
||||
pub fn bench_diamond(c: &mut Criterion) {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
|
||||
@@ -12,24 +12,23 @@ pub fn bench_diamond(c: &mut Criterion) {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
for i in 0..50 {
|
||||
let source = dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
let source: TaskHandle<_> =
|
||||
dag.add_task(task_fn::<(), _, _>(move |_: ()| i)).into();
|
||||
|
||||
let left = dag
|
||||
.add_task(task_fn(|x: i32| async move { x * 2 }))
|
||||
.depends_on(&source);
|
||||
.add_task(task_fn::<i32, _, _>(|&x: &i32| x * 2))
|
||||
.depends_on(source);
|
||||
let right = dag
|
||||
.add_task(task_fn(|x: i32| async move { x * 3 }))
|
||||
.depends_on(&source);
|
||||
.add_task(task_fn::<i32, _, _>(|&x: &i32| x * 3))
|
||||
.depends_on(source);
|
||||
|
||||
dag.add_task(task_fn(|(l, r): (i32, i32)| async move { l + r }))
|
||||
dag.add_task(task_fn::<(i32, i32), _, _>(|(l, r): (&i32, &i32)| l + r))
|
||||
.depends_on((&left, &right));
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
+19
-48
@@ -1,8 +1,8 @@
|
||||
//! Fan-out pattern benchmarks (1 → N dependencies)
|
||||
|
||||
use criterion::Criterion;
|
||||
use dagx::{task_fn, DagRunner};
|
||||
|
||||
use dagx::{DagRunner, TaskHandle};
|
||||
use dagx_test::task_fn;
|
||||
pub fn bench_fanout(c: &mut Criterion) {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
|
||||
@@ -11,18 +11,16 @@ pub fn bench_fanout(c: &mut Criterion) {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
let dag = DagRunner::new();
|
||||
let source = dag.add_task(task_fn(|_: ()| async { 42 }));
|
||||
let source: TaskHandle<_> = dag.add_task(task_fn::<(), _, _>(|_: ()| 42)).into();
|
||||
|
||||
for i in 0..100 {
|
||||
dag.add_task(task_fn(move |x: i32| async move { x + i }))
|
||||
.depends_on(&source);
|
||||
dag.add_task(task_fn::<i32, _, _>(move |&x: &i32| x + i))
|
||||
.depends_on(source);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
@@ -32,51 +30,24 @@ pub fn bench_fanout(c: &mut Criterion) {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
let dag = DagRunner::new();
|
||||
let source = dag.add_task(task_fn(|_: ()| async {
|
||||
// Create 1000 strings - framework wraps in Arc automatically
|
||||
(0..1000).map(|i| format!("Item {}", i)).collect::<Vec<_>>()
|
||||
}));
|
||||
let source: TaskHandle<_> = dag
|
||||
.add_task(task_fn::<(), _, _>(|_: ()| {
|
||||
// Create 1000 strings - framework wraps in Arc automatically
|
||||
(0..1000).map(|i| format!("Item {}", i)).collect::<Vec<_>>()
|
||||
}))
|
||||
.into();
|
||||
|
||||
for i in 0..100 {
|
||||
dag.add_task(task_fn(move |data: Vec<String>| async move {
|
||||
dag.add_task(task_fn::<Vec<_>, _, _>(move |data: &Vec<String>| {
|
||||
// Each consumer gets data extracted from Arc
|
||||
data.iter().filter(|s| s.contains(&i.to_string())).count()
|
||||
}))
|
||||
.depends_on(&source);
|
||||
.depends_on(source);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
// Arc overhead benchmark - shows cost on Copy types
|
||||
// For Copy types like usize, Arc adds overhead (atomic refcounting + indirection)
|
||||
// For heap types like Vec, Arc overhead is negligible compared to clone cost
|
||||
c.bench_function("arc_overhead_copy_type", |b| {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Source produces a simple Copy type (usize)
|
||||
// Framework wraps in Arc<usize> internally
|
||||
let source = dag.add_task(task_fn(|_: ()| async { 42usize }));
|
||||
|
||||
// 100 dependents - Arc overhead visible on Copy types
|
||||
for i in 0..100 {
|
||||
dag.add_task(task_fn(move |x: usize| async move { x + i }))
|
||||
.depends_on(&source);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
//! Configuration broadcast pattern benchmark - one config shared across many workers
|
||||
|
||||
use criterion::{BenchmarkId, Criterion};
|
||||
use dagx::{task_fn, DagRunner};
|
||||
use std::collections::HashMap;
|
||||
use std::hint::black_box;
|
||||
|
||||
pub fn bench_config_broadcast(c: &mut Criterion) {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
let mut group = c.benchmark_group("config_broadcast");
|
||||
|
||||
for worker_count in [10, 50, 100, 200].iter() {
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("config", worker_count),
|
||||
worker_count,
|
||||
|b, &workers| {
|
||||
b.iter(|| {
|
||||
rt.block_on(async {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Configuration source - framework wraps HashMap in Arc automatically
|
||||
let config = dag.add_task(task_fn(|_: ()| async {
|
||||
HashMap::from([
|
||||
("max_connections".to_string(), "100".to_string()),
|
||||
("timeout_ms".to_string(), "5000".to_string()),
|
||||
("batch_size".to_string(), "50".to_string()),
|
||||
("retry_count".to_string(), "3".to_string()),
|
||||
("worker_threads".to_string(), "4".to_string()),
|
||||
("buffer_size".to_string(), "8192".to_string()),
|
||||
("compression".to_string(), "true".to_string()),
|
||||
("log_level".to_string(), "info".to_string()),
|
||||
])
|
||||
}));
|
||||
|
||||
// Workers receive and process based on shared config
|
||||
for worker_id in 0..workers {
|
||||
dag.add_task(task_fn(move |cfg: HashMap<String, String>| async move {
|
||||
// Simulate work based on configuration
|
||||
let batch_size: usize = cfg
|
||||
.get("batch_size")
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(10);
|
||||
let timeout: usize = cfg
|
||||
.get("timeout_ms")
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(1000);
|
||||
|
||||
// Simulate some computation
|
||||
black_box(worker_id * batch_size + timeout / 100)
|
||||
}))
|
||||
.depends_on(&config);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
+28
-28
@@ -1,8 +1,8 @@
|
||||
//! ETL (Extract, Transform, Load) pipeline benchmark
|
||||
|
||||
use criterion::{BenchmarkId, Criterion};
|
||||
use dagx::{task_fn, DagRunner};
|
||||
|
||||
use dagx::{DagRunner, TaskHandle};
|
||||
use dagx_test::task_fn;
|
||||
pub fn bench_etl_pipeline(c: &mut Criterion) {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
let mut group = c.benchmark_group("etl_scaling");
|
||||
@@ -20,24 +20,26 @@ pub fn bench_etl_pipeline(c: &mut Criterion) {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Extract: Load JSON-like data (framework wraps in Arc automatically)
|
||||
let extract = dag.add_task(task_fn(move |_: ()| async move {
|
||||
(0..count)
|
||||
.map(|i| {
|
||||
format!(
|
||||
"{{\"id\":{},\"name\":\"User{}\",\"score\":{}}}",
|
||||
i,
|
||||
i,
|
||||
i * 10
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
}));
|
||||
let extract: TaskHandle<_> = dag
|
||||
.add_task(task_fn::<(), _, _>(move |_: ()| {
|
||||
(0..count)
|
||||
.map(|i| {
|
||||
format!(
|
||||
"{{\"id\":{},\"name\":\"User{}\",\"score\":{}}}",
|
||||
i,
|
||||
i,
|
||||
i * 10
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
}))
|
||||
.into();
|
||||
|
||||
// Transform: Three parallel transformations
|
||||
|
||||
// T1: Parse and validate
|
||||
let validate = dag
|
||||
.add_task(task_fn(move |data: std::sync::Arc<Vec<String>>| async move {
|
||||
.add_task(task_fn::<Vec<_>, _, _>(move |data: &Vec<String>| {
|
||||
data.iter()
|
||||
.filter(|s| {
|
||||
s.contains("\"id\"")
|
||||
@@ -46,11 +48,11 @@ pub fn bench_etl_pipeline(c: &mut Criterion) {
|
||||
})
|
||||
.count()
|
||||
}))
|
||||
.depends_on(&extract);
|
||||
.depends_on(extract);
|
||||
|
||||
// T2: Extract scores
|
||||
let scores = dag
|
||||
.add_task(task_fn(move |data: std::sync::Arc<Vec<String>>| async move {
|
||||
.add_task(task_fn::<Vec<_>, _, _>(move |data: &Vec<String>| {
|
||||
data.iter()
|
||||
.filter_map(|s| {
|
||||
s.split("\"score\":").nth(1).and_then(|part| {
|
||||
@@ -59,11 +61,11 @@ pub fn bench_etl_pipeline(c: &mut Criterion) {
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
}))
|
||||
.depends_on(&extract);
|
||||
.depends_on(extract);
|
||||
|
||||
// T3: Extract names
|
||||
let names = dag
|
||||
.add_task(task_fn(move |data: std::sync::Arc<Vec<String>>| async move {
|
||||
.add_task(task_fn::<Vec<_>, _, _>(move |data: &Vec<String>| {
|
||||
data.iter()
|
||||
.filter_map(|s| {
|
||||
s.split("\"name\":\"")
|
||||
@@ -73,16 +75,16 @@ pub fn bench_etl_pipeline(c: &mut Criterion) {
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
}))
|
||||
.depends_on(&extract);
|
||||
.depends_on(extract);
|
||||
|
||||
// Load: Aggregate all transformations
|
||||
dag.add_task(task_fn(
|
||||
move |(valid, scores, names): (std::sync::Arc<usize>, std::sync::Arc<Vec<i32>>, std::sync::Arc<Vec<String>>)| async move {
|
||||
dag.add_task(task_fn::<(usize, Vec<_>, Vec<_>), _, _>(
|
||||
move |(valid, scores, names): (&usize, &Vec<i32>, &Vec<String>)| {
|
||||
let avg_score =
|
||||
scores.iter().sum::<i32>() / scores.len().max(1) as i32;
|
||||
format!(
|
||||
"Processed {} valid records, {} names, avg score: {}",
|
||||
*valid,
|
||||
valid,
|
||||
names.len(),
|
||||
avg_score
|
||||
)
|
||||
@@ -90,11 +92,9 @@ pub fn bench_etl_pipeline(c: &mut Criterion) {
|
||||
))
|
||||
.depends_on((&validate, &scores, &names));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
},
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
//! Memory efficiency benchmark with large data structures
|
||||
|
||||
use criterion::{BenchmarkId, Criterion};
|
||||
use dagx::{task_fn, DagRunner};
|
||||
use dagx::{DagRunner, TaskHandle};
|
||||
use dagx_test::task_fn;
|
||||
use std::hint::black_box;
|
||||
|
||||
pub fn bench_memory_efficiency(c: &mut Criterion) {
|
||||
@@ -18,14 +19,16 @@ pub fn bench_memory_efficiency(c: &mut Criterion) {
|
||||
rt.block_on(async {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let source = dag.add_task(task_fn(move |_: ()| async move {
|
||||
// Create MB of data - framework wraps in Arc automatically
|
||||
vec![0u8; mb * 1024 * 1024]
|
||||
}));
|
||||
let source: TaskHandle<_> = dag
|
||||
.add_task(task_fn::<(), _, _>(move |_: ()| {
|
||||
// Create MB of data - framework wraps in Arc automatically
|
||||
vec![0u8; mb * 1024 * 1024]
|
||||
}))
|
||||
.into();
|
||||
|
||||
// 20 consumers share the data via automatic Arc wrapping
|
||||
for i in 0..20 {
|
||||
dag.add_task(task_fn(move |data: Vec<u8>| async move {
|
||||
dag.add_task(task_fn::<Vec<_>, _, _>(move |data: &Vec<u8>| {
|
||||
// Process a small portion (data extracted from Arc)
|
||||
let sample_sum: u32 = data
|
||||
.iter()
|
||||
@@ -35,14 +38,12 @@ pub fn bench_memory_efficiency(c: &mut Criterion) {
|
||||
.sum();
|
||||
black_box(sample_sum)
|
||||
}))
|
||||
.depends_on(&source);
|
||||
.depends_on(source);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
},
|
||||
|
||||
+24
-22
@@ -1,8 +1,8 @@
|
||||
//! Mixed realistic patterns - fan-out, fan-in, and processing
|
||||
|
||||
use criterion::Criterion;
|
||||
use dagx::{task_fn, DagRunner};
|
||||
|
||||
use dagx::{DagRunner, TaskHandle};
|
||||
use dagx_test::task_fn;
|
||||
pub fn bench_mixed_patterns(c: &mut Criterion) {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
|
||||
@@ -12,49 +12,51 @@ pub fn bench_mixed_patterns(c: &mut Criterion) {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Stage 1: Load data (framework wraps in Arc automatically)
|
||||
let data = dag.add_task(task_fn(|_: ()| async {
|
||||
(0..1000)
|
||||
.map(|i| format!("Record-{:04}", i))
|
||||
.collect::<Vec<_>>()
|
||||
}));
|
||||
let data: TaskHandle<_> = dag
|
||||
.add_task(task_fn::<(), _, _>(|_: ()| {
|
||||
(0..1000)
|
||||
.map(|i| format!("Record-{:04}", i))
|
||||
.collect::<Vec<_>>()
|
||||
}))
|
||||
.into();
|
||||
|
||||
// Stage 2: Fan-out for parallel analysis
|
||||
let analysis1 = dag
|
||||
.add_task(task_fn(|d: Vec<String>| async move {
|
||||
.add_task(task_fn::<Vec<_>, _, _>(|d: &Vec<String>| {
|
||||
d.iter().filter(|s| s.contains("00")).count()
|
||||
}))
|
||||
.depends_on(&data);
|
||||
.depends_on(data);
|
||||
|
||||
let analysis2 = dag
|
||||
.add_task(task_fn(|d: Vec<String>| async move {
|
||||
.add_task(task_fn::<Vec<_>, _, _>(|d: &Vec<String>| {
|
||||
d.iter().map(|s| s.len()).sum::<usize>()
|
||||
}))
|
||||
.depends_on(&data);
|
||||
.depends_on(data);
|
||||
|
||||
let analysis3 = dag
|
||||
.add_task(task_fn(|d: Vec<String>| async move { d.len() }))
|
||||
.depends_on(&data);
|
||||
.add_task(task_fn::<Vec<_>, _, _>(|d: &Vec<String>| d.len()))
|
||||
.depends_on(data);
|
||||
|
||||
// Stage 3: Fan-in to aggregate
|
||||
let summary = dag
|
||||
.add_task(task_fn(|(a1, a2, a3): (usize, usize, usize)| async move {
|
||||
format!("Matches: {}, Total bytes: {}, Count: {}", a1, a2, a3)
|
||||
}))
|
||||
.add_task(task_fn::<(usize, usize, usize), _, _>(
|
||||
|(a1, a2, a3): (&usize, &usize, &usize)| {
|
||||
format!("Matches: {}, Total bytes: {}, Count: {}", a1, a2, a3)
|
||||
},
|
||||
))
|
||||
.depends_on((&analysis1, &analysis2, &analysis3));
|
||||
|
||||
// Stage 4: Fan-out final report to multiple destinations
|
||||
for i in 0..3 {
|
||||
dag.add_task(task_fn(move |report: String| async move {
|
||||
dag.add_task(task_fn::<String, _, _>(move |report: &String| {
|
||||
format!("Destination {}: {}", i, report)
|
||||
}))
|
||||
.depends_on(summary);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
//! Realistic workload benchmarks
|
||||
|
||||
pub mod config_broadcast;
|
||||
pub mod etl;
|
||||
pub mod memory_efficiency;
|
||||
pub mod mixed;
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "dagx-macros"
|
||||
version = "0.3.0"
|
||||
version = "0.3.1"
|
||||
edition = "2021"
|
||||
rust-version = "1.81"
|
||||
authors = ["Stephen Waits <steve@waits.net>"]
|
||||
|
||||
+132
-152
@@ -1,21 +1,26 @@
|
||||
//! Procedural macros for dagx
|
||||
//!
|
||||
//! This crate provides the `#[task]` attribute macro that automatically generates
|
||||
//! an `execute()` method by deriving Input and Output types from the `run()` method signature.
|
||||
//! This crate provides the `#[task]` attribute macro that automatically implements
|
||||
//! the `Task` trait by deriving Input and Output types from the `run()` method signature.
|
||||
|
||||
use proc_macro::TokenStream;
|
||||
use proc_macro2::Span;
|
||||
use quote::quote;
|
||||
use syn::{parse_macro_input, FnArg, ImplItem, ItemImpl, Pat, PatType, ReturnType, Type};
|
||||
use syn::{parse_macro_input, FnArg, Ident, ImplItem, ItemImpl, Pat, PatType, ReturnType, Type};
|
||||
|
||||
/// Attribute macro to automatically generate the execution logic for a task.
|
||||
/// Attribute macro to automatically implement the `Task` trait.
|
||||
///
|
||||
/// Apply this to an `impl` block containing a `run()` method (sync or async). The macro:
|
||||
/// - Derives input types from the `run()` signature parameters
|
||||
/// - Generates an `execute()` method that handles listener extraction and announcement
|
||||
/// - Derives `Input` and `Output` types from the `run()` signature
|
||||
/// - Automatically implements the `Task` trait
|
||||
/// - **Generates type-specific extraction logic** - works with ANY type (Send + Sync + 'static)!
|
||||
/// - Supports both sync and async run methods
|
||||
/// - Supports stateless (no self) and stateful (&self, &mut self) tasks
|
||||
/// - Handles various input patterns (no inputs, single input, multiple inputs)
|
||||
///
|
||||
/// **Key Feature**: Custom types work automatically without implementing any traits!
|
||||
/// The macro generates inline extraction logic specific to your task's parameter types.
|
||||
///
|
||||
/// # Task Patterns
|
||||
///
|
||||
/// The `#[task]` macro supports three patterns based on state requirements:
|
||||
@@ -25,7 +30,7 @@ use syn::{parse_macro_input, FnArg, ImplItem, ItemImpl, Pat, PatType, ReturnType
|
||||
/// Unit structs for pure computations. Use **no `self` parameter**:
|
||||
///
|
||||
/// ```ignore
|
||||
/// use dagx::task;
|
||||
/// use dagx::{task, Task};
|
||||
///
|
||||
/// struct Add;
|
||||
///
|
||||
@@ -42,7 +47,7 @@ use syn::{parse_macro_input, FnArg, ImplItem, ItemImpl, Pat, PatType, ReturnType
|
||||
/// Tasks that read configuration or constant data. Use **`&self`**:
|
||||
///
|
||||
/// ```ignore
|
||||
/// use dagx::task;
|
||||
/// use dagx::{task, Task};
|
||||
///
|
||||
/// struct Multiplier {
|
||||
/// factor: i32,
|
||||
@@ -61,7 +66,7 @@ use syn::{parse_macro_input, FnArg, ImplItem, ItemImpl, Pat, PatType, ReturnType
|
||||
/// Tasks that accumulate or modify state. Use **`&mut self`**:
|
||||
///
|
||||
/// ```ignore
|
||||
/// use dagx::task;
|
||||
/// use dagx::{task, Task};
|
||||
///
|
||||
/// struct Counter {
|
||||
/// count: i32,
|
||||
@@ -81,7 +86,7 @@ use syn::{parse_macro_input, FnArg, ImplItem, ItemImpl, Pat, PatType, ReturnType
|
||||
/// ## No Inputs (Source Tasks)
|
||||
///
|
||||
/// ```ignore
|
||||
/// use dagx::task;
|
||||
/// use dagx::{task, Task};
|
||||
///
|
||||
/// struct LoadData {
|
||||
/// value: i32,
|
||||
@@ -98,13 +103,13 @@ use syn::{parse_macro_input, FnArg, ImplItem, ItemImpl, Pat, PatType, ReturnType
|
||||
/// ## Single Input
|
||||
///
|
||||
/// ```ignore
|
||||
/// use dagx::task;
|
||||
/// use dagx::{task, Task};
|
||||
///
|
||||
/// struct Double;
|
||||
///
|
||||
/// #[task]
|
||||
/// impl Double {
|
||||
/// async fn run(input: &i32) -> i32 {
|
||||
/// async fn run(&mut self, input: &i32) -> i32 {
|
||||
/// input * 2
|
||||
/// }
|
||||
/// }
|
||||
@@ -113,13 +118,13 @@ use syn::{parse_macro_input, FnArg, ImplItem, ItemImpl, Pat, PatType, ReturnType
|
||||
/// ## Multiple Inputs (up to 8)
|
||||
///
|
||||
/// ```ignore
|
||||
/// use dagx::task;
|
||||
/// use dagx::{task, Task};
|
||||
///
|
||||
/// struct Combine;
|
||||
///
|
||||
/// #[task]
|
||||
/// impl Combine {
|
||||
/// async fn run(a: &i32, b: &String, c: &bool) -> String {
|
||||
/// async fn run(&mut self, a: &i32, b: &String, c: &bool) -> String {
|
||||
/// format!("{}: {} ({})", b, a, c)
|
||||
/// }
|
||||
/// }
|
||||
@@ -133,11 +138,68 @@ use syn::{parse_macro_input, FnArg, ImplItem, ItemImpl, Pat, PatType, ReturnType
|
||||
///
|
||||
/// # Generated Code
|
||||
///
|
||||
/// The macro transforms your implementation into an `execute()` method that:
|
||||
/// - Extracts inputs from announcement listeners
|
||||
/// - Calls your run() method
|
||||
/// - Announces the result to dependents
|
||||
/// - Returns the Arc-wrapped output
|
||||
/// The macro transforms your implementation into a full `Task` trait implementation.
|
||||
///
|
||||
/// For stateless tasks (no self parameter):
|
||||
///
|
||||
/// ```ignore
|
||||
/// // Your code:
|
||||
/// #[task]
|
||||
/// impl Add {
|
||||
/// async fn run(a: &i32, b: &i32) -> i32 {
|
||||
/// a + b
|
||||
/// }
|
||||
/// }
|
||||
///
|
||||
/// // Generated:
|
||||
/// impl Task for Add {
|
||||
/// type Input = (i32, i32);
|
||||
/// type Output = i32;
|
||||
///
|
||||
/// async fn run(&mut self, input: Self::Input) -> Self::Output {
|
||||
/// let (a, b) = input;
|
||||
/// Self::run_impl(&a, &b).await
|
||||
/// }
|
||||
/// }
|
||||
///
|
||||
/// impl Add {
|
||||
/// #[inline]
|
||||
/// async fn run_impl(a: &i32, b: &i32) -> i32 {
|
||||
/// a + b
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// For stateful tasks (with &mut self):
|
||||
///
|
||||
/// ```ignore
|
||||
/// // Your code:
|
||||
/// #[task]
|
||||
/// impl Counter {
|
||||
/// async fn run(&mut self, inc: &i32) -> i32 {
|
||||
/// self.count += inc;
|
||||
/// self.count
|
||||
/// }
|
||||
/// }
|
||||
///
|
||||
/// // Generated:
|
||||
/// impl Task for Counter {
|
||||
/// type Input = i32;
|
||||
/// type Output = i32;
|
||||
///
|
||||
/// async fn run(&mut self, input: Self::Input) -> Self::Output {
|
||||
/// let inc = input;
|
||||
/// self.run_impl(&inc).await
|
||||
/// }
|
||||
/// }
|
||||
///
|
||||
/// impl Counter {
|
||||
/// async fn run_impl(&mut self, inc: &i32) -> i32 {
|
||||
/// self.count += inc;
|
||||
/// self.count
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
#[proc_macro_attribute]
|
||||
pub fn task(_attr: TokenStream, item: TokenStream) -> TokenStream {
|
||||
let impl_block = parse_macro_input!(item as ItemImpl);
|
||||
@@ -161,16 +223,13 @@ pub fn task(_attr: TokenStream, item: TokenStream) -> TokenStream {
|
||||
"impl block must contain a run() method\n\n\
|
||||
Expected signature: fn run(&mut self, ...) -> OutputType\n\
|
||||
or: async fn run(&mut self, ...) -> OutputType\n\
|
||||
The #[task] macro requires a run() method to implement task execution.",
|
||||
The #[task] macro requires a run() method to implement the Task trait.",
|
||||
)
|
||||
.to_compile_error()
|
||||
.into();
|
||||
}
|
||||
};
|
||||
|
||||
// Check if the method is async or sync
|
||||
let is_async = run_method.sig.asyncness.is_some();
|
||||
|
||||
// Extract parameters (excluding self)
|
||||
let params_result: Result<Vec<_>, _> = run_method
|
||||
.sig
|
||||
@@ -215,17 +274,40 @@ pub fn task(_attr: TokenStream, item: TokenStream) -> TokenStream {
|
||||
|
||||
// Extract return type
|
||||
let output_type = match &run_method.sig.output {
|
||||
ReturnType::Default => {
|
||||
return syn::Error::new_spanned(
|
||||
&run_method.sig,
|
||||
"run() method must have an explicit return type\n\n\
|
||||
Specify the output type: async fn run(...) -> OutputType\n\
|
||||
For tasks that don't return a value, use '-> ()'.",
|
||||
)
|
||||
.to_compile_error()
|
||||
.into();
|
||||
ReturnType::Default => syn::parse_quote!(()),
|
||||
ReturnType::Type(_, ty) => {
|
||||
if let Type::Tuple(_tuple) = &**ty {
|
||||
return syn::Error::new_spanned(
|
||||
ty,
|
||||
"Returning bare tuples from task functions is currently not supported.\n\n\
|
||||
Please wrap this in a newtype or struct.",
|
||||
)
|
||||
.into_compile_error()
|
||||
.into();
|
||||
}
|
||||
ty.clone()
|
||||
}
|
||||
ReturnType::Type(_, ty) => ty.clone(),
|
||||
};
|
||||
|
||||
// Build Input type based on parameter count
|
||||
let types: Vec<_> = params.iter().map(|(_, ty)| ty).collect();
|
||||
let input_type = quote! { ( #(#types,)* ) };
|
||||
|
||||
// Generate parameter references for the wrapper run() method
|
||||
let (param_destructure, param_refs) = {
|
||||
let (destructure, refs): (Vec<_>, Vec<_>) = params
|
||||
.iter()
|
||||
.map(|(ident, _ty)| {
|
||||
let ident = Ident::new(&format!("__dagx_extract_{}", ident), Span::mixed_site());
|
||||
(
|
||||
quote! {
|
||||
let (#ident, input) = input.next();
|
||||
},
|
||||
ident,
|
||||
)
|
||||
})
|
||||
.unzip();
|
||||
(quote! { #(#destructure)* }, quote! { #(#refs),* })
|
||||
};
|
||||
|
||||
// Clone the run method and rename it to run_impl
|
||||
@@ -239,132 +321,30 @@ pub fn task(_attr: TokenStream, item: TokenStream) -> TokenStream {
|
||||
.iter()
|
||||
.any(|arg| matches!(arg, FnArg::Receiver(_)));
|
||||
|
||||
// Generate listener extraction code
|
||||
let listener_extractions: Vec<_> = params
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (name, ty))| {
|
||||
quote! {
|
||||
let #name = {
|
||||
if listeners.len() <= #i {
|
||||
return Err(format!(
|
||||
"Expected at least {} listener(s), got {}",
|
||||
#i + 1,
|
||||
listeners.len()
|
||||
));
|
||||
}
|
||||
let listener = listeners[#i]
|
||||
.downcast_ref::<announcement::Listener<std::sync::Arc<#ty>>>()
|
||||
.ok_or_else(|| {
|
||||
format!("Type mismatch: expected Listener<Arc<{}>> at index {}",
|
||||
stringify!(#ty), #i)
|
||||
})?;
|
||||
listener.listen().await
|
||||
.map_err(|_| "Listener closed before receiving value")?
|
||||
};
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let is_async = run_method.sig.asyncness.is_some();
|
||||
|
||||
// Generate parameter refs for the run call
|
||||
// Dereference the Arc to get &T for the user's function
|
||||
let param_refs: Vec<_> = params.iter().map(|(name, _)| quote! { &*#name }).collect();
|
||||
|
||||
// Generate the execute method body based on whether we have self and async/sync
|
||||
let run_call = if has_self_receiver {
|
||||
if is_async {
|
||||
quote! { self.run_impl(#(#param_refs),*).await }
|
||||
} else {
|
||||
quote! { self.run_impl(#(#param_refs),*) }
|
||||
}
|
||||
} else if is_async {
|
||||
quote! { Self::run_impl(#(#param_refs),*).await }
|
||||
} else {
|
||||
quote! { Self::run_impl(#(#param_refs),*) }
|
||||
let run_call = match (has_self_receiver, is_async) {
|
||||
(true, true) => quote! { self.run_impl(#param_refs).await },
|
||||
(true, false) => quote! { self.run_impl(#param_refs) },
|
||||
(false, true) => quote! { Self::run_impl(#param_refs).await },
|
||||
(false, false) => quote! { Self::run_impl(#param_refs) },
|
||||
};
|
||||
|
||||
// Generate the self parameter for execute method
|
||||
let self_param = if has_self_receiver {
|
||||
quote! { mut self }
|
||||
} else {
|
||||
quote! { self }
|
||||
};
|
||||
|
||||
// Generate the Input type based on parameter count
|
||||
let input_type = match params.len() {
|
||||
0 => quote! { () },
|
||||
1 => {
|
||||
let (_, ty) = ¶ms[0];
|
||||
quote! { #ty }
|
||||
}
|
||||
_ => {
|
||||
let types: Vec<_> = params.iter().map(|(_, ty)| ty).collect();
|
||||
quote! { (#(#types,)*) }
|
||||
}
|
||||
};
|
||||
|
||||
// Generate the complete execute method and TaskExecute trait implementation
|
||||
let expanded = quote! {
|
||||
impl #struct_name {
|
||||
#[doc(hidden)]
|
||||
pub async fn execute(
|
||||
#self_param,
|
||||
listeners: Vec<Box<dyn std::any::Any + Send>>,
|
||||
announcer: Box<dyn std::any::Any + Send>,
|
||||
) -> Result<std::sync::Arc<dyn std::any::Any + Send + Sync>, String> {
|
||||
// Extract inputs from listeners
|
||||
#(#listener_extractions)*
|
||||
|
||||
// Call the user's run method
|
||||
let output: #output_type = #run_call;
|
||||
|
||||
// Wrap in Arc for sharing
|
||||
let output_arc = std::sync::Arc::new(output);
|
||||
|
||||
// Announce the result (announce the Arc, not a clone of the data)
|
||||
let announcer_typed = announcer
|
||||
.downcast::<announcement::Announcer<std::sync::Arc<#output_type>>>()
|
||||
.map_err(|_| "Type mismatch: announcer type does not match task output type")?;
|
||||
let _ = announcer_typed.announce(std::sync::Arc::clone(&output_arc));
|
||||
|
||||
// Return Arc-wrapped output
|
||||
Ok(output_arc as std::sync::Arc<dyn std::any::Any + Send + Sync>)
|
||||
}
|
||||
|
||||
#run_impl_method
|
||||
}
|
||||
|
||||
// Implement TaskExecute trait for type erasure
|
||||
impl dagx::TaskExecute for #struct_name {
|
||||
// Generate the Task trait implementation based on whether we have self and async/sync
|
||||
quote! {
|
||||
impl ::dagx::Task for #struct_name {
|
||||
type Input = #input_type;
|
||||
type Output = #output_type;
|
||||
|
||||
fn execute(
|
||||
self,
|
||||
listeners: Vec<Box<dyn std::any::Any + Send>>,
|
||||
announcer: Box<dyn std::any::Any + Send>,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<std::sync::Arc<dyn std::any::Any + Send + Sync>, String>> + Send>> {
|
||||
Box::pin(async move {
|
||||
self.execute(listeners, announcer).await
|
||||
})
|
||||
}
|
||||
|
||||
fn create_announcement_erased() -> (Box<dyn std::any::Any + Send>, Box<dyn std::any::Any + Send>) {
|
||||
let (announcer, announcement) = announcement::Announcement::<std::sync::Arc<#output_type>>::new();
|
||||
(
|
||||
Box::new(announcer) as Box<dyn std::any::Any + Send>,
|
||||
Box::new(announcement) as Box<dyn std::any::Any + Send>,
|
||||
)
|
||||
}
|
||||
|
||||
fn create_listener_erased(announcement: &dyn std::any::Any) -> Box<dyn std::any::Any + Send> {
|
||||
let announcement = announcement
|
||||
.downcast_ref::<announcement::Announcement<std::sync::Arc<#output_type>>>()
|
||||
.expect("Type mismatch: announcement type does not match task output type");
|
||||
Box::new(announcement.listener()) as Box<dyn std::any::Any + Send>
|
||||
async fn run(mut self, mut input: ::dagx::TaskInput<'_, Self::Input>) -> Self::Output {
|
||||
#param_destructure
|
||||
#run_call
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
TokenStream::from(expanded)
|
||||
impl #struct_name {
|
||||
#run_impl_method
|
||||
}
|
||||
}
|
||||
.into()
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
[package]
|
||||
name = "dagx-test"
|
||||
version = "0.3.1"
|
||||
edition = "2021"
|
||||
rust-version = "1.81"
|
||||
authors = ["TechnoPorg"]
|
||||
license = "MIT"
|
||||
description = "Utilities for dagx's internal tests"
|
||||
repository = "https://github.com/swaits/dagx"
|
||||
homepage = "https://github.com/swaits/dagx"
|
||||
documentation = "https://docs.rs/dagx-test"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
dagx = { version = "0.3.1", path = "../" }
|
||||
@@ -0,0 +1,158 @@
|
||||
//! Input extraction for tasks from a list of dependencies.
|
||||
|
||||
use std::{collections::HashMap, hash::Hash, sync::Arc};
|
||||
|
||||
use crate::TaskInput;
|
||||
|
||||
/// Helper trait for input extraction from list of dependencies.
|
||||
pub trait ExtractInput: Sized + Clone {
|
||||
type Input: Send;
|
||||
type Retv<'input>;
|
||||
|
||||
fn extract_from_task_input<'input>(
|
||||
input: TaskInput<'input, Self::Input>,
|
||||
) -> Result<Self::Retv<'input>, String>;
|
||||
}
|
||||
|
||||
// Unit type - no dependencies
|
||||
impl ExtractInput for () {
|
||||
type Input = ();
|
||||
type Retv<'input> = ();
|
||||
|
||||
fn extract_from_task_input(_input: TaskInput<'_, Self>) -> Result<Self, String> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Implementations for common primitive and standard library types.
|
||||
// These allow tasks to receive these types as single inputs without wrapping in tuples.
|
||||
// Receives Arc<T> from runner, clones the inner value for task consumption.
|
||||
// For custom types, implement ExtractInput following the same pattern.
|
||||
macro_rules! impl_extract_single {
|
||||
($($t:ty),+) => {
|
||||
$(
|
||||
impl ExtractInput for $t {
|
||||
type Input = ($t,);
|
||||
type Retv<'input> = &'input $t;
|
||||
fn extract_from_task_input<'input>(input: TaskInput<'input, Self::Input>) -> Result<Self::Retv<'input>, String> {
|
||||
Ok(input.next().0)
|
||||
}
|
||||
}
|
||||
)+
|
||||
};
|
||||
}
|
||||
|
||||
impl_extract_single!(
|
||||
i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize, f32, f64, bool, char, String
|
||||
);
|
||||
|
||||
// Implement ExtractInput for Result types to support error handling patterns
|
||||
impl<T, E> ExtractInput for Result<T, E>
|
||||
where
|
||||
T: 'static + Clone + Send + Sync,
|
||||
E: 'static + Clone + Send + Sync,
|
||||
{
|
||||
type Input = (Self,);
|
||||
type Retv<'input> = &'input Result<T, E>;
|
||||
|
||||
fn extract_from_task_input<'input>(
|
||||
input: TaskInput<'input, (Self,)>,
|
||||
) -> Result<Self::Retv<'input>, String> {
|
||||
Ok(input.next().0)
|
||||
}
|
||||
}
|
||||
|
||||
// Implement ExtractInput for Option types
|
||||
impl<T> ExtractInput for Option<T>
|
||||
where
|
||||
T: 'static + Clone + Send + Sync,
|
||||
{
|
||||
type Input = (Self,);
|
||||
type Retv<'input> = &'input Option<T>;
|
||||
|
||||
fn extract_from_task_input<'input>(
|
||||
input: TaskInput<'input, (Self,)>,
|
||||
) -> Result<Self::Retv<'input>, String> {
|
||||
Ok(input.next().0)
|
||||
}
|
||||
}
|
||||
|
||||
// Implement ExtractInput for Vec types
|
||||
impl<T> ExtractInput for Vec<T>
|
||||
where
|
||||
T: 'static + Clone + Send + Sync,
|
||||
{
|
||||
type Input = (Self,);
|
||||
type Retv<'input> = &'input Vec<T>;
|
||||
|
||||
fn extract_from_task_input<'input>(
|
||||
input: TaskInput<'input, (Self,)>,
|
||||
) -> Result<Self::Retv<'input>, String> {
|
||||
Ok(input.next().0)
|
||||
}
|
||||
}
|
||||
|
||||
// Implement ExtractInput for HashMap types
|
||||
impl<K, V> ExtractInput for HashMap<K, V>
|
||||
where
|
||||
K: 'static + Clone + Send + Sync + Eq + Hash,
|
||||
V: 'static + Clone + Send + Sync,
|
||||
{
|
||||
type Input = (Self,);
|
||||
type Retv<'input> = &'input HashMap<K, V>;
|
||||
|
||||
fn extract_from_task_input<'input>(
|
||||
input: TaskInput<'input, (Self,)>,
|
||||
) -> Result<Self::Retv<'input>, String> {
|
||||
Ok(input.next().0)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ExtractInput for Arc<T>
|
||||
where
|
||||
T: 'static + Clone + Send + Sync,
|
||||
{
|
||||
type Input = (Self,);
|
||||
type Retv<'input> = &'input Arc<T>;
|
||||
|
||||
fn extract_from_task_input<'input>(
|
||||
input: TaskInput<'input, (Self,)>,
|
||||
) -> Result<Self::Retv<'input>, String> {
|
||||
Ok(input.next().0)
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! impl_extract_tuple {
|
||||
($($T:ident),+) => {
|
||||
impl<$($T: 'static + Clone + Send + Sync),+> ExtractInput for ($($T,)+) {
|
||||
type Input = Self;
|
||||
type Retv<'input> = ($(&'input $T,)+);
|
||||
|
||||
fn extract_from_task_input<'input>(input: TaskInput<'input, Self::Input>) -> Result<Self::Retv<'input>, String> {
|
||||
$(
|
||||
let ($T, input): (&'input $T, _) = input.next();
|
||||
)+
|
||||
|
||||
let _input: TaskInput<'input, ()> = input;
|
||||
|
||||
Ok(($($T,)+))
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Generate ExtractInput implementations for tuples of size 2-8.
|
||||
// Each invocation creates an impl for a specific tuple size.
|
||||
// Supporting up to 8 elements covers the vast majority of use cases.
|
||||
#[allow(non_snake_case)]
|
||||
mod impls {
|
||||
use super::*;
|
||||
|
||||
impl_extract_tuple!(A, B);
|
||||
impl_extract_tuple!(A, B, C);
|
||||
impl_extract_tuple!(A, B, C, D);
|
||||
impl_extract_tuple!(A, B, C, D, E);
|
||||
impl_extract_tuple!(A, B, C, D, E, F);
|
||||
impl_extract_tuple!(A, B, C, D, E, F, G);
|
||||
impl_extract_tuple!(A, B, C, D, E, F, G, H);
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
//! DAG builder helpers for internal use in dagx tests and benchmarks.
|
||||
//!
|
||||
//! This crate is not meant for public use and offers no stability guarantees.
|
||||
//! The ergonomics are also such that you don't want to use it anyways. **Use the `#[task]` macro instead.**
|
||||
|
||||
#![cfg(not(tarpaulin_include))]
|
||||
|
||||
use crate::extract::ExtractInput;
|
||||
|
||||
use dagx::{Task, TaskInput};
|
||||
|
||||
mod extract;
|
||||
|
||||
pub struct TaskFn<I, O, F>
|
||||
where
|
||||
for<'input> F: FnMut(I::Retv<'input>) -> O + Send,
|
||||
I: ExtractInput + Send + Sync + 'static,
|
||||
O: Send,
|
||||
{
|
||||
f: F,
|
||||
_phantom: std::marker::PhantomData<fn(I) -> O>,
|
||||
}
|
||||
|
||||
impl<I, O, F> Task for TaskFn<I, O, F>
|
||||
where
|
||||
for<'input> F: FnMut(I::Retv<'input>) -> O + Send,
|
||||
I: ExtractInput + Send + Sync + 'static,
|
||||
O: Send + Sync,
|
||||
{
|
||||
type Input = I::Input;
|
||||
type Output = O;
|
||||
|
||||
async fn run(mut self, input: TaskInput<'_, I::Input>) -> O {
|
||||
(self.f)(I::extract_from_task_input(input).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience function to create a task from a closure.
|
||||
pub fn task_fn<I, O, F>(f: F) -> TaskFn<I, O, F>
|
||||
where
|
||||
for<'input> F: FnMut(I::Retv<'input>) -> O + Send,
|
||||
I: ExtractInput + Send + Sync + 'static,
|
||||
I::Input: Send + Sync + 'static,
|
||||
O: Send + Sync + 'static,
|
||||
{
|
||||
TaskFn {
|
||||
f,
|
||||
_phantom: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -18,8 +18,8 @@ impl Add { async fn run(a: &i32, b: &i32) -> i32 { a + b } }
|
||||
let dag = DagRunner::new();
|
||||
let x = dag.add_task(Value(2));
|
||||
let y = dag.add_task(Value(3));
|
||||
let sum = dag.add_task(Add).depends_on((&x, &y));
|
||||
dag.run(|fut| tokio::spawn(fut)).await?;
|
||||
let sum = dag.add_task(Add).depends_on((x, y));
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await?;
|
||||
```
|
||||
|
||||
**dagrs** (Complex: traits + channels + IDs + manual wiring):
|
||||
|
||||
@@ -13,7 +13,7 @@ let a = dag.add_task(TaskA);
|
||||
let b = dag.add_task(TaskB);
|
||||
a.depends_on(b);
|
||||
b.depends_on(a); // Compiles fine...
|
||||
dag.run().await?; // ❌ Error: "Cycle detected!" (at runtime)
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
```
|
||||
|
||||
**dagx prevents this at compile time:**
|
||||
|
||||
+23
-26
@@ -8,28 +8,23 @@ Traditional DAG executors contain substantial scheduling code—algorithms that
|
||||
|
||||
dagx eliminates this entirely:
|
||||
|
||||
1. **Wire up tasks with primitives**: During DAG construction, tasks are connected using channels and organized into topological layers based on their dependencies
|
||||
1. **Wire up tasks with primitives**: During DAG construction, tasks are organized into topological layers based on their dependencies
|
||||
2. **Start everything simultaneously**: When you call `run()`, all tasks spawn at once—there's no scheduler deciding when each task should start
|
||||
3. **Let the primitives handle coordination**: Channels naturally enforce execution order—tasks wait on their input channels until upstream tasks send data. No custom orchestration needed.
|
||||
4. **Runtime joins on completion**: The runtime simply spawns all tasks and waits for the completion channel to close. That's it.
|
||||
3. **Let the primitives handle coordination**: Layers naturally enforce execution order—tasks wait on their dependencies until . No custom orchestration needed.
|
||||
4. **Runtime joins on completion**: The runtime simply spawns all tasks and waits for them to finish. That's it.
|
||||
|
||||
## The Implementation
|
||||
|
||||
Under the hood, dagx uses:
|
||||
|
||||
- **Oneshot channels**: Each edge in the DAG gets a `futures::channel::oneshot` - producer sends once, consumer receives once
|
||||
- **Ownership model**: Tasks take ownership (`self`) and are consumed during execution - no Mutex needed for task state
|
||||
- **Direct data flow**: Outputs flow from producer to consumer via channels, never stored in shared memory during execution
|
||||
- **Direct data flow**: Outputs flow from producer(s) to consumer(s) through a shared hash map
|
||||
- **Fast-path optimization**: Single-task layers execute inline without spawning overhead
|
||||
|
||||
The core execution logic in `run()` is remarkably simple:
|
||||
|
||||
```rust
|
||||
// Create oneshot channel for each edge
|
||||
for (producer, consumer) in edges {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
// Wire tx to producer, rx to consumer
|
||||
}
|
||||
let outputs = HashMap::new();
|
||||
|
||||
// Execute layer by layer
|
||||
for layer in layers {
|
||||
@@ -40,25 +35,27 @@ for layer in layers {
|
||||
// Convert panic to error to match spawned task behavior
|
||||
} else {
|
||||
// Parallel path: Spawn all tasks in layer
|
||||
for task in layer {
|
||||
spawner(async {
|
||||
let input = await_on_input_channels(); // Blocks until deps complete
|
||||
let tasks = layers
|
||||
.map(async {
|
||||
let input = get_inputs(map); // Deps are guaranteed to be in the map
|
||||
let output = task.run(input); // Consumes task
|
||||
send_to_output_channels(output); // Sends to dependents
|
||||
});
|
||||
output
|
||||
})
|
||||
.collect::<FuturesUnordered<_>>(); // Run all tasks in parallel
|
||||
|
||||
while let Some(out) = tasks.next().await {
|
||||
outputs.insert(out.task_id, out.output); // Make result available for dependents
|
||||
}
|
||||
// Wait for layer completion
|
||||
}
|
||||
// Wait for layer completion
|
||||
}
|
||||
|
||||
// Tasks coordinate themselves via channels - no scheduler needed
|
||||
// FuturesUnordered coordinates task execution - no custom scheduler needed
|
||||
```
|
||||
|
||||
No state machines. No task queues. No wake-up logic. No Mutex locks. Just channels doing what channels do, with an inline fast-path for sequential execution.
|
||||
|
||||
## Inline Execution Fast-Path
|
||||
|
||||
**Performance optimization for sequential workloads**: When a layer contains only a single task (common in deep chains and linear pipelines), dagx executes it inline rather than spawning it. This eliminates spawning overhead, context switching, and channel creation, resulting in 10-100x performance improvements for sequential patterns.
|
||||
**Performance optimization for sequential workloads**: When a layer contains only a single task (common in deep chains and linear pipelines), dagx executes it inline rather than spawning it. This eliminates spawning overhead and context switching, resulting in a 10-100x performance improvements for sequential patterns.
|
||||
|
||||
**Panic handling guarantee**: To maintain behavioral consistency between inline and spawned execution, panics in inline tasks are caught using `FutureExt::catch_unwind()` and converted to errors. This matches the behavior of all major async runtimes (Tokio, async-std, smol, embassy-rs), which catch panics in spawned tasks and convert them to `JoinError` or equivalent.
|
||||
|
||||
@@ -72,19 +69,19 @@ This ensures your code behaves the same whether a task runs inline or spawned, m
|
||||
|
||||
## Benefits
|
||||
|
||||
**Simplicity**: The runtime is straightforward: create channels, spawn tasks, let them coordinate via awaiting. No complex scheduler code to maintain, debug, or optimize.
|
||||
**Simplicity**: The runtime is straightforward: spawn tasks, coordinate via compile-time dependency guarantees. No complex scheduler code to maintain, debug, or optimize.
|
||||
|
||||
**Reliability**: Built on battle-tested primitives (oneshot channels, async/await) from Rust's standard library and the futures crate. These have been used in production by thousands of projects and are orders of magnitude more reliable than custom scheduling logic.
|
||||
**Reliability**: Built on battle-tested primitives from Rust's standard library and the futures crate. These have been used in production by thousands of projects and are orders of magnitude more reliable than custom scheduling logic.
|
||||
|
||||
**Bug resistance**: Fewer moving parts means fewer places for bugs to hide. The type system enforces correct wiring at compile time. Channels handle synchronization. The ownership model prevents data races. What's left to break?
|
||||
**Bug resistance**: Fewer moving parts means fewer places for bugs to hide. The type system enforces correct wiring at compile time. The ownership model prevents data races. What's left to break?
|
||||
|
||||
**Performance**: Near zero-overhead. No Mutex locks during execution. Arc reference counting for efficient fanout (atomic operations, not locks). Oneshot channels are often lock-free. Tasks start as soon as their dependencies complete - maximum parallelism.
|
||||
**Performance**: Near zero-overhead. No Mutex locks during execution. Arc reference counting for efficient fanout (atomic operations, not locks). Tasks start as soon as their dependencies complete - maximum parallelism.
|
||||
|
||||
**Auditability**: Want to verify correctness? Check the channel wiring, verify tasks await their inputs, done. No need to trace through complex state machine transitions or wake-up cascades.
|
||||
**Auditability**: Want to verify correctness? Check the dependency wiring, verify tasks await their inputs, done. No need to trace through complex state machine transitions or wake-up cascades.
|
||||
|
||||
## The Insight
|
||||
|
||||
The key insight is that **dependencies ARE the schedule**. If task B depends on task A's output, a channel naturally enforces that B waits for A. The dependency graph already encodes all the scheduling information—we just need to wire up channels to match it.
|
||||
The key insight is that **dependencies ARE the schedule**. If task B depends on task A's output, the topological sort naturally enforces that B waits for A. The dependency graph already encodes all the scheduling information—we just need to wire outputs to match it.
|
||||
|
||||
This is dagx's core philosophy: leverage the type system for correctness, use primitives for coordination, and let the compiler optimize everything else away.
|
||||
|
||||
|
||||
@@ -41,8 +41,6 @@ RUST_LOG=dagx=debug cargo run # Task and layer details
|
||||
RUST_LOG=dagx=trace cargo run # All execution details
|
||||
```
|
||||
|
||||
See [`examples/tracing_example.rs`](../examples/tracing_example.rs) for a complete working example.
|
||||
|
||||
## Zero-Cost Guarantee
|
||||
|
||||
When the `tracing` feature is disabled (the default), there is **literally 0ns overhead**:
|
||||
|
||||
@@ -82,8 +82,8 @@ async fn main() {
|
||||
// Step 4: Add source tasks (tasks with no dependencies)
|
||||
//
|
||||
// add_task() returns a TaskHandle<T> where T is the task's output type
|
||||
let x = dag.add_task(Value::new(2)); // TaskHandle<i32>
|
||||
let y = dag.add_task(Value::new(3)); // TaskHandle<i32>
|
||||
let x = dag.add_task(Value::new(2)).into(); // TaskHandle<i32>
|
||||
let y = dag.add_task(Value::new(3)).into(); // TaskHandle<i32>
|
||||
|
||||
// Step 5: Add a task with dependencies
|
||||
//
|
||||
@@ -105,20 +105,17 @@ async fn main() {
|
||||
// 1. Tasks with no dependencies run first (x and y in parallel)
|
||||
// 2. Once x and y complete, sum runs (it depends on both)
|
||||
println!("Running DAG...\n");
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Step 7: Retrieve results
|
||||
//
|
||||
// Use .get() with a TaskHandle to retrieve the output.
|
||||
// Results are cloned, so you can retrieve them multiple times.
|
||||
// Results are not cloned, so you can only retrieve them once.
|
||||
println!("\nResult: {}", dag.get(sum).unwrap());
|
||||
|
||||
// Note: You can also retrieve intermediate results
|
||||
assert_eq!(dag.get(x).unwrap(), 2);
|
||||
assert_eq!(dag.get(y).unwrap(), 3);
|
||||
assert_eq!(dag.get(sum).unwrap(), 5);
|
||||
}
|
||||
+8
-10
@@ -55,7 +55,7 @@
|
||||
//! Squared: 100
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagRunner};
|
||||
use dagx::{task, DagRunner, TaskHandle};
|
||||
|
||||
// Source task with a value field
|
||||
struct Source {
|
||||
@@ -130,21 +130,19 @@ async fn main() {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Single source task - constructed with ::new()
|
||||
let source = dag.add_task(Source::new(10));
|
||||
let source: TaskHandle<_> = dag.add_task(Source::new(10)).into();
|
||||
|
||||
// Multiple downstream tasks consuming the same value
|
||||
// Each constructed differently to show flexibility
|
||||
let plus_one = dag.add_task(AddOffset::new(1)).depends_on(&source);
|
||||
let times_two = dag.add_task(Multiply::new(2)).depends_on(&source);
|
||||
let squared = dag.add_task(Power { exponent: 2 }).depends_on(&source);
|
||||
let plus_one = dag.add_task(AddOffset::new(1)).depends_on(source);
|
||||
let times_two = dag.add_task(Multiply::new(2)).depends_on(source);
|
||||
let squared = dag.add_task(Power { exponent: 2 }).depends_on(source);
|
||||
|
||||
// Run the DAG
|
||||
println!("Running fan-out DAG...\n");
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Print all results
|
||||
println!("\nResults:");
|
||||
|
||||
@@ -129,15 +129,13 @@ async fn main() {
|
||||
// Single downstream task consuming all values
|
||||
let profile = dag
|
||||
.add_task(BuildProfile)
|
||||
.depends_on((&name, &age, &city, &active));
|
||||
.depends_on((name, age, city, active));
|
||||
|
||||
// Run the DAG
|
||||
println!("Running fan-in DAG...\n");
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Print result
|
||||
println!("\n{}", dag.get(profile).unwrap());
|
||||
|
||||
@@ -52,7 +52,8 @@
|
||||
//! This proves dagx runs tasks on multiple CPU cores simultaneously.
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagResult, DagRunner};
|
||||
use dagx::{task, DagResult, DagRunner, Task, TaskInput};
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
|
||||
@@ -122,14 +123,10 @@ async fn main() -> DagResult<()> {
|
||||
let sum3 = dag.add_task(ComputeSum::new(501, 751, WORK_DELAY));
|
||||
let sum4 = dag.add_task(ComputeSum::new(751, 1001, WORK_DELAY));
|
||||
|
||||
let total = dag
|
||||
.add_task(Aggregate)
|
||||
.depends_on((&sum1, &sum2, &sum3, &sum4));
|
||||
let total = dag.add_task(Aggregate).depends_on((sum1, sum2, sum3, sum4));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
let result = dag.get(total)?;
|
||||
let elapsed = start.elapsed();
|
||||
@@ -144,32 +141,22 @@ async fn main() -> DagResult<()> {
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// SEQUENTIAL EXECUTION: Run tasks one at a time in separate DAG executions
|
||||
// SEQUENTIAL EXECUTION: Run the same 4 tasks one after another
|
||||
// ============================================================================
|
||||
println!("Running SEQUENTIAL computation (same work, one at a time)...");
|
||||
let (sequential_time, sequential_result) = {
|
||||
let start = Instant::now();
|
||||
|
||||
// Run each task separately to force sequential execution
|
||||
let dag1 = DagRunner::new();
|
||||
let sum1_handle = dag1.add_task(ComputeSum::new(1, 251, WORK_DELAY));
|
||||
dag1.run(|fut| { tokio::spawn(fut); }).await?;
|
||||
let r1 = dag1.get(sum1_handle)?;
|
||||
// Run tasks sequentially (not using DAG, just raw execution)
|
||||
let task1 = ComputeSum::new(1, 251, WORK_DELAY);
|
||||
let task2 = ComputeSum::new(251, 501, WORK_DELAY);
|
||||
let task3 = ComputeSum::new(501, 751, WORK_DELAY);
|
||||
let task4 = ComputeSum::new(751, 1001, WORK_DELAY);
|
||||
|
||||
let dag2 = DagRunner::new();
|
||||
let sum2_handle = dag2.add_task(ComputeSum::new(251, 501, WORK_DELAY));
|
||||
dag2.run(|fut| { tokio::spawn(fut); }).await?;
|
||||
let r2 = dag2.get(sum2_handle)?;
|
||||
|
||||
let dag3 = DagRunner::new();
|
||||
let sum3_handle = dag3.add_task(ComputeSum::new(501, 751, WORK_DELAY));
|
||||
dag3.run(|fut| { tokio::spawn(fut); }).await?;
|
||||
let r3 = dag3.get(sum3_handle)?;
|
||||
|
||||
let dag4 = DagRunner::new();
|
||||
let sum4_handle = dag4.add_task(ComputeSum::new(751, 1001, WORK_DELAY));
|
||||
dag4.run(|fut| { tokio::spawn(fut); }).await?;
|
||||
let r4 = dag4.get(sum4_handle)?;
|
||||
let r1 = task1.run(TaskInput::empty()).await;
|
||||
let r2 = task2.run(TaskInput::empty()).await;
|
||||
let r3 = task3.run(TaskInput::empty()).await;
|
||||
let r4 = task4.run(TaskInput::empty()).await;
|
||||
|
||||
let result = r1 + r2 + r3 + r4;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
+23
-36
@@ -151,6 +151,7 @@
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagRunner};
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::Mutex;
|
||||
@@ -389,11 +390,9 @@ async fn main() {
|
||||
false, // success
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result 1: {:?}\n", dag.get(call1).unwrap());
|
||||
|
||||
@@ -405,11 +404,9 @@ async fn main() {
|
||||
true, // fail
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result 2: {:?}\n", dag.get(call2).unwrap());
|
||||
|
||||
@@ -421,11 +418,9 @@ async fn main() {
|
||||
true, // fail
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result 3: {:?}\n", dag.get(call3).unwrap());
|
||||
|
||||
@@ -437,11 +432,9 @@ async fn main() {
|
||||
true, // fail
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result 4: {:?}\n", dag.get(call4).unwrap());
|
||||
|
||||
@@ -455,13 +448,11 @@ async fn main() {
|
||||
|
||||
let fallback5 = dag
|
||||
.add_task(FallbackService::new("ServiceCall-5"))
|
||||
.depends_on(&call5);
|
||||
.depends_on(call5);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result 5: {:?}\n", dag.get(fallback5).unwrap());
|
||||
|
||||
@@ -478,11 +469,9 @@ async fn main() {
|
||||
false, // success
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result 6: {:?}\n", dag.get(call6).unwrap());
|
||||
|
||||
@@ -494,11 +483,9 @@ async fn main() {
|
||||
false, // success
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result 7: {:?}\n", dag.get(call7).unwrap());
|
||||
}
|
||||
|
||||
@@ -1,217 +0,0 @@
|
||||
//! # Building Complex Workflows: Multi-Layer DAG
|
||||
//!
|
||||
//! This example demonstrates how to build complex, real-world DAGs by combining
|
||||
//! multiple dependency patterns into a cohesive workflow.
|
||||
//!
|
||||
//! ## What You'll Learn
|
||||
//! - How to combine fan-out, fan-in, and diamond patterns in one DAG
|
||||
//! - How to build multi-layer workflows with intermediate results
|
||||
//! - How to structure an ETL (Extract-Transform-Load) pipeline
|
||||
//! - How to pass results through multiple transformation stages
|
||||
//!
|
||||
//! ## Workflow Diagram
|
||||
//! ```text
|
||||
//! EXTRACT TRANSFORM COMBINE AGGREGATE
|
||||
//! ┌──────────┐ ┌──────────┐
|
||||
//! │ LoadData │────►│Transform │────┐
|
||||
//! │ A │ │ (×2) │ │
|
||||
//! └──────────┘ └──────────┘ │
|
||||
//! ├──►┌─────────┐───►┌─────────┐───►┌─────────┐
|
||||
//! │ │ Combine │ │ Average │ │ Report │
|
||||
//! └──►└─────────┘ └─────────┘ └─────────┘
|
||||
//! ┌──────────┐ ┌──────────┐ ┌──► │ │
|
||||
//! │ LoadData │────►│Transform │────┘ │ │
|
||||
//! │ B │ │ (×3) │ └──────────────┘
|
||||
//! └──────────┘ └──────────┘
|
||||
//! ```
|
||||
//!
|
||||
//! ## Key Concepts
|
||||
//! - **Multi-layer execution**: Tasks organized into logical stages (Extract → Transform → Combine)
|
||||
//! - **Many-to-many dependencies**: The Combine task depends on both Transform tasks
|
||||
//! - **Result reuse**: The combined result is used by both Average and Report tasks
|
||||
//! - **Diamond pattern**: Multiple paths from sources converge and then diverge again
|
||||
//!
|
||||
//! ## Use Cases
|
||||
//! - ETL pipelines with multiple data sources
|
||||
//! - Multi-stage data processing workflows
|
||||
//! - Complex business logic with branching and merging
|
||||
//! - Data aggregation and reporting systems
|
||||
//!
|
||||
//! ## Running This Example
|
||||
//! ```bash
|
||||
//! cargo run --example complex_dag
|
||||
//! ```
|
||||
//!
|
||||
//! ## Expected Output
|
||||
//! ```text
|
||||
//! Building ETL pipeline...
|
||||
//!
|
||||
//! Executing pipeline...
|
||||
//!
|
||||
//! Loading data from source A
|
||||
//! Loading data from source B
|
||||
//! Transforming: 6 * 2 = 12
|
||||
//! Transforming: 15 * 3 = 45
|
||||
//! Combining datasets: 12 + 45 = 57
|
||||
//! Computing average: 57 / 6 = 9.50
|
||||
//!
|
||||
//! ETL Pipeline Results:
|
||||
//! Items: 6
|
||||
//! Total: 57
|
||||
//! Average: 9.50
|
||||
//!
|
||||
//! Calculation:
|
||||
//! Source A: sum=6, transformed (x2) = 12
|
||||
//! Source B: sum=15, transformed (x3) = 45
|
||||
//! Combined: 6 items, sum=57, avg=9.50
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagResult, DagRunner};
|
||||
|
||||
// Extract: Load data from different sources
|
||||
struct LoadData {
|
||||
source: &'static str,
|
||||
}
|
||||
|
||||
impl LoadData {
|
||||
fn new(source: &'static str) -> Self {
|
||||
Self { source }
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl LoadData {
|
||||
async fn run(&mut self) -> i32 {
|
||||
// Simulate loading and summing data
|
||||
println!("Loading data from source {}", self.source);
|
||||
match self.source {
|
||||
"A" => 6, // sum of [1, 2, 3]
|
||||
"B" => 15, // sum of [4, 5, 6]
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Transform: Process data with a multiplier
|
||||
struct Transform {
|
||||
multiplier: i32,
|
||||
}
|
||||
|
||||
impl Transform {
|
||||
fn new(multiplier: i32) -> Self {
|
||||
Self { multiplier }
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl Transform {
|
||||
async fn run(&mut self, sum: &i32) -> i32 {
|
||||
let result = sum * self.multiplier;
|
||||
println!("Transforming: {} * {} = {}", sum, self.multiplier, result);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
// Combine: Merge two datasets
|
||||
struct Combine;
|
||||
|
||||
#[task]
|
||||
impl Combine {
|
||||
async fn run(sum_a: &i32, sum_b: &i32) -> i32 {
|
||||
println!(
|
||||
"Combining datasets: {} + {} = {}",
|
||||
sum_a,
|
||||
sum_b,
|
||||
sum_a + sum_b
|
||||
);
|
||||
sum_a + sum_b
|
||||
}
|
||||
}
|
||||
|
||||
// Compute average (using a fixed count for simplicity)
|
||||
struct ComputeAverage {
|
||||
total_count: i32,
|
||||
}
|
||||
|
||||
impl ComputeAverage {
|
||||
fn new(total_count: i32) -> Self {
|
||||
Self { total_count }
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl ComputeAverage {
|
||||
async fn run(&mut self, sum: &i32) -> f64 {
|
||||
let avg = *sum as f64 / self.total_count as f64;
|
||||
println!(
|
||||
"Computing average: {} / {} = {:.2}",
|
||||
sum, self.total_count, avg
|
||||
);
|
||||
avg
|
||||
}
|
||||
}
|
||||
|
||||
// Format report
|
||||
struct FormatReport {
|
||||
count: i32,
|
||||
}
|
||||
|
||||
impl FormatReport {
|
||||
fn new(count: i32) -> Self {
|
||||
Self { count }
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl FormatReport {
|
||||
async fn run(&mut self, sum: &i32, avg: &f64) -> String {
|
||||
format!(
|
||||
"ETL Pipeline Results:\n Items: {}\n Total: {}\n Average: {:.2}",
|
||||
self.count, sum, avg
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> DagResult<()> {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
println!("Building ETL pipeline...\n");
|
||||
|
||||
// Extract: Load from two sources (parallel)
|
||||
let source_a = dag.add_task(LoadData::new("A"));
|
||||
let source_b = dag.add_task(LoadData::new("B"));
|
||||
|
||||
// Transform: Process each source (parallel)
|
||||
let transform_a = dag.add_task(Transform::new(2)).depends_on(&source_a);
|
||||
let transform_b = dag.add_task(Transform::new(3)).depends_on(&source_b);
|
||||
|
||||
// Combine: Merge the transformed datasets
|
||||
let combined = dag
|
||||
.add_task(Combine)
|
||||
.depends_on((&transform_a, &transform_b));
|
||||
|
||||
// Compute average on combined data (we know we have 6 items total)
|
||||
let avg = dag.add_task(ComputeAverage::new(6)).depends_on(combined);
|
||||
|
||||
// Format final report (needs both sum and average)
|
||||
let report = dag
|
||||
.add_task(FormatReport::new(6))
|
||||
.depends_on((&combined, &avg));
|
||||
|
||||
// Execute the entire ETL pipeline
|
||||
println!("Executing pipeline...\n");
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Display final report
|
||||
println!("\n{}", dag.get(report)?);
|
||||
println!("\nCalculation:");
|
||||
println!(" Source A: sum=6, transformed (x2) = 12");
|
||||
println!(" Source B: sum=15, transformed (x3) = 45");
|
||||
println!(" Combined: 6 items, sum=57, avg=9.50");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
//! # Pattern: Conditional Execution with Data Validation
|
||||
//!
|
||||
//! This example demonstrates how to implement conditional logic in DAGs by using
|
||||
//! simple data transformations rather than traditional Result types.
|
||||
//!
|
||||
//! ## What You'll Learn
|
||||
//! - How to implement validation logic in parallel tasks
|
||||
//! - How to use return values for conditional behavior (e.g., 0 for invalid)
|
||||
//! - How to categorize and branch based on computed values
|
||||
//! - How to aggregate results from multiple conditional branches
|
||||
//!
|
||||
//! ## Workflow Diagram
|
||||
//! ```text
|
||||
//! VALIDATE CATEGORIZE SUMMARIZE
|
||||
//! ┌─────────────┐ ┌──────────────┐
|
||||
//! │ Validate(10)│────20──►│Categorize(25)│────"low"────┐
|
||||
//! └─────────────┘ └──────────────┘ │
|
||||
//! │
|
||||
//! ┌─────────────┐ ┌──────────────┐ │
|
||||
//! │ Validate(-5)│────0───►│Categorize(25)│──"invalid"──┼──►┌────────────┐
|
||||
//! └─────────────┘ └──────────────┘ │ │ Summarize │
|
||||
//! │ └────────────┘
|
||||
//! ┌─────────────┐ ┌──────────────┐ │
|
||||
//! │ Validate(20)│────40──►│Categorize(25)│────"high"───┘
|
||||
//! └─────────────┘ └──────────────┘
|
||||
//! ```
|
||||
//!
|
||||
//! ## Key Concepts
|
||||
//! - **Validation as transformation**: Invalid inputs become sentinel values (0)
|
||||
//! - **Parallel validation**: All inputs validated simultaneously
|
||||
//! - **Data-driven branching**: Categorization based on computed values
|
||||
//! - **Type-safe aggregation**: All branches produce the same type (String)
|
||||
//!
|
||||
//! ## Use Cases
|
||||
//! - Input validation pipelines
|
||||
//! - Multi-stage data cleaning
|
||||
//! - Classification and categorization workflows
|
||||
//! - Quality control systems
|
||||
//! - Data preprocessing for ML pipelines
|
||||
//!
|
||||
//! ## Running This Example
|
||||
//! ```bash
|
||||
//! cargo run --example conditional_workflow
|
||||
//! ```
|
||||
//!
|
||||
//! ## Expected Output
|
||||
//! ```text
|
||||
//! Processing and categorizing inputs in parallel...
|
||||
//!
|
||||
//! Valid input 10: doubling to 20
|
||||
//! Invalid input -5: returning 0
|
||||
//! Valid input 20: doubling to 40
|
||||
//!
|
||||
//! Final summary: Categories: [low, invalid, high]
|
||||
//! Expected: Categories: [low, invalid, high]
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagResult, DagRunner};
|
||||
|
||||
// Task that validates and processes input
|
||||
struct ValidateAndDouble {
|
||||
value: i32,
|
||||
}
|
||||
|
||||
impl ValidateAndDouble {
|
||||
fn new(value: i32) -> Self {
|
||||
Self { value }
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl ValidateAndDouble {
|
||||
async fn run(&mut self) -> i32 {
|
||||
if self.value > 0 {
|
||||
println!("Valid input {}: doubling to {}", self.value, self.value * 2);
|
||||
self.value * 2
|
||||
} else {
|
||||
println!("Invalid input {}: returning 0", self.value);
|
||||
0 // Return 0 for invalid inputs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Task that categorizes a number
|
||||
struct Categorize {
|
||||
threshold: i32,
|
||||
}
|
||||
|
||||
impl Categorize {
|
||||
fn new(threshold: i32) -> Self {
|
||||
Self { threshold }
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl Categorize {
|
||||
async fn run(&mut self, value: &i32) -> String {
|
||||
if *value == 0 {
|
||||
"invalid".to_string()
|
||||
} else if *value >= self.threshold {
|
||||
"high".to_string()
|
||||
} else {
|
||||
"low".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Task that combines categorization results
|
||||
struct SummarizeResults;
|
||||
|
||||
#[task]
|
||||
impl SummarizeResults {
|
||||
async fn run(cat1: &String, cat2: &String, cat3: &String) -> String {
|
||||
format!("Categories: [{}, {}, {}]", cat1, cat2, cat3)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> DagResult<()> {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
println!("Processing and categorizing inputs in parallel...\n");
|
||||
|
||||
// Validate and process multiple inputs in parallel
|
||||
let processed1 = dag.add_task(ValidateAndDouble::new(10)); // Valid: 20
|
||||
let processed2 = dag.add_task(ValidateAndDouble::new(-5)); // Invalid: 0
|
||||
let processed3 = dag.add_task(ValidateAndDouble::new(20)); // Valid: 40
|
||||
|
||||
// Categorize each result (also parallel, threshold = 25)
|
||||
let cat1 = dag.add_task(Categorize::new(25)).depends_on(&processed1); // 20 < 25 = "low"
|
||||
let cat2 = dag.add_task(Categorize::new(25)).depends_on(&processed2); // 0 = "invalid"
|
||||
let cat3 = dag.add_task(Categorize::new(25)).depends_on(&processed3); // 40 >= 25 = "high"
|
||||
|
||||
// Combine results
|
||||
let summary = dag
|
||||
.add_task(SummarizeResults)
|
||||
.depends_on((&cat1, &cat2, &cat3));
|
||||
|
||||
// Execute
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Display results
|
||||
println!("\nFinal summary: {}", dag.get(summary)?);
|
||||
println!("Expected: Categories: [low, invalid, high]");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
//! # Custom Types: Using Your Own Structs
|
||||
//!
|
||||
//! This example demonstrates that **ANY custom type** works in dagx as long as it implements
|
||||
//! `Clone + Send + Sync + 'static`. You don't need to implement any special traits!
|
||||
//!
|
||||
//! ## What You'll Learn
|
||||
//! - How to use custom structs as task outputs
|
||||
//! - How custom types flow through the DAG automatically
|
||||
//! - How one task's output can be consumed by multiple downstream tasks
|
||||
//! - No trait implementations required (just derive Clone!)
|
||||
//!
|
||||
//! ## Key Insight
|
||||
//!
|
||||
//! **dagx works with ANY type automatically!** The `#[task]` macro generates type-specific
|
||||
//! extraction logic, so you don't need to implement any special traits. Just make sure your
|
||||
//! type implements:
|
||||
//! - `Clone` (so it can be extracted from Arc)
|
||||
//! - `Send + Sync` (required for async tasks)
|
||||
//! - `'static` (no lifetime constraints)
|
||||
//!
|
||||
//! That's it! No special traits, no annotations, no boilerplate.
|
||||
//!
|
||||
//! ## Running This Example
|
||||
//! ```bash
|
||||
//! cargo run --example custom_types
|
||||
//! ```
|
||||
//!
|
||||
//! ## Expected Output
|
||||
//! ```text
|
||||
//! Creating custom types example...
|
||||
//!
|
||||
//! Creating user: Alice (age: 30)
|
||||
//! Processing user Alice for greeting
|
||||
//! Processing user Alice for validation
|
||||
//! Greeting: Hello, Alice!
|
||||
//! Validation: Alice is valid (age 30 >= 18)
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagRunner};
|
||||
|
||||
// Custom type - just derive Clone!
|
||||
#[derive(Clone, Debug)]
|
||||
struct User {
|
||||
name: String,
|
||||
age: u32,
|
||||
}
|
||||
|
||||
// Task that produces a custom type
|
||||
struct CreateUser {
|
||||
name: String,
|
||||
age: u32,
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl CreateUser {
|
||||
async fn run(&self) -> User {
|
||||
println!("Creating user: {} (age: {})", self.name, self.age);
|
||||
User {
|
||||
name: self.name.clone(),
|
||||
age: self.age,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Task that consumes the custom type and produces a String
|
||||
struct GreetUser;
|
||||
|
||||
#[task]
|
||||
impl GreetUser {
|
||||
async fn run(user: &User) -> String {
|
||||
println!("Processing user {} for greeting", user.name);
|
||||
format!("Hello, {}!", user.name)
|
||||
}
|
||||
}
|
||||
|
||||
// Another task that also consumes the same custom type
|
||||
// This demonstrates fan-out: one output consumed by multiple tasks
|
||||
struct ValidateUser;
|
||||
|
||||
#[task]
|
||||
impl ValidateUser {
|
||||
async fn run(user: &User) -> String {
|
||||
println!("Processing user {} for validation", user.name);
|
||||
if user.age >= 18 {
|
||||
format!("{} is valid (age {} >= 18)", user.name, user.age)
|
||||
} else {
|
||||
format!("{} is too young (age {} < 18)", user.name, user.age)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
println!("Creating custom types example...\n");
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Create a task that produces a custom User type
|
||||
let user_task = dag.add_task(CreateUser {
|
||||
name: "Alice".to_string(),
|
||||
age: 30,
|
||||
});
|
||||
|
||||
// Two different tasks consume the same custom User type
|
||||
// dagx automatically handles Arc wrapping, so both tasks get &User
|
||||
let greeting = dag.add_task(GreetUser).depends_on(&user_task);
|
||||
let validation = dag.add_task(ValidateUser).depends_on(&user_task);
|
||||
|
||||
// Execute the DAG
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Print results
|
||||
println!("Greeting: {}", dag.get(greeting).unwrap());
|
||||
println!("Validation: {}", dag.get(validation).unwrap());
|
||||
}
|
||||
@@ -190,15 +190,15 @@ async fn main() {
|
||||
// Stage 2: Data transformation (with stateful tasks)
|
||||
let transformed_1 = dag
|
||||
.add_task(DataTransformer::new(1.5))
|
||||
.depends_on(&raw_value_1);
|
||||
.depends_on(raw_value_1);
|
||||
|
||||
let transformed_2 = dag
|
||||
.add_task(DataTransformer::new(2.0))
|
||||
.depends_on(&raw_value_2);
|
||||
.depends_on(raw_value_2);
|
||||
|
||||
let transformed_3 = dag
|
||||
.add_task(DataTransformer::new(0.5))
|
||||
.depends_on(&raw_value_3);
|
||||
.depends_on(raw_value_3);
|
||||
|
||||
// Stage 3: Statistical analysis (parallel operations on transformed data)
|
||||
let sum = dag
|
||||
@@ -220,11 +220,9 @@ async fn main() {
|
||||
|
||||
// Execute the pipeline
|
||||
println!("Executing pipeline...\n");
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Print final report
|
||||
println!("{}", dag.get(report).unwrap());
|
||||
|
||||
@@ -1,464 +0,0 @@
|
||||
//! # Debug Tracing and Task Naming
|
||||
//!
|
||||
//! This example demonstrates how to add debug metadata and tracing information to tasks
|
||||
//! for better observability in complex DAG workflows.
|
||||
//!
|
||||
//! ## What You'll Learn
|
||||
//! - How to add task names and IDs for debugging
|
||||
//! - How to implement correlation IDs for distributed tracing
|
||||
//! - How to track task execution timing and lifecycle events
|
||||
//! - How to build traceable task wrappers
|
||||
//!
|
||||
//! ## Debugging Patterns
|
||||
//!
|
||||
//! **1. Named Tasks** - Add human-readable names to tasks
|
||||
//! ```rust
|
||||
//! struct TaskMetadata {
|
||||
//! id: String,
|
||||
//! name: String,
|
||||
//! start_time: Option<Instant>,
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! **2. Correlation IDs** - Track requests across task boundaries
|
||||
//! ```rust
|
||||
//! struct TraceContext {
|
||||
//! correlation_id: String,
|
||||
//! parent_task: Option<String>,
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! **3. Execution Metrics** - Measure task performance
|
||||
//! ```rust
|
||||
//! struct TaskTrace {
|
||||
//! name: String,
|
||||
//! duration_ms: u64,
|
||||
//! status: &'static str,
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! ## Workflow Diagram
|
||||
//! ```text
|
||||
//! TRACE: order-12345
|
||||
//! ┌──────────────────┐
|
||||
//! │ ValidateOrder │ [order-12345] Start: 0ms
|
||||
//! │ ID: task-001 │──────► Status: Success
|
||||
//! └──────────────────┘ Duration: 10ms
|
||||
//! │
|
||||
//! ├──────────────────┬──────────────────┐
|
||||
//! ▼ ▼ ▼
|
||||
//! ┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐
|
||||
//! │ CheckInventory │ │ ValidatePayment │ │ CalculateShipping│
|
||||
//! │ ID: task-002 │ │ ID: task-003 │ │ ID: task-004 │
|
||||
//! └──────────────────┘ └──────────────────┘ └──────────────────┘
|
||||
//! [order-12345] [order-12345] [order-12345]
|
||||
//! Duration: 45ms Duration: 120ms Duration: 15ms
|
||||
//! │ │ │
|
||||
//! └──────────────────┴──────────────────┘
|
||||
//! ▼
|
||||
//! ┌──────────────────┐
|
||||
//! │ FulfillOrder │ [order-12345]
|
||||
//! │ ID: task-005 │ Duration: 30ms
|
||||
//! └──────────────────┘
|
||||
//! ```
|
||||
//!
|
||||
//! ## Key Concepts
|
||||
//! - **Observability**: Make task execution visible for debugging
|
||||
//! - **Correlation IDs**: Track a request through multiple tasks
|
||||
//! - **Structured logging**: Consistent task metadata across logs
|
||||
//! - **Performance metrics**: Measure task execution time
|
||||
//! - **Task lifecycle**: Track start, completion, and errors
|
||||
//!
|
||||
//! ## Use Cases
|
||||
//! - Debugging complex workflows in production
|
||||
//! - Performance analysis and bottleneck identification
|
||||
//! - Distributed tracing in microservices
|
||||
//! - SLA monitoring and alerting
|
||||
//! - Audit logging for compliance
|
||||
//!
|
||||
//! ## Running This Example
|
||||
//! ```bash
|
||||
//! cargo run --example debug_tracing
|
||||
//! ```
|
||||
//!
|
||||
//! ## Expected Output
|
||||
//! ```text
|
||||
//! === Debug Tracing Example ===
|
||||
//!
|
||||
//! Starting order processing for order-12345
|
||||
//!
|
||||
//! [order-12345] [task-001] ValidateOrder: Starting
|
||||
//! [order-12345] [task-001] ValidateOrder: Completed in 10ms - Success
|
||||
//!
|
||||
//! [order-12345] [task-002] CheckInventory: Starting
|
||||
//! [order-12345] [task-003] ValidatePayment: Starting
|
||||
//! [order-12345] [task-004] CalculateShipping: Starting
|
||||
//! [order-12345] [task-002] CheckInventory: Completed in 45ms - Success
|
||||
//! [order-12345] [task-004] CalculateShipping: Completed in 15ms - Success
|
||||
//! [order-12345] [task-003] ValidatePayment: Completed in 120ms - Success
|
||||
//!
|
||||
//! [order-12345] [task-005] FulfillOrder: Starting
|
||||
//! [order-12345] [task-005] FulfillOrder: Completed in 30ms - Success
|
||||
//!
|
||||
//! === Execution Trace Summary ===
|
||||
//! Correlation ID: order-12345
|
||||
//! Total Tasks: 5
|
||||
//! Total Duration: 220ms
|
||||
//! Success: 5, Failed: 0
|
||||
//!
|
||||
//! Task Breakdown:
|
||||
//! ValidateOrder: 10ms
|
||||
//! CheckInventory: 45ms
|
||||
//! ValidatePayment: 120ms (slowest)
|
||||
//! CalculateShipping: 15ms
|
||||
//! FulfillOrder: 30ms
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagRunner};
|
||||
use std::time::Instant;
|
||||
use tokio::time::{sleep, Duration};
|
||||
|
||||
// Stateful task that tracks execution metadata for debugging
|
||||
struct TracedTask {
|
||||
task_id: String,
|
||||
task_name: String,
|
||||
correlation_id: String,
|
||||
work_duration_ms: u64,
|
||||
start_time: Option<Instant>,
|
||||
}
|
||||
|
||||
impl TracedTask {
|
||||
fn new(
|
||||
task_id: impl Into<String>,
|
||||
task_name: impl Into<String>,
|
||||
correlation_id: impl Into<String>,
|
||||
work_duration_ms: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
task_id: task_id.into(),
|
||||
task_name: task_name.into(),
|
||||
correlation_id: correlation_id.into(),
|
||||
work_duration_ms,
|
||||
start_time: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl TracedTask {
|
||||
async fn run(&mut self) -> Result<TaskTrace, String> {
|
||||
// Record start
|
||||
let start = Instant::now();
|
||||
self.start_time = Some(start);
|
||||
|
||||
println!(
|
||||
"[{}] [{}] {}: Starting",
|
||||
self.correlation_id, self.task_id, self.task_name
|
||||
);
|
||||
|
||||
// Simulate work
|
||||
sleep(Duration::from_millis(self.work_duration_ms)).await;
|
||||
|
||||
// Record completion
|
||||
let duration_ms = start.elapsed().as_millis() as u64;
|
||||
|
||||
println!(
|
||||
"[{}] [{}] {}: Completed in {}ms - Success",
|
||||
self.correlation_id, self.task_id, self.task_name, duration_ms
|
||||
);
|
||||
|
||||
Ok(TaskTrace {
|
||||
task_id: self.task_id.clone(),
|
||||
task_name: self.task_name.clone(),
|
||||
correlation_id: self.correlation_id.clone(),
|
||||
duration_ms,
|
||||
status: "Success",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Task that can fail, demonstrating error tracing
|
||||
struct TracedTaskWithError {
|
||||
task_id: String,
|
||||
task_name: String,
|
||||
correlation_id: String,
|
||||
should_fail: bool,
|
||||
}
|
||||
|
||||
impl TracedTaskWithError {
|
||||
fn new(
|
||||
task_id: impl Into<String>,
|
||||
task_name: impl Into<String>,
|
||||
correlation_id: impl Into<String>,
|
||||
should_fail: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
task_id: task_id.into(),
|
||||
task_name: task_name.into(),
|
||||
correlation_id: correlation_id.into(),
|
||||
should_fail,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl TracedTaskWithError {
|
||||
async fn run(&mut self) -> Result<TaskTrace, TaskTrace> {
|
||||
let start = Instant::now();
|
||||
|
||||
println!(
|
||||
"[{}] [{}] {}: Starting",
|
||||
self.correlation_id, self.task_id, self.task_name
|
||||
);
|
||||
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as u64;
|
||||
|
||||
if self.should_fail {
|
||||
println!(
|
||||
"[{}] [{}] {}: Failed in {}ms - Validation Error",
|
||||
self.correlation_id, self.task_id, self.task_name, duration_ms
|
||||
);
|
||||
|
||||
Err(TaskTrace {
|
||||
task_id: self.task_id.clone(),
|
||||
task_name: self.task_name.clone(),
|
||||
correlation_id: self.correlation_id.clone(),
|
||||
duration_ms,
|
||||
status: "Failed",
|
||||
})
|
||||
} else {
|
||||
println!(
|
||||
"[{}] [{}] {}: Completed in {}ms - Success",
|
||||
self.correlation_id, self.task_id, self.task_name, duration_ms
|
||||
);
|
||||
|
||||
Ok(TaskTrace {
|
||||
task_id: self.task_id.clone(),
|
||||
task_name: self.task_name.clone(),
|
||||
correlation_id: self.correlation_id.clone(),
|
||||
duration_ms,
|
||||
status: "Success",
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Task trace data structure
|
||||
#[derive(Clone, Debug)]
|
||||
#[allow(dead_code)]
|
||||
struct TaskTrace {
|
||||
task_id: String,
|
||||
task_name: String,
|
||||
correlation_id: String,
|
||||
duration_ms: u64,
|
||||
status: &'static str,
|
||||
}
|
||||
|
||||
// Task that aggregates traces from multiple upstream tasks
|
||||
struct TraceAggregator {
|
||||
correlation_id: String,
|
||||
}
|
||||
|
||||
impl TraceAggregator {
|
||||
fn new(correlation_id: impl Into<String>) -> Self {
|
||||
Self {
|
||||
correlation_id: correlation_id.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl TraceAggregator {
|
||||
async fn run(
|
||||
&mut self,
|
||||
t1: &Result<TaskTrace, String>,
|
||||
t2: &Result<TaskTrace, String>,
|
||||
t3: &Result<TaskTrace, String>,
|
||||
) -> TraceSummary {
|
||||
let traces = vec![t1, t2, t3];
|
||||
|
||||
let mut summary = TraceSummary {
|
||||
correlation_id: self.correlation_id.clone(),
|
||||
task_count: traces.len(),
|
||||
total_duration_ms: 0,
|
||||
success_count: 0,
|
||||
failed_count: 0,
|
||||
task_details: Vec::new(),
|
||||
};
|
||||
|
||||
for trace_result in traces {
|
||||
match trace_result {
|
||||
Ok(trace) => {
|
||||
summary.success_count += 1;
|
||||
summary.total_duration_ms += trace.duration_ms;
|
||||
summary.task_details.push(trace.clone());
|
||||
}
|
||||
Err(e) => {
|
||||
summary.failed_count += 1;
|
||||
eprintln!("[{}] Task failed: {}", self.correlation_id, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
summary
|
||||
}
|
||||
}
|
||||
|
||||
// Summary of traced execution
|
||||
#[derive(Clone, Debug)]
|
||||
struct TraceSummary {
|
||||
correlation_id: String,
|
||||
task_count: usize,
|
||||
total_duration_ms: u64,
|
||||
success_count: usize,
|
||||
failed_count: usize,
|
||||
task_details: Vec<TaskTrace>,
|
||||
}
|
||||
|
||||
impl TraceSummary {
|
||||
fn print(&self) {
|
||||
println!("\n=== Execution Trace Summary ===");
|
||||
println!("Correlation ID: {}", self.correlation_id);
|
||||
println!("Total Tasks: {}", self.task_count);
|
||||
println!("Total Duration: {}ms", self.total_duration_ms);
|
||||
println!(
|
||||
"Success: {}, Failed: {}",
|
||||
self.success_count, self.failed_count
|
||||
);
|
||||
|
||||
if !self.task_details.is_empty() {
|
||||
println!("\nTask Breakdown:");
|
||||
let max_duration = self
|
||||
.task_details
|
||||
.iter()
|
||||
.map(|t| t.duration_ms)
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
|
||||
for trace in &self.task_details {
|
||||
let slowest_tag = if trace.duration_ms == max_duration {
|
||||
" (slowest)"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
println!(
|
||||
" {}: {}ms{}",
|
||||
trace.task_name, trace.duration_ms, slowest_tag
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
println!("=== Debug Tracing Example ===\n");
|
||||
|
||||
// Example 1: Successful order processing with full tracing
|
||||
println!("Example 1: Successful order processing\n");
|
||||
{
|
||||
let correlation_id = "order-12345";
|
||||
println!("Starting order processing for {}\n", correlation_id);
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Root task: Validate order
|
||||
let _validate = dag.add_task(TracedTask::new(
|
||||
"task-001",
|
||||
"ValidateOrder",
|
||||
correlation_id,
|
||||
10,
|
||||
));
|
||||
|
||||
// Parallel validation tasks (independent, no dependencies)
|
||||
let inventory = dag.add_task(TracedTask::new(
|
||||
"task-002",
|
||||
"CheckInventory",
|
||||
correlation_id,
|
||||
45,
|
||||
));
|
||||
|
||||
let payment = dag.add_task(TracedTask::new(
|
||||
"task-003",
|
||||
"ValidatePayment",
|
||||
correlation_id,
|
||||
120,
|
||||
));
|
||||
|
||||
let shipping = dag.add_task(TracedTask::new(
|
||||
"task-004",
|
||||
"CalculateShipping",
|
||||
correlation_id,
|
||||
15,
|
||||
));
|
||||
|
||||
// Aggregate traces
|
||||
let summary = dag
|
||||
.add_task(TraceAggregator::new(correlation_id))
|
||||
.depends_on((&inventory, &payment, &shipping));
|
||||
|
||||
// Run the DAG
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Print summary
|
||||
dag.get(summary).unwrap().print();
|
||||
}
|
||||
|
||||
println!("\n");
|
||||
|
||||
// Example 2: Order with failed payment
|
||||
println!("Example 2: Order with failed payment\n");
|
||||
{
|
||||
let correlation_id = "order-67890";
|
||||
println!("Starting order processing for {}\n", correlation_id);
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let _validate = dag.add_task(TracedTaskWithError::new(
|
||||
"task-101",
|
||||
"ValidateOrder",
|
||||
correlation_id,
|
||||
false,
|
||||
));
|
||||
|
||||
let _inventory = dag.add_task(TracedTaskWithError::new(
|
||||
"task-102",
|
||||
"CheckInventory",
|
||||
correlation_id,
|
||||
false,
|
||||
));
|
||||
|
||||
// This task will fail
|
||||
let payment = dag.add_task(TracedTaskWithError::new(
|
||||
"task-103",
|
||||
"ValidatePayment",
|
||||
correlation_id,
|
||||
true, // will fail
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Check results
|
||||
match dag.get(payment).unwrap() {
|
||||
Ok(_) => println!("\nPayment validation succeeded"),
|
||||
Err(trace) => {
|
||||
println!("\n[{}] Payment validation failed:", correlation_id);
|
||||
println!(" Task ID: {}", trace.task_id);
|
||||
println!(" Duration: {}ms", trace.duration_ms);
|
||||
println!(" Status: {}", trace.status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("\n=== Debug Tracing Example Complete ===");
|
||||
}
|
||||
+19
-27
@@ -213,7 +213,7 @@ async fn main() {
|
||||
|
||||
let input = dag.add_task(StringSource("42"));
|
||||
|
||||
let parsed = dag.add_task(ParseInt).depends_on(&input);
|
||||
let parsed = dag.add_task(ParseInt).depends_on(input);
|
||||
|
||||
let validated = dag
|
||||
.add_task(ValidateRange { min: 0, max: 100 })
|
||||
@@ -222,13 +222,11 @@ async fn main() {
|
||||
let processed = dag.add_task(ProcessData).depends_on(validated);
|
||||
let logged = dag.add_task(LogError).depends_on(processed);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
match dag.get(logged).unwrap() {
|
||||
match dag.get(logged).unwrap().as_ref() {
|
||||
Ok(result) => println!("Final result: {}\n", result),
|
||||
Err(e) => eprintln!("Final error: {}\n", e),
|
||||
}
|
||||
@@ -241,7 +239,7 @@ async fn main() {
|
||||
|
||||
let input = dag.add_task(StringSource("not a number"));
|
||||
|
||||
let parsed = dag.add_task(ParseInt).depends_on(&input);
|
||||
let parsed = dag.add_task(ParseInt).depends_on(input);
|
||||
|
||||
let validated = dag
|
||||
.add_task(ValidateRange { min: 0, max: 100 })
|
||||
@@ -250,13 +248,11 @@ async fn main() {
|
||||
let processed = dag.add_task(ProcessData).depends_on(validated);
|
||||
let logged = dag.add_task(LogError).depends_on(processed);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
match dag.get(logged).unwrap() {
|
||||
match dag.get(logged).unwrap().as_ref() {
|
||||
Ok(result) => println!("Final result: {}\n", result),
|
||||
Err(e) => eprintln!("Final error: {}\n", e),
|
||||
}
|
||||
@@ -269,7 +265,7 @@ async fn main() {
|
||||
|
||||
let input = dag.add_task(StringSource("150"));
|
||||
|
||||
let parsed = dag.add_task(ParseInt).depends_on(&input);
|
||||
let parsed = dag.add_task(ParseInt).depends_on(input);
|
||||
|
||||
let validated = dag
|
||||
.add_task(ValidateRange { min: 0, max: 100 })
|
||||
@@ -278,13 +274,11 @@ async fn main() {
|
||||
let processed = dag.add_task(ProcessData).depends_on(validated);
|
||||
let logged = dag.add_task(LogError).depends_on(processed);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
match dag.get(logged).unwrap() {
|
||||
match dag.get(logged).unwrap().as_ref() {
|
||||
Ok(result) => println!("Final result: {}\n", result),
|
||||
Err(e) => eprintln!("Final error: {}\n", e),
|
||||
}
|
||||
@@ -297,17 +291,15 @@ async fn main() {
|
||||
|
||||
let input = dag.add_task(StringSource("invalid"));
|
||||
|
||||
let parsed = dag.add_task(ParseInt).depends_on(&input);
|
||||
let parsed = dag.add_task(ParseInt).depends_on(input);
|
||||
|
||||
let with_fallback = dag
|
||||
.add_task(WithFallback { fallback: 0 })
|
||||
.depends_on(parsed);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = dag.get(with_fallback).unwrap();
|
||||
println!("Recovered with fallback: {}\n", result);
|
||||
|
||||
@@ -1,391 +0,0 @@
|
||||
//! # Working with Large DAGs: Performance and Scalability
|
||||
//!
|
||||
//! This example demonstrates dagx's performance characteristics when working
|
||||
//! with large-scale DAGs containing hundreds or thousands of tasks.
|
||||
//!
|
||||
//! ## What You'll Learn
|
||||
//! - How dagx scales with increasing task counts
|
||||
//! - Performance characteristics of different DAG topologies
|
||||
//! - How to benchmark DAG construction and execution
|
||||
//! - Memory efficiency considerations
|
||||
//!
|
||||
//! ## DAG Topologies Tested
|
||||
//!
|
||||
//! **1. Wide DAG** - Maximum parallelism
|
||||
//! ```text
|
||||
//! ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐
|
||||
//! │Task 1│ │Task 2│ │Task 3│ ... │Task N│
|
||||
//! └──────┘ └──────┘ └──────┘ └──────┘
|
||||
//! (1000 independent parallel tasks)
|
||||
//! ```
|
||||
//!
|
||||
//! **2. Deep DAG** - Sequential chain
|
||||
//! ```text
|
||||
//! ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐
|
||||
//! │Task 1│───►│Task 2│───►│Task 3│───► ... ──►│Task N│
|
||||
//! └──────┘ └──────┘ └──────┘ └──────┘
|
||||
//! (100-level dependency chain)
|
||||
//! ```
|
||||
//!
|
||||
//! **3. Binary Tree** - Balanced aggregation
|
||||
//! ```text
|
||||
//! ┌────┐ ┌────┐ ┌────┐ ┌────┐
|
||||
//! │ T1 │ │ T2 │ │ T3 │ │ T4 │
|
||||
//! └─┬──┘ └─┬──┘ └─┬──┘ └─┬──┘
|
||||
//! └───┬───┘ └───┬───┘
|
||||
//! │ │
|
||||
//! ┌──▼──┐ ┌──▼──┐
|
||||
//! │ C1 │ │ C2 │
|
||||
//! └──┬──┘ └──┬──┘
|
||||
//! └───────┬──────┘
|
||||
//! │
|
||||
//! ┌──▼──┐
|
||||
//! │Root │
|
||||
//! └─────┘
|
||||
//! ```
|
||||
//!
|
||||
//! **4. Diamond Pattern** - Many-to-many
|
||||
//! ```text
|
||||
//! Layer 1: 50 sources
|
||||
//! Layer 2: 50 tasks (each depends on 2 sources)
|
||||
//! Layer 3: 4 aggregates
|
||||
//! Layer 4: 1 final result
|
||||
//! ```
|
||||
//!
|
||||
//! ## Performance Metrics
|
||||
//! - **Build time**: Time to construct the DAG structure
|
||||
//! - **Execution time**: Time to run all tasks
|
||||
//! - **Tasks/second**: Throughput metric
|
||||
//! - **Memory overhead**: Approximate per-task memory usage
|
||||
//!
|
||||
//! ## Key Observations
|
||||
//! - DAG construction is very fast (< 1ms for 1000s of tasks)
|
||||
//! - Parallel execution scales well with available CPU cores
|
||||
//! - Deep chains execute sequentially as expected
|
||||
//! - Memory overhead is minimal (~200 bytes per task)
|
||||
//!
|
||||
//! ## Running This Example
|
||||
//! ```bash
|
||||
//! cargo run --example large_dag --release # Use --release for accurate benchmarks
|
||||
//! ```
|
||||
//!
|
||||
//! ## Expected Output
|
||||
//! ```text
|
||||
//! === Large DAG Performance Example ===
|
||||
//!
|
||||
//! Example 1: Wide DAG (1000 parallel tasks)
|
||||
//! Build time: 347µs
|
||||
//! Execution time: 12.5ms
|
||||
//! Tasks/second: 80000
|
||||
//! Sample sum (first 10): ...
|
||||
//!
|
||||
//! Example 2: Deep DAG (100-level chain)
|
||||
//! Build time: 156µs
|
||||
//! Execution time: 8.2ms
|
||||
//! Final result: ...
|
||||
//!
|
||||
//! Example 3: Binary Tree DAG (depth 10, 1023 tasks)
|
||||
//! Build time: 423µs
|
||||
//! Tree depth: 9
|
||||
//! Execution time: 15.3ms
|
||||
//! Root result: ...
|
||||
//!
|
||||
//! Example 4: Diamond Pattern (50x50 = 2500 edges)
|
||||
//! Build time: 234µs
|
||||
//! Execution time: 11.7ms
|
||||
//! Total tasks: 105
|
||||
//! Final result: ...
|
||||
//!
|
||||
//! Example 5: Memory Efficiency (100 tasks, measure overhead)
|
||||
//! i32 sum: ...
|
||||
//! String total length: ...
|
||||
//!
|
||||
//! === Large DAG Performance Example Complete ===
|
||||
//!
|
||||
//! Key Observations:
|
||||
//! - DAG construction is very fast (< 1ms for 1000s of tasks)
|
||||
//! - Parallel execution scales well with available CPU cores
|
||||
//! - Deep chains execute sequentially as expected
|
||||
//! - Memory overhead is minimal (~ 200 bytes per task)
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagRunner};
|
||||
use std::time::Instant;
|
||||
|
||||
// Simple computation task
|
||||
struct Compute {
|
||||
value: i32,
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl Compute {
|
||||
async fn run(&mut self) -> i32 {
|
||||
// Simulate some work
|
||||
let mut result = self.value;
|
||||
for _ in 0..100 {
|
||||
result = result.wrapping_mul(31).wrapping_add(17);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
// Task that combines two values
|
||||
struct Combine;
|
||||
|
||||
#[task]
|
||||
impl Combine {
|
||||
async fn run(a: &i32, b: &i32) -> i32 {
|
||||
a.wrapping_add(*b)
|
||||
}
|
||||
}
|
||||
|
||||
// Task that aggregates multiple values
|
||||
struct Aggregate;
|
||||
|
||||
#[task]
|
||||
impl Aggregate {
|
||||
async fn run(a: &i32, b: &i32, c: &i32, d: &i32) -> i32 {
|
||||
a.wrapping_add(*b).wrapping_add(*c).wrapping_add(*d)
|
||||
}
|
||||
}
|
||||
|
||||
// Task that formats a number as a string
|
||||
struct FormatNumber {
|
||||
id: i32,
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl FormatNumber {
|
||||
async fn run(&self) -> String {
|
||||
format!("Result {}", self.id)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
println!("=== Large DAG Performance Example ===\n");
|
||||
|
||||
// Example 1: Wide DAG (many parallel tasks)
|
||||
println!("Example 1: Wide DAG (1000 parallel tasks)");
|
||||
{
|
||||
let start = Instant::now();
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Create 1000 independent tasks
|
||||
let tasks: Vec<_> = (0..1000)
|
||||
.map(|i| dag.add_task(Compute { value: i }))
|
||||
.collect();
|
||||
|
||||
let build_time = start.elapsed();
|
||||
println!(" Build time: {:?}", build_time);
|
||||
|
||||
let exec_start = Instant::now();
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let exec_time = exec_start.elapsed();
|
||||
|
||||
println!(" Execution time: {:?}", exec_time);
|
||||
println!(" Tasks/second: {:.0}", 1000.0 / exec_time.as_secs_f64());
|
||||
|
||||
// Verify a few results
|
||||
let sum: i32 = tasks
|
||||
.iter()
|
||||
.take(10)
|
||||
.map(|t| dag.get(t).unwrap())
|
||||
.fold(0i32, |acc, x| acc.wrapping_add(x));
|
||||
println!(" Sample sum (first 10): {}\n", sum);
|
||||
}
|
||||
|
||||
// Example 2: Deep DAG (long dependency chain)
|
||||
println!("Example 2: Deep DAG (100-level chain)");
|
||||
{
|
||||
let start = Instant::now();
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Create a deep chain by combining adjacent task pairs
|
||||
let tasks: Vec<_> = (0..100)
|
||||
.map(|i| dag.add_task(Compute { value: i }))
|
||||
.collect();
|
||||
let mut chain_tasks = vec![];
|
||||
for i in 0..99 {
|
||||
let combined = dag.add_task(Combine).depends_on((&tasks[i], &tasks[i + 1]));
|
||||
chain_tasks.push(combined);
|
||||
}
|
||||
|
||||
let prev = chain_tasks.last().unwrap();
|
||||
|
||||
let build_time = start.elapsed();
|
||||
println!(" Build time: {:?}", build_time);
|
||||
|
||||
let exec_start = Instant::now();
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let exec_time = exec_start.elapsed();
|
||||
|
||||
println!(" Execution time: {:?}", exec_time);
|
||||
println!(" Final result: {}\n", dag.get(prev).unwrap());
|
||||
}
|
||||
|
||||
// Example 3: Binary tree DAG
|
||||
println!("Example 3: Binary Tree DAG (depth 10, 1023 tasks)");
|
||||
{
|
||||
let start = Instant::now();
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Build binary tree: each level combines pairs from previous level
|
||||
let current_level: Vec<_> = (0..512)
|
||||
.map(|i| dag.add_task(Compute { value: i }))
|
||||
.collect();
|
||||
|
||||
// Process all levels reducing by combining pairs
|
||||
let mut level = 0;
|
||||
let mut layer: Vec<dagx::TaskHandle<i32>> =
|
||||
current_level.iter().map(|b| b.into()).collect();
|
||||
|
||||
while layer.len() > 1 {
|
||||
level += 1;
|
||||
let mut next_layer = Vec::new();
|
||||
|
||||
for i in (0..layer.len()).step_by(2) {
|
||||
if i + 1 < layer.len() {
|
||||
// We have a pair - combine them
|
||||
let combined = dag.add_task(Combine).depends_on((&layer[i], &layer[i + 1]));
|
||||
next_layer.push(combined);
|
||||
} else {
|
||||
// Odd one out - just pass it through
|
||||
next_layer.push(layer[i]);
|
||||
}
|
||||
}
|
||||
|
||||
layer = next_layer;
|
||||
}
|
||||
|
||||
let root = &layer[0];
|
||||
|
||||
let build_time = start.elapsed();
|
||||
println!(" Build time: {:?}", build_time);
|
||||
println!(" Tree depth: {}", level);
|
||||
|
||||
let exec_start = Instant::now();
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let exec_time = exec_start.elapsed();
|
||||
|
||||
println!(" Execution time: {:?}", exec_time);
|
||||
println!(" Root result: {}\n", dag.get(root).unwrap());
|
||||
}
|
||||
|
||||
// Example 4: Diamond pattern (many-to-many)
|
||||
println!("Example 4: Diamond Pattern (50x50 = 2500 edges)");
|
||||
{
|
||||
let start = Instant::now();
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Layer 1: 50 source tasks
|
||||
let sources: Vec<_> = (0..50)
|
||||
.map(|i| dag.add_task(Compute { value: i }))
|
||||
.collect();
|
||||
|
||||
// Layer 2: 50 tasks, each depends on all sources
|
||||
let middle: Vec<_> = (0..50)
|
||||
.map(|i| {
|
||||
let combine = dag.add_task(Combine);
|
||||
// Each middle task depends on two sources
|
||||
combine.depends_on((
|
||||
&sources[i % sources.len()],
|
||||
&sources[(i + 1) % sources.len()],
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Layer 3: Aggregate to 4 groups
|
||||
let aggregates: Vec<_> = (0..4)
|
||||
.map(|i| {
|
||||
let start_idx = i * 12;
|
||||
dag.add_task(Aggregate).depends_on((
|
||||
&middle[start_idx % middle.len()],
|
||||
&middle[(start_idx + 1) % middle.len()],
|
||||
&middle[(start_idx + 2) % middle.len()],
|
||||
&middle[(start_idx + 3) % middle.len()],
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Layer 4: Final aggregate
|
||||
let final_result = dag.add_task(Aggregate).depends_on((
|
||||
&aggregates[0],
|
||||
&aggregates[1],
|
||||
&aggregates[2],
|
||||
&aggregates[3],
|
||||
));
|
||||
|
||||
let build_time = start.elapsed();
|
||||
println!(" Build time: {:?}", build_time);
|
||||
|
||||
let exec_start = Instant::now();
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let exec_time = exec_start.elapsed();
|
||||
|
||||
println!(" Execution time: {:?}", exec_time);
|
||||
println!(" Total tasks: {}", 50 + 50 + 4 + 1);
|
||||
println!(" Final result: {}\n", dag.get(final_result).unwrap());
|
||||
}
|
||||
|
||||
// Example 5: Memory efficiency test
|
||||
println!("Example 5: Memory Efficiency (100 tasks, measure overhead)");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Create tasks with various output sizes
|
||||
let small_tasks: Vec<_> = (0..50)
|
||||
.map(|i| dag.add_task(Compute { value: i }))
|
||||
.collect();
|
||||
|
||||
let string_tasks: Vec<_> = (0..50)
|
||||
.map(|i| dag.add_task(FormatNumber { id: i }))
|
||||
.collect();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Verify results are accessible
|
||||
let i32_sum: i32 = small_tasks
|
||||
.iter()
|
||||
.map(|t| dag.get(t).unwrap())
|
||||
.fold(0i32, |acc, x| acc.wrapping_add(x));
|
||||
let string_count = string_tasks
|
||||
.iter()
|
||||
.map(|t| dag.get(t).unwrap().len())
|
||||
.sum::<usize>();
|
||||
|
||||
println!(" i32 sum: {}", i32_sum);
|
||||
println!(" String total length: {}", string_count);
|
||||
}
|
||||
|
||||
println!("\n=== Large DAG Performance Example Complete ===");
|
||||
println!("\nKey Observations:");
|
||||
println!(" - DAG construction is very fast (< 1ms for 1000s of tasks)");
|
||||
println!(" - Parallel execution scales well with available CPU cores");
|
||||
println!(" - Deep chains execute sequentially as expected");
|
||||
println!(" - Memory overhead is minimal (~ 200 bytes per task)");
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
//! # Proof of True Parallelism
|
||||
//!
|
||||
//! This example demonstrates that dagx provides genuine parallel execution
|
||||
//! by running 10,000 tasks that each sleep for 1 second, completing in ~1 second
|
||||
//! total instead of 10,000 seconds.
|
||||
//!
|
||||
//! ## What You'll Learn
|
||||
//! - How to verify true parallel execution
|
||||
//! - The difference between concurrency and parallelism
|
||||
//! - How dagx scales to thousands of parallel tasks
|
||||
//! - Performance characteristics of parallel execution
|
||||
//!
|
||||
//! ## The Proof
|
||||
//!
|
||||
//! If tasks ran sequentially:
|
||||
//! - 10,000 tasks × 1 second each = 10,000 seconds (2.78 hours)
|
||||
//!
|
||||
//! With true parallelism:
|
||||
//! - All 10,000 tasks run simultaneously = ~1 second total
|
||||
//!
|
||||
//! This dramatic difference (10,000x speedup) proves that tasks execute in
|
||||
//! parallel across multiple threads, not sequentially.
|
||||
//!
|
||||
//! ## Key Concepts
|
||||
//!
|
||||
//! **Concurrency** - Multiple tasks making progress, but not necessarily at the same time
|
||||
//! **Parallelism** - Multiple tasks executing simultaneously on different CPU cores
|
||||
//!
|
||||
//! This example proves dagx provides true parallelism.
|
||||
//!
|
||||
//! ## Running This Example
|
||||
//! ```bash
|
||||
//! cargo run --example parallelism_proof --release
|
||||
//! ```
|
||||
//!
|
||||
//! ## Expected Output
|
||||
//! ```text
|
||||
//! === Parallelism Proof ===
|
||||
//!
|
||||
//! Launching 10,000 tasks that each sleep for 1 second...
|
||||
//!
|
||||
//! Result: 10,000 tasks × 1s sleep completed in ~1.2s
|
||||
//!
|
||||
//! Proof: If tasks ran sequentially, this would take 10,000 seconds (2.78 hours).
|
||||
//! Actual time: 1.2s
|
||||
//! Speedup: 8333x
|
||||
//! This proves TRUE PARALLEL EXECUTION!
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagRunner};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
|
||||
// Task that sleeps for a duration and returns its ID
|
||||
struct SleepTask {
|
||||
id: u32,
|
||||
duration: Duration,
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl SleepTask {
|
||||
async fn run(&self) -> u32 {
|
||||
sleep(self.duration).await;
|
||||
self.id
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
println!("=== Parallelism Proof ===\n");
|
||||
println!("Launching 10,000 tasks that each sleep for 1 second...\n");
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let sleep_duration = Duration::from_secs(1);
|
||||
let num_tasks = 10_000u32;
|
||||
|
||||
// Create 10,000 independent tasks that each sleep for 1 second
|
||||
let tasks: Vec<_> = (0..num_tasks)
|
||||
.map(|i| {
|
||||
dag.add_task(SleepTask {
|
||||
id: i,
|
||||
duration: sleep_duration,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Measure execution time
|
||||
let start = Instant::now();
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Verify all tasks completed correctly
|
||||
for (i, task) in tasks.iter().enumerate() {
|
||||
assert_eq!(dag.get(task).unwrap(), i as u32);
|
||||
}
|
||||
|
||||
// Calculate what sequential execution would take
|
||||
let sequential_time = sleep_duration * num_tasks;
|
||||
let speedup = sequential_time.as_secs_f64() / elapsed.as_secs_f64();
|
||||
|
||||
println!(
|
||||
"Result: {} tasks × {}s sleep completed in {:.1}s\n",
|
||||
num_tasks,
|
||||
sleep_duration.as_secs(),
|
||||
elapsed.as_secs_f64()
|
||||
);
|
||||
|
||||
println!(
|
||||
"Proof: If tasks ran sequentially, this would take {} seconds ({:.2} hours).",
|
||||
sequential_time.as_secs(),
|
||||
sequential_time.as_secs_f64() / 3600.0
|
||||
);
|
||||
println!(" Actual time: {:.1}s", elapsed.as_secs_f64());
|
||||
println!(" Speedup: {:.0}x", speedup);
|
||||
|
||||
// The proof: execution time should be close to 1 second, not 10,000 seconds
|
||||
if elapsed < Duration::from_secs(3) {
|
||||
println!(" This proves TRUE PARALLEL EXECUTION!");
|
||||
} else {
|
||||
panic!(
|
||||
"FAILED: Tasks appear to be running sequentially! Took {:.1}s instead of ~1s",
|
||||
elapsed.as_secs_f64()
|
||||
);
|
||||
}
|
||||
|
||||
println!("\n=== Parallelism Proof Complete ===");
|
||||
}
|
||||
@@ -1,470 +0,0 @@
|
||||
//! # Retry Strategies for Resilient Workflows
|
||||
//!
|
||||
//! This example demonstrates various retry patterns for handling transient failures
|
||||
//! in DAG workflows, such as network errors, rate limits, or temporary service unavailability.
|
||||
//!
|
||||
//! ## What You'll Learn
|
||||
//! - How to implement immediate, linear, and exponential backoff retry strategies
|
||||
//! - How to distinguish retriable vs non-retriable errors
|
||||
//! - How to track retry attempts and provide diagnostics
|
||||
//! - How to implement jitter to prevent thundering herd problems
|
||||
//!
|
||||
//! ## Retry Patterns
|
||||
//!
|
||||
//! **1. Immediate Retry** - Retry without delay (for quick transient failures)
|
||||
//! ```text
|
||||
//! Attempt 1: ✗ Error
|
||||
//! Attempt 2: ✗ Error (no delay)
|
||||
//! Attempt 3: ✓ Success
|
||||
//! ```
|
||||
//!
|
||||
//! **2. Linear Backoff** - Fixed delay between retries
|
||||
//! ```text
|
||||
//! Attempt 1: ✗ Error
|
||||
//! Wait 100ms
|
||||
//! Attempt 2: ✗ Error
|
||||
//! Wait 100ms
|
||||
//! Attempt 3: ✓ Success
|
||||
//! ```
|
||||
//!
|
||||
//! **3. Exponential Backoff** - Increasing delays between retries
|
||||
//! ```text
|
||||
//! Attempt 1: ✗ Error
|
||||
//! Wait 100ms
|
||||
//! Attempt 2: ✗ Error
|
||||
//! Wait 200ms
|
||||
//! Attempt 3: ✗ Error
|
||||
//! Wait 400ms
|
||||
//! Attempt 4: ✓ Success
|
||||
//! ```
|
||||
//!
|
||||
//! **4. Exponential Backoff with Jitter** - Randomized delays to prevent coordinated retries
|
||||
//! ```text
|
||||
//! Attempt 1: ✗ Error
|
||||
//! Wait 87ms (50-150ms range)
|
||||
//! Attempt 2: ✗ Error
|
||||
//! Wait 234ms (100-300ms range)
|
||||
//! Attempt 3: ✓ Success
|
||||
//! ```
|
||||
//!
|
||||
//! ## Workflow Diagram
|
||||
//! ```text
|
||||
//! ┌──────────────────────┐
|
||||
//! │ ApiCallWithRetry │
|
||||
//! │ Max attempts: 3 │
|
||||
//! │ Strategy: Exp │
|
||||
//! └──────────────────────┘
|
||||
//! │
|
||||
//! ├─► Attempt 1: ✗ (Connection timeout)
|
||||
//! │ Wait 100ms
|
||||
//! │
|
||||
//! ├─► Attempt 2: ✗ (Connection timeout)
|
||||
//! │ Wait 200ms
|
||||
//! │
|
||||
//! └─► Attempt 3: ✓ Success
|
||||
//!
|
||||
//! ┌──────────────────────┐
|
||||
//! │ ApiCallWithRetry │
|
||||
//! │ Max attempts: 3 │
|
||||
//! │ Strategy: Linear │
|
||||
//! └──────────────────────┘
|
||||
//! │
|
||||
//! ├─► Attempt 1: ✗ (Rate limit)
|
||||
//! │ Wait 500ms
|
||||
//! │
|
||||
//! ├─► Attempt 2: ✗ (Rate limit)
|
||||
//! │ Wait 500ms
|
||||
//! │
|
||||
//! ├─► Attempt 3: ✗ (Rate limit)
|
||||
//! │
|
||||
//! └─► Final: ✗ Retry exhausted
|
||||
//! ```
|
||||
//!
|
||||
//! ## Key Concepts
|
||||
//! - **Idempotency**: Retried operations should be safe to repeat
|
||||
//! - **Retriable errors**: Network timeouts, rate limits, temporary service errors
|
||||
//! - **Non-retriable errors**: Authentication failures, invalid input, logical errors
|
||||
//! - **Backoff strategies**: Balance between recovery speed and system load
|
||||
//! - **Jitter**: Randomization to avoid thundering herd when many clients retry simultaneously
|
||||
//! - **Retry budgets**: Limit total retries to prevent cascading failures
|
||||
//!
|
||||
//! ## Use Cases
|
||||
//! - External API calls with transient network failures
|
||||
//! - Database operations with temporary connection issues
|
||||
//! - Distributed systems with eventual consistency
|
||||
//! - Rate-limited API integrations
|
||||
//! - Cloud service calls with intermittent availability
|
||||
//!
|
||||
//! ## Running This Example
|
||||
//! ```bash
|
||||
//! cargo run --example retry_strategies
|
||||
//! ```
|
||||
//!
|
||||
//! ## Expected Output
|
||||
//! ```text
|
||||
//! === Retry Strategies Example ===
|
||||
//!
|
||||
//! Example 1: Immediate retry strategy
|
||||
//! [ApiCall-1] Attempt 1/3: Calling external API...
|
||||
//! [ApiCall-1] ✗ Transient error: Connection timeout
|
||||
//! [ApiCall-1] Retrying immediately (attempt 2/3)
|
||||
//! [ApiCall-1] Attempt 2/3: Calling external API...
|
||||
//! [ApiCall-1] ✗ Transient error: Connection timeout
|
||||
//! [ApiCall-1] Retrying immediately (attempt 3/3)
|
||||
//! [ApiCall-1] Attempt 3/3: Calling external API...
|
||||
//! [ApiCall-1] ✓ Success: API response data
|
||||
//! Result: Ok(RetryResult { attempts: 3, success: true })
|
||||
//!
|
||||
//! Example 2: Linear backoff (500ms delay)
|
||||
//! [ApiCall-2] Attempt 1/4: Calling external API...
|
||||
//! [ApiCall-2] ✗ Transient error: Rate limit exceeded
|
||||
//! [ApiCall-2] Waiting 500ms before retry (attempt 2/4)
|
||||
//! [ApiCall-2] Attempt 2/4: Calling external API...
|
||||
//! [ApiCall-2] ✓ Success: API response data
|
||||
//! Result: Ok(RetryResult { attempts: 2, success: true })
|
||||
//!
|
||||
//! Example 3: Exponential backoff
|
||||
//! [ApiCall-3] Attempt 1/5: Calling external API...
|
||||
//! [ApiCall-3] ✗ Transient error: Service unavailable
|
||||
//! [ApiCall-3] Waiting 100ms before retry (attempt 2/5)
|
||||
//! [ApiCall-3] Attempt 2/5: Calling external API...
|
||||
//! [ApiCall-3] ✗ Transient error: Service unavailable
|
||||
//! [ApiCall-3] Waiting 200ms before retry (attempt 3/5)
|
||||
//! [ApiCall-3] Attempt 3/5: Calling external API...
|
||||
//! [ApiCall-3] ✓ Success: API response data
|
||||
//! Result: Ok(RetryResult { attempts: 3, success: true })
|
||||
//!
|
||||
//! Example 4: Non-retriable error (fails immediately)
|
||||
//! [ApiCall-4] Attempt 1/3: Calling external API...
|
||||
//! [ApiCall-4] ✗ Non-retriable error: Authentication failed
|
||||
//! Result: Err("Authentication failed - not retrying")
|
||||
//!
|
||||
//! Example 5: Retry exhausted
|
||||
//! [ApiCall-5] Attempt 1/3: Calling external API...
|
||||
//! [ApiCall-5] ✗ Transient error: Connection timeout
|
||||
//! [ApiCall-5] Waiting 100ms before retry (attempt 2/3)
|
||||
//! [ApiCall-5] Attempt 2/3: Calling external API...
|
||||
//! [ApiCall-5] ✗ Transient error: Connection timeout
|
||||
//! [ApiCall-5] Waiting 200ms before retry (attempt 3/3)
|
||||
//! [ApiCall-5] Attempt 3/3: Calling external API...
|
||||
//! [ApiCall-5] ✗ Transient error: Connection timeout
|
||||
//! [ApiCall-5] All retry attempts exhausted
|
||||
//! Result: Err("Retry exhausted after 3 attempts: Connection timeout")
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagRunner};
|
||||
use tokio::time::{sleep, Duration};
|
||||
|
||||
// Retry strategy configuration
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum RetryStrategy {
|
||||
Immediate,
|
||||
Linear { delay_ms: u64 },
|
||||
Exponential { base_ms: u64, factor: f64 },
|
||||
}
|
||||
|
||||
impl RetryStrategy {
|
||||
fn calculate_delay(&self, attempt: u32) -> Option<Duration> {
|
||||
match self {
|
||||
RetryStrategy::Immediate => None,
|
||||
RetryStrategy::Linear { delay_ms } => Some(Duration::from_millis(*delay_ms)),
|
||||
RetryStrategy::Exponential { base_ms, factor } => {
|
||||
let delay = *base_ms as f64 * factor.powi(attempt as i32 - 1);
|
||||
Some(Duration::from_millis(delay as u64))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Simulated error types
|
||||
#[derive(Clone, Debug)]
|
||||
enum ApiError {
|
||||
// Retriable errors
|
||||
ConnectionTimeout,
|
||||
ServiceUnavailable,
|
||||
RateLimitExceeded,
|
||||
|
||||
// Non-retriable errors
|
||||
AuthenticationFailed,
|
||||
#[allow(dead_code)]
|
||||
InvalidInput,
|
||||
}
|
||||
|
||||
impl ApiError {
|
||||
fn is_retriable(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
ApiError::ConnectionTimeout
|
||||
| ApiError::ServiceUnavailable
|
||||
| ApiError::RateLimitExceeded
|
||||
)
|
||||
}
|
||||
|
||||
fn message(&self) -> &'static str {
|
||||
match self {
|
||||
ApiError::ConnectionTimeout => "Connection timeout",
|
||||
ApiError::ServiceUnavailable => "Service unavailable",
|
||||
ApiError::RateLimitExceeded => "Rate limit exceeded",
|
||||
ApiError::AuthenticationFailed => "Authentication failed",
|
||||
ApiError::InvalidInput => "Invalid input",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Result of retry operation
|
||||
#[derive(Clone, Debug)]
|
||||
#[allow(dead_code)]
|
||||
struct RetryResult {
|
||||
attempts: u32,
|
||||
success: bool,
|
||||
final_error: Option<String>,
|
||||
}
|
||||
|
||||
// Task that simulates an API call with configurable failure pattern
|
||||
struct ApiCallWithRetry {
|
||||
task_id: String,
|
||||
max_attempts: u32,
|
||||
strategy: RetryStrategy,
|
||||
// Simulated failure pattern: errors to return before success
|
||||
failure_sequence: Vec<ApiError>,
|
||||
current_attempt: u32,
|
||||
}
|
||||
|
||||
impl ApiCallWithRetry {
|
||||
fn new(
|
||||
task_id: impl Into<String>,
|
||||
max_attempts: u32,
|
||||
strategy: RetryStrategy,
|
||||
failure_sequence: Vec<ApiError>,
|
||||
) -> Self {
|
||||
Self {
|
||||
task_id: task_id.into(),
|
||||
max_attempts,
|
||||
strategy,
|
||||
failure_sequence,
|
||||
current_attempt: 0,
|
||||
}
|
||||
}
|
||||
|
||||
async fn attempt_api_call(&mut self) -> Result<String, ApiError> {
|
||||
println!(
|
||||
"[{}] Attempt {}/{}: Calling external API...",
|
||||
self.task_id,
|
||||
self.current_attempt + 1,
|
||||
self.max_attempts
|
||||
);
|
||||
|
||||
// Simulate network delay
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
|
||||
// Check if we should fail this attempt
|
||||
if (self.current_attempt as usize) < self.failure_sequence.len() {
|
||||
let error = self.failure_sequence[self.current_attempt as usize].clone();
|
||||
Err(error)
|
||||
} else {
|
||||
Ok("API response data".to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl ApiCallWithRetry {
|
||||
async fn run(&mut self) -> Result<RetryResult, String> {
|
||||
loop {
|
||||
self.current_attempt += 1;
|
||||
|
||||
match self.attempt_api_call().await {
|
||||
Ok(data) => {
|
||||
println!("[{}] ✓ Success: {}", self.task_id, data);
|
||||
return Ok(RetryResult {
|
||||
attempts: self.current_attempt,
|
||||
success: true,
|
||||
final_error: None,
|
||||
});
|
||||
}
|
||||
Err(error) => {
|
||||
println!(
|
||||
"[{}] ✗ {} error: {}",
|
||||
self.task_id,
|
||||
if error.is_retriable() {
|
||||
"Transient"
|
||||
} else {
|
||||
"Non-retriable"
|
||||
},
|
||||
error.message()
|
||||
);
|
||||
|
||||
// Check if error is retriable
|
||||
if !error.is_retriable() {
|
||||
return Err(format!("{} - not retrying", error.message()));
|
||||
}
|
||||
|
||||
// Check if we have attempts remaining
|
||||
if self.current_attempt >= self.max_attempts {
|
||||
println!("[{}] All retry attempts exhausted", self.task_id);
|
||||
return Err(format!(
|
||||
"Retry exhausted after {} attempts: {}",
|
||||
self.current_attempt,
|
||||
error.message()
|
||||
));
|
||||
}
|
||||
|
||||
// Calculate and apply backoff delay
|
||||
if let Some(delay) = self.strategy.calculate_delay(self.current_attempt) {
|
||||
println!(
|
||||
"[{}] Waiting {}ms before retry (attempt {}/{})",
|
||||
self.task_id,
|
||||
delay.as_millis(),
|
||||
self.current_attempt + 1,
|
||||
self.max_attempts
|
||||
);
|
||||
sleep(delay).await;
|
||||
} else {
|
||||
println!(
|
||||
"[{}] Retrying immediately (attempt {}/{})",
|
||||
self.task_id,
|
||||
self.current_attempt + 1,
|
||||
self.max_attempts
|
||||
);
|
||||
}
|
||||
|
||||
// Continue loop to retry
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
println!("=== Retry Strategies Example ===\n");
|
||||
|
||||
// Example 1: Immediate retry - succeeds on 3rd attempt
|
||||
println!("Example 1: Immediate retry strategy");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let api_call = dag.add_task(ApiCallWithRetry::new(
|
||||
"ApiCall-1",
|
||||
3,
|
||||
RetryStrategy::Immediate,
|
||||
vec![
|
||||
ApiError::ConnectionTimeout,
|
||||
ApiError::ConnectionTimeout,
|
||||
// 3rd attempt will succeed
|
||||
],
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result: {:?}\n", dag.get(api_call).unwrap());
|
||||
}
|
||||
|
||||
// Example 2: Linear backoff - succeeds on 2nd attempt
|
||||
println!("Example 2: Linear backoff (500ms delay)");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let api_call = dag.add_task(ApiCallWithRetry::new(
|
||||
"ApiCall-2",
|
||||
4,
|
||||
RetryStrategy::Linear { delay_ms: 500 },
|
||||
vec![
|
||||
ApiError::RateLimitExceeded,
|
||||
// 2nd attempt will succeed
|
||||
],
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result: {:?}\n", dag.get(api_call).unwrap());
|
||||
}
|
||||
|
||||
// Example 3: Exponential backoff - succeeds on 3rd attempt
|
||||
println!("Example 3: Exponential backoff");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let api_call = dag.add_task(ApiCallWithRetry::new(
|
||||
"ApiCall-3",
|
||||
5,
|
||||
RetryStrategy::Exponential {
|
||||
base_ms: 100,
|
||||
factor: 2.0,
|
||||
},
|
||||
vec![
|
||||
ApiError::ServiceUnavailable,
|
||||
ApiError::ServiceUnavailable,
|
||||
// 3rd attempt will succeed
|
||||
],
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result: {:?}\n", dag.get(api_call).unwrap());
|
||||
}
|
||||
|
||||
// Example 4: Non-retriable error - fails immediately
|
||||
println!("Example 4: Non-retriable error (fails immediately)");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let api_call = dag.add_task(ApiCallWithRetry::new(
|
||||
"ApiCall-4",
|
||||
3,
|
||||
RetryStrategy::Immediate,
|
||||
vec![ApiError::AuthenticationFailed],
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result: {:?}\n", dag.get(api_call).unwrap());
|
||||
}
|
||||
|
||||
// Example 5: Retry exhausted - all attempts fail
|
||||
println!("Example 5: Retry exhausted");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let api_call = dag.add_task(ApiCallWithRetry::new(
|
||||
"ApiCall-5",
|
||||
3,
|
||||
RetryStrategy::Exponential {
|
||||
base_ms: 100,
|
||||
factor: 2.0,
|
||||
},
|
||||
vec![
|
||||
ApiError::ConnectionTimeout,
|
||||
ApiError::ConnectionTimeout,
|
||||
ApiError::ConnectionTimeout,
|
||||
// All attempts will fail
|
||||
],
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result: {:?}\n", dag.get(api_call).unwrap());
|
||||
}
|
||||
|
||||
println!("=== Retry Strategies Example Complete ===");
|
||||
}
|
||||
@@ -1,303 +0,0 @@
|
||||
//! # Managing Timeouts in Tasks
|
||||
//!
|
||||
//! This example demonstrates how to implement timeout patterns in DAG tasks using
|
||||
//! tokio's timeout utilities for time-bounded execution.
|
||||
//!
|
||||
//! ## What You'll Learn
|
||||
//! - How to implement timeouts using `tokio::time::timeout`
|
||||
//! - How to build tasks with built-in timeout logic
|
||||
//! - How to handle timeout errors as Result types
|
||||
//! - How to apply timeouts at both task-level and DAG-level
|
||||
//!
|
||||
//! ## Timeout Patterns
|
||||
//!
|
||||
//! **1. Task-Level Timeout** - Individual task with timeout
|
||||
//! ```rust
|
||||
//! async fn run(&mut self) -> Result<String, String> {
|
||||
//! let work = async { /* ... */ };
|
||||
//! match timeout(Duration::from_millis(self.timeout_ms), work).await {
|
||||
//! Ok(result) => Ok(result),
|
||||
//! Err(_) => Err("Task timed out".to_string()),
|
||||
//! }
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! **2. DAG-Level Timeout** - Entire workflow with timeout
|
||||
//! ```rust
|
||||
//! match timeout(dag_timeout, dag.run(spawner)).await {
|
||||
//! Ok(result) => { /* DAG completed */ },
|
||||
//! Err(_) => { /* DAG timed out */ },
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! ## Workflow Diagram
|
||||
//! ```text
|
||||
//! ┌─────────────┐
|
||||
//! │ FastTask │ (completes in 100ms)
|
||||
//! │ (timeout │ ───► Ok("Fast task completed")
|
||||
//! │ 500ms) │
|
||||
//! └─────────────┘
|
||||
//!
|
||||
//! ┌─────────────┐
|
||||
//! │ SlowTask │ (takes 1000ms)
|
||||
//! │ (timeout │ ───► Err("Task timed out after 200ms")
|
||||
//! │ 200ms) │
|
||||
//! └─────────────┘
|
||||
//! │
|
||||
//! └──────►┌──────────────┐
|
||||
//! │ProcessResult │ ───► "✗ Task timed out..."
|
||||
//! └──────────────┘
|
||||
//! ```
|
||||
//!
|
||||
//! ## Key Concepts
|
||||
//! - **Timeout as error**: Timeouts produce `Err` results that propagate through the DAG
|
||||
//! - **Error as data**: Use Result<T, E> to represent timeout states
|
||||
//! - **Graceful degradation**: Downstream tasks can handle timeout errors
|
||||
//! - **Time-bounded execution**: Enforce SLAs and prevent runaway tasks
|
||||
//!
|
||||
//! ## Use Cases
|
||||
//! - API calls with timeout limits
|
||||
//! - SLA enforcement in workflows
|
||||
//! - Preventing resource exhaustion from slow tasks
|
||||
//! - Circuit breaker patterns
|
||||
//! - External service integration with timeouts
|
||||
//!
|
||||
//! ## Running This Example
|
||||
//! ```bash
|
||||
//! cargo run --example timeout
|
||||
//! ```
|
||||
//!
|
||||
//! ## Expected Output
|
||||
//! ```text
|
||||
//! === Timeout Example ===
|
||||
//!
|
||||
//! Example 1: All tasks complete successfully
|
||||
//! Fast task: Ok("Fast task completed")
|
||||
//! Timed task: Ok("Work completed in 100ms")
|
||||
//!
|
||||
//! Example 2: Task exceeds timeout
|
||||
//! Result: ✗ Task timed out after 200ms (work would take 1000ms)
|
||||
//!
|
||||
//! Example 3: Parallel tasks with different speeds
|
||||
//! Fast task 1: Ok("Fast task completed")
|
||||
//! Fast task 2: Ok("Fast task completed")
|
||||
//! Slow task: Ok("Slow task completed after 500ms")
|
||||
//! Timed task 1: Ok("Work completed in 100ms")
|
||||
//! Timed task 2: Err("Task timed out after 200ms...")
|
||||
//!
|
||||
//! Example 4: Downstream tasks handling timeout errors
|
||||
//! Processed 1: ✗ Task timed out after 100ms (work would take 500ms)
|
||||
//! Processed 2: ✗ Task timed out after 100ms (work would take 500ms)
|
||||
//!
|
||||
//! Example 5: Whole-DAG timeout
|
||||
//! DAG completed within timeout
|
||||
//! Task 1: Ok("Slow task completed after 200ms")
|
||||
//! Task 2: Ok("Slow task completed after 200ms")
|
||||
//! Task 3: Ok("Slow task completed after 200ms")
|
||||
//!
|
||||
//! === Timeout Example Complete ===
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagRunner};
|
||||
use tokio::time::{sleep, timeout, Duration};
|
||||
|
||||
// Fast task that completes quickly
|
||||
struct FastTask;
|
||||
|
||||
#[task]
|
||||
impl FastTask {
|
||||
async fn run() -> Result<String, String> {
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
Ok("Fast task completed".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
// Slow task that takes a long time
|
||||
struct SlowTask {
|
||||
duration_ms: u64,
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl SlowTask {
|
||||
async fn run(&mut self) -> Result<String, String> {
|
||||
sleep(Duration::from_millis(self.duration_ms)).await;
|
||||
Ok(format!("Slow task completed after {}ms", self.duration_ms))
|
||||
}
|
||||
}
|
||||
|
||||
// Task with built-in timeout
|
||||
struct TaskWithTimeout {
|
||||
work_duration_ms: u64,
|
||||
timeout_ms: u64,
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl TaskWithTimeout {
|
||||
async fn run(&mut self) -> Result<String, String> {
|
||||
let work = async {
|
||||
sleep(Duration::from_millis(self.work_duration_ms)).await;
|
||||
format!("Work completed in {}ms", self.work_duration_ms)
|
||||
};
|
||||
|
||||
match timeout(Duration::from_millis(self.timeout_ms), work).await {
|
||||
Ok(result) => Ok(result),
|
||||
Err(_) => Err(format!(
|
||||
"Task timed out after {}ms (work would take {}ms)",
|
||||
self.timeout_ms, self.work_duration_ms
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Task that processes results (including timeouts)
|
||||
struct ProcessResult;
|
||||
|
||||
#[task]
|
||||
impl ProcessResult {
|
||||
async fn run(input: &Result<String, String>) -> String {
|
||||
match input {
|
||||
Ok(msg) => format!("✓ {}", msg),
|
||||
Err(err) => format!("✗ {}", err),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
println!("=== Timeout Example ===\n");
|
||||
|
||||
// Example 1: All tasks complete within timeout
|
||||
println!("Example 1: All tasks complete successfully");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let fast = dag.add_task(FastTask);
|
||||
let timed = dag.add_task(TaskWithTimeout {
|
||||
work_duration_ms: 100,
|
||||
timeout_ms: 500,
|
||||
});
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Fast task: {:?}", dag.get(fast).unwrap());
|
||||
println!("Timed task: {:?}\n", dag.get(timed).unwrap());
|
||||
}
|
||||
|
||||
// Example 2: Task exceeds timeout
|
||||
println!("Example 2: Task exceeds timeout");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let timed = dag.add_task(TaskWithTimeout {
|
||||
work_duration_ms: 1000,
|
||||
timeout_ms: 200,
|
||||
});
|
||||
|
||||
let processed = dag.add_task(ProcessResult).depends_on(&timed);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Result: {}\n", dag.get(processed).unwrap());
|
||||
}
|
||||
|
||||
// Example 3: Mixed fast and slow tasks
|
||||
println!("Example 3: Parallel tasks with different speeds");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let fast1 = dag.add_task(FastTask);
|
||||
let fast2 = dag.add_task(FastTask);
|
||||
|
||||
let slow = dag.add_task(SlowTask { duration_ms: 500 });
|
||||
|
||||
let timed1 = dag.add_task(TaskWithTimeout {
|
||||
work_duration_ms: 100,
|
||||
timeout_ms: 200,
|
||||
});
|
||||
|
||||
let timed2 = dag.add_task(TaskWithTimeout {
|
||||
work_duration_ms: 300,
|
||||
timeout_ms: 200,
|
||||
});
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Fast task 1: {:?}", dag.get(fast1).unwrap());
|
||||
println!("Fast task 2: {:?}", dag.get(fast2).unwrap());
|
||||
println!("Slow task: {:?}", dag.get(slow).unwrap());
|
||||
println!("Timed task 1: {:?}", dag.get(timed1).unwrap());
|
||||
println!("Timed task 2: {:?}\n", dag.get(timed2).unwrap());
|
||||
}
|
||||
|
||||
// Example 4: Cascading timeout errors
|
||||
println!("Example 4: Downstream tasks handling timeout errors");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let source = dag.add_task(TaskWithTimeout {
|
||||
work_duration_ms: 500,
|
||||
timeout_ms: 100,
|
||||
});
|
||||
|
||||
let processed1 = dag.add_task(ProcessResult).depends_on(&source);
|
||||
let processed2 = dag.add_task(ProcessResult).depends_on(&source);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Processed 1: {}", dag.get(processed1).unwrap());
|
||||
println!("Processed 2: {}\n", dag.get(processed2).unwrap());
|
||||
}
|
||||
|
||||
// Example 5: Implementing whole-DAG timeout
|
||||
println!("Example 5: Whole-DAG timeout");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let t1 = dag.add_task(SlowTask { duration_ms: 200 });
|
||||
let t2 = dag.add_task(SlowTask { duration_ms: 200 });
|
||||
let t3 = dag.add_task(SlowTask { duration_ms: 200 });
|
||||
|
||||
// Run the entire DAG with a timeout
|
||||
let dag_timeout = Duration::from_millis(500);
|
||||
match timeout(
|
||||
dag_timeout,
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
}),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
result.unwrap();
|
||||
println!("DAG completed within timeout");
|
||||
println!("Task 1: {:?}", dag.get(t1).unwrap());
|
||||
println!("Task 2: {:?}", dag.get(t2).unwrap());
|
||||
println!("Task 3: {:?}", dag.get(t3).unwrap());
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!(
|
||||
"DAG execution timed out after {}ms",
|
||||
dag_timeout.as_millis()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("\n=== Timeout Example Complete ===");
|
||||
}
|
||||
@@ -1,167 +0,0 @@
|
||||
//! # Tracing Example
|
||||
//!
|
||||
//! This example demonstrates how to use the optional tracing feature to gain
|
||||
//! visibility into DAG execution for debugging and performance analysis.
|
||||
//!
|
||||
//! ## Running This Example
|
||||
//!
|
||||
//! ```bash
|
||||
//! # With default log level (info)
|
||||
//! cargo run --example tracing_example --features tracing
|
||||
//!
|
||||
//! # With debug level (shows more details)
|
||||
//! RUST_LOG=dagx=debug cargo run --example tracing_example --features tracing
|
||||
//!
|
||||
//! # With trace level (shows all execution details)
|
||||
//! RUST_LOG=dagx=trace cargo run --example tracing_example --features tracing
|
||||
//! ```
|
||||
//!
|
||||
//! ## What You'll See
|
||||
//!
|
||||
//! - Task additions to the DAG
|
||||
//! - Dependency wiring between tasks
|
||||
//! - Topological layer computation
|
||||
//! - Layer-by-layer execution
|
||||
//! - Inline vs spawned task execution
|
||||
//! - Completion status
|
||||
//!
|
||||
//! ## Log Levels
|
||||
//!
|
||||
//! - **INFO**: High-level DAG execution start/completion
|
||||
//! - **DEBUG**: Task additions, dependency wiring, layer computation
|
||||
//! - **TRACE**: Individual task execution details (inline vs spawned)
|
||||
//! - **ERROR**: Errors, panics, cycle detection
|
||||
//!
|
||||
|
||||
use dagx::{task, DagRunner};
|
||||
use tracing_subscriber::{fmt, EnvFilter};
|
||||
|
||||
// Simple source task
|
||||
struct Value(i32);
|
||||
|
||||
#[task]
|
||||
impl Value {
|
||||
async fn run(&self) -> i32 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
// Computation task
|
||||
struct Add;
|
||||
|
||||
#[task]
|
||||
impl Add {
|
||||
async fn run(a: &i32, b: &i32) -> i32 {
|
||||
a + b
|
||||
}
|
||||
}
|
||||
|
||||
// Another computation task
|
||||
struct Multiply;
|
||||
|
||||
#[task]
|
||||
impl Multiply {
|
||||
async fn run(a: &i32, b: &i32) -> i32 {
|
||||
a * b
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
// Initialize tracing subscriber with environment filter
|
||||
// Use RUST_LOG environment variable to control log level
|
||||
fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("dagx=info")),
|
||||
)
|
||||
.init();
|
||||
|
||||
println!("=== DAG Execution with Tracing ===\n");
|
||||
println!("Run with RUST_LOG=dagx=debug or RUST_LOG=dagx=trace for more details\n");
|
||||
|
||||
// Example 1: Simple diamond pattern
|
||||
println!("Example 1: Diamond Pattern");
|
||||
println!("---------------------------");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Layer 0: Source
|
||||
let source = dag.add_task(Value(10));
|
||||
|
||||
// Layer 1: Two parallel computations
|
||||
let left = dag.add_task(Add).depends_on((&source, &source));
|
||||
let right = dag.add_task(Multiply).depends_on((&source, &source));
|
||||
|
||||
// Layer 2: Combine results
|
||||
let result = dag.add_task(Add).depends_on((&left, &right));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("\nResult: {}", dag.get(result).unwrap());
|
||||
println!("(Expected: 10+10 = 20, 10*10 = 100, 20+100 = 120)\n");
|
||||
}
|
||||
|
||||
// Example 2: Linear chain (demonstrates inline execution)
|
||||
println!("\nExample 2: Linear Chain (Inline Execution)");
|
||||
println!("-------------------------------------------");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Create a linear chain - each layer has only one task
|
||||
// This should trigger inline execution optimization
|
||||
let a = dag.add_task(Value(1));
|
||||
let b = dag.add_task(Add).depends_on((&a, &a));
|
||||
let c = dag.add_task(Multiply).depends_on((&b, &b));
|
||||
let d = dag.add_task(Add).depends_on((&c, &c));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("\nResult: {}", dag.get(d).unwrap());
|
||||
println!("(Expected: 1+1 = 2, 2*2 = 4, 4+4 = 8)\n");
|
||||
}
|
||||
|
||||
// Example 3: Wide pattern (demonstrates spawned execution)
|
||||
println!("\nExample 3: Wide Pattern (Spawned Execution)");
|
||||
println!("--------------------------------------------");
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Layer 0: Multiple sources
|
||||
let v1 = dag.add_task(Value(1));
|
||||
let v2 = dag.add_task(Value(2));
|
||||
let v3 = dag.add_task(Value(3));
|
||||
let v4 = dag.add_task(Value(4));
|
||||
|
||||
// Layer 1: Multiple parallel computations
|
||||
let sum12 = dag.add_task(Add).depends_on((&v1, &v2));
|
||||
let sum23 = dag.add_task(Add).depends_on((&v2, &v3));
|
||||
let sum34 = dag.add_task(Add).depends_on((&v3, &v4));
|
||||
|
||||
// Layer 2: Final combination
|
||||
let sum_all = dag.add_task(Add).depends_on((&sum12, &sum23));
|
||||
let final_result = dag.add_task(Add).depends_on((&sum_all, &sum34));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("\nResult: {}", dag.get(final_result).unwrap());
|
||||
println!("(Expected: (1+2)+(2+3) = 8, 8+(3+4) = 15)\n");
|
||||
}
|
||||
|
||||
println!("\n=== Tracing Example Complete ===");
|
||||
println!("\nTry running with different log levels:");
|
||||
println!(" RUST_LOG=dagx=info - High-level execution info");
|
||||
println!(" RUST_LOG=dagx=debug - Task additions and layer details");
|
||||
println!(" RUST_LOG=dagx=trace - All execution details including inline/spawned");
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
target
|
||||
corpus
|
||||
artifacts
|
||||
Cargo.lock
|
||||
@@ -1,43 +0,0 @@
|
||||
[package]
|
||||
name = "dagx-fuzz"
|
||||
version = "0.0.0"
|
||||
publish = false
|
||||
edition = "2021"
|
||||
|
||||
[package.metadata]
|
||||
cargo-fuzz = true
|
||||
|
||||
[dependencies]
|
||||
libfuzzer-sys = "0.4"
|
||||
tokio = { version = "1", features = ["rt", "macros"] }
|
||||
|
||||
[dependencies.dagx]
|
||||
path = ".."
|
||||
|
||||
[[bin]]
|
||||
name = "fuzz_dag_execution"
|
||||
path = "fuzz_targets/fuzz_dag_execution.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "fuzz_dependencies"
|
||||
path = "fuzz_targets/fuzz_dependencies.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "fuzz_type_safety"
|
||||
path = "fuzz_targets/fuzz_type_safety.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "fuzz_concurrent"
|
||||
path = "fuzz_targets/fuzz_concurrent.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
-191
@@ -1,191 +0,0 @@
|
||||
# Fuzz Testing for dagx
|
||||
|
||||
This directory contains fuzz testing infrastructure using [cargo-fuzz](https://github.com/rust-fuzz/cargo-fuzz) and libFuzzer.
|
||||
|
||||
## Overview
|
||||
|
||||
We have 4 fuzz targets that test different aspects of the `dagx` library:
|
||||
|
||||
1. **fuzz_dag_execution** - Tests DAG execution with various task configurations
|
||||
2. **fuzz_dependencies** - Tests dependency patterns (linear, diamond, fan-out, complex)
|
||||
3. **fuzz_type_safety** - Tests type-safe output retrieval with various types
|
||||
4. **fuzz_concurrent** - Tests concurrent task execution and thread safety
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Fuzzing requires nightly Rust and cargo-fuzz:
|
||||
|
||||
```bash
|
||||
rustup install nightly
|
||||
cargo install cargo-fuzz
|
||||
```
|
||||
|
||||
## Running Fuzz Tests
|
||||
|
||||
Run a single fuzz target:
|
||||
|
||||
```bash
|
||||
cargo +nightly fuzz run fuzz_dag_execution
|
||||
```
|
||||
|
||||
Run with a time limit (recommended for CI):
|
||||
|
||||
```bash
|
||||
cargo +nightly fuzz run fuzz_dag_execution -- -max_total_time=60
|
||||
```
|
||||
|
||||
Run with a specific number of iterations:
|
||||
|
||||
```bash
|
||||
cargo +nightly fuzz run fuzz_dag_execution -- -runs=10000
|
||||
```
|
||||
|
||||
Run all fuzz targets sequentially:
|
||||
|
||||
```bash
|
||||
for target in fuzz_dag_execution fuzz_dependencies fuzz_type_safety fuzz_concurrent; do
|
||||
echo "Running $target..."
|
||||
cargo +nightly fuzz run $target -- -runs=10000
|
||||
done
|
||||
```
|
||||
|
||||
## Continuous Fuzzing
|
||||
|
||||
For deeper testing, run fuzz targets for extended periods:
|
||||
|
||||
```bash
|
||||
# Run overnight (8 hours)
|
||||
cargo +nightly fuzz run fuzz_concurrent -- -max_total_time=28800
|
||||
|
||||
# Run until crash (or Ctrl+C)
|
||||
cargo +nightly fuzz run fuzz_dag_execution
|
||||
```
|
||||
|
||||
## Understanding Output
|
||||
|
||||
Fuzzer output includes:
|
||||
|
||||
- `cov: X` - Code coverage (number of edges covered)
|
||||
- `ft: X` - Feature coverage (number of unique code paths)
|
||||
- `corp: X/Yb` - Corpus size (X inputs, Y bytes total)
|
||||
- `exec/s: X` - Executions per second
|
||||
|
||||
If a crash is found:
|
||||
|
||||
- Crash input is saved to `artifacts/FUZZ_TARGET/crash-*`
|
||||
- Reproduce with: `cargo +nightly fuzz run FUZZ_TARGET artifacts/FUZZ_TARGET/crash-*`
|
||||
|
||||
## Fuzz Target Details
|
||||
|
||||
### fuzz_dag_execution
|
||||
|
||||
Tests basic DAG execution:
|
||||
- Simple linear task chains
|
||||
- Fan-out patterns (one task → multiple dependents)
|
||||
- Various data types (i32, Vec<u8>, custom structs)
|
||||
- DAG execution correctness
|
||||
- Output value verification
|
||||
|
||||
### fuzz_dependencies
|
||||
|
||||
Tests dependency management:
|
||||
- Linear dependency chains
|
||||
- Diamond patterns (source → two paths → merge)
|
||||
- Multiple independent sources
|
||||
- Complex dependency graphs
|
||||
- Dependency tracking correctness
|
||||
|
||||
### fuzz_type_safety
|
||||
|
||||
Tests type system correctness:
|
||||
- Various output types (i32, String, Vec, custom structs)
|
||||
- Correct type retrieval with dag.get()
|
||||
- Option<T> and Result<T, E> handling
|
||||
- Custom struct preservation
|
||||
- Type mismatch error handling
|
||||
|
||||
### fuzz_concurrent
|
||||
|
||||
Tests thread safety and concurrency:
|
||||
- Multiple independent concurrent tasks
|
||||
- Concurrent dependent tasks (fan-out)
|
||||
- Arc<DagRunner> sharing across threads
|
||||
- Task timing variations
|
||||
- Result access from multiple threads
|
||||
|
||||
## Adding New Fuzz Targets
|
||||
|
||||
1. Create a new file in `fuzz_targets/` (e.g., `fuzz_new_feature.rs`)
|
||||
2. Use the template:
|
||||
|
||||
```rust
|
||||
#![no_main]
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use dagx::{task, DagRunner};
|
||||
|
||||
fuzz_target!(|data: &[u8]| {
|
||||
// Your fuzz test here
|
||||
});
|
||||
```
|
||||
|
||||
3. Add to `fuzz/Cargo.toml`:
|
||||
|
||||
```toml
|
||||
[[bin]]
|
||||
name = "fuzz_new_feature"
|
||||
path = "fuzz_targets/fuzz_new_feature.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
```
|
||||
|
||||
4. Build and run:
|
||||
|
||||
```bash
|
||||
cargo +nightly fuzz build
|
||||
cargo +nightly fuzz run fuzz_new_feature
|
||||
```
|
||||
|
||||
## CI Integration
|
||||
|
||||
Fuzzing can run automatically in CI on every push/PR:
|
||||
- Each target runs for 60-120 seconds
|
||||
- Targets run 1,000-10,000 iterations minimum
|
||||
- Any crashes fail the build
|
||||
|
||||
See your CI configuration for details.
|
||||
|
||||
## Corpus Management
|
||||
|
||||
The fuzzer builds a corpus of interesting inputs in `corpus/FUZZ_TARGET/`. These inputs:
|
||||
- Are automatically discovered during fuzzing
|
||||
- Can be committed to Git for regression testing
|
||||
- Help future fuzzing sessions start with good coverage
|
||||
|
||||
To minimize the corpus:
|
||||
|
||||
```bash
|
||||
cargo +nightly fuzz cmin fuzz_dag_execution
|
||||
```
|
||||
|
||||
## Tips
|
||||
|
||||
1. **Start with short runs** - Use `-max_total_time=60` to catch obvious bugs quickly
|
||||
2. **Run overnight** - For thorough testing, run for hours or days
|
||||
3. **Check coverage** - Monitor `cov:` values to ensure good code coverage
|
||||
4. **Save crashes** - Commit crash artifacts as regression tests
|
||||
5. **Update regularly** - Re-run fuzzing when adding new features
|
||||
6. **Limit data size** - Fuzz targets filter out oversized inputs to avoid timeouts
|
||||
|
||||
## Performance Notes
|
||||
|
||||
- DAG execution involves async runtime overhead
|
||||
- Each fuzz iteration creates a new Tokio runtime
|
||||
- Typical execution rate: 100-1000 runs/second
|
||||
- Adjust `-runs` or `-max_total_time` based on your needs
|
||||
|
||||
## References
|
||||
|
||||
- [cargo-fuzz book](https://rust-fuzz.github.io/book/cargo-fuzz.html)
|
||||
- [libFuzzer documentation](https://llvm.org/docs/LibFuzzer.html)
|
||||
- [Fuzzing in Rust](https://rust-fuzz.github.io/book/)
|
||||
@@ -1,137 +0,0 @@
|
||||
#![no_main]
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use dagx::{task, DagRunner};
|
||||
use std::sync::Arc;
|
||||
|
||||
fuzz_target!(|data: &[u8]| {
|
||||
if data.is_empty() || data.len() > 50 {
|
||||
return;
|
||||
}
|
||||
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
|
||||
// Test concurrent task execution
|
||||
struct SlowTask { val: i32, delay_ms: u64 }
|
||||
#[task]
|
||||
impl SlowTask {
|
||||
async fn run(&self) -> i32 {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(self.delay_ms)).await;
|
||||
self.val
|
||||
}
|
||||
}
|
||||
|
||||
struct FastTask { val: i32 }
|
||||
#[task]
|
||||
impl FastTask {
|
||||
async fn run(&self) -> i32 { self.val }
|
||||
}
|
||||
|
||||
// Test concurrent independent tasks
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Create multiple independent tasks
|
||||
let num_tasks = (data[0] % 5 + 1) as usize;
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for i in 0..num_tasks {
|
||||
let val = data.get(i).copied().unwrap_or(0) as i32;
|
||||
let delay = (val.abs() as u64 % 10) + 1;
|
||||
|
||||
let task = dag.add_task(SlowTask { val, delay_ms: delay });
|
||||
tasks.push((task, val));
|
||||
}
|
||||
|
||||
if rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}).is_ok() {
|
||||
// Verify all tasks completed with correct values
|
||||
for (handle, expected) in tasks {
|
||||
if let Ok(result) = dag.get(handle) {
|
||||
assert_eq!(result, expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test concurrent dependent tasks
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let source = dag.add_task(FastTask { val: data[0] as i32 });
|
||||
|
||||
// Create multiple tasks that depend on source
|
||||
struct AddOne;
|
||||
#[task]
|
||||
impl AddOne {
|
||||
async fn run(input: &i32) -> i32 {
|
||||
input.wrapping_add(1)
|
||||
}
|
||||
}
|
||||
|
||||
struct AddTwo;
|
||||
#[task]
|
||||
impl AddTwo {
|
||||
async fn run(input: &i32) -> i32 {
|
||||
input.wrapping_add(2)
|
||||
}
|
||||
}
|
||||
|
||||
struct AddThree;
|
||||
#[task]
|
||||
impl AddThree {
|
||||
async fn run(input: &i32) -> i32 {
|
||||
input.wrapping_add(3)
|
||||
}
|
||||
}
|
||||
|
||||
let t1 = dag.add_task(AddOne).depends_on(&source);
|
||||
let t2 = dag.add_task(AddTwo).depends_on(&source);
|
||||
let t3 = dag.add_task(AddThree).depends_on(&source);
|
||||
|
||||
if rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}).is_ok() {
|
||||
let base = data[0] as i32;
|
||||
if let Ok(r1) = dag.get(t1) {
|
||||
assert_eq!(r1, base.wrapping_add(1));
|
||||
}
|
||||
if let Ok(r2) = dag.get(t2) {
|
||||
assert_eq!(r2, base.wrapping_add(2));
|
||||
}
|
||||
if let Ok(r3) = dag.get(t3) {
|
||||
assert_eq!(r3, base.wrapping_add(3));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test Arc sharing across concurrent tasks
|
||||
{
|
||||
let dag = Arc::new(DagRunner::new());
|
||||
|
||||
struct SharedTask { val: i32 }
|
||||
#[task]
|
||||
impl SharedTask {
|
||||
async fn run(&self) -> i32 { self.val }
|
||||
}
|
||||
|
||||
let task = dag.add_task(SharedTask { val: data[0] as i32 });
|
||||
let task_copy = task;
|
||||
|
||||
let dag1 = Arc::clone(&dag);
|
||||
let dag2 = Arc::clone(&dag);
|
||||
|
||||
if rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}).is_ok() {
|
||||
// Multiple threads can access result
|
||||
if let Ok(r1) = dag1.get(task_copy) {
|
||||
if let Ok(r2) = dag2.get(task_copy) {
|
||||
assert_eq!(r1, r2);
|
||||
assert_eq!(r1, data[0] as i32);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -1,120 +0,0 @@
|
||||
#![no_main]
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use dagx::{task, DagRunner};
|
||||
|
||||
fuzz_target!(|data: &[u8]| {
|
||||
// Only fuzz with reasonable data sizes to avoid timeouts
|
||||
if data.is_empty() || data.len() > 1000 {
|
||||
return;
|
||||
}
|
||||
|
||||
// Convert bytes to values for tasks
|
||||
let value1 = data.get(0).copied().unwrap_or(0) as i32;
|
||||
let value2 = data.get(1).copied().unwrap_or(0) as i32;
|
||||
let value3 = data.get(2).copied().unwrap_or(0) as i32;
|
||||
|
||||
// Test simple linear DAG execution
|
||||
{
|
||||
struct Source1 { val: i32 }
|
||||
#[task]
|
||||
impl Source1 {
|
||||
async fn run(&self) -> i32 { self.val }
|
||||
}
|
||||
|
||||
struct Source2 { val: i32 }
|
||||
#[task]
|
||||
impl Source2 {
|
||||
async fn run(&self) -> i32 { self.val }
|
||||
}
|
||||
|
||||
struct Compute;
|
||||
#[task]
|
||||
impl Compute {
|
||||
async fn run(a: &i32, b: &i32) -> i32 {
|
||||
a.wrapping_add(*b)
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let s1 = dag.add_task(Source1 { val: value1 });
|
||||
let s2 = dag.add_task(Source2 { val: value2 });
|
||||
let compute = dag.add_task(Compute).depends_on((&s1, &s2));
|
||||
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
if let Ok(()) = rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}) {
|
||||
// Verify output
|
||||
if let Ok(result) = dag.get(compute) {
|
||||
assert_eq!(result, value1.wrapping_add(value2));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test fan-out pattern
|
||||
{
|
||||
struct Source { val: i32 }
|
||||
#[task]
|
||||
impl Source {
|
||||
async fn run(&self) -> i32 { self.val }
|
||||
}
|
||||
|
||||
struct Double;
|
||||
#[task]
|
||||
impl Double {
|
||||
async fn run(input: &i32) -> i32 {
|
||||
input.wrapping_mul(2)
|
||||
}
|
||||
}
|
||||
|
||||
struct Triple;
|
||||
#[task]
|
||||
impl Triple {
|
||||
async fn run(input: &i32) -> i32 {
|
||||
input.wrapping_mul(3)
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let source = dag.add_task(Source { val: value3 });
|
||||
let double = dag.add_task(Double).depends_on(&source);
|
||||
let triple = dag.add_task(Triple).depends_on(&source);
|
||||
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
if let Ok(()) = rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}) {
|
||||
// Verify outputs
|
||||
if let (Ok(d), Ok(t)) = (dag.get(double), dag.get(triple)) {
|
||||
assert_eq!(d, value3.wrapping_mul(2));
|
||||
assert_eq!(t, value3.wrapping_mul(3));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test error-free execution with various data
|
||||
if data.len() >= 4 {
|
||||
struct DataTask { data: Vec<u8> }
|
||||
#[task]
|
||||
impl DataTask {
|
||||
async fn run(&self) -> Vec<u8> {
|
||||
self.data.clone()
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let task = dag.add_task(DataTask {
|
||||
data: data[..4.min(data.len())].to_vec()
|
||||
});
|
||||
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
if let Ok(()) = rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}) {
|
||||
if let Ok(result) = dag.get(task) {
|
||||
assert_eq!(result.len(), 4.min(data.len()));
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -1,115 +0,0 @@
|
||||
#![no_main]
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use dagx::{task, DagRunner};
|
||||
|
||||
fuzz_target!(|data: &[u8]| {
|
||||
// Only test with reasonable sizes
|
||||
if data.is_empty() || data.len() > 100 {
|
||||
return;
|
||||
}
|
||||
|
||||
// Test various dependency patterns based on input data
|
||||
let pattern = data[0] % 4;
|
||||
|
||||
struct SimpleTask { val: i32 }
|
||||
#[task]
|
||||
impl SimpleTask {
|
||||
async fn run(&self) -> i32 { self.val }
|
||||
}
|
||||
|
||||
struct DependentTask;
|
||||
#[task]
|
||||
impl DependentTask {
|
||||
async fn run(input: &i32) -> i32 {
|
||||
input.wrapping_add(1)
|
||||
}
|
||||
}
|
||||
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
|
||||
match pattern {
|
||||
0 => {
|
||||
// Linear dependency chain
|
||||
let dag = DagRunner::new();
|
||||
let t1 = dag.add_task(SimpleTask { val: data[0] as i32 });
|
||||
let t2 = dag.add_task(DependentTask).depends_on(&t1);
|
||||
let t3 = dag.add_task(DependentTask).depends_on(&t2);
|
||||
|
||||
if rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}).is_ok() {
|
||||
if let Ok(result) = dag.get(t3) {
|
||||
assert_eq!(result, (data[0] as i32).wrapping_add(2));
|
||||
}
|
||||
}
|
||||
}
|
||||
1 => {
|
||||
// Diamond pattern
|
||||
let dag = DagRunner::new();
|
||||
let source = dag.add_task(SimpleTask { val: data[0] as i32 });
|
||||
let left = dag.add_task(DependentTask).depends_on(&source);
|
||||
let right = dag.add_task(DependentTask).depends_on(&source);
|
||||
|
||||
struct MergeTask;
|
||||
#[task]
|
||||
impl MergeTask {
|
||||
async fn run(a: &i32, b: &i32) -> i32 {
|
||||
a.wrapping_add(*b)
|
||||
}
|
||||
}
|
||||
|
||||
let merge = dag.add_task(MergeTask).depends_on((&left, &right));
|
||||
|
||||
if rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}).is_ok() {
|
||||
if let Ok(result) = dag.get(merge) {
|
||||
let expected = (data[0] as i32).wrapping_add(1);
|
||||
assert_eq!(result, expected.wrapping_add(expected));
|
||||
}
|
||||
}
|
||||
}
|
||||
2 => {
|
||||
// Multiple independent sources
|
||||
let dag = DagRunner::new();
|
||||
let t1 = dag.add_task(SimpleTask { val: data[0] as i32 });
|
||||
let t2 = dag.add_task(SimpleTask {
|
||||
val: data.get(1).copied().unwrap_or(0) as i32
|
||||
});
|
||||
let t3 = dag.add_task(SimpleTask {
|
||||
val: data.get(2).copied().unwrap_or(0) as i32
|
||||
});
|
||||
|
||||
if rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}).is_ok() {
|
||||
if let (Ok(r1), Ok(r2), Ok(r3)) = (dag.get(t1), dag.get(t2), dag.get(t3)) {
|
||||
assert_eq!(r1, data[0] as i32);
|
||||
assert_eq!(r2, data.get(1).copied().unwrap_or(0) as i32);
|
||||
assert_eq!(r3, data.get(2).copied().unwrap_or(0) as i32);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Complex dependency graph
|
||||
let dag = DagRunner::new();
|
||||
let s1 = dag.add_task(SimpleTask { val: data[0] as i32 });
|
||||
let s2 = dag.add_task(SimpleTask {
|
||||
val: data.get(1).copied().unwrap_or(0) as i32
|
||||
});
|
||||
|
||||
let d1 = dag.add_task(DependentTask).depends_on(&s1);
|
||||
let d2 = dag.add_task(DependentTask).depends_on(&s2);
|
||||
let d3 = dag.add_task(DependentTask).depends_on(&d1);
|
||||
|
||||
if rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}).is_ok() {
|
||||
let _ = dag.get(d1);
|
||||
let _ = dag.get(d2);
|
||||
let _ = dag.get(d3);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -1,138 +0,0 @@
|
||||
#![no_main]
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use dagx::{task, DagRunner};
|
||||
|
||||
fuzz_target!(|data: &[u8]| {
|
||||
if data.is_empty() || data.len() > 100 {
|
||||
return;
|
||||
}
|
||||
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
|
||||
// Test type safety with various output types
|
||||
struct IntTask { val: i32 }
|
||||
#[task]
|
||||
impl IntTask {
|
||||
async fn run(&self) -> i32 { self.val }
|
||||
}
|
||||
|
||||
struct StringTask { val: String }
|
||||
#[task]
|
||||
impl StringTask {
|
||||
async fn run(&self) -> String { self.val.clone() }
|
||||
}
|
||||
|
||||
struct VecTask { data: Vec<u8> }
|
||||
#[task]
|
||||
impl VecTask {
|
||||
async fn run(&self) -> Vec<u8> { self.data.clone() }
|
||||
}
|
||||
|
||||
// Test correct type retrieval
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
let int_task = dag.add_task(IntTask { val: data[0] as i32 });
|
||||
|
||||
if let Ok(string_data) = std::str::from_utf8(data) {
|
||||
let string_task = dag.add_task(StringTask {
|
||||
val: string_data.to_string()
|
||||
});
|
||||
|
||||
if rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}).is_ok() {
|
||||
// Correct type should work
|
||||
let _ = dag.get(int_task);
|
||||
let _ = dag.get(string_task);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test Vec type
|
||||
{
|
||||
let dag = DagRunner::new();
|
||||
let vec_task = dag.add_task(VecTask {
|
||||
data: data.to_vec()
|
||||
});
|
||||
|
||||
if rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}).is_ok() {
|
||||
if let Ok(result) = dag.get(vec_task) {
|
||||
assert_eq!(result, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test custom struct types
|
||||
{
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
struct CustomData {
|
||||
value: i32,
|
||||
flag: bool,
|
||||
}
|
||||
|
||||
struct CustomTask { data: CustomData }
|
||||
#[task]
|
||||
impl CustomTask {
|
||||
async fn run(&self) -> CustomData {
|
||||
self.data.clone()
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let custom = CustomData {
|
||||
value: data[0] as i32,
|
||||
flag: data.get(1).map(|&b| b % 2 == 0).unwrap_or(false),
|
||||
};
|
||||
let task = dag.add_task(CustomTask { data: custom.clone() });
|
||||
|
||||
if rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}).is_ok() {
|
||||
if let Ok(result) = dag.get(task) {
|
||||
assert_eq!(result, custom);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test Option and Result types
|
||||
{
|
||||
struct OptionTask { val: Option<i32> }
|
||||
#[task]
|
||||
impl OptionTask {
|
||||
async fn run(&self) -> Option<i32> { self.val }
|
||||
}
|
||||
|
||||
struct ResultTask { val: Result<i32, String> }
|
||||
#[task]
|
||||
impl ResultTask {
|
||||
async fn run(&self) -> Result<i32, String> { self.val.clone() }
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let opt_task = dag.add_task(OptionTask {
|
||||
val: if data[0] % 2 == 0 {
|
||||
Some(data[0] as i32)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
let result_task = dag.add_task(ResultTask {
|
||||
val: if data.get(1).map(|&b| b % 2 == 0).unwrap_or(false) {
|
||||
Ok(data.get(1).copied().unwrap_or(0) as i32)
|
||||
} else {
|
||||
Err("error".to_string())
|
||||
}
|
||||
});
|
||||
|
||||
if rt.block_on(async {
|
||||
dag.run(|fut| { tokio::spawn(fut); }).await
|
||||
}).is_ok() {
|
||||
let _ = dag.get(opt_task);
|
||||
let _ = dag.get(result_task);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -1,257 +0,0 @@
|
||||
# justfile for dagx crate
|
||||
# High-level development and CI workflows
|
||||
|
||||
# =============================================================================
|
||||
# Configuration
|
||||
# =============================================================================
|
||||
|
||||
# Treat all warnings as errors
|
||||
rustflags := "-D warnings"
|
||||
|
||||
# Code coverage target percentage (fail if below this threshold)
|
||||
coverage_target := "80"
|
||||
|
||||
# Timeout for coverage analysis in seconds
|
||||
coverage_timeout := "600"
|
||||
|
||||
# Benchmark timeout in seconds
|
||||
bench_timeout := "300"
|
||||
|
||||
# =============================================================================
|
||||
|
||||
# Default recipe - show available recipes
|
||||
default:
|
||||
@just --list
|
||||
|
||||
# =============================================================================
|
||||
# Primary Development Workflows
|
||||
# =============================================================================
|
||||
|
||||
# Quick development loop: format + lint + fast tests
|
||||
dev:
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
echo "🔄 Running development checks..."
|
||||
cargo fmt
|
||||
RUSTFLAGS="{{rustflags}}" cargo clippy --all-targets --all-features &
|
||||
RUSTFLAGS="{{rustflags}}" cargo nextest run --lib --all-features &
|
||||
wait
|
||||
echo "✅ Development checks passed"
|
||||
|
||||
# Pre-commit checks: format-check + lint + typecheck
|
||||
check:
|
||||
@echo "Running format check..."
|
||||
cargo fmt -- --check
|
||||
@echo ""
|
||||
@echo "Running clippy..."
|
||||
RUSTFLAGS="{{rustflags}}" cargo clippy --all-targets --all-features
|
||||
RUSTFLAGS="{{rustflags}}" cargo clippy --all-targets --no-default-features
|
||||
@echo ""
|
||||
@echo "Verifying compilation..."
|
||||
RUSTFLAGS="{{rustflags}}" cargo test --lib --all-features --no-run
|
||||
RUSTFLAGS="{{rustflags}}" cargo test --lib --no-default-features --no-run
|
||||
@echo ""
|
||||
@echo "✅ All checks passed"
|
||||
|
||||
# Run all tests
|
||||
test:
|
||||
@echo "Running all tests..."
|
||||
RUSTFLAGS="{{rustflags}}" cargo nextest run --lib --all-features
|
||||
@echo "✅ All tests passed"
|
||||
|
||||
# Before pushing: check + test + coverage
|
||||
pre-push: check test coverage
|
||||
@echo ""
|
||||
@echo "✅ Pre-push checks passed - safe to push!"
|
||||
|
||||
# Comprehensive pre-release validation
|
||||
pre-release: check test coverage audit examples bench-check
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
# Verify documentation version matches crate version
|
||||
echo ""
|
||||
echo "📋 Verifying documentation version..."
|
||||
VERSION=$(grep '^version = ' Cargo.toml | head -1 | sed 's/.*"\(.*\)".*/\1/')
|
||||
MAJOR_MINOR=$(echo $VERSION | cut -d. -f1-2)
|
||||
|
||||
if ! grep -q "dagx = \"$MAJOR_MINOR\"" README.md; then
|
||||
echo "❌ ERROR: README.md should reference version $MAJOR_MINOR (crate is $VERSION)"
|
||||
echo " Update installation examples in README.md"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ Documentation version correct: $MAJOR_MINOR"
|
||||
|
||||
# Package verification
|
||||
echo ""
|
||||
echo "📦 Verifying package..."
|
||||
cargo publish --dry-run
|
||||
cargo package --list > /tmp/dagx-files.txt
|
||||
echo "✅ Package verified"
|
||||
|
||||
echo ""
|
||||
echo "✅ All pre-release checks passed!"
|
||||
echo " - Code formatted and linted"
|
||||
echo " - All tests passing"
|
||||
echo " - Code coverage ≥ {{coverage_target}}%"
|
||||
echo " - No security vulnerabilities"
|
||||
echo " - Dependencies verified"
|
||||
echo " - All examples run successfully"
|
||||
echo " - All benchmarks compile"
|
||||
echo " - Documentation version verified"
|
||||
echo " - Package ready for publication"
|
||||
echo ""
|
||||
echo "📦 Ready to publish! Run: cargo publish"
|
||||
|
||||
# =============================================================================
|
||||
# Quality & Security
|
||||
# =============================================================================
|
||||
|
||||
# Run code coverage analysis
|
||||
coverage:
|
||||
@echo "Running code coverage analysis (target: {{coverage_target}}%)..."
|
||||
cargo llvm-cov nextest --lib --html --output-dir coverage --fail-under-lines {{coverage_target}}
|
||||
@echo "Coverage report: coverage/html/index.html"
|
||||
|
||||
# Mutation testing with cargo-mutants
|
||||
mutants:
|
||||
@echo "Running mutation testing..."
|
||||
@echo "⚠️ This will use significant CPU and memory resources!"
|
||||
cargo mutants \
|
||||
--no-shuffle \
|
||||
--test-tool nextest \
|
||||
--jobs 3 \
|
||||
--all-features \
|
||||
-vV
|
||||
|
||||
# Security audit + dependency checks
|
||||
audit:
|
||||
@echo "🔒 Running security checks..."
|
||||
cargo audit
|
||||
@echo "✅ Security checks passed"
|
||||
|
||||
# Run Miri for undefined behavior detection
|
||||
miri:
|
||||
@echo "Running Miri tests for UB detection..."
|
||||
cargo +nightly miri test --lib
|
||||
|
||||
# =============================================================================
|
||||
# Benchmarks
|
||||
# =============================================================================
|
||||
|
||||
# Verify benchmarks compile
|
||||
bench-check:
|
||||
@echo "Verifying benchmarks compile..."
|
||||
cargo bench --no-run
|
||||
@echo "✅ All benchmarks compile"
|
||||
|
||||
# Run all benchmarks
|
||||
bench:
|
||||
cargo bench
|
||||
|
||||
# Run benchmarks and save as baseline
|
||||
bench-baseline:
|
||||
cargo bench -- --save-baseline baseline
|
||||
|
||||
# Compare benchmarks against baseline
|
||||
bench-compare:
|
||||
cargo bench -- --baseline baseline
|
||||
|
||||
# Run specific benchmark suite
|
||||
bench-suite suite:
|
||||
cargo bench --bench {{suite}}
|
||||
|
||||
# =============================================================================
|
||||
# Fuzzing
|
||||
# =============================================================================
|
||||
|
||||
# Run a single fuzz target for 60 seconds
|
||||
fuzz target:
|
||||
@echo "Fuzzing {{target}} for 60 seconds..."
|
||||
cargo +nightly fuzz run {{target}} -- -max_total_time=60
|
||||
|
||||
# Run all fuzz targets sequentially for quick smoke test
|
||||
fuzz-all:
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
echo "🔬 Running all fuzz targets..."
|
||||
for target in fuzz_dag_execution fuzz_dependencies fuzz_type_safety fuzz_concurrent; do
|
||||
echo ""
|
||||
echo "Running $target..."
|
||||
cargo +nightly fuzz run $target -- -max_total_time=30 -runs=10000
|
||||
done
|
||||
echo ""
|
||||
echo "✅ All fuzz targets completed"
|
||||
|
||||
# Run extended fuzzing session (for deeper testing)
|
||||
fuzz-extended target:
|
||||
@echo "Running extended fuzzing session for {{target}}..."
|
||||
@echo "⚠️ This will run for 1 hour. Press Ctrl+C to stop early."
|
||||
cargo +nightly fuzz run {{target}} -- -max_total_time=3600
|
||||
|
||||
# Build all fuzz targets
|
||||
fuzz-build:
|
||||
@echo "Building fuzz targets..."
|
||||
cargo +nightly fuzz build
|
||||
@echo "✅ All fuzz targets built"
|
||||
|
||||
# Minimize corpus for a fuzz target
|
||||
fuzz-cmin target:
|
||||
@echo "Minimizing corpus for {{target}}..."
|
||||
cargo +nightly fuzz cmin {{target}}
|
||||
|
||||
# Run property-based tests
|
||||
proptest:
|
||||
@echo "Running property-based tests..."
|
||||
cargo nextest run property_tests --all-features
|
||||
@echo "✅ Property tests passed"
|
||||
|
||||
# =============================================================================
|
||||
# Documentation & Utilities
|
||||
# =============================================================================
|
||||
|
||||
# Build documentation and open in browser
|
||||
doc:
|
||||
cargo doc --no-deps --all-features --open
|
||||
|
||||
# Format code
|
||||
fmt:
|
||||
cargo fmt
|
||||
|
||||
# Show project information
|
||||
info:
|
||||
@echo "dagx crate - Type-safe DAG execution framework for async Rust"
|
||||
@echo ""
|
||||
@echo "Rust: $(rustc --version)"
|
||||
@echo "Cargo: $(cargo --version)"
|
||||
@echo ""
|
||||
@echo "Dependencies:"
|
||||
@cargo tree --depth 1
|
||||
|
||||
# Run all examples
|
||||
examples:
|
||||
@echo "Running all examples..."
|
||||
cargo run --example 01_basic
|
||||
cargo run --example 02_fan_out
|
||||
cargo run --example 03_fan_in
|
||||
cargo run --example 04_parallel_computation
|
||||
cargo run --example circuit_breaker
|
||||
cargo run --example complex_dag
|
||||
cargo run --example conditional_workflow
|
||||
cargo run --example custom_types
|
||||
cargo run --example data_pipeline
|
||||
cargo run --example debug_tracing
|
||||
cargo run --example error_handling
|
||||
cargo run --example large_dag
|
||||
cargo run --example parallelism_proof
|
||||
cargo run --example retry_strategies
|
||||
cargo run --example timeout
|
||||
cargo run --example tracing_example
|
||||
@echo "✅ All examples completed successfully"
|
||||
|
||||
# Clean build artifacts
|
||||
clean:
|
||||
cargo clean
|
||||
rm -rf mutants.out/ mutants.out.old/ mutation_history.json
|
||||
rm -rf target/criterion
|
||||
rm -rf coverage/
|
||||
Whitespace-only changes.
@@ -1,5 +0,0 @@
|
||||
0.038627406s DEBUG cargo_mutants::copy_tree: /home/swaits/.local/share/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/cargo-mutants-25.3.1/src/copy_tree.rs:69: walk_builder=WalkBuilder { paths: ["/home/swaits/Code/dagx"], ig_builder: IgnoreBuilder { dir: "", overrides: Override(Gitignore { set: GlobSet { len: 0, strats: [] }, root: "", globs: [], num_ignores: 0, num_whitelists: 0, matches: None }), types: Types { defs: [], selections: [], has_selected: false, glob_to_selection: [], set: GlobSet { len: 0, strats: [] }, matches: Pool(Pool { stacks: [CacheLine(Mutex { data: [], poisoned: false, .. }), CacheLine(Mutex { data: [], poisoned: false, .. }), CacheLine(Mutex { data: [], poisoned: false, .. }), CacheLine(Mutex { data: [], poisoned: false, .. }), CacheLine(Mutex { data: [], poisoned: false, .. }), CacheLine(Mutex { data: [], poisoned: false, .. }), CacheLine(Mutex { data: [], poisoned: false, .. }), CacheLine(Mutex { data: [], poisoned: false, .. })], owner: 0, owner_val: UnsafeCell { .. } }) }, explicit_ignores: [], custom_ignore_filenames: [], opts: IgnoreOptions { hidden: false, ignore: false, parents: true, git_global: false, git_ignore: false, git_exclude: false, ignore_case_insensitive: false, require_git: true } }, max_depth: None, max_filesize: None, follow_links: false, threads: 0, skip: None }
|
||||
0.041495573s DEBUG cargo_mutants::copy_tree: /home/swaits/.local/share/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/cargo-mutants-25.3.1/src/copy_tree.rs:115: Copied source tree total_bytes=967004 total_files=153 temp_dir="/tmp/cargo-mutants-dagx-o2Mc3y.tmp"
|
||||
0.041758047s DEBUG cargo_mutants::lab: /home/swaits/.local/share/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/cargo-mutants-25.3.1/src/lab.rs:55: starting jobserver n_tasks=16
|
||||
0.041839390s DEBUG cargo_mutants::lab: /home/swaits/.local/share/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/cargo-mutants-25.3.1/src/lab.rs:267: test_packages=Explicit([Package { name: "dagx", version: "0.3.0", relative_dir: "", top_sources: ["src/lib.rs"] }])
|
||||
0.041870839s DEBUG run{phase=Build}: cargo_mutants::process: /home/swaits/.local/share/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/cargo-mutants-25.3.1/src/process.rs:81: start process quoted_argv=/home/swaits/.local/share/rustup/toolchains/1.90.0-x86_64-unknown-linux-gnu/bin/cargo nextest run --no-run --verbose --package=dagx@0.3.0 --all-features
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"cargo_mutants_version": "25.3.1",
|
||||
"start_time": "2025-10-14T16:44:19.225601093Z",
|
||||
"hostname": "swaits-fw13",
|
||||
"username": "swaits"
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
Whitespace-only changes.
File diff suppressed because it is too large.
Load diff
Whitespace-only changes.
Whitespace-only changes.
+36
-36
@@ -7,7 +7,7 @@ set -e
|
||||
# =============================================================================
|
||||
# CONFIGURATION
|
||||
# =============================================================================
|
||||
VERSION="0.3.0"
|
||||
VERSION="0.3.1"
|
||||
|
||||
echo "=================="
|
||||
echo "dagx Release Check"
|
||||
@@ -22,23 +22,23 @@ BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
check_pass() {
|
||||
echo -e "${GREEN}✓${NC} $1"
|
||||
echo -e "${GREEN}✓${NC} $1"
|
||||
}
|
||||
|
||||
check_fail() {
|
||||
echo -e "${RED}✗${NC} $1"
|
||||
exit 1
|
||||
echo -e "${RED}✗${NC} $1"
|
||||
exit 1
|
||||
}
|
||||
|
||||
check_warn() {
|
||||
echo -e "${YELLOW}⚠${NC} $1"
|
||||
echo -e "${YELLOW}⚠${NC} $1"
|
||||
}
|
||||
|
||||
section() {
|
||||
echo ""
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${BLUE}$1${NC}"
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo ""
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${BLUE}$1${NC}"
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
@@ -57,18 +57,18 @@ check_pass "Rust version compatible"
|
||||
echo ""
|
||||
echo "→ Checking for uncommitted changes..."
|
||||
if jj status 2>&1 | grep -q "Working copy changes:"; then
|
||||
check_warn "Uncommitted changes found - you'll need to commit before publishing"
|
||||
check_warn "Uncommitted changes found - you'll need to commit before publishing"
|
||||
else
|
||||
check_pass "No uncommitted changes"
|
||||
check_pass "No uncommitted changes"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "→ Checking for old 'dagrunner' references..."
|
||||
OLD_REFS=$(grep -ri "dagrunner" --exclude-dir=target --exclude-dir=.git --exclude-dir=.jj --exclude="*.lock" --exclude="release_check.sh" . 2>/dev/null | grep -v "DagRunner" | grep -v "dag-runner" | wc -l)
|
||||
if [ "$OLD_REFS" -gt 0 ]; then
|
||||
check_fail "Found $OLD_REFS old 'dagrunner' references (excluding DagRunner struct)"
|
||||
check_fail "Found $OLD_REFS old 'dagrunner' references (excluding DagRunner struct)"
|
||||
else
|
||||
check_pass "No old 'dagrunner' references found"
|
||||
check_pass "No old 'dagrunner' references found"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
@@ -90,9 +90,9 @@ echo ""
|
||||
echo "→ Checking for TODO/FIXME/XXX/HACK in source..."
|
||||
TODO_COUNT=$(grep -r "TODO\|FIXME\|XXX\|HACK" src/ dagx-macros/src/ 2>/dev/null | wc -l)
|
||||
if [ "$TODO_COUNT" -gt 0 ]; then
|
||||
check_warn "Found $TODO_COUNT TODO/FIXME/HACK comments in source"
|
||||
check_warn "Found $TODO_COUNT TODO/FIXME/HACK comments in source"
|
||||
else
|
||||
check_pass "No TODO/FIXME/HACK comments in source"
|
||||
check_pass "No TODO/FIXME/HACK comments in source"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
@@ -218,16 +218,16 @@ check_pass "LICENSE files present in both crates"
|
||||
section "COVERAGE CHECK"
|
||||
|
||||
echo "→ Running coverage analysis..."
|
||||
COVERAGE=$(cargo tarpaulin --lib --all-features --out Stdout 2>&1 | grep -oP '^\d+\.\d+%' | head -1 | sed 's/%//')
|
||||
COVERAGE=$(cargo tarpaulin --out Stdout 2>&1 | grep -oP '^\d+\.\d+%' | head -1 | sed 's/%//')
|
||||
if [ -z "$COVERAGE" ]; then
|
||||
check_warn "Could not determine coverage percentage"
|
||||
check_warn "Could not determine coverage percentage"
|
||||
else
|
||||
echo " Current coverage: ${COVERAGE}%"
|
||||
if (( $(echo "$COVERAGE < 80" | bc -l) )); then
|
||||
check_warn "Coverage is ${COVERAGE}% (below 80% target)"
|
||||
else
|
||||
check_pass "Coverage is ${COVERAGE}% (meets 80% target)"
|
||||
fi
|
||||
echo " Current coverage: ${COVERAGE}%"
|
||||
if (($(echo "$COVERAGE < 80" | bc -l))); then
|
||||
check_warn "Coverage is ${COVERAGE}% (below 80% target)"
|
||||
else
|
||||
check_pass "Coverage is ${COVERAGE}% (meets 80% target)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
@@ -248,10 +248,10 @@ check_pass "All examples build successfully"
|
||||
echo ""
|
||||
echo "→ Checking examples run without panics..."
|
||||
if [ -f ./scripts/run_examples.sh ]; then
|
||||
./scripts/run_examples.sh || check_fail "Some examples failed to run"
|
||||
check_pass "All examples run successfully"
|
||||
./scripts/run_examples.sh || check_fail "Some examples failed to run"
|
||||
check_pass "All examples run successfully"
|
||||
else
|
||||
check_warn "run_examples.sh not found, skipping runtime check"
|
||||
check_warn "run_examples.sh not found, skipping runtime check"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
@@ -297,9 +297,9 @@ echo "→ Verifying version consistency..."
|
||||
DAGX_VERSION=$(grep '^version = ' Cargo.toml | head -1 | cut -d'"' -f2)
|
||||
MACROS_VERSION=$(grep '^version = ' dagx-macros/Cargo.toml | head -1 | cut -d'"' -f2)
|
||||
if [ "$DAGX_VERSION" != "$MACROS_VERSION" ]; then
|
||||
check_fail "Version mismatch: dagx=$DAGX_VERSION, dagx-macros=$MACROS_VERSION"
|
||||
check_fail "Version mismatch: dagx=$DAGX_VERSION, dagx-macros=$MACROS_VERSION"
|
||||
else
|
||||
check_pass "Versions match: both are $DAGX_VERSION"
|
||||
check_pass "Versions match: both are $DAGX_VERSION"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
@@ -316,10 +316,10 @@ echo ""
|
||||
echo "→ Testing dagx package (dry-run with --allow-dirty)..."
|
||||
echo " Note: This will fail until dagx-macros is published to crates.io"
|
||||
if cargo publish --dry-run --allow-dirty 2>&1 | grep -q "failed to select a version for the requirement.*dagx-macros"; then
|
||||
check_warn "dagx package check shows dependency on unpublished dagx-macros v$VERSION (expected)"
|
||||
check_warn "dagx package check shows dependency on unpublished dagx-macros v$VERSION (expected)"
|
||||
else
|
||||
cargo publish --dry-run --allow-dirty || check_fail "dagx package check failed unexpectedly"
|
||||
check_pass "dagx package is ready (dagx-macros must be published first)"
|
||||
cargo publish --dry-run --allow-dirty || check_fail "dagx package check failed unexpectedly"
|
||||
check_pass "dagx package is ready (dagx-macros must be published first)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
@@ -332,9 +332,9 @@ echo "→ Checking package size..."
|
||||
PACKAGE_SIZE=$(cargo package --list --allow-dirty 2>/dev/null | wc -l)
|
||||
echo " Files to be packaged: $PACKAGE_SIZE"
|
||||
if [ "$PACKAGE_SIZE" -lt 5 ]; then
|
||||
check_fail "Suspiciously few files to package ($PACKAGE_SIZE)"
|
||||
check_fail "Suspiciously few files to package ($PACKAGE_SIZE)"
|
||||
else
|
||||
check_pass "Package contains $PACKAGE_SIZE files"
|
||||
check_pass "Package contains $PACKAGE_SIZE files"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
@@ -345,16 +345,16 @@ section "SECURITY CHECKS"
|
||||
|
||||
echo "→ Checking for sensitive data..."
|
||||
if grep -r "api[_-]key\|secret\|password\|token" src/ dagx-macros/src/ 2>/dev/null | grep -v "^Binary" | grep -v "token for a node" | grep -q .; then
|
||||
check_fail "Potential sensitive data found in source"
|
||||
check_fail "Potential sensitive data found in source"
|
||||
else
|
||||
check_pass "No sensitive data found"
|
||||
check_pass "No sensitive data found"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "→ Verifying .gitignore..."
|
||||
[ -f .gitignore ] || check_fail ".gitignore missing"
|
||||
if ! grep -qE "^/?target(/)?$" .gitignore; then
|
||||
check_fail ".gitignore doesn't exclude target/"
|
||||
check_fail ".gitignore doesn't exclude target/"
|
||||
fi
|
||||
check_pass ".gitignore looks good"
|
||||
|
||||
|
||||
+12
-22
@@ -4,22 +4,13 @@
|
||||
//! The type parameter tracks whether dependencies have been specified yet.
|
||||
|
||||
use crate::deps::DepsTuple;
|
||||
use crate::node::TaskExecute;
|
||||
use crate::runner::DagRunner;
|
||||
use crate::task::Task;
|
||||
use crate::types::{NodeId, TaskHandle};
|
||||
|
||||
#[cfg(feature = "tracing")]
|
||||
use tracing::debug;
|
||||
|
||||
/// Marker trait to identify the unit type at compile time.
|
||||
///
|
||||
/// This is used to enforce that tasks with non-unit inputs MUST call `.depends_on()`,
|
||||
/// while tasks with unit inputs can be used directly.
|
||||
pub trait IsUnitType {}
|
||||
|
||||
/// Only the unit type implements IsUnitType
|
||||
impl IsUnitType for () {}
|
||||
|
||||
/// Node builder that tracks dependency completion via type state.
|
||||
///
|
||||
/// A `TaskBuilder<Tk, Deps>` is returned from [`DagRunner::add_task`] and uses the type-state
|
||||
@@ -31,6 +22,7 @@ impl IsUnitType for () {}
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use dagx::{task, DagRunner, Task};
|
||||
/// #
|
||||
/// // Using tuple struct for simple constants
|
||||
/// struct Constant(i32);
|
||||
///
|
||||
@@ -62,20 +54,20 @@ impl IsUnitType for () {}
|
||||
/// let b = dag.add_task(Multiplier::new(2));
|
||||
/// // b is TaskBuilder<_, Pending> until we call depends_on()
|
||||
///
|
||||
/// let b = b.depends_on(&a);
|
||||
/// let b = b.depends_on(a);
|
||||
/// // Now b is a TaskHandle<i32>
|
||||
///
|
||||
/// dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
/// assert_eq!(dag.get(&b).unwrap(), 20);
|
||||
/// dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
/// assert_eq!(dag.get(b).unwrap(), 20);
|
||||
/// # };
|
||||
/// ```
|
||||
pub struct TaskBuilder<'a, Tk: TaskExecute, Deps> {
|
||||
pub struct TaskBuilder<'a, Tk: Task, Deps> {
|
||||
pub(crate) id: NodeId,
|
||||
pub(crate) dag: &'a DagRunner,
|
||||
pub(crate) _phantom: std::marker::PhantomData<(Tk, Deps)>,
|
||||
}
|
||||
|
||||
impl<'a, Tk: TaskExecute, Deps> TaskBuilder<'a, Tk, Deps> {
|
||||
impl<'a, Tk: Task, Deps> TaskBuilder<'a, Tk, Deps> {
|
||||
/// Provide all dependencies exactly once as a tuple.
|
||||
///
|
||||
/// The dependencies must match the task's `Input` type exactly:
|
||||
@@ -89,6 +81,7 @@ impl<'a, Tk: TaskExecute, Deps> TaskBuilder<'a, Tk, Deps> {
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use dagx::{task, DagRunner, Task};
|
||||
/// #
|
||||
/// // Tuple struct
|
||||
/// struct Value(i32);
|
||||
///
|
||||
@@ -116,18 +109,19 @@ impl<'a, Tk: TaskExecute, Deps> TaskBuilder<'a, Tk, Deps> {
|
||||
/// # async {
|
||||
/// let dag = DagRunner::new();
|
||||
///
|
||||
/// let x = dag.add_task(Value(2));
|
||||
/// let x = dag.add_task(Value(2)).into();
|
||||
/// let y = dag.add_task(Value(3));
|
||||
///
|
||||
/// // Single dependency
|
||||
/// let double = dag.add_task(Scale(2)).depends_on(&x);
|
||||
///
|
||||
/// // Multiple dependencies: tuple form
|
||||
/// let sum = dag.add_task(Add).depends_on((&x, &y));
|
||||
/// let sum = dag.add_task(Add).depends_on((&x, y));
|
||||
///
|
||||
/// dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
/// dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
/// # };
|
||||
/// ```
|
||||
#[allow(private_bounds)]
|
||||
pub fn depends_on<D>(self, deps: D) -> TaskHandle<Tk::Output>
|
||||
where
|
||||
D: DepsTuple<Tk::Input>,
|
||||
@@ -167,7 +161,3 @@ impl<'a, Tk: TaskExecute, Deps> TaskBuilder<'a, Tk, Deps> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
// Note: We cannot safely implement AsRef<TaskHandle<Tk::Output>> for TaskBuilder<Tk, Deps>
|
||||
// because Handle is Copy and returning a reference to a temporary would be unsound.
|
||||
// Instead, users should use From/Into conversions or call .into() explicitly.
|
||||
+8
-36
@@ -1,9 +1,5 @@
|
||||
//! Unit tests for builder module
|
||||
|
||||
// Allow tests to reference dagx:: paths (for proc macro generated code)
|
||||
extern crate self as dagx;
|
||||
|
||||
use crate::builder::IsUnitType;
|
||||
use crate::runner::DagRunner;
|
||||
use crate::types::TaskHandle;
|
||||
|
||||
@@ -55,49 +51,25 @@ fn test_task_builder_from_conversion() {
|
||||
let builder = dag.add_task(TestTask { value: 42 });
|
||||
|
||||
// Convert builder to TaskHandle
|
||||
let handle: TaskHandle<i32> = builder.into();
|
||||
let handle: TaskHandle<_> = builder.into();
|
||||
|
||||
// Check that the ID is preserved
|
||||
assert_eq!(handle.id.0, 0); // First task should have ID 0
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_unit_type_trait() {
|
||||
// Test that IsUnitType is correctly implemented for ()
|
||||
fn assert_is_unit_type<T: IsUnitType>() {}
|
||||
|
||||
// This should compile - () implements IsUnitType
|
||||
assert_is_unit_type::<()>();
|
||||
|
||||
// We can't test the negative case (non-unit types don't implement it)
|
||||
// because that would be a compile error
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_task_builder_depends_on_returns_handle() {
|
||||
init_tracing();
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let source = dag.add_task(TestTask { value: 10 });
|
||||
let dependent = dag.add_task(TestTaskWithInput).depends_on(&source);
|
||||
|
||||
// depends_on should return a TaskHandle
|
||||
let _handle: TaskHandle<i32> = dependent;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_task_builder_chain() {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Test chaining multiple tasks
|
||||
let t1 = dag.add_task(TestTask { value: 1 });
|
||||
let t2 = dag.add_task(TestTaskWithInput).depends_on(&t1);
|
||||
let t1 = dag.add_task(TestTask { value: 1 }).into();
|
||||
let t2 = dag.add_task(TestTaskWithInput).depends_on(t1);
|
||||
let t3 = dag.add_task(TestTaskWithInput).depends_on(t2);
|
||||
|
||||
// All should return TaskHandles
|
||||
let _: TaskHandle<i32> = t1.into();
|
||||
let _: TaskHandle<i32> = t2;
|
||||
let _: TaskHandle<i32> = t3;
|
||||
let _: TaskHandle<_> = t1;
|
||||
let _: TaskHandle<_> = t2;
|
||||
let _: TaskHandle<_> = t3;
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -118,10 +90,10 @@ fn test_multiple_dependencies() {
|
||||
let b = dag.add_task(TestTask { value: 20 });
|
||||
|
||||
// Test multiple dependencies
|
||||
let sum = dag.add_task(AddTask).depends_on((&a, &b));
|
||||
let sum = dag.add_task(AddTask).depends_on((a, b));
|
||||
|
||||
// Should return a TaskHandle
|
||||
let _: TaskHandle<i32> = sum;
|
||||
let _: TaskHandle<_> = sum;
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
//! # Compile-Time Cycle Prevention
|
||||
//!
|
||||
//! This module documents how the type system prevents cycles in the DAG.
|
||||
//! **Cycles are impossible via the public API** due to the type-state pattern.
|
||||
//!
|
||||
//! ## Why Cycles Can't Happen
|
||||
//!
|
||||
//! 1. [`crate::builder::TaskBuilder::depends_on`] **consumes** the builder (takes `self` by value)
|
||||
//! 2. It returns a [`crate::types::TaskHandle`] - an immutable, opaque reference
|
||||
//! 3. [`crate::types::TaskHandle`] has **no methods** to add or modify dependencies
|
||||
//! 4. You can only wire dependencies **once** per task
|
||||
//!
|
||||
//! ## The Catch-22
|
||||
//!
|
||||
//! To create a cycle A→B→A, you would need:
|
||||
//! - Task A to depend on Task B's handle (requires B finalized)
|
||||
//! - Task B to depend on Task A's handle (requires A finalized)
|
||||
//!
|
||||
//! But you can't finalize both before wiring them together!
|
||||
//!
|
||||
//! ## Proof by Compilation Failure
|
||||
//!
|
||||
//! The following examples demonstrate that cycles fail to compile:
|
||||
//!
|
||||
//! ### Proof 1: Self-Loop Prevention via Move Semantics
|
||||
//!
|
||||
//! ```compile_fail
|
||||
//! use dagx::*;
|
||||
//!
|
||||
//! struct TaskA;
|
||||
//! #[dagx::task]
|
||||
//! impl TaskA {
|
||||
//! async fn run(&self, input: &i32) -> i32 {
|
||||
//! input + 1
|
||||
//! }
|
||||
//! }
|
||||
//!
|
||||
//! async fn self_loop() {
|
||||
//! let dag = DagRunner::new();
|
||||
//! let a_builder = dag.add_task(TaskA);
|
||||
//!
|
||||
//! // ERROR: a_builder moved in the call to depends_on
|
||||
//! // Can't use 'a_builder' as both the builder and the dependency
|
||||
//! let a = a_builder.depends_on(&a_builder);
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! ### Proof 2: TaskHandle is Immutable
|
||||
//!
|
||||
//! ```compile_fail,E0599
|
||||
//! use dagx::*;
|
||||
//!
|
||||
//! struct NoInput;
|
||||
//! #[dagx::task]
|
||||
//! impl NoInput {
|
||||
//! async fn run(&self) -> i32 { 42 }
|
||||
//! }
|
||||
//!
|
||||
//! async fn unit_cycle() {
|
||||
//! let dag = DagRunner::new();
|
||||
//! let a_builder = dag.add_task(NoInput);
|
||||
//! let b_builder = dag.add_task(NoInput);
|
||||
//!
|
||||
//! // We can convert to handles (unit input = no dependencies required)
|
||||
//! let a_handle: TaskHandle<i32> = a_builder.into();
|
||||
//! let b_handle: TaskHandle<i32> = b_builder.into();
|
||||
//!
|
||||
//! // But TaskHandle has no methods to add dependencies!
|
||||
//! // ERROR: no method named `depends_on` found for type `TaskHandle<i32>`
|
||||
//! let _ = a_handle.depends_on(&b_handle);
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! ## Implications
|
||||
//!
|
||||
//! Because cycles are impossible at compile-time:
|
||||
//! - No runtime cycle detection is needed
|
||||
//! - Topological sort will always succeed
|
||||
//! - The DAG structure is guaranteed by the type system
|
||||
//!
|
||||
//! This is an example of **zero-cost abstraction** - the type system
|
||||
//! provides safety guarantees without runtime overhead.
|
||||
+54
-60
@@ -8,55 +8,63 @@
|
||||
//! use the provided implementations.
|
||||
|
||||
use crate::builder::TaskBuilder;
|
||||
use crate::node::TaskExecute;
|
||||
use crate::task::Task;
|
||||
use crate::types::{NodeId, TaskHandle};
|
||||
|
||||
trait IntoNodeRef {
|
||||
type Output;
|
||||
fn into_node_ref(self) -> NodeId;
|
||||
}
|
||||
|
||||
impl<T> IntoNodeRef for &TaskHandle<T> {
|
||||
type Output = T;
|
||||
fn into_node_ref(self) -> NodeId {
|
||||
self.id
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> IntoNodeRef for TaskHandle<T> {
|
||||
type Output = T;
|
||||
fn into_node_ref(self) -> NodeId {
|
||||
self.id
|
||||
}
|
||||
}
|
||||
|
||||
impl<'runner, Tk: Task, Deps> IntoNodeRef for TaskBuilder<'runner, Tk, Deps> {
|
||||
type Output = Tk::Output;
|
||||
|
||||
fn into_node_ref(self) -> NodeId {
|
||||
self.id
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) trait DepsTuple<Input> {
|
||||
fn to_node_ids(&self) -> Vec<NodeId>;
|
||||
fn to_node_ids(self) -> Vec<NodeId>;
|
||||
}
|
||||
|
||||
// Implementation for unit (no dependencies)
|
||||
impl DepsTuple<()> for () {
|
||||
fn to_node_ids(&self) -> Vec<NodeId> {
|
||||
fn to_node_ids(self) -> Vec<NodeId> {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
// Implementation for single dependency (all forms)
|
||||
impl<T> DepsTuple<T> for &TaskHandle<T> {
|
||||
fn to_node_ids(&self) -> Vec<NodeId> {
|
||||
vec![self.id]
|
||||
impl<T> DepsTuple<(T::Output,)> for T
|
||||
where
|
||||
T: IntoNodeRef,
|
||||
{
|
||||
fn to_node_ids(self) -> Vec<NodeId> {
|
||||
vec![self.into_node_ref()]
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> DepsTuple<T> for TaskHandle<T> {
|
||||
fn to_node_ids(&self) -> Vec<NodeId> {
|
||||
vec![self.id]
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> DepsTuple<T> for (&TaskHandle<T>,) {
|
||||
fn to_node_ids(&self) -> Vec<NodeId> {
|
||||
vec![self.0.id]
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> DepsTuple<T> for (TaskHandle<T>,) {
|
||||
fn to_node_ids(&self) -> Vec<NodeId> {
|
||||
vec![self.0.id]
|
||||
}
|
||||
}
|
||||
|
||||
// Support for &TaskBuilder
|
||||
impl<'a, Tk: TaskExecute, Deps> DepsTuple<Tk::Output> for &TaskBuilder<'a, Tk, Deps> {
|
||||
fn to_node_ids(&self) -> Vec<NodeId> {
|
||||
vec![self.id]
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, Tk: TaskExecute, Deps> DepsTuple<Tk::Output> for (&TaskBuilder<'a, Tk, Deps>,) {
|
||||
fn to_node_ids(&self) -> Vec<NodeId> {
|
||||
vec![self.0.id]
|
||||
impl<T> DepsTuple<T::Output> for (T,)
|
||||
where
|
||||
T: IntoNodeRef,
|
||||
{
|
||||
fn to_node_ids(self) -> Vec<NodeId> {
|
||||
vec![self.0.into_node_ref()]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,10 +73,6 @@ impl<'a, Tk: TaskExecute, Deps> DepsTuple<Tk::Output> for (&TaskBuilder<'a, Tk,
|
||||
/// This macro exists because Rust lacks variadic generics - we need separate implementations
|
||||
/// for each tuple size.
|
||||
///
|
||||
/// For each tuple size, generates two implementations:
|
||||
/// - `(&TaskHandle<A>, &TaskHandle<B>, ...)` → `DepsTuple<(A, B, ...)>`
|
||||
/// - `(&TaskBuilder<TkA, _>, &TaskBuilder<TkB, _>, ...)` → `DepsTuple<(TkA::Output, TkB::Output, ...)>`
|
||||
///
|
||||
/// This allows flexible dependency specification:
|
||||
/// ```ignore
|
||||
/// // Using handles
|
||||
@@ -78,37 +82,27 @@ impl<'a, Tk: TaskExecute, Deps> DepsTuple<Tk::Output> for (&TaskBuilder<'a, Tk,
|
||||
/// task.depends_on((&dag.add_task(TaskA), &dag.add_task(TaskB)))
|
||||
/// ```
|
||||
macro_rules! impl_deps_tuple {
|
||||
($($T:ident : $Tk:ident : $D:ident),+) => {
|
||||
// For &TaskHandle tuples
|
||||
impl<$($T),+> DepsTuple<($($T,)+)> for ($(&TaskHandle<$T>,)+) {
|
||||
($($T:ident),+) => {
|
||||
// For tuples of any combination of &TaskHandle, TaskHandle, and TaskBuilder
|
||||
impl<$($T: IntoNodeRef),+> DepsTuple<($($T::Output,)+)> for ($($T,)+) {
|
||||
#[allow(non_snake_case)]
|
||||
fn to_node_ids(&self) -> Vec<NodeId> {
|
||||
fn to_node_ids(self) -> Vec<NodeId> {
|
||||
let ($($T,)+) = self;
|
||||
vec![$($T.id,)+]
|
||||
}
|
||||
}
|
||||
|
||||
// For &TaskBuilder tuples
|
||||
impl<'a, $($Tk: TaskExecute, $D),+> DepsTuple<($($Tk::Output,)+)> for ($(&TaskBuilder<'a, $Tk, $D>,)+) {
|
||||
#[allow(non_snake_case)]
|
||||
fn to_node_ids(&self) -> Vec<NodeId> {
|
||||
let ($($T,)+) = self;
|
||||
vec![$($T.id,)+]
|
||||
vec![$($T.into_node_ref(),)+]
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Generate DepsTuple implementations for tuples of size 2-8.
|
||||
// Each invocation generates 2 implementations (TaskHandle refs and TaskBuilder refs).
|
||||
// Supporting up to 8 elements covers the vast majority of use cases.
|
||||
impl_deps_tuple!(A:TkA:DA, B:TkB:DB);
|
||||
impl_deps_tuple!(A:TkA:DA, B:TkB:DB, C:TkC:DC);
|
||||
impl_deps_tuple!(A:TkA:DA, B:TkB:DB, C:TkC:DC, D:TkD:DD);
|
||||
impl_deps_tuple!(A:TkA:DA, B:TkB:DB, C:TkC:DC, D:TkD:DD, E:TkE:DE);
|
||||
impl_deps_tuple!(A:TkA:DA, B:TkB:DB, C:TkC:DC, D:TkD:DD, E:TkE:DE, F:TkF:DF);
|
||||
impl_deps_tuple!(A:TkA:DA, B:TkB:DB, C:TkC:DC, D:TkD:DD, E:TkE:DE, F:TkF:DF, G:TkG:DG);
|
||||
impl_deps_tuple!(A:TkA:DA, B:TkB:DB, C:TkC:DC, D:TkD:DD, E:TkE:DE, F:TkF:DF, G:TkG:DG, H:TkH:DH);
|
||||
impl_deps_tuple!(T1, T2);
|
||||
impl_deps_tuple!(T1, T2, T3);
|
||||
impl_deps_tuple!(T1, T2, T3, T4);
|
||||
impl_deps_tuple!(T1, T2, T3, T4, T5);
|
||||
impl_deps_tuple!(T1, T2, T3, T4, T5, T6);
|
||||
impl_deps_tuple!(T1, T2, T3, T4, T5, T6, T7);
|
||||
impl_deps_tuple!(T1, T2, T3, T4, T5, T6, T7, T8);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
+16
-31
@@ -1,8 +1,5 @@
|
||||
//! Unit tests for deps module
|
||||
|
||||
// Allow tests to reference dagx:: paths (for proc macro generated code)
|
||||
extern crate self as dagx;
|
||||
|
||||
use crate::deps::DepsTuple;
|
||||
use crate::runner::DagRunner;
|
||||
use crate::types::{NodeId, TaskHandle};
|
||||
@@ -20,17 +17,6 @@ impl TestTask {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
struct TestTaskWithInput;
|
||||
|
||||
#[crate::task]
|
||||
impl TestTaskWithInput {
|
||||
#[cfg(not(tarpaulin_include))]
|
||||
async fn run(input: &i32) -> i32 {
|
||||
input * 2
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deps_tuple_unit() {
|
||||
// Test lines 20-22 in deps.rs - unit type implementation
|
||||
@@ -94,12 +80,13 @@ fn test_deps_tuple_task_builder_ref() {
|
||||
// Test lines 51-55 in deps.rs - &TaskBuilder
|
||||
let dag = DagRunner::new();
|
||||
let builder = dag.add_task(TestTask { value: 42 });
|
||||
let builder_id = builder.id;
|
||||
|
||||
// We need to test the DepsTuple implementation directly
|
||||
// The builder has an id field we can check
|
||||
let node_ids = (&builder).to_node_ids();
|
||||
let node_ids = builder.to_node_ids();
|
||||
assert_eq!(node_ids.len(), 1);
|
||||
assert_eq!(node_ids[0], builder.id);
|
||||
assert_eq!(node_ids[0], builder_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -107,10 +94,11 @@ fn test_deps_tuple_task_builder_tuple() {
|
||||
// Test lines 57-61 in deps.rs - (&TaskBuilder,)
|
||||
let dag = DagRunner::new();
|
||||
let builder = dag.add_task(TestTask { value: 100 });
|
||||
let builder_id = builder.id;
|
||||
|
||||
let node_ids = (&builder,).to_node_ids();
|
||||
let node_ids = (builder,).to_node_ids();
|
||||
assert_eq!(node_ids.len(), 1);
|
||||
assert_eq!(node_ids[0], builder.id);
|
||||
assert_eq!(node_ids[0], builder_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -148,21 +136,18 @@ fn test_deps_tuple_multiple_builders() {
|
||||
// Test macro-generated implementations for multiple builders
|
||||
let dag = DagRunner::new();
|
||||
let builder1 = dag.add_task(TestTask { value: 10 });
|
||||
let builder1_id = builder1.id;
|
||||
let builder2 = dag.add_task(TestTask { value: 20 });
|
||||
let builder2_id = builder2.id;
|
||||
let builder3 = dag.add_task(TestTask { value: 30 });
|
||||
|
||||
// Test 2-tuple of builders
|
||||
let node_ids = (&builder1, &builder2).to_node_ids();
|
||||
assert_eq!(node_ids.len(), 2);
|
||||
assert_eq!(node_ids[0], builder1.id);
|
||||
assert_eq!(node_ids[1], builder2.id);
|
||||
let builder3_id = builder3.id;
|
||||
|
||||
// Test 3-tuple of builders
|
||||
let node_ids = (&builder1, &builder2, &builder3).to_node_ids();
|
||||
let node_ids = (builder1, builder2, builder3).to_node_ids();
|
||||
assert_eq!(node_ids.len(), 3);
|
||||
assert_eq!(node_ids[0], builder1.id);
|
||||
assert_eq!(node_ids[1], builder2.id);
|
||||
assert_eq!(node_ids[2], builder3.id);
|
||||
assert_eq!(node_ids[0], builder1_id);
|
||||
assert_eq!(node_ids[1], builder2_id);
|
||||
assert_eq!(node_ids[2], builder3_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -180,7 +165,7 @@ fn test_deps_tuple_large_tuples() {
|
||||
let node_ids = (&handles[0], &handles[1], &handles[2], &handles[3]).to_node_ids();
|
||||
assert_eq!(node_ids.len(), 4);
|
||||
for (i, node_id) in node_ids.iter().enumerate().take(4) {
|
||||
assert_eq!(*node_id, NodeId(i));
|
||||
assert_eq!(*node_id, NodeId(i as u32));
|
||||
}
|
||||
|
||||
// Test 8-tuple
|
||||
@@ -197,7 +182,7 @@ fn test_deps_tuple_large_tuples() {
|
||||
.to_node_ids();
|
||||
assert_eq!(node_ids.len(), 8);
|
||||
for (i, node_id) in node_ids.iter().enumerate().take(8) {
|
||||
assert_eq!(*node_id, NodeId(i));
|
||||
assert_eq!(*node_id, NodeId(i as u32));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -211,7 +196,7 @@ fn test_deps_handle_ref_explicit() {
|
||||
|
||||
// Create a reference and explicitly call to_node_ids
|
||||
let handle_ref = &handle;
|
||||
let node_ids = DepsTuple::<i32>::to_node_ids(handle_ref);
|
||||
let node_ids = DepsTuple::<(i32,)>::to_node_ids(handle_ref);
|
||||
|
||||
assert_eq!(node_ids.len(), 1);
|
||||
assert_eq!(node_ids[0], NodeId(77));
|
||||
|
||||
+5
-7
@@ -4,21 +4,19 @@
|
||||
|
||||
/// Errors that can occur during DAG construction and execution
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
pub enum DagError {
|
||||
/// Invalid dependency: task does not exist
|
||||
InvalidDependency { task_id: usize },
|
||||
InvalidDependency { task_id: u32 },
|
||||
/// Type mismatch in task dependencies
|
||||
TypeMismatch {
|
||||
expected: &'static str,
|
||||
found: &'static str,
|
||||
},
|
||||
/// Task panicked during execution
|
||||
TaskPanicked {
|
||||
task_id: usize,
|
||||
panic_message: String,
|
||||
},
|
||||
TaskPanicked { task_id: u32, panic_message: String },
|
||||
/// Result not found for task
|
||||
ResultNotFound { task_id: usize },
|
||||
ResultNotFound { task_id: u32 },
|
||||
/// Attempted to run the DAG while it's already running
|
||||
ConcurrentExecution,
|
||||
}
|
||||
@@ -63,7 +61,7 @@ impl std::fmt::Display for DagError {
|
||||
f,
|
||||
"Result not found for task #{}.\n\
|
||||
\n\
|
||||
Call dag.run() before accessing results with get().",
|
||||
Call dag.run(|fut| async move {{ tokio::spawn(fut).await.unwrap() }}).",
|
||||
task_id
|
||||
)
|
||||
}
|
||||
|
||||
+2
-51
@@ -1,6 +1,6 @@
|
||||
//! Unit tests for error module
|
||||
|
||||
use crate::error::{DagError, DagResult};
|
||||
use crate::error::DagError;
|
||||
|
||||
#[test]
|
||||
fn test_dag_error_display_invalid_dependency() {
|
||||
@@ -51,7 +51,7 @@ fn test_dag_error_display_result_not_found() {
|
||||
|
||||
assert!(display.contains("Result not found"));
|
||||
assert!(display.contains("task #7"));
|
||||
assert!(display.contains("Call dag.run()"));
|
||||
assert!(display.contains("Call dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -74,52 +74,3 @@ fn test_dag_error_std_error_impl() {
|
||||
let _ = err_ref.to_string();
|
||||
assert!(err_ref.source().is_none()); // DagError doesn't chain errors
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dag_error_equality() {
|
||||
let err1 = DagError::InvalidDependency { task_id: 42 };
|
||||
let err2 = DagError::InvalidDependency { task_id: 42 };
|
||||
let err3 = DagError::InvalidDependency { task_id: 99 };
|
||||
|
||||
assert_eq!(err1, err2);
|
||||
assert_ne!(err1, err3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dag_error_clone() {
|
||||
let err = DagError::TaskPanicked {
|
||||
task_id: 10,
|
||||
panic_message: "test panic".to_string(),
|
||||
};
|
||||
|
||||
let cloned = err.clone();
|
||||
assert_eq!(err, cloned);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dag_error_debug() {
|
||||
let err = DagError::TypeMismatch {
|
||||
expected: "i32",
|
||||
found: "bool",
|
||||
};
|
||||
|
||||
let debug_str = format!("{:?}", err);
|
||||
assert!(debug_str.contains("TypeMismatch"));
|
||||
assert!(debug_str.contains("expected"));
|
||||
assert!(debug_str.contains("found"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dag_result_type_alias() {
|
||||
// Test that DagResult works as expected
|
||||
fn returns_dag_result() -> DagResult<i32> {
|
||||
Ok(42)
|
||||
}
|
||||
|
||||
fn returns_dag_error() -> DagResult<String> {
|
||||
Err(DagError::ResultNotFound { task_id: 1 })
|
||||
}
|
||||
|
||||
assert!(returns_dag_result().is_ok());
|
||||
assert!(returns_dag_error().is_err());
|
||||
}
|
||||
+77
-35
@@ -6,10 +6,13 @@
|
||||
//! # Features
|
||||
//!
|
||||
//! - **Compile-time cycle prevention**: The type system makes cycles **impossible**—no runtime
|
||||
//! cycle detection needed! See [`cycle_prevention`] module for detailed explanation and proof.
|
||||
//! cycle detection needed! See [the repository documentation](https://github.com/swaits/dagx/blob/main/docs/CYCLE_PREVENTION.md) for a detailed explanation.
|
||||
//! - **Compile-time type safety**: Dependencies are validated at compile time through the type
|
||||
//! system. The public API is fully type-safe with no runtime type errors. Internal execution
|
||||
//! uses type erasure for heterogeneous task storage, but this is never exposed to users.
|
||||
//! - **Works with ANY type**: Custom types work automatically with the `#[task]` macro—just
|
||||
//! implement `Send + Sync`, no trait implementations needed! The macro generates
|
||||
//! type-specific extraction logic for seamless integration.
|
||||
//! - **Runtime-agnostic**: Works with any async runtime (Tokio, async-std, smol, Embassy, etc.)
|
||||
//! - **Type-state pattern**: The API guides you with compile-time errors if you wire dependencies
|
||||
//! incorrectly. This is what prevents cycles at compile time.
|
||||
@@ -20,7 +23,7 @@
|
||||
//! # Quick Start
|
||||
//!
|
||||
//! ```no_run
|
||||
//! use dagx::{task, DagRunner, Task};
|
||||
//! use dagx::{task, DagRunner, Task, TaskHandle};
|
||||
//!
|
||||
//! // Source task with read-only state (tuple struct)
|
||||
//! struct Value(i32);
|
||||
@@ -43,13 +46,13 @@
|
||||
//!
|
||||
//! // Add source tasks
|
||||
//! let x = dag.add_task(Value(2));
|
||||
//! let y = dag.add_task(Value(3));
|
||||
//! let y: TaskHandle<_> = dag.add_task(Value(3)).into(); // Allows y to be reused as a dependency
|
||||
//!
|
||||
//! // Add task with dependencies
|
||||
//! let sum = dag.add_task(Add).depends_on((&x, &y));
|
||||
//! let sum = dag.add_task(Add).depends_on((x, &y));
|
||||
//!
|
||||
//! // Execute and retrieve results
|
||||
//! dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
//! dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
//! assert_eq!(dag.get(sum).unwrap(), 5);
|
||||
//! # };
|
||||
//! ```
|
||||
@@ -61,7 +64,7 @@
|
||||
//! dagx automatically optimizes execution for sequential workloads using an inline fast-path.
|
||||
//! When a layer contains only a single task (common in deep chains and linear pipelines),
|
||||
//! that task executes inline rather than being spawned. This eliminates spawning overhead,
|
||||
//! context switching, and channel creation.
|
||||
//! and context switching.
|
||||
//!
|
||||
//! **Panic handling**: To maintain consistent behavior between inline and spawned execution,
|
||||
//! panics in inline tasks are caught using `FutureExt::catch_unwind()` and converted to
|
||||
@@ -113,7 +116,6 @@
|
||||
//! use dagx::{task, Task};
|
||||
//!
|
||||
//! // Define a struct to hold your multiple values
|
||||
//! #[derive(Clone)]
|
||||
//! struct UserData {
|
||||
//! name: String,
|
||||
//! age: i32,
|
||||
@@ -252,7 +254,7 @@
|
||||
//! # async {
|
||||
//! # let dag = DagRunner::new();
|
||||
//! # let source = dag.add_task(Value(42));
|
||||
//! let doubled = dag.add_task(Double).depends_on(&source);
|
||||
//! let doubled = dag.add_task(Double).depends_on(source);
|
||||
//! # };
|
||||
//! ```
|
||||
//!
|
||||
@@ -276,7 +278,7 @@
|
||||
//! # let dag = DagRunner::new();
|
||||
//! # let x = dag.add_task(Value(2));
|
||||
//! # let y = dag.add_task(Value(3));
|
||||
//! let sum = dag.add_task(Add).depends_on((&x, &y));
|
||||
//! let sum = dag.add_task(Add).depends_on((x, y));
|
||||
//! # };
|
||||
//! ```
|
||||
//!
|
||||
@@ -289,6 +291,7 @@
|
||||
//! ```no_run
|
||||
//! use dagx::{task, DagRunner, Task};
|
||||
//!
|
||||
//!
|
||||
//! // Source task (tuple struct)
|
||||
//! struct Value(i32);
|
||||
//!
|
||||
@@ -316,11 +319,11 @@
|
||||
//! # async {
|
||||
//! let dag = DagRunner::new();
|
||||
//!
|
||||
//! let base = dag.add_task(Value(10));
|
||||
//! let base = dag.add_task(Value(10)).into();
|
||||
//! let plus1 = dag.add_task(Add(1)).depends_on(&base);
|
||||
//! let times2 = dag.add_task(Scale(2)).depends_on(&base);
|
||||
//!
|
||||
//! dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
//! dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
//!
|
||||
//! assert_eq!(dag.get(plus1).unwrap(), 11);
|
||||
//! assert_eq!(dag.get(times2).unwrap(), 20);
|
||||
@@ -334,6 +337,7 @@
|
||||
//! ```no_run
|
||||
//! use dagx::{task, DagRunner, Task};
|
||||
//!
|
||||
//!
|
||||
//! // Source task for String (tuple struct)
|
||||
//! struct Name(String);
|
||||
//!
|
||||
@@ -374,9 +378,9 @@
|
||||
//! let name = dag.add_task(Name("Alice".to_string()));
|
||||
//! let age = dag.add_task(Age(30));
|
||||
//! let active = dag.add_task(Active(true));
|
||||
//! let result = dag.add_task(FormatUser).depends_on((&name, &age, &active));
|
||||
//! let result = dag.add_task(FormatUser).depends_on((name, age, active));
|
||||
//!
|
||||
//! dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
//! dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
//!
|
||||
//! assert_eq!(dag.get(result).unwrap(), "User: Alice, Age: 30, Active: true");
|
||||
//! # };
|
||||
@@ -389,6 +393,7 @@
|
||||
//! ```no_run
|
||||
//! use dagx::{task, DagRunner, Task};
|
||||
//!
|
||||
//!
|
||||
//! // Source task (tuple struct)
|
||||
//! struct Value(i32);
|
||||
//!
|
||||
@@ -418,22 +423,67 @@
|
||||
//!
|
||||
//! // Layer 1: Sources
|
||||
//! let x = dag.add_task(Value(2));
|
||||
//! let y = dag.add_task(Value(3));
|
||||
//! let y = dag.add_task(Value(3)).into();
|
||||
//! let z = dag.add_task(Value(5));
|
||||
//!
|
||||
//! // Layer 2: Intermediate computations
|
||||
//! let sum_xy = dag.add_task(Add).depends_on((&x, &y)); // 2 + 3 = 5
|
||||
//! let prod_yz = dag.add_task(Multiply).depends_on((&y, &z)); // 3 * 5 = 15
|
||||
//! let sum_xy = dag.add_task(Add).depends_on((x, &y)); // 2 + 3 = 5
|
||||
//! let prod_yz = dag.add_task(Multiply).depends_on((&y, z)); // 3 * 5 = 15
|
||||
//!
|
||||
//! // Layer 3: Final result
|
||||
//! let total = dag.add_task(Add).depends_on((&sum_xy, &prod_yz)); // 5 + 15 = 20
|
||||
//!
|
||||
//! dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
//! dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
//!
|
||||
//! assert_eq!(dag.get(total).unwrap(), 20);
|
||||
//! # };
|
||||
//! ```
|
||||
//!
|
||||
//! ## Custom Types
|
||||
//!
|
||||
//! **dagx works with ANY type automatically!** As long as your type is `Send + Sync + 'static`,
|
||||
//! the `#[task]` macro generates the necessary extraction logic:
|
||||
//!
|
||||
//! ```no_run
|
||||
//! # use dagx::{task, DagRunner, Task};
|
||||
//! #
|
||||
//!
|
||||
//! struct User {
|
||||
//! name: String,
|
||||
//! age: u32,
|
||||
//! }
|
||||
//!
|
||||
//! struct CreateUser;
|
||||
//! #[task]
|
||||
//! impl CreateUser {
|
||||
//! async fn run() -> User {
|
||||
//! User { name: "Alice".to_string(), age: 30 }
|
||||
//! }
|
||||
//! }
|
||||
//!
|
||||
//! struct FormatUser;
|
||||
//! #[task]
|
||||
//! impl FormatUser {
|
||||
//! async fn run(user: &User) -> String {
|
||||
//! format!("{} is {} years old", user.name, user.age)
|
||||
//! }
|
||||
//! }
|
||||
//!
|
||||
//! # async {
|
||||
//! let dag = DagRunner::new();
|
||||
//!
|
||||
//! let user = dag.add_task(CreateUser);
|
||||
//! let formatted = dag.add_task(FormatUser).depends_on(user);
|
||||
//!
|
||||
//! dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
//!
|
||||
//! assert_eq!(dag.get(formatted).unwrap(), "Alice is 30 years old");
|
||||
//! # };
|
||||
//! ```
|
||||
//!
|
||||
//! **No trait implementations needed!** Works with nested structs, collections, enums, and any
|
||||
//! other custom type.
|
||||
//!
|
||||
//! # Runtime Agnostic
|
||||
//!
|
||||
//! dagx works with any async runtime. The library has been tested with:
|
||||
@@ -481,6 +531,7 @@
|
||||
//!
|
||||
//! ```no_run
|
||||
//! # use dagx::{task, DagRunner, Task};
|
||||
//! #
|
||||
//! # struct Value(i32);
|
||||
//! # #[task]
|
||||
//! # impl Value {
|
||||
@@ -490,11 +541,11 @@
|
||||
//! # let dag = DagRunner::new();
|
||||
//! # let node = dag.add_task(Value(42));
|
||||
//! // Simple approach with .unwrap()
|
||||
//! dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
//! dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
//! let result = dag.get(node).unwrap();
|
||||
//!
|
||||
//! // Or handle errors explicitly
|
||||
//! match dag.run(|fut| { tokio::spawn(fut); }).await {
|
||||
//! match dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await {
|
||||
//! Ok(_) => println!("DAG executed successfully"),
|
||||
//! Err(e) => eprintln!("DAG execution failed: {}", e),
|
||||
//! }
|
||||
@@ -577,7 +628,8 @@
|
||||
//! You output `T`, the framework handles the Arc wrapping:
|
||||
//!
|
||||
//! ```
|
||||
//! use dagx::{task, DagRunner, Task};
|
||||
//! # use dagx::{task, DagRunner, Task};
|
||||
//! #
|
||||
//!
|
||||
//! // ✅ CORRECT: Just output Vec<String>, framework wraps in Arc internally
|
||||
//! struct FetchData;
|
||||
@@ -589,7 +641,7 @@
|
||||
//! }
|
||||
//!
|
||||
//! // Downstream tasks receive &Vec<String> as normal
|
||||
//! // Behind the scenes: Arc<Vec<String>> is cloned cheaply, then inner Vec extracted
|
||||
//! // Behind the scenes: Arc<Vec<String>> is cloned cheaply, then a reference to the inner Vec is extracted
|
||||
//! struct ProcessData;
|
||||
//! #[task]
|
||||
//! impl ProcessData {
|
||||
@@ -600,14 +652,14 @@
|
||||
//!
|
||||
//! # async {
|
||||
//! let dag = DagRunner::new();
|
||||
//! let data = dag.add_task(FetchData);
|
||||
//! let data = dag.add_task(FetchData).into();
|
||||
//!
|
||||
//! // All three tasks get efficient Arc-wrapped sharing automatically
|
||||
//! let task1 = dag.add_task(ProcessData).depends_on(&data);
|
||||
//! let task2 = dag.add_task(ProcessData).depends_on(&data);
|
||||
//! let task3 = dag.add_task(ProcessData).depends_on(&data);
|
||||
//!
|
||||
//! dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
//! dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
//! # };
|
||||
//! ```
|
||||
//!
|
||||
@@ -711,16 +763,11 @@
|
||||
//!
|
||||
//! This follows the same zero-cost pattern used by Tokio, Hyper, and other performance-critical
|
||||
//! Rust async libraries.
|
||||
//!
|
||||
//! See [`examples/tracing_example.rs`](https://github.com/swaits/dagx/blob/main/examples/tracing_example.rs)
|
||||
//! for a complete working example.
|
||||
//!
|
||||
|
||||
#![allow(private_bounds, private_interfaces)]
|
||||
extern crate self as dagx;
|
||||
|
||||
// Module declarations
|
||||
mod builder;
|
||||
pub mod cycle_prevention;
|
||||
mod deps;
|
||||
mod error;
|
||||
mod node;
|
||||
@@ -731,14 +778,9 @@ mod types;
|
||||
// Public re-exports
|
||||
pub use builder::TaskBuilder;
|
||||
pub use error::{DagError, DagResult};
|
||||
pub use node::TaskExecute; // For macro-generated trait implementations
|
||||
pub use runner::DagRunner;
|
||||
pub use task::Task;
|
||||
pub use task::{Task, TaskInput};
|
||||
pub use types::{Pending, TaskHandle};
|
||||
|
||||
// Re-export the procedural macro
|
||||
pub use dagx_macros::task;
|
||||
|
||||
// Internal testing support - not part of the public API
|
||||
#[doc(hidden)]
|
||||
pub use task::task_fn;
|
||||
+30
-94
@@ -12,145 +12,81 @@ use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::error::DagError;
|
||||
use crate::types::NodeId;
|
||||
|
||||
/// Type-erased listeners (for receiving inputs from dependencies)
|
||||
type ErasedListeners = Vec<Box<dyn Any + Send>>;
|
||||
use crate::task::{Task, TaskInput};
|
||||
|
||||
/// Type-erased async execution result (Arc-wrapped for efficient fanout)
|
||||
type ExecuteFuture =
|
||||
Pin<Box<dyn Future<Output = Result<Arc<dyn Any + Send + Sync>, DagError>> + Send>>;
|
||||
|
||||
/// Type-erased async execution result for TaskExecute trait (returns String errors)
|
||||
type TaskExecuteFuture =
|
||||
Pin<Box<dyn Future<Output = Result<Arc<dyn Any + Send + Sync>, String>> + Send>>;
|
||||
|
||||
/// Internal trait for executing heterogeneous tasks.
|
||||
///
|
||||
/// This trait provides type erasure necessary for storing different task types in a single
|
||||
/// collection. While the public API (Task, Handle, Node) is fully type-safe, internally we
|
||||
/// need dynamic dispatch to execute tasks with different input/output types.
|
||||
///
|
||||
/// This is the ONLY place where type erasure occurs - announcements are created with full type
|
||||
/// This is the ONLY place where type erasure occurs - tasks are created with full type
|
||||
/// information and only erased at this trait boundary.
|
||||
///
|
||||
/// This is an implementation detail and not part of the public API.
|
||||
pub(crate) trait ExecutableNode: Send {
|
||||
/// Create a type-erased announcement for this node's output.
|
||||
///
|
||||
/// Returns a tuple of (Announcer, Announcement) both type-erased.
|
||||
/// The Announcer will be used by this node to broadcast its output.
|
||||
/// The Announcement can be used to create listeners for dependents.
|
||||
fn create_announcement(&self) -> (Box<dyn Any + Send>, Box<dyn Any + Send>);
|
||||
|
||||
/// Create a type-erased listener from an announcement.
|
||||
///
|
||||
/// This allows dependents to listen to this node's output.
|
||||
fn create_listener(&self, announcement: &dyn Any) -> Box<dyn Any + Send>;
|
||||
|
||||
/// Execute the task with type-erased input listeners and an output announcer.
|
||||
/// Execute the task with type-erased inputs and output
|
||||
///
|
||||
/// The TypedNode implementation knows the concrete types and can safely downcast.
|
||||
/// Consumes the node and returns the output value.
|
||||
fn execute_with_listeners(
|
||||
fn execute_with_deps(
|
||||
self: Box<Self>,
|
||||
listeners: ErasedListeners,
|
||||
announcer: Box<dyn Any + Send>,
|
||||
dependencies: Vec<Arc<dyn Any + Send + Sync>>,
|
||||
) -> ExecuteFuture;
|
||||
}
|
||||
|
||||
/// Fully-typed node storage for a single task.
|
||||
///
|
||||
/// Stores a task that has an `execute()` method (generated by the #[task] macro).
|
||||
/// Implements ExecutableNode for type erasure, allowing heterogeneous tasks to be
|
||||
/// stored in a single Vec.
|
||||
/// Stores a task of type T. Implements ExecutableNode for type erasure,
|
||||
/// allowing heterogeneous tasks to be stored in a single Vec.
|
||||
///
|
||||
/// # Ownership Model
|
||||
///
|
||||
/// The task is owned directly (no Mutex needed) and consumed during execution.
|
||||
/// Each task executes exactly once, taking ownership and producing an output.
|
||||
///
|
||||
/// # Announcement Broadcasting
|
||||
///
|
||||
/// Outputs are communicated via announcement channels for inter-task communication.
|
||||
/// Each task has ONE announcer that broadcasts to multiple listeners.
|
||||
/// The output is also returned from execute_with_listeners for central storage.
|
||||
pub(crate) struct TypedNode<T> {
|
||||
pub(crate) id: NodeId,
|
||||
/// The output is also returned from execute_with_deps for central storage.
|
||||
pub(crate) struct TypedNode<T: Task> {
|
||||
pub(crate) task: T,
|
||||
pub(crate) _output_type: std::marker::PhantomData<fn() -> ()>,
|
||||
}
|
||||
|
||||
impl<T> TypedNode<T> {
|
||||
pub(crate) fn new(id: NodeId, task: T) -> Self {
|
||||
Self {
|
||||
id,
|
||||
task,
|
||||
_output_type: std::marker::PhantomData,
|
||||
}
|
||||
impl<T: Task + 'static> TypedNode<T> {
|
||||
pub(crate) fn new(task: T) -> Self {
|
||||
Self { task }
|
||||
}
|
||||
}
|
||||
|
||||
// Implementation for tasks created with the #[task] macro
|
||||
// These tasks have an execute() method generated by the macro
|
||||
impl<T: 'static + Send> ExecutableNode for TypedNode<T>
|
||||
impl<T: Task + 'static> ExecutableNode for TypedNode<T>
|
||||
where
|
||||
T: TaskExecute,
|
||||
T::Input: 'static,
|
||||
T::Output: 'static,
|
||||
{
|
||||
fn create_announcement(&self) -> (Box<dyn Any + Send>, Box<dyn Any + Send>) {
|
||||
T::create_announcement_erased()
|
||||
}
|
||||
|
||||
fn create_listener(&self, announcement: &dyn Any) -> Box<dyn Any + Send> {
|
||||
T::create_listener_erased(announcement)
|
||||
}
|
||||
|
||||
fn execute_with_listeners(
|
||||
fn execute_with_deps(
|
||||
self: Box<Self>,
|
||||
listeners: ErasedListeners,
|
||||
announcer: Box<dyn Any + Send>,
|
||||
dependencies: Vec<Arc<dyn Any + Send + Sync>>,
|
||||
) -> ExecuteFuture {
|
||||
Box::pin(async move {
|
||||
// Call the task's execute() method (generated by the #[task] macro)
|
||||
// This handles listener extraction, calling run(), and announcing
|
||||
let result = self.task.execute(listeners, announcer).await;
|
||||
// Use the task's run method which has inline extraction logic
|
||||
// generated by the #[task] macro. This allows ANY type to work without
|
||||
// requiring ExtractInput trait implementations.
|
||||
let output = self.task.run(TaskInput::new(dependencies.iter())).await;
|
||||
|
||||
match result {
|
||||
Ok(output_arc) => Ok(output_arc),
|
||||
Err(err_msg) => {
|
||||
// Execution failed - convert error message to DagError
|
||||
eprintln!("Task execution failed: {}", err_msg);
|
||||
Err(DagError::InvalidDependency { task_id: self.id.0 })
|
||||
}
|
||||
}
|
||||
// Arc Wrapping Strategy:
|
||||
// Wrap output in Arc ONCE for efficient fan-out sharing.
|
||||
// For 1:N dependencies, this enables O(1) sharing (Arc clone = atomic increment)
|
||||
// instead of O(N) cloning (N deep copies of output data).
|
||||
// Downstream tasks receive &T after extraction from Arc.
|
||||
let arc_output = Arc::new(output);
|
||||
|
||||
// Return Arc-wrapped output (for storage by runner)
|
||||
Ok(arc_output as Arc<dyn Any + Send + Sync>)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait for tasks that have an execute() method (generated by #[task] macro).
|
||||
///
|
||||
/// This trait is automatically implemented by the macro and should not be
|
||||
/// implemented manually.
|
||||
pub trait TaskExecute: Send {
|
||||
/// Input type as a tuple of parameter types (for dependency checking)
|
||||
/// - `()` for no parameters
|
||||
/// - `T` for single parameter
|
||||
/// - `(A, B, C, ...)` for multiple parameters
|
||||
type Input: Send;
|
||||
|
||||
/// Output type of the task
|
||||
type Output: Send + Sync + Clone;
|
||||
|
||||
fn execute(
|
||||
self,
|
||||
listeners: Vec<Box<dyn Any + Send>>,
|
||||
announcer: Box<dyn Any + Send>,
|
||||
) -> TaskExecuteFuture;
|
||||
|
||||
fn create_announcement_erased() -> (Box<dyn Any + Send>, Box<dyn Any + Send>);
|
||||
|
||||
fn create_listener_erased(announcement: &dyn Any) -> Box<dyn Any + Send>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
+26
-147
@@ -1,10 +1,8 @@
|
||||
//! Unit tests for node module
|
||||
|
||||
// Allow tests to reference dagx:: paths (for proc macro generated code)
|
||||
extern crate self as dagx;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::node::{ExecutableNode, TypedNode};
|
||||
use crate::types::NodeId;
|
||||
|
||||
// Simple test task
|
||||
struct TestTask {
|
||||
@@ -18,36 +16,6 @@ impl TestTask {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_create_announcement() {
|
||||
let node = TypedNode::new(NodeId(0), TestTask { value: 42 });
|
||||
|
||||
// Create ONE announcement that can broadcast to multiple listeners
|
||||
let (announcer, announcement) = node.create_announcement();
|
||||
|
||||
// Verify both are created (type-erased)
|
||||
assert!(std::any::Any::type_id(&*announcer) != std::any::TypeId::of::<()>());
|
||||
assert!(std::any::Any::type_id(&*announcement) != std::any::TypeId::of::<()>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_create_multiple_listeners() {
|
||||
let node = TypedNode::new(NodeId(0), TestTask { value: 42 });
|
||||
|
||||
// Create announcement
|
||||
let (_announcer, announcement) = node.create_announcement();
|
||||
|
||||
// Create multiple listeners from the same announcement (broadcast pattern)
|
||||
let listener1 = node.create_listener(announcement.as_ref());
|
||||
let listener2 = node.create_listener(announcement.as_ref());
|
||||
let listener3 = node.create_listener(announcement.as_ref());
|
||||
|
||||
// All listeners are created from the same announcement
|
||||
assert!(std::any::Any::type_id(&*listener1) != std::any::TypeId::of::<()>());
|
||||
assert!(std::any::Any::type_id(&*listener2) != std::any::TypeId::of::<()>());
|
||||
assert!(std::any::Any::type_id(&*listener3) != std::any::TypeId::of::<()>());
|
||||
}
|
||||
|
||||
struct TaskWithDependency;
|
||||
|
||||
#[crate::task]
|
||||
@@ -58,85 +26,38 @@ impl TaskWithDependency {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_listeners_missing_input() {
|
||||
// Test when task expects input but doesn't receive it
|
||||
let node = TypedNode::new(NodeId(0), TaskWithDependency);
|
||||
async fn test_execute_with_deps_success() {
|
||||
// Test successful execution with channels
|
||||
let dependency_node = Box::new(TypedNode::new(TestTask { value: 21 }));
|
||||
let dependent_node = Box::new(TypedNode::new(TaskWithDependency));
|
||||
|
||||
// Create a real announcer even though we won't use it
|
||||
let (announcer, _announcement) = node.create_announcement();
|
||||
|
||||
let node = Box::new(node);
|
||||
|
||||
// TaskWithDependency expects an i32 input, but we provide no listeners
|
||||
let listeners = vec![];
|
||||
|
||||
let result = node.execute_with_listeners(listeners, announcer).await;
|
||||
assert!(result.is_err());
|
||||
|
||||
if let Err(crate::error::DagError::InvalidDependency { task_id }) = result {
|
||||
assert_eq!(task_id, 0);
|
||||
} else {
|
||||
panic!("Expected InvalidDependency error, got: {:?}", result);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_listeners_success() {
|
||||
// Test successful execution with announcements and listeners
|
||||
let dependency_node = TypedNode::new(NodeId(0), TestTask { value: 21 });
|
||||
let dependent_node = Box::new(TypedNode::new(NodeId(1), TaskWithDependency));
|
||||
|
||||
// Create announcement from dependency
|
||||
let (dep_announcer, dep_announcement) = dependency_node.create_announcement();
|
||||
|
||||
// Create listener for dependent from dependency's announcement
|
||||
let listener = dependency_node.create_listener(dep_announcement.as_ref());
|
||||
|
||||
// Execute dependency node with announcer
|
||||
let dependency_node = Box::new(dependency_node);
|
||||
let dep_listeners = vec![];
|
||||
// Execute dependency node with sender
|
||||
let dep_dependencies = vec![];
|
||||
|
||||
// Execute in background
|
||||
let dep_handle = tokio::spawn(async move {
|
||||
dependency_node
|
||||
.execute_with_listeners(dep_listeners, dep_announcer)
|
||||
let dep_result =
|
||||
tokio::spawn(async move { dependency_node.execute_with_deps(dep_dependencies).await })
|
||||
.await
|
||||
});
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
// Execute dependent node with listener
|
||||
// Create a real announcer for the dependent (even though no one listens to it)
|
||||
let dep_node_for_announcer = TypedNode::new(NodeId(1), TaskWithDependency);
|
||||
let (dependent_announcer, _dep_announcement) = dep_node_for_announcer.create_announcement();
|
||||
// Execute dependent node with receiver
|
||||
let dependencies = vec![dep_result as Arc<dyn std::any::Any + Send + Sync>];
|
||||
|
||||
let listeners = vec![listener];
|
||||
|
||||
let result = dependent_node
|
||||
.execute_with_listeners(listeners, dependent_announcer)
|
||||
.await;
|
||||
|
||||
// Wait for dependency to complete
|
||||
let _ = dep_handle.await;
|
||||
let result = dependent_node.execute_with_deps(dependencies).await;
|
||||
|
||||
assert!(result.is_ok());
|
||||
let output = result.unwrap();
|
||||
let arc_value = output.downcast::<i32>().unwrap();
|
||||
assert_eq!(*arc_value, 42); // 21 * 2 = 42
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_listeners_sink_node() {
|
||||
async fn test_execute_with_deps_sink_node() {
|
||||
// Test that sink nodes (no dependents) return their output
|
||||
let node = TypedNode::new(NodeId(0), TestTask { value: 42 });
|
||||
let node = Box::new(TypedNode::new(TestTask { value: 42 }));
|
||||
|
||||
// Create a real announcer (even though no one will listen)
|
||||
let (announcer, _announcement) = node.create_announcement();
|
||||
// No senders means this is a sink node
|
||||
let dependencies = vec![];
|
||||
|
||||
let node = Box::new(node);
|
||||
|
||||
// Sink node has no listeners
|
||||
let listeners = vec![];
|
||||
|
||||
let result = node.execute_with_listeners(listeners, announcer).await;
|
||||
let result = node.execute_with_deps(dependencies).await;
|
||||
assert!(result.is_ok());
|
||||
|
||||
// Check output was returned (Arc-wrapped)
|
||||
@@ -146,60 +67,18 @@ async fn test_execute_with_listeners_sink_node() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_listeners_non_sink_node() {
|
||||
// Test that non-sink nodes broadcast output via announcement
|
||||
let node = TypedNode::new(NodeId(0), TestTask { value: 42 });
|
||||
async fn test_execute_with_deps_non_sink_node() {
|
||||
// Test that non-sink nodes return their output
|
||||
let node = TypedNode::new(TestTask { value: 42 });
|
||||
|
||||
// Create announcement and listener
|
||||
let (announcer, announcement) = node.create_announcement();
|
||||
let listener = node.create_listener(announcement.as_ref());
|
||||
|
||||
// Execute with announcer (non-sink node)
|
||||
// Execute with senders (non-sink node)
|
||||
let node = Box::new(node);
|
||||
let listeners = vec![];
|
||||
let result = node.execute_with_listeners(listeners, announcer).await;
|
||||
let result = node.execute_with_deps(vec![]).await;
|
||||
assert!(result.is_ok());
|
||||
|
||||
// Verify output was broadcast through announcement
|
||||
let listener = *listener
|
||||
.downcast::<announcement::Listener<std::sync::Arc<i32>>>()
|
||||
.unwrap();
|
||||
|
||||
let received = listener.listen().await;
|
||||
assert!(received.is_ok());
|
||||
assert_eq!(*received.unwrap(), 42);
|
||||
// Verify output was sent through channel (Arc-wrapped)
|
||||
let arc_value = result.unwrap().downcast::<i32>().unwrap();
|
||||
|
||||
// Also verify output was returned (Arc-wrapped)
|
||||
let output = result.unwrap();
|
||||
let arc_value = output.downcast::<i32>().unwrap();
|
||||
assert_eq!(*arc_value, 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_multiple_listeners() {
|
||||
// Test that output is broadcast to multiple listeners
|
||||
let node = TypedNode::new(NodeId(0), TestTask { value: 100 });
|
||||
|
||||
// Create announcement and multiple listeners
|
||||
let (announcer, announcement) = node.create_announcement();
|
||||
let listener1 = node.create_listener(announcement.as_ref());
|
||||
let listener2 = node.create_listener(announcement.as_ref());
|
||||
let listener3 = node.create_listener(announcement.as_ref());
|
||||
|
||||
// Execute node
|
||||
let node = Box::new(node);
|
||||
let listeners = vec![];
|
||||
let result = node.execute_with_listeners(listeners, announcer).await;
|
||||
assert!(result.is_ok());
|
||||
|
||||
// Verify all three listeners receive the output
|
||||
for listener in [listener1, listener2, listener3] {
|
||||
let listener = *listener
|
||||
.downcast::<announcement::Listener<std::sync::Arc<i32>>>()
|
||||
.unwrap();
|
||||
|
||||
let received = listener.listen().await;
|
||||
assert!(received.is_ok());
|
||||
assert_eq!(*received.unwrap(), 100);
|
||||
}
|
||||
}
|
||||
+170
-171
@@ -5,16 +5,18 @@
|
||||
//!
|
||||
//! Uses Mutex for interior mutability to enable builder pattern (`&self` instead of `&mut self`).
|
||||
|
||||
use std::any::Any;
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::future::Future;
|
||||
use std::hash::{BuildHasher, Hasher};
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc,
|
||||
};
|
||||
|
||||
use futures::channel::mpsc;
|
||||
use futures::future::BoxFuture;
|
||||
use futures::{FutureExt, StreamExt};
|
||||
use futures_util::future::BoxFuture;
|
||||
use futures_util::{stream::FuturesUnordered, FutureExt, StreamExt, TryFutureExt};
|
||||
use parking_lot::Mutex;
|
||||
|
||||
#[cfg(feature = "tracing")]
|
||||
@@ -23,6 +25,7 @@ use tracing::{debug, error, info, trace};
|
||||
use crate::builder::TaskBuilder;
|
||||
use crate::error::{DagError, DagResult};
|
||||
use crate::node::{ExecutableNode, TypedNode};
|
||||
use crate::task::Task;
|
||||
use crate::types::{NodeId, Pending, TaskHandle};
|
||||
|
||||
// Guard to ensure run_lock is released even on early return or panic
|
||||
@@ -36,6 +39,35 @@ impl<'a> Drop for RunGuard<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub(crate) struct PassThroughHasher {
|
||||
hash: u64,
|
||||
}
|
||||
|
||||
impl Hasher for PassThroughHasher {
|
||||
fn finish(&self) -> u64 {
|
||||
self.hash
|
||||
}
|
||||
|
||||
fn write_u32(&mut self, i: u32) {
|
||||
self.hash = i as u64;
|
||||
}
|
||||
|
||||
fn write(&mut self, _bytes: &[u8]) {
|
||||
panic!("PassThroughHasher used on invalid type");
|
||||
}
|
||||
}
|
||||
|
||||
impl BuildHasher for PassThroughHasher {
|
||||
type Hasher = PassThroughHasher;
|
||||
|
||||
fn build_hasher(&self) -> Self::Hasher {
|
||||
PassThroughHasher::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) type PassThroughHashMap<K, V> = HashMap<K, V, PassThroughHasher>;
|
||||
|
||||
/// Build and execute a typed DAG of tasks.
|
||||
///
|
||||
/// A `DagRunner` is the main orchestrator for building and executing a directed acyclic graph
|
||||
@@ -53,6 +85,7 @@ impl<'a> Drop for RunGuard<'a> {
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use dagx::{task, DagRunner, Task};
|
||||
/// #
|
||||
/// // Task with state constructed via ::new()
|
||||
/// struct LoadValue { value: i32 }
|
||||
///
|
||||
@@ -79,9 +112,9 @@ impl<'a> Drop for RunGuard<'a> {
|
||||
/// // Construct instances using ::new() pattern
|
||||
/// let x = dag.add_task(LoadValue::new(2));
|
||||
/// let y = dag.add_task(LoadValue::new(3));
|
||||
/// let sum = dag.add_task(Add).depends_on((&x, &y));
|
||||
/// let sum = dag.add_task(Add).depends_on((x, y));
|
||||
///
|
||||
/// dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
/// dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
///
|
||||
/// assert_eq!(dag.get(sum).unwrap(), 5);
|
||||
/// # };
|
||||
@@ -94,11 +127,12 @@ impl<'a> Drop for RunGuard<'a> {
|
||||
/// Outputs are Arc-wrapped and stored separately for retrieval via get().
|
||||
/// Arc enables efficient sharing during fanout without cloning data.
|
||||
pub struct DagRunner {
|
||||
pub(crate) nodes: Mutex<Vec<Option<Box<dyn ExecutableNode>>>>,
|
||||
pub(crate) outputs: Mutex<HashMap<NodeId, std::sync::Arc<dyn std::any::Any + Send + Sync>>>,
|
||||
pub(crate) edges: Mutex<HashMap<NodeId, Vec<NodeId>>>, // node -> dependencies
|
||||
pub(crate) dependents: Mutex<HashMap<NodeId, Vec<NodeId>>>, // node -> tasks that depend on it
|
||||
pub(crate) next_id: Mutex<usize>,
|
||||
pub(crate) nodes: Mutex<Vec<Option<Box<dyn ExecutableNode + Sync>>>>,
|
||||
pub(crate) outputs:
|
||||
Mutex<PassThroughHashMap<NodeId, std::sync::Arc<dyn std::any::Any + Send + Sync>>>,
|
||||
pub(crate) edges: Mutex<PassThroughHashMap<NodeId, Vec<NodeId>>>, // node -> dependencies
|
||||
pub(crate) dependents: Mutex<PassThroughHashMap<NodeId, Vec<NodeId>>>, // node -> tasks that depend on it
|
||||
pub(crate) next_id: Mutex<u32>,
|
||||
pub(crate) run_lock: AtomicBool, // Ensures only one run() at a time
|
||||
}
|
||||
|
||||
@@ -121,9 +155,9 @@ impl DagRunner {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
nodes: Mutex::new(Vec::new()),
|
||||
outputs: Mutex::new(HashMap::new()),
|
||||
edges: Mutex::new(HashMap::new()),
|
||||
dependents: Mutex::new(HashMap::new()),
|
||||
outputs: Mutex::new(HashMap::default()),
|
||||
edges: Mutex::new(HashMap::default()),
|
||||
dependents: Mutex::new(HashMap::default()),
|
||||
next_id: Mutex::new(0),
|
||||
run_lock: AtomicBool::new(false),
|
||||
}
|
||||
@@ -146,6 +180,7 @@ impl DagRunner {
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use dagx::{task, DagRunner, Task};
|
||||
/// #
|
||||
/// // Task with state - shows you construct with specific value
|
||||
/// struct LoadValue {
|
||||
/// initial: i32,
|
||||
@@ -185,15 +220,17 @@ impl DagRunner {
|
||||
/// let base = dag.add_task(LoadValue::new(10));
|
||||
///
|
||||
/// // Construct task with offset of 1
|
||||
/// let inc = dag.add_task(AddOffset::new(1)).depends_on(&base);
|
||||
/// let inc = dag.add_task(AddOffset::new(1)).depends_on(base);
|
||||
///
|
||||
/// dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
/// assert_eq!(dag.get(&inc).unwrap(), 11);
|
||||
/// dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
/// assert_eq!(dag.get(inc).unwrap(), 11);
|
||||
/// # };
|
||||
/// ```
|
||||
pub fn add_task<Tk>(&self, task: Tk) -> TaskBuilder<'_, Tk, Pending>
|
||||
where
|
||||
Tk: crate::node::TaskExecute + 'static,
|
||||
Tk: Task + Sync + 'static,
|
||||
Tk::Input: 'static,
|
||||
Tk::Output: 'static,
|
||||
{
|
||||
let id = self.alloc_id();
|
||||
|
||||
@@ -204,7 +241,7 @@ impl DagRunner {
|
||||
"adding task to DAG"
|
||||
);
|
||||
|
||||
let node = TypedNode::new(id, task);
|
||||
let node = TypedNode::new(task);
|
||||
self.nodes.lock().push(Some(Box::new(node)));
|
||||
self.edges.lock().insert(id, Vec::new());
|
||||
self.dependents.lock().insert(id, Vec::new());
|
||||
@@ -227,25 +264,24 @@ impl DagRunner {
|
||||
///
|
||||
/// # Parameters
|
||||
///
|
||||
/// - `spawner`: A function that spawns futures on the async runtime. Examples:
|
||||
/// - Tokio: `|fut| { tokio::spawn(fut); }`
|
||||
/// - Smol: `|fut| { smol::spawn(fut).detach(); }`
|
||||
/// - Async-std: `|fut| { async_std::task::spawn(fut); }`
|
||||
/// - `spawner`: A function that spawns futures on the async runtime
|
||||
/// and returns a handle to the task. This is the only way to run tasks on separate threads. Examples:
|
||||
/// - Tokio: `|fut| { tokio::spawn(fut).await.unwrap() }`
|
||||
/// - Smol: `|fut| { smol::spawn(fut) }`
|
||||
/// - Async-std: `|fut| { async_std::task::spawn(fut) }`
|
||||
/// - Single-threaded: `|fut| fut`
|
||||
/// - For computationally light tasks, concurrency without parallelism can be significantly faster.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// - Returns `DagError::ConcurrentExecution` if the DAG is already executing
|
||||
/// - Returns `DagError::TaskPanicked` if any task panics during execution
|
||||
///
|
||||
/// # Note on Cycles
|
||||
///
|
||||
/// Cycles are impossible via the public API due to compile-time prevention.
|
||||
/// See the [`crate::cycle_prevention`] module for details.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// # use dagx::{task, DagRunner, Task};
|
||||
/// #
|
||||
/// // Tuple struct
|
||||
/// struct Value(i32);
|
||||
///
|
||||
@@ -267,23 +303,17 @@ impl DagRunner {
|
||||
///
|
||||
/// let a = dag.add_task(Value(1));
|
||||
/// let b = dag.add_task(Value(2));
|
||||
/// let sum = dag.add_task(Add).depends_on((&a, &b));
|
||||
/// let sum = dag.add_task(Add).depends_on((a, b));
|
||||
///
|
||||
/// dag.run(|fut| { tokio::spawn(fut); }).await.unwrap(); // Executes all tasks
|
||||
/// dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap(); // Executes all tasks
|
||||
/// # };
|
||||
/// ```
|
||||
///
|
||||
/// # Implementation Note
|
||||
///
|
||||
/// Tasks communicate via oneshot channels created fresh during run().
|
||||
/// This eliminates Mutex contention on outputs and enables true streaming execution.
|
||||
/// Type erasure occurs only at the ExecutableNode trait boundary - channels are created
|
||||
/// with full type information and type-erased just before passing to execute_with_channels.
|
||||
#[inline]
|
||||
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self, spawner)))]
|
||||
pub async fn run<S>(&self, spawner: S) -> DagResult<()>
|
||||
pub async fn run<S, F>(&self, spawner: S) -> DagResult<()>
|
||||
where
|
||||
S: Fn(BoxFuture<'static, ()>),
|
||||
S: Fn(BoxFuture<'static, DagResult<Arc<dyn Any + Send + Sync>>>) -> F,
|
||||
F: Future<Output = DagResult<Arc<dyn Any + Send + Sync>>>,
|
||||
{
|
||||
#[cfg(feature = "tracing")]
|
||||
info!("starting DAG execution");
|
||||
@@ -308,58 +338,19 @@ impl DagRunner {
|
||||
// Build topological layers
|
||||
let layers = self.compute_layers()?;
|
||||
|
||||
let total_tasks = layers.iter().map(|l| l.len()).sum::<usize>();
|
||||
|
||||
#[cfg(feature = "tracing")]
|
||||
debug!(
|
||||
layer_count = layers.len(),
|
||||
total_tasks = layers.iter().map(|l| l.len()).sum::<usize>(),
|
||||
"computed topological layers"
|
||||
total_tasks, "computed topological layers"
|
||||
);
|
||||
|
||||
let edges = self.edges.lock().clone();
|
||||
|
||||
// Create ONE announcement per task
|
||||
// Each task broadcasts its output to all dependents via announcement
|
||||
let mut announcements: HashMap<NodeId, Box<dyn std::any::Any + Send>> = HashMap::new();
|
||||
let mut announcers: HashMap<NodeId, Box<dyn std::any::Any + Send>> = HashMap::new();
|
||||
|
||||
// Create announcements for all tasks
|
||||
{
|
||||
let nodes_lock = self.nodes.lock();
|
||||
for (node_id, node_opt) in nodes_lock.iter().enumerate() {
|
||||
if let Some(node) = node_opt {
|
||||
let (announcer, announcement) = node.create_announcement();
|
||||
announcers.insert(NodeId(node_id), announcer);
|
||||
announcements.insert(NodeId(node_id), announcement);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create listeners for each consumer from their dependencies
|
||||
let mut consumer_listeners: HashMap<NodeId, Vec<Box<dyn std::any::Any + Send>>> =
|
||||
HashMap::new();
|
||||
|
||||
{
|
||||
let nodes_lock = self.nodes.lock();
|
||||
for (consumer_id, producer_ids) in &edges {
|
||||
let mut listeners = Vec::new();
|
||||
for &producer_id in producer_ids {
|
||||
if let Some(announcement) = announcements.get(&producer_id) {
|
||||
if let Some(Some(producer_node)) = nodes_lock.get(producer_id.0) {
|
||||
// Producer node creates listener from its own announcement
|
||||
let listener = producer_node.create_listener(announcement.as_ref());
|
||||
listeners.push(listener);
|
||||
}
|
||||
}
|
||||
}
|
||||
consumer_listeners.insert(*consumer_id, listeners);
|
||||
}
|
||||
}
|
||||
|
||||
// Create a channel to collect Arc-wrapped outputs from all tasks
|
||||
let (output_tx, mut output_rx) = mpsc::unbounded::<(
|
||||
NodeId,
|
||||
Result<std::sync::Arc<dyn std::any::Any + Send + Sync>, DagError>,
|
||||
)>();
|
||||
let mut outputs: PassThroughHashMap<NodeId, Arc<dyn Any + Send + Sync>> =
|
||||
HashMap::with_capacity_and_hasher(total_tasks, PassThroughHasher::default());
|
||||
let mut first_error = None;
|
||||
|
||||
for layer in layers {
|
||||
#[cfg(feature = "tracing")]
|
||||
@@ -374,7 +365,6 @@ impl DagRunner {
|
||||
// pipelines), we execute it inline rather than spawning it. This provides
|
||||
// 10-100x performance improvements for sequential workloads by eliminating:
|
||||
// - Task spawning overhead
|
||||
// - Channel creation/destruction for layer coordination
|
||||
// - Context switching to/from the runtime
|
||||
//
|
||||
// CRITICAL: Panic handling is required to maintain behavioral consistency.
|
||||
@@ -401,23 +391,18 @@ impl DagRunner {
|
||||
"executing task inline (single-task layer optimization)"
|
||||
);
|
||||
|
||||
let out_tx = output_tx.clone();
|
||||
|
||||
// Take ownership of the node
|
||||
let node = {
|
||||
let mut nodes_lock = self.nodes.lock();
|
||||
nodes_lock[node_id.0].take()
|
||||
nodes_lock[node_id.0 as usize].take()
|
||||
};
|
||||
|
||||
if let Some(node) = node {
|
||||
// Take the listeners for this consumer
|
||||
let listeners = consumer_listeners.remove(&node_id).unwrap_or_default();
|
||||
|
||||
// Take the announcer for this producer
|
||||
let announcer = announcers.remove(&node_id).unwrap_or_else(|| {
|
||||
// Create dummy announcer for tasks with no dependents
|
||||
Box::new(()) as Box<dyn std::any::Any + Send>
|
||||
});
|
||||
let dependencies: Vec<_> = edges[&node_id]
|
||||
.iter()
|
||||
.flat_map(|dep| outputs.get(dep))
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
// Execute inline with panic handling.
|
||||
//
|
||||
@@ -425,7 +410,7 @@ impl DagRunner {
|
||||
// DagError::TaskPanicked, matching the behavior of async runtimes when
|
||||
// they spawn tasks. This guarantees consistent error handling whether
|
||||
// a task executes inline (single-task layer) or spawned (multi-task layer).
|
||||
let result = AssertUnwindSafe(node.execute_with_listeners(listeners, announcer))
|
||||
let result = AssertUnwindSafe(node.execute_with_deps(dependencies))
|
||||
.catch_unwind()
|
||||
.await
|
||||
.unwrap_or_else(|panic_payload| {
|
||||
@@ -452,83 +437,97 @@ impl DagRunner {
|
||||
})
|
||||
});
|
||||
|
||||
// Send the output (ignore send errors - receiver may be dropped)
|
||||
let _ = out_tx.unbounded_send((node_id, result));
|
||||
match result {
|
||||
Ok(output) => {
|
||||
outputs.insert(node_id, output);
|
||||
}
|
||||
Err(e) => {
|
||||
first_error.get_or_insert(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Slow path: Multiple tasks require spawning and coordination
|
||||
// Create a channel to track task completion for this layer
|
||||
let (tx, mut rx) = mpsc::channel::<()>(layer.len());
|
||||
|
||||
// Spawn each task in this layer
|
||||
for &node_id in &layer {
|
||||
#[cfg(feature = "tracing")]
|
||||
trace!(task_id = node_id.0, "spawning task");
|
||||
let mut task_tx = tx.clone();
|
||||
let out_tx = output_tx.clone();
|
||||
let mut futures: FuturesUnordered<_> = layer
|
||||
.into_iter()
|
||||
.filter_map(|node_id| {
|
||||
#[cfg(feature = "tracing")]
|
||||
trace!(task_id = node_id.0, "spawning task");
|
||||
|
||||
// Take ownership of the node
|
||||
let node = {
|
||||
let mut nodes_lock = self.nodes.lock();
|
||||
nodes_lock[node_id.0].take()
|
||||
};
|
||||
|
||||
if let Some(node) = node {
|
||||
// Take the listeners for this consumer
|
||||
let listeners = consumer_listeners.remove(&node_id).unwrap_or_default();
|
||||
|
||||
// Take the announcer for this producer
|
||||
let announcer = announcers.remove(&node_id).unwrap_or_else(|| {
|
||||
// Create dummy announcer for tasks with no dependents
|
||||
Box::new(()) as Box<dyn std::any::Any + Send>
|
||||
});
|
||||
|
||||
// Create a 'static future by taking ownership
|
||||
let inner_future = async move {
|
||||
let task_future = node.execute_with_listeners(listeners, announcer);
|
||||
|
||||
let result = task_future.await;
|
||||
|
||||
// Send the output (ignore send errors - receiver may be dropped)
|
||||
let _ = out_tx.unbounded_send((node_id, result));
|
||||
|
||||
// Signal completion (ignore send errors - receiver may be dropped)
|
||||
let _ = task_tx.try_send(());
|
||||
// Take ownership of the node
|
||||
let node = {
|
||||
let mut nodes_lock = self.nodes.lock();
|
||||
nodes_lock[node_id.0 as usize].take()
|
||||
};
|
||||
if let Some(node) = node {
|
||||
let dependencies: Vec<_> = edges[&node_id]
|
||||
.iter()
|
||||
.flat_map(|dep| outputs.get(dep))
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
// Spawn the task using the provided spawner
|
||||
spawner(Box::pin(inner_future));
|
||||
let inner_future = spawner(node.execute_with_deps(dependencies));
|
||||
// Spawn the task using the provided spawner
|
||||
let inner_future = async move {
|
||||
let result = inner_future.await?;
|
||||
Ok((node_id, result))
|
||||
};
|
||||
|
||||
Some(
|
||||
AssertUnwindSafe(inner_future)
|
||||
.catch_unwind()
|
||||
.unwrap_or_else(move |panic_payload| {
|
||||
// Convert panic to error
|
||||
let panic_message =
|
||||
if let Some(s) = panic_payload.downcast_ref::<&str>() {
|
||||
s.to_string()
|
||||
} else if let Some(s) =
|
||||
panic_payload.downcast_ref::<String>()
|
||||
{
|
||||
s.clone()
|
||||
} else {
|
||||
"unknown panic".to_string()
|
||||
};
|
||||
|
||||
#[cfg(feature = "tracing")]
|
||||
error!(
|
||||
task_id = node_id.0,
|
||||
panic_message = %panic_message,
|
||||
"task panicked during inline execution"
|
||||
);
|
||||
|
||||
Err(DagError::TaskPanicked {
|
||||
task_id: node_id.0,
|
||||
panic_message,
|
||||
})
|
||||
}),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
while let Some(out) = futures.next().await {
|
||||
match out {
|
||||
Ok(output) => {
|
||||
outputs.insert(output.0, output.1);
|
||||
}
|
||||
Err(e) => {
|
||||
first_error.get_or_insert(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Drop the original sender so the channel closes when all tasks complete
|
||||
drop(tx);
|
||||
|
||||
// Wait for all tasks in this layer to complete
|
||||
while rx.next().await.is_some() {}
|
||||
}
|
||||
}
|
||||
|
||||
// Drop the output sender so the channel closes
|
||||
drop(output_tx);
|
||||
|
||||
// Collect all outputs and check for errors
|
||||
let mut collected = Vec::new();
|
||||
let mut first_error = None;
|
||||
while let Some((node_id, result)) = output_rx.next().await {
|
||||
match result {
|
||||
Ok(output) => collected.push((node_id, output)),
|
||||
Err(e) if first_error.is_none() => first_error = Some(e),
|
||||
Err(_) => {} // Ignore subsequent errors
|
||||
}
|
||||
}
|
||||
|
||||
// Insert all successful outputs (avoiding holding lock across await)
|
||||
let mut outputs = self.outputs.lock();
|
||||
for (node_id, output) in collected {
|
||||
outputs.insert(node_id, output);
|
||||
let mut outputs_lock = self.outputs.lock();
|
||||
for (node_id, output) in outputs {
|
||||
outputs_lock.insert(node_id, output);
|
||||
}
|
||||
drop(outputs);
|
||||
drop(outputs_lock);
|
||||
|
||||
// Return first error if any
|
||||
if let Some(err) = first_error {
|
||||
@@ -548,6 +547,7 @@ impl DagRunner {
|
||||
/// # Behavior
|
||||
///
|
||||
/// All task outputs are stored after execution and can be retrieved via get().
|
||||
/// Each task's output can only be taken once, after which it will return `DagError::ResultNotFound`.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
@@ -559,6 +559,7 @@ impl DagRunner {
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use dagx::{task, DagRunner, Task};
|
||||
/// #
|
||||
/// struct Configuration {
|
||||
/// setting: i32,
|
||||
/// }
|
||||
@@ -580,31 +581,31 @@ impl DagRunner {
|
||||
/// // Construct task with specific setting value
|
||||
/// let task = dag.add_task(Configuration::new(42));
|
||||
///
|
||||
/// dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
/// dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
///
|
||||
/// assert_eq!(dag.get(task).unwrap(), 42);
|
||||
/// # };
|
||||
/// ```
|
||||
pub fn get<T: 'static + Clone + Send + Sync, H>(&self, handle: H) -> DagResult<T>
|
||||
pub fn get<T: 'static + Send + Sync, H>(&self, handle: H) -> DagResult<T>
|
||||
where
|
||||
H: Into<TaskHandle<T>>,
|
||||
{
|
||||
let handle: TaskHandle<T> = handle.into();
|
||||
let outputs = self.outputs.lock();
|
||||
let mut outputs = self.outputs.lock();
|
||||
|
||||
let arc_output = outputs.get(&handle.id).ok_or(DagError::ResultNotFound {
|
||||
let arc_output = outputs.remove(&handle.id).ok_or(DagError::ResultNotFound {
|
||||
task_id: handle.id.0,
|
||||
})?;
|
||||
|
||||
// Downcast Arc<dyn Any> to Arc<T>, then clone the inner value
|
||||
// The Arc itself is stored for efficient sharing, but get() clones the data
|
||||
Arc::clone(arc_output)
|
||||
// Downcast Arc<dyn Any> to Arc<T>
|
||||
let output = arc_output
|
||||
.downcast::<T>()
|
||||
.map(|arc| (*arc).clone())
|
||||
.map_err(|_| DagError::TypeMismatch {
|
||||
expected: std::any::type_name::<T>(),
|
||||
found: "unknown",
|
||||
})
|
||||
})?;
|
||||
|
||||
Ok(Arc::into_inner(output).unwrap())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -612,7 +613,7 @@ impl DagRunner {
|
||||
#[cfg(feature = "tracing")]
|
||||
debug!("computing topological layers");
|
||||
|
||||
let mut in_degree: HashMap<NodeId, usize> = HashMap::new();
|
||||
let mut in_degree: PassThroughHashMap<NodeId, usize> = HashMap::default();
|
||||
let mut layers = Vec::new();
|
||||
|
||||
// Calculate in-degrees: for each node, count how many dependencies it has
|
||||
@@ -668,13 +669,11 @@ impl DagRunner {
|
||||
}
|
||||
|
||||
if !current_layer.is_empty() {
|
||||
current_layer.sort(); // Deterministic ordering
|
||||
layers.push(current_layer);
|
||||
}
|
||||
}
|
||||
|
||||
// Cycle detection removed: cycles are impossible via the public API.
|
||||
// See src/cycle_prevention.rs for proof that the type system prevents cycles.
|
||||
//
|
||||
// The type-state pattern enforces acyclic structure at compile time through:
|
||||
// 1. TaskBuilder::depends_on() consumes the builder (move semantics)
|
||||
|
||||
+30
-244
@@ -1,10 +1,10 @@
|
||||
//! Unit tests for runner module
|
||||
|
||||
// Allow tests to reference dagx:: paths (for proc macro generated code)
|
||||
extern crate self as dagx;
|
||||
use std::collections::HashMap;
|
||||
use std::hash::Hasher;
|
||||
|
||||
use crate::error::DagError;
|
||||
use crate::runner::DagRunner;
|
||||
use crate::runner::{DagRunner, PassThroughHasher};
|
||||
use crate::types::TaskHandle;
|
||||
|
||||
// Initialize tracing subscriber for tests (idempotent)
|
||||
@@ -70,11 +70,9 @@ async fn test_get_wrong_type() {
|
||||
let dag = DagRunner::new();
|
||||
let handle = dag.add_task(TestTask { value: 42 });
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Try to get with wrong type - downcast will fail
|
||||
let fake_handle: TaskHandle<String> = TaskHandle {
|
||||
@@ -147,18 +145,14 @@ async fn test_concurrent_run_protection() {
|
||||
// Start two runs concurrently
|
||||
let handle1 = tokio::spawn(async move {
|
||||
barrier1.wait().await;
|
||||
dag1.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
dag1.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
});
|
||||
|
||||
let handle2 = tokio::spawn(async move {
|
||||
barrier2.wait().await;
|
||||
dag2.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
dag2.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
});
|
||||
|
||||
// One should succeed, one should fail (concurrent execution not supported)
|
||||
@@ -169,60 +163,14 @@ async fn test_concurrent_run_protection() {
|
||||
assert!(result1.is_ok() != result2.is_ok());
|
||||
|
||||
// The error should be about concurrent execution
|
||||
if result1.is_err() {
|
||||
assert!(matches!(
|
||||
result1.unwrap_err(),
|
||||
crate::error::DagError::ConcurrentExecution
|
||||
));
|
||||
if let Err(e) = result1 {
|
||||
assert!(matches!(e, crate::error::DagError::ConcurrentExecution));
|
||||
}
|
||||
if result2.is_err() {
|
||||
assert!(matches!(
|
||||
result2.unwrap_err(),
|
||||
crate::error::DagError::ConcurrentExecution
|
||||
));
|
||||
if let Err(e) = result2 {
|
||||
assert!(matches!(e, crate::error::DagError::ConcurrentExecution));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_task_increments_id() {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let handle1 = dag.add_task(TestTask { value: 1 });
|
||||
let handle2 = dag.add_task(TestTask { value: 2 });
|
||||
let handle3 = dag.add_task(TestTask { value: 3 });
|
||||
|
||||
// Node IDs should be sequential
|
||||
assert_eq!(handle1.id.0, 0);
|
||||
assert_eq!(handle2.id.0, 1);
|
||||
assert_eq!(handle3.id.0, 2);
|
||||
|
||||
// Check that nodes were actually added
|
||||
assert_eq!(dag.nodes.lock().len(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_get_calls() {
|
||||
init_tracing();
|
||||
// Test that get() can be called multiple times for the same handle
|
||||
let dag = DagRunner::new();
|
||||
let handle = dag.add_task(TestTask { value: 100 });
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Multiple get calls should all return the same value
|
||||
let result1 = dag.get(&handle).unwrap();
|
||||
let result2 = dag.get(&handle).unwrap();
|
||||
let result3 = dag.get(&handle).unwrap();
|
||||
|
||||
assert_eq!(result1, 100);
|
||||
assert_eq!(result2, 100);
|
||||
assert_eq!(result3, 100);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_invalid_node_id_in_get() {
|
||||
init_tracing();
|
||||
@@ -235,11 +183,9 @@ async fn test_invalid_node_id_in_get() {
|
||||
_phantom: std::marker::PhantomData,
|
||||
};
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = dag.get(invalid_handle);
|
||||
assert!(result.is_err());
|
||||
@@ -283,190 +229,30 @@ async fn test_task_panic_in_multi_task_layer() {
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let source = dag.add_task(Source);
|
||||
let source: TaskHandle<_> = dag.add_task(Source).into();
|
||||
|
||||
// Create two tasks in the same layer - one panics, one doesn't
|
||||
let _panic_task = dag.add_task(PanicTask).depends_on(&source);
|
||||
let _good_task = dag.add_task(GoodTask).depends_on(&source);
|
||||
let _panic_task = dag.add_task(PanicTask).depends_on(source);
|
||||
let _good_task = dag.add_task(GoodTask).depends_on(source);
|
||||
|
||||
// Run the DAG - the panic should be caught
|
||||
let result = dag
|
||||
.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await;
|
||||
|
||||
// The run should fail due to the panic
|
||||
// Note: Tokio catches panics in spawned tasks, so the behavior depends on the runtime
|
||||
// This test mainly ensures we don't crash
|
||||
assert!(result.is_ok() || result.is_err());
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Coverage tests for DagRunner::get() type mismatch errors
|
||||
// These tests target the error path at src/runner.rs:601-607
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_type_mismatch_i32_to_vec() {
|
||||
init_tracing();
|
||||
// Test type mismatch: i32 to Vec<u8>
|
||||
|
||||
struct IntTask;
|
||||
#[crate::task]
|
||||
impl IntTask {
|
||||
async fn run(&self) -> i32 {
|
||||
123
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let int_task = dag.add_task(IntTask);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Try to get as Vec<u8> - TYPE MISMATCH!
|
||||
let wrong_handle: TaskHandle<Vec<u8>> = TaskHandle {
|
||||
id: int_task.id,
|
||||
_phantom: std::marker::PhantomData,
|
||||
};
|
||||
|
||||
let result = dag.get(wrong_handle);
|
||||
assert!(result.is_err());
|
||||
match result.unwrap_err() {
|
||||
DagError::TypeMismatch { expected, found } => {
|
||||
assert_eq!(expected, std::any::type_name::<Vec<u8>>());
|
||||
assert_eq!(found, "unknown");
|
||||
}
|
||||
_ => panic!("Expected TypeMismatch error"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_type_mismatch_struct_to_hashmap() {
|
||||
init_tracing();
|
||||
// Test type mismatch: custom struct to HashMap
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CustomOutput {
|
||||
#[allow(dead_code)]
|
||||
value: i32,
|
||||
}
|
||||
|
||||
struct CustomTask;
|
||||
#[crate::task]
|
||||
impl CustomTask {
|
||||
async fn run(&self) -> CustomOutput {
|
||||
CustomOutput { value: 42 }
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let task_handle = dag.add_task(CustomTask);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Try to get as HashMap - TYPE MISMATCH!
|
||||
let wrong_handle: TaskHandle<HashMap<String, i32>> = TaskHandle {
|
||||
id: task_handle.id,
|
||||
_phantom: std::marker::PhantomData,
|
||||
};
|
||||
|
||||
let result = dag.get(wrong_handle);
|
||||
assert!(result.is_err());
|
||||
match result.unwrap_err() {
|
||||
DagError::TypeMismatch { expected, found } => {
|
||||
assert!(expected.contains("HashMap"));
|
||||
assert_eq!(found, "unknown");
|
||||
}
|
||||
_ => panic!("Expected TypeMismatch error"),
|
||||
}
|
||||
#[test]
|
||||
fn test_passthrough_hasher() {
|
||||
let mut passthrough_hashmap = HashMap::with_hasher(PassThroughHasher::default());
|
||||
passthrough_hashmap.insert(64u32, "test string");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_type_mismatch_vec_different_types() {
|
||||
init_tracing();
|
||||
// Test type mismatch: Vec<i32> to Vec<String>
|
||||
|
||||
struct VecTask;
|
||||
#[crate::task]
|
||||
impl VecTask {
|
||||
async fn run(&self) -> Vec<i32> {
|
||||
vec![1, 2, 3]
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let vec_task = dag.add_task(VecTask);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Try to get as Vec<String> - TYPE MISMATCH!
|
||||
let wrong_handle: TaskHandle<Vec<String>> = TaskHandle {
|
||||
id: vec_task.id,
|
||||
_phantom: std::marker::PhantomData,
|
||||
};
|
||||
|
||||
let result = dag.get(wrong_handle);
|
||||
assert!(result.is_err());
|
||||
match result.unwrap_err() {
|
||||
DagError::TypeMismatch { expected, found } => {
|
||||
assert_eq!(expected, std::any::type_name::<Vec<String>>());
|
||||
assert_eq!(found, "unknown");
|
||||
}
|
||||
_ => panic!("Expected TypeMismatch error"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_type_mismatch_option_types() {
|
||||
init_tracing();
|
||||
// Test type mismatch: Option<i32> to Option<String>
|
||||
|
||||
struct OptionTask;
|
||||
#[crate::task]
|
||||
impl OptionTask {
|
||||
async fn run(&self) -> Option<i32> {
|
||||
Some(42)
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let opt_task = dag.add_task(OptionTask);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Try to get as Option<String> - TYPE MISMATCH!
|
||||
let wrong_handle: TaskHandle<Option<String>> = TaskHandle {
|
||||
id: opt_task.id,
|
||||
_phantom: std::marker::PhantomData,
|
||||
};
|
||||
|
||||
let result = dag.get(wrong_handle);
|
||||
assert!(result.is_err());
|
||||
match result.unwrap_err() {
|
||||
DagError::TypeMismatch { expected, found } => {
|
||||
assert_eq!(expected, std::any::type_name::<Option<String>>());
|
||||
assert_eq!(found, "unknown");
|
||||
}
|
||||
_ => panic!("Expected TypeMismatch error"),
|
||||
}
|
||||
#[test]
|
||||
#[should_panic]
|
||||
fn test_passthrough_hasher_only_u32() {
|
||||
let mut passthrough_hasher = PassThroughHasher::default();
|
||||
passthrough_hasher.write(&[0; 4]);
|
||||
}
|
||||
+120
-279
@@ -3,306 +3,147 @@
|
||||
//! This module defines the core Task trait that all DAG nodes must implement,
|
||||
//! along with convenience functions for creating tasks from closures.
|
||||
|
||||
use std::any::Any;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Helper trait for extracting inputs from listeners (internal use only)
|
||||
trait ExtractFromListeners: Sized + Send + Sync + Clone + 'static {
|
||||
fn extract(
|
||||
listeners: Vec<Box<dyn Any + Send>>,
|
||||
) -> impl Future<Output = Result<Self, String>> + Send;
|
||||
}
|
||||
|
||||
// Implementation for unit type (no dependencies)
|
||||
impl ExtractFromListeners for () {
|
||||
async fn extract(_listeners: Vec<Box<dyn Any + Send>>) -> Result<Self, String> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Implementation for single input types (not tuples)
|
||||
// Note: These now return Arc<T> instead of T for efficiency
|
||||
macro_rules! impl_extract_single {
|
||||
($($t:ty),+) => {
|
||||
$(
|
||||
impl ExtractFromListeners for std::sync::Arc<$t> {
|
||||
async fn extract(mut listeners: Vec<Box<dyn Any + Send>>) -> Result<Self, String> {
|
||||
if listeners.len() != 1 {
|
||||
return Err(format!("Expected 1 dependency, got {}", listeners.len()));
|
||||
}
|
||||
let listener = *listeners.pop()
|
||||
.unwrap()
|
||||
.downcast::<announcement::Listener<std::sync::Arc<$t>>>()
|
||||
.map_err(|_| format!("Type mismatch: expected Listener<Arc<{}>>", std::any::type_name::<$t>()))?;
|
||||
listener.listen().await.map_err(|_| "Listener closed before receiving value".to_string())
|
||||
}
|
||||
}
|
||||
)+
|
||||
};
|
||||
}
|
||||
|
||||
impl_extract_single!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize, f32, f64, bool, char, String);
|
||||
|
||||
// Implementation for Result types (wrapped in Arc)
|
||||
impl<T, E> ExtractFromListeners for std::sync::Arc<Result<T, E>>
|
||||
where
|
||||
T: 'static + Clone + Send + Sync,
|
||||
E: 'static + Clone + Send + Sync,
|
||||
{
|
||||
async fn extract(mut listeners: Vec<Box<dyn Any + Send>>) -> Result<Self, String> {
|
||||
if listeners.len() != 1 {
|
||||
return Err(format!("Expected 1 dependency, got {}", listeners.len()));
|
||||
}
|
||||
let listener = *listeners.pop()
|
||||
.unwrap()
|
||||
.downcast::<announcement::Listener<std::sync::Arc<Result<T, E>>>>()
|
||||
.map_err(|_| format!("Type mismatch: expected Listener<Arc<Result<{}, {}>>>", std::any::type_name::<T>(), std::any::type_name::<E>()))?;
|
||||
listener.listen().await.map_err(|_| "Listener closed before receiving value".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
// Implementation for Option types (wrapped in Arc)
|
||||
impl<T> ExtractFromListeners for std::sync::Arc<Option<T>>
|
||||
where
|
||||
T: 'static + Clone + Send + Sync,
|
||||
{
|
||||
async fn extract(mut listeners: Vec<Box<dyn Any + Send>>) -> Result<Self, String> {
|
||||
if listeners.len() != 1 {
|
||||
return Err(format!("Expected 1 dependency, got {}", listeners.len()));
|
||||
}
|
||||
let listener = *listeners.pop()
|
||||
.unwrap()
|
||||
.downcast::<announcement::Listener<std::sync::Arc<Option<T>>>>()
|
||||
.map_err(|_| format!("Type mismatch: expected Listener<Arc<Option<{}>>>", std::any::type_name::<T>()))?;
|
||||
listener.listen().await.map_err(|_| "Listener closed before receiving value".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
// Implementation for Vec types (wrapped in Arc)
|
||||
impl<T> ExtractFromListeners for std::sync::Arc<Vec<T>>
|
||||
where
|
||||
T: 'static + Clone + Send + Sync,
|
||||
{
|
||||
async fn extract(mut listeners: Vec<Box<dyn Any + Send>>) -> Result<Self, String> {
|
||||
if listeners.len() != 1 {
|
||||
return Err(format!("Expected 1 dependency, got {}", listeners.len()));
|
||||
}
|
||||
let listener = *listeners.pop()
|
||||
.unwrap()
|
||||
.downcast::<announcement::Listener<std::sync::Arc<Vec<T>>>>()
|
||||
.map_err(|_| format!("Type mismatch: expected Listener<Arc<Vec<{}>>>", std::any::type_name::<T>()))?;
|
||||
listener.listen().await.map_err(|_| "Listener closed before receiving value".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
// Implementation for Arc types (Arc<Arc<T>> - useful when T itself is Arc-wrapped by user)
|
||||
impl<T> ExtractFromListeners for std::sync::Arc<std::sync::Arc<T>>
|
||||
where
|
||||
T: 'static + Clone + Send + Sync,
|
||||
{
|
||||
async fn extract(mut listeners: Vec<Box<dyn Any + Send>>) -> Result<Self, String> {
|
||||
if listeners.len() != 1 {
|
||||
return Err(format!("Expected 1 dependency, got {}", listeners.len()));
|
||||
}
|
||||
let listener = *listeners.pop()
|
||||
.unwrap()
|
||||
.downcast::<announcement::Listener<std::sync::Arc<std::sync::Arc<T>>>>()
|
||||
.map_err(|_| format!("Type mismatch: expected Listener<Arc<Arc<{}>>>", std::any::type_name::<T>()))?;
|
||||
listener.listen().await.map_err(|_| "Listener closed before receiving value".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
// Implementation for HashMap types (wrapped in Arc)
|
||||
impl<K, V> ExtractFromListeners for std::sync::Arc<std::collections::HashMap<K, V>>
|
||||
where
|
||||
K: 'static + Clone + Send + Sync + Eq + std::hash::Hash,
|
||||
V: 'static + Clone + Send + Sync,
|
||||
{
|
||||
async fn extract(mut listeners: Vec<Box<dyn Any + Send>>) -> Result<Self, String> {
|
||||
if listeners.len() != 1 {
|
||||
return Err(format!("Expected 1 dependency, got {}", listeners.len()));
|
||||
}
|
||||
let listener = *listeners.pop()
|
||||
.unwrap()
|
||||
.downcast::<announcement::Listener<std::sync::Arc<std::collections::HashMap<K, V>>>>()
|
||||
.map_err(|_| format!("Type mismatch: expected Listener<Arc<HashMap<{}, {}>>>", std::any::type_name::<K>(), std::any::type_name::<V>()))?;
|
||||
listener.listen().await.map_err(|_| "Listener closed before receiving value".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
// Macro to implement ExtractFromListeners for tuple types
|
||||
// Note: Tuples now contain Arc-wrapped elements for efficiency
|
||||
macro_rules! impl_extract_tuple {
|
||||
($($T:ident : $idx:tt),+) => {
|
||||
#[allow(non_snake_case)]
|
||||
impl<$($T),+> ExtractFromListeners for ($(std::sync::Arc<$T>,)+)
|
||||
where
|
||||
$($T: 'static + Clone + Send + Sync),+
|
||||
{
|
||||
async fn extract(listeners: Vec<Box<dyn Any + Send>>) -> Result<Self, String> {
|
||||
let expected_count = [$($idx),+].len();
|
||||
if listeners.len() != expected_count {
|
||||
return Err(format!("Expected {} dependencies, got {}", expected_count, listeners.len()));
|
||||
}
|
||||
|
||||
let mut iter = listeners.into_iter();
|
||||
$(
|
||||
let $T = *iter.next().unwrap()
|
||||
.downcast::<announcement::Listener<std::sync::Arc<$T>>>()
|
||||
.map_err(|_| format!("Type mismatch at index {}: expected Listener<Arc<{}>>", $idx, std::any::type_name::<$T>()))?;
|
||||
)+
|
||||
|
||||
let results = futures::join!(
|
||||
$(
|
||||
async move {
|
||||
$T.listen().await.map_err(|_| format!("Listener {} closed", $idx))
|
||||
}
|
||||
),+
|
||||
);
|
||||
|
||||
Ok(($(
|
||||
results.$idx?
|
||||
,)+))
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
impl_extract_tuple!(A:0, B:1);
|
||||
impl_extract_tuple!(A:0, B:1, C:2);
|
||||
impl_extract_tuple!(A:0, B:1, C:2, D:3);
|
||||
impl_extract_tuple!(A:0, B:1, C:2, D:3, E:4);
|
||||
impl_extract_tuple!(A:0, B:1, C:2, D:3, E:4, F:5);
|
||||
impl_extract_tuple!(A:0, B:1, C:2, D:3, E:4, F:5, G:6);
|
||||
impl_extract_tuple!(A:0, B:1, C:2, D:3, E:4, F:5, G:6, H:7);
|
||||
use std::{any::Any, future::Future, marker::PhantomData, slice::Iter, sync::Arc};
|
||||
|
||||
/// A unit of async work with typed inputs and outputs.
|
||||
///
|
||||
/// **NOTE**: For new code, use the [`task`](crate::task) macro which automatically generates
|
||||
/// an `execute()` method and handles all the wiring. This trait is primarily for internal use
|
||||
/// and backward compatibility with `task_fn`.
|
||||
/// Use the [`task`](crate::task) macro to implement this trait. The macro automatically
|
||||
/// derives `Input` and `Output` types from your `run()` method signature and generates
|
||||
/// type-specific extraction logic in `run()`.
|
||||
///
|
||||
/// The `#[task]` macro generates an `execute()` method on your struct that:
|
||||
/// - Extracts inputs from announcement listeners
|
||||
/// - Calls your `run()` method
|
||||
/// - Announces the result to dependents
|
||||
/// - Returns Arc-wrapped output
|
||||
/// **Custom types work automatically!** As long as your output type implements
|
||||
/// `Send + Sync + 'static`, the macro handles everything. No trait implementations needed!
|
||||
///
|
||||
/// # Task Patterns
|
||||
///
|
||||
/// dagx supports three main task patterns:
|
||||
///
|
||||
/// ## 1. Stateless Tasks (Pure Functions)
|
||||
///
|
||||
/// Tasks with no internal state - just transform inputs to outputs:
|
||||
///
|
||||
/// ```
|
||||
/// use dagx::{task, Task};
|
||||
///
|
||||
/// struct Add;
|
||||
///
|
||||
/// #[task]
|
||||
/// impl Add {
|
||||
/// async fn run(a: &i32, b: &i32) -> i32 {
|
||||
/// a + b
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// ## 2. Stateful Tasks with Shared State (`&self`)
|
||||
///
|
||||
/// Tasks that read from internal state but don't modify it:
|
||||
///
|
||||
/// ```
|
||||
/// use dagx::{task, Task};
|
||||
///
|
||||
/// struct Multiplier {
|
||||
/// factor: i32,
|
||||
/// }
|
||||
///
|
||||
/// #[task]
|
||||
/// impl Multiplier {
|
||||
/// async fn run(&self, input: &i32) -> i32 {
|
||||
/// input * self.factor
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// ## 3. Stateful Tasks with Mutable State (`&mut self`)
|
||||
///
|
||||
/// Tasks that modify internal state during execution:
|
||||
///
|
||||
/// ```
|
||||
/// use dagx::{task, Task};
|
||||
///
|
||||
/// struct Counter {
|
||||
/// count: i32,
|
||||
/// }
|
||||
///
|
||||
/// #[task]
|
||||
/// impl Counter {
|
||||
/// async fn run(&mut self, increment: &i32) -> i32 {
|
||||
/// self.count += increment;
|
||||
/// self.count
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// # Input Patterns
|
||||
///
|
||||
/// - `()`: No dependencies (source task)
|
||||
/// - `&T`: Single dependency
|
||||
/// - `&A, &B, &C, ...`: Multiple dependencies (up to 8, order matters)
|
||||
///
|
||||
/// # The Underlying Trait
|
||||
///
|
||||
/// The `#[task]` macro generates an implementation of this trait:
|
||||
///
|
||||
/// # Output Wrapping
|
||||
///
|
||||
/// **Task outputs are automatically wrapped in `Arc<T>` internally** for efficient fan-out.
|
||||
/// You define output as `T`, but the framework wraps it in `Arc<T>` when announcing.
|
||||
/// You define `Output = T`, but the framework wraps it in `Arc<T>` before sending to dependents.
|
||||
/// This enables O(1) sharing for 1:N dependencies - Arc is cloned (cheap), not your data.
|
||||
///
|
||||
/// Downstream tasks receive `&T` after ExtractInput extracts from the Arc. This is transparent
|
||||
/// to your task code - you work with `T`, the framework handles Arc wrapping/unwrapping.
|
||||
pub trait Task: Send {
|
||||
type Input: Send;
|
||||
type Output: Send + Sync + Clone; // Sync and Clone required for Arc-wrapping and announcement
|
||||
type Output: Send + Sync; // Sync required for Arc-wrapping and cross-thread sharing
|
||||
|
||||
fn run(self, input: Self::Input) -> impl Future<Output = Self::Output> + Send;
|
||||
fn run(self, input: TaskInput<Self::Input>) -> impl Future<Output = Self::Output> + Send;
|
||||
}
|
||||
|
||||
/// Convenience function to create a task from a closure.
|
||||
///
|
||||
/// **INTERNAL USE ONLY**: This function is for testing purposes only and should
|
||||
/// not be used in production code. Use the `#[task]` macro instead.
|
||||
#[doc(hidden)]
|
||||
pub fn task_fn<I, O, Fut, F>(f: F) -> impl Task<Input = I, Output = O> + crate::node::TaskExecute<Input = I, Output = O>
|
||||
where
|
||||
F: FnMut(I) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = O> + Send + 'static,
|
||||
I: ExtractFromListeners,
|
||||
O: Send + Sync + Clone + 'static,
|
||||
{
|
||||
struct TaskFn<I, O, Fut, F>
|
||||
where
|
||||
F: FnMut(I) -> Fut + Send,
|
||||
Fut: Future<Output = O> + Send,
|
||||
I: Send,
|
||||
O: Send,
|
||||
{
|
||||
f: F,
|
||||
_phantom: std::marker::PhantomData<fn(I) -> O>,
|
||||
}
|
||||
pub struct TaskInput<'inputs, Input: Send> {
|
||||
inputs: Iter<'inputs, Arc<dyn Any + Send + Sync + 'static>>,
|
||||
phantom: PhantomData<Input>,
|
||||
}
|
||||
|
||||
impl<I, O, Fut, F> Task for TaskFn<I, O, Fut, F>
|
||||
where
|
||||
F: FnMut(I) -> Fut + Send,
|
||||
Fut: Future<Output = O> + Send,
|
||||
I: Send,
|
||||
O: Send + Sync + Clone,
|
||||
{
|
||||
type Input = I;
|
||||
type Output = O;
|
||||
|
||||
async fn run(mut self, input: I) -> O {
|
||||
(self.f)(input).await
|
||||
impl<'inputs, Input: Send> TaskInput<'inputs, Input> {
|
||||
pub(crate) fn new(inputs: Iter<'inputs, Arc<dyn Any + Send + Sync + 'static>>) -> Self {
|
||||
Self {
|
||||
inputs,
|
||||
phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, O, Fut, F> crate::node::TaskExecute for TaskFn<I, O, Fut, F>
|
||||
where
|
||||
F: FnMut(I) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = O> + Send + 'static,
|
||||
I: ExtractFromListeners,
|
||||
O: Send + Sync + Clone + 'static,
|
||||
{
|
||||
type Input = I;
|
||||
type Output = O;
|
||||
|
||||
fn execute(
|
||||
mut self,
|
||||
listeners: Vec<Box<dyn std::any::Any + Send>>,
|
||||
announcer: Box<dyn std::any::Any + Send>,
|
||||
) -> std::pin::Pin<
|
||||
Box<
|
||||
dyn Future<Output = Result<std::sync::Arc<dyn std::any::Any + Send + Sync>, String>>
|
||||
+ Send,
|
||||
>,
|
||||
> {
|
||||
Box::pin(async move {
|
||||
use std::sync::Arc;
|
||||
|
||||
// Extract the input using the ExtractFromListeners trait
|
||||
let input = I::extract(listeners).await?;
|
||||
|
||||
// Call the closure
|
||||
let output: O = (self.f)(input).await;
|
||||
|
||||
// Wrap in Arc
|
||||
let output_arc = Arc::new(output);
|
||||
|
||||
// Announce the result (announce the Arc, not a clone of the data)
|
||||
let announcer_typed = announcer
|
||||
.downcast::<announcement::Announcer<Arc<O>>>()
|
||||
.map_err(|_| {
|
||||
"Type mismatch: announcer type does not match task output type".to_string()
|
||||
})?;
|
||||
let _ = announcer_typed.announce(Arc::clone(&output_arc));
|
||||
|
||||
// Return Arc-wrapped output
|
||||
Ok(output_arc as Arc<dyn std::any::Any + Send + Sync>)
|
||||
})
|
||||
macro_rules! impl_task_input {
|
||||
($First:ident $(, $T:ident)*) => {
|
||||
impl<'inputs, $First: 'static + Send, $($T: 'static + Send),*> TaskInput<'inputs, ($First, $($T,)*)> {
|
||||
#[must_use]
|
||||
pub fn next(mut self) -> (&'inputs $First, TaskInput<'inputs, ($($T,)*)>) {
|
||||
let value = self.inputs.next().and_then(|value| value.downcast_ref()).unwrap();
|
||||
let next_inputs = TaskInput {
|
||||
inputs: self.inputs,
|
||||
phantom: PhantomData,
|
||||
};
|
||||
(value, next_inputs)
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
fn create_announcement_erased() -> (Box<dyn std::any::Any + Send>, Box<dyn std::any::Any + Send>) {
|
||||
let (announcer, announcement) = announcement::Announcement::<Arc<O>>::new();
|
||||
(
|
||||
Box::new(announcer) as Box<dyn std::any::Any + Send>,
|
||||
Box::new(announcement) as Box<dyn std::any::Any + Send>,
|
||||
)
|
||||
impl_task_input!(A);
|
||||
impl_task_input!(A, B);
|
||||
impl_task_input!(A, B, C);
|
||||
impl_task_input!(A, B, C, D);
|
||||
impl_task_input!(A, B, C, D, E);
|
||||
impl_task_input!(A, B, C, D, E, F);
|
||||
impl_task_input!(A, B, C, D, E, F, G);
|
||||
impl_task_input!(A, B, C, D, E, F, G, H);
|
||||
|
||||
impl TaskInput<'static, ()> {
|
||||
pub fn empty() -> Self {
|
||||
Self {
|
||||
inputs: [].iter(),
|
||||
phantom: PhantomData,
|
||||
}
|
||||
|
||||
fn create_listener_erased(announcement: &dyn std::any::Any) -> Box<dyn std::any::Any + Send> {
|
||||
let announcement = announcement
|
||||
.downcast_ref::<announcement::Announcement<Arc<O>>>()
|
||||
.expect("Type mismatch: announcement type does not match task output type");
|
||||
Box::new(announcement.listener()) as Box<dyn std::any::Any + Send>
|
||||
}
|
||||
}
|
||||
|
||||
TaskFn {
|
||||
f,
|
||||
_phantom: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+15
-25
@@ -1,8 +1,8 @@
|
||||
//! Unit tests for task module
|
||||
|
||||
use crate::task::Task;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use crate::task::{Task, TaskInput};
|
||||
use std::any::Any;
|
||||
use std::sync::Arc;
|
||||
|
||||
// Test custom Task implementation
|
||||
struct CustomTask {
|
||||
@@ -10,15 +10,12 @@ struct CustomTask {
|
||||
}
|
||||
|
||||
impl Task for CustomTask {
|
||||
type Input = i32;
|
||||
type Input = (i32,);
|
||||
type Output = i32;
|
||||
|
||||
#[allow(refining_impl_trait)]
|
||||
fn run(mut self, input: Self::Input) -> Pin<Box<dyn Future<Output = Self::Output> + Send>> {
|
||||
Box::pin(async move {
|
||||
self.state += input;
|
||||
self.state
|
||||
})
|
||||
async fn run(mut self, input: TaskInput<'_, Self::Input>) -> i32 {
|
||||
self.state += input.next().0;
|
||||
self.state
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,26 +23,19 @@ impl Task for CustomTask {
|
||||
async fn test_custom_task_implementation() {
|
||||
let task = CustomTask { state: 10 };
|
||||
|
||||
let result1 = task.run(5).await;
|
||||
let result1 = task
|
||||
.run(TaskInput::new(
|
||||
[Arc::new(5) as Arc<dyn Any + Send + Sync + 'static>].iter(),
|
||||
))
|
||||
.await;
|
||||
assert_eq!(result1, 15);
|
||||
|
||||
// Can't call run again since task was consumed
|
||||
// This demonstrates the ownership model
|
||||
}
|
||||
|
||||
// Test that Task trait bounds are correct
|
||||
fn assert_task_bounds<T: Task>() {
|
||||
// This function just checks that T implements Task with the right bounds
|
||||
fn assert_send<T: Send>() {}
|
||||
|
||||
// These should compile if bounds are correct
|
||||
assert_send::<T>();
|
||||
assert_send::<T::Input>();
|
||||
assert_send::<T::Output>();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_task_trait_bounds() {
|
||||
// Verify that our test tasks meet the Task trait bounds
|
||||
assert_task_bounds::<CustomTask>();
|
||||
fn test_empty_task_input() {
|
||||
let empty = TaskInput::empty();
|
||||
assert_eq!(empty.inputs.len(), 0);
|
||||
}
|
||||
+11
-22
@@ -3,12 +3,12 @@
|
||||
//! This module defines the fundamental types used throughout the DAG system,
|
||||
//! including node identifiers, task handles, and type-state markers.
|
||||
|
||||
use crate::builder::{IsUnitType, TaskBuilder};
|
||||
use crate::node::TaskExecute;
|
||||
use crate::builder::TaskBuilder;
|
||||
use crate::task::Task;
|
||||
|
||||
/// Opaque node identifier (internal use)
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub(crate) struct NodeId(pub usize);
|
||||
/// Opaque node identifier
|
||||
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
|
||||
pub struct NodeId(pub u32);
|
||||
|
||||
/// Opaque, typed token for a node's output.
|
||||
///
|
||||
@@ -23,6 +23,7 @@ pub(crate) struct NodeId(pub usize);
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use dagx::{task, DagRunner, Task};
|
||||
/// #
|
||||
/// # struct LoadValue { value: i32 }
|
||||
/// # impl LoadValue { pub fn new(v: i32) -> Self { Self { value: v } } }
|
||||
/// # #[task]
|
||||
@@ -33,7 +34,7 @@ pub(crate) struct NodeId(pub usize);
|
||||
/// let dag = DagRunner::new();
|
||||
/// let node = dag.add_task(LoadValue::new(42));
|
||||
///
|
||||
/// dag.run(|fut| { tokio::spawn(fut); }).await.unwrap();
|
||||
/// dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await.unwrap();
|
||||
///
|
||||
/// assert_eq!(dag.get(node).unwrap(), 42);
|
||||
/// # };
|
||||
@@ -62,10 +63,7 @@ pub struct Pending;
|
||||
|
||||
// TaskBuilder can be converted to a TaskHandle ONLY if Input is () (unit type)
|
||||
// This enforces at compile time that tasks with non-unit inputs MUST call .depends_on()
|
||||
impl<'a, Tk: TaskExecute, Deps> From<TaskBuilder<'a, Tk, Deps>> for TaskHandle<Tk::Output>
|
||||
where
|
||||
Tk::Input: IsUnitType,
|
||||
{
|
||||
impl<'a, Tk: Task<Input = ()>, Deps> From<TaskBuilder<'a, Tk, Deps>> for TaskHandle<Tk::Output> {
|
||||
fn from(node: TaskBuilder<'a, Tk, Deps>) -> Self {
|
||||
TaskHandle {
|
||||
id: node.id,
|
||||
@@ -74,18 +72,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, Tk: TaskExecute, Deps> From<&TaskBuilder<'a, Tk, Deps>> for TaskHandle<Tk::Output>
|
||||
where
|
||||
Tk::Input: IsUnitType,
|
||||
{
|
||||
fn from(node: &TaskBuilder<'a, Tk, Deps>) -> Self {
|
||||
TaskHandle {
|
||||
id: node.id,
|
||||
_phantom: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TaskHandle can be converted from &TaskHandle (for .get() calls)
|
||||
impl<T> From<&TaskHandle<T>> for TaskHandle<T> {
|
||||
fn from(handle: &TaskHandle<T>) -> Self {
|
||||
@@ -95,3 +81,6 @@ impl<T> From<&TaskHandle<T>> for TaskHandle<T> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod coverage_tests;
|
||||
@@ -0,0 +1,63 @@
|
||||
//! Coverage tests for types module - specifically targeting the TaskBuilder to TaskHandle conversion
|
||||
//!
|
||||
//! These tests ensure we hit line 71 in the From<TaskBuilder> implementation
|
||||
|
||||
use crate::{task, DagRunner, TaskHandle};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_unit_struct_with_state() {
|
||||
// Even structs with fields can have unit input type
|
||||
|
||||
struct StatefulTask {
|
||||
value: i32,
|
||||
}
|
||||
|
||||
impl StatefulTask {
|
||||
fn new(value: i32) -> Self {
|
||||
Self { value }
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl StatefulTask {
|
||||
async fn run(&self) -> i32 {
|
||||
self.value * 2
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Create task with initial state
|
||||
let builder = dag.add_task(StatefulTask::new(21));
|
||||
|
||||
// Explicit conversion - triggers line 71
|
||||
let handle: TaskHandle<_> = builder.into();
|
||||
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(dag.get(handle).unwrap(), 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_taskbuilder_conversion_without_running() {
|
||||
// Test that conversion works even without running the DAG
|
||||
struct TestTask;
|
||||
#[task]
|
||||
impl TestTask {
|
||||
async fn run(&self) -> String {
|
||||
"test".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let builder = dag.add_task(TestTask);
|
||||
let builder_id = builder.id;
|
||||
|
||||
// Convert without running the DAG
|
||||
let handle: TaskHandle<String> = builder.into();
|
||||
|
||||
// Verify the ID was transferred correctly
|
||||
assert_eq!(handle.id, builder_id);
|
||||
}
|
||||
+6
-285
@@ -1,60 +1,22 @@
|
||||
//! Unit tests for types module
|
||||
|
||||
// Allow tests to reference dagx:: paths (for proc macro generated code)
|
||||
extern crate self as dagx;
|
||||
|
||||
use crate::types::{NodeId, TaskHandle};
|
||||
use std::marker::PhantomData;
|
||||
|
||||
#[test]
|
||||
fn test_node_id_operations() {
|
||||
let id1 = NodeId(1);
|
||||
let id2 = NodeId(2);
|
||||
let id3 = NodeId(1);
|
||||
|
||||
// Test equality
|
||||
assert_eq!(id1, id3);
|
||||
assert_ne!(id1, id2);
|
||||
|
||||
// Test ordering
|
||||
assert!(id1 < id2);
|
||||
assert!(id2 > id1);
|
||||
|
||||
// Test Debug
|
||||
let debug_str = format!("{:?}", id1);
|
||||
assert!(debug_str.contains("NodeId"));
|
||||
assert!(debug_str.contains("1"));
|
||||
|
||||
// Test Clone
|
||||
let cloned = id1;
|
||||
assert_eq!(id1, cloned);
|
||||
|
||||
// Test Copy
|
||||
let copied = id1;
|
||||
assert_eq!(id1, copied);
|
||||
|
||||
// Test Hash (compile-time check that it implements Hash)
|
||||
use std::collections::HashSet;
|
||||
let mut set = HashSet::new();
|
||||
set.insert(id1);
|
||||
set.insert(id2);
|
||||
assert_eq!(set.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::clone_on_copy)]
|
||||
fn test_task_handle_clone() {
|
||||
let handle: TaskHandle<i32> = TaskHandle {
|
||||
let handle: TaskHandle<(i32,)> = TaskHandle {
|
||||
id: NodeId(42),
|
||||
_phantom: PhantomData,
|
||||
};
|
||||
|
||||
// Test Clone implementation (lines 44-46 in types.rs)
|
||||
let cloned = handle;
|
||||
// Test Clone implementation
|
||||
let cloned = handle.clone();
|
||||
assert_eq!(handle.id.0, cloned.id.0);
|
||||
|
||||
// Test that clone is actually a copy (same memory semantics)
|
||||
let handle2 = handle;
|
||||
let handle3 = handle2;
|
||||
let handle2 = handle.clone();
|
||||
let handle3 = handle2.clone();
|
||||
assert_eq!(handle3.id.0, 42);
|
||||
}
|
||||
|
||||
@@ -72,70 +34,6 @@ fn test_task_handle_copy() {
|
||||
assert_eq!(copied2.id.0, 99);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_task_handle_with_different_types() {
|
||||
// Test that TaskHandle works with various types
|
||||
let int_handle: TaskHandle<i32> = TaskHandle {
|
||||
id: NodeId(1),
|
||||
_phantom: PhantomData,
|
||||
};
|
||||
|
||||
let string_handle: TaskHandle<String> = TaskHandle {
|
||||
id: NodeId(2),
|
||||
_phantom: PhantomData,
|
||||
};
|
||||
|
||||
let vec_handle: TaskHandle<Vec<u8>> = TaskHandle {
|
||||
id: NodeId(3),
|
||||
_phantom: PhantomData,
|
||||
};
|
||||
|
||||
// Clone and copy should work for all
|
||||
let _ = int_handle;
|
||||
let _ = string_handle;
|
||||
let _ = vec_handle;
|
||||
|
||||
let _ = int_handle;
|
||||
let _ = string_handle;
|
||||
let _ = vec_handle;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_task_handle_explicit_clone() {
|
||||
// Test explicit .clone() call to cover line 47-48
|
||||
let handle: TaskHandle<i32> = TaskHandle {
|
||||
id: NodeId(42),
|
||||
_phantom: PhantomData,
|
||||
};
|
||||
|
||||
#[allow(clippy::clone_on_copy)]
|
||||
let cloned = handle.clone();
|
||||
assert_eq!(handle.id.0, cloned.id.0);
|
||||
assert_eq!(cloned.id.0, 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_task_builder_from_conversion() {
|
||||
use crate::runner::DagRunner;
|
||||
|
||||
// Test From<TaskBuilder> for TaskHandle (line 69-74)
|
||||
struct SimpleTask;
|
||||
#[crate::task]
|
||||
impl SimpleTask {
|
||||
async fn run(&self) -> i32 {
|
||||
42
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let builder = dag.add_task(SimpleTask);
|
||||
let builder_id = builder.id;
|
||||
|
||||
// This conversion uses the From implementation
|
||||
let handle: TaskHandle<i32> = builder.into();
|
||||
assert_eq!(handle.id, builder_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_task_handle_ref_from_conversion() {
|
||||
// Test From<&TaskHandle<T>> for TaskHandle<T> (line 90-93)
|
||||
@@ -150,180 +48,3 @@ fn test_task_handle_ref_from_conversion() {
|
||||
assert_eq!(converted.id.0, 123);
|
||||
assert_eq!(handle.id, converted.id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_task_builder_ref_from_conversion() {
|
||||
use crate::runner::DagRunner;
|
||||
|
||||
// Test From<&TaskBuilder> for TaskHandle (line 77-87, specifically line 83)
|
||||
struct SimpleTask2;
|
||||
#[crate::task]
|
||||
impl SimpleTask2 {
|
||||
async fn run(&self) -> String {
|
||||
"test".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let builder = dag.add_task(SimpleTask2);
|
||||
let builder_id = builder.id;
|
||||
|
||||
// This conversion uses the From<&TaskBuilder> implementation
|
||||
let handle: TaskHandle<String> = (&builder).into();
|
||||
assert_eq!(handle.id, builder_id);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Additional TaskBuilder conversion coverage tests
|
||||
// Port from commit sq (sqpzloqrlykp) - src/types/coverage_tests.rs
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_taskbuilder_conversion_with_multiple_tasks() {
|
||||
// Test conversion with multiple unit-type tasks to ensure IDs are preserved
|
||||
use crate::runner::DagRunner;
|
||||
|
||||
struct FirstTask;
|
||||
#[crate::task]
|
||||
impl FirstTask {
|
||||
async fn run(&self) -> String {
|
||||
"first".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
struct SecondTask;
|
||||
#[crate::task]
|
||||
impl SecondTask {
|
||||
async fn run(&self) -> i32 {
|
||||
100
|
||||
}
|
||||
}
|
||||
|
||||
struct ThirdTask;
|
||||
#[crate::task]
|
||||
impl ThirdTask {
|
||||
async fn run(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Create builders for all tasks
|
||||
let first_builder = dag.add_task(FirstTask);
|
||||
let second_builder = dag.add_task(SecondTask);
|
||||
let third_builder = dag.add_task(ThirdTask);
|
||||
|
||||
// Save IDs before conversion
|
||||
let first_id = first_builder.id;
|
||||
let second_id = second_builder.id;
|
||||
let third_id = third_builder.id;
|
||||
|
||||
// Convert each builder to handle explicitly
|
||||
let first_handle: TaskHandle<String> = first_builder.into();
|
||||
let second_handle: TaskHandle<i32> = second_builder.into();
|
||||
let third_handle: TaskHandle<bool> = third_builder.into();
|
||||
|
||||
// Verify IDs are preserved through conversion
|
||||
assert_eq!(first_handle.id, first_id);
|
||||
assert_eq!(second_handle.id, second_id);
|
||||
assert_eq!(third_handle.id, third_id);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Verify all outputs are correct
|
||||
assert_eq!(dag.get(first_handle).unwrap(), "first");
|
||||
assert_eq!(dag.get(second_handle).unwrap(), 100);
|
||||
assert!(dag.get(third_handle).unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_taskbuilder_conversion_with_stateful_task() {
|
||||
// Test that stateful tasks (with fields) can have unit input and convert correctly
|
||||
use crate::runner::DagRunner;
|
||||
|
||||
struct StatefulTask {
|
||||
value: i32,
|
||||
}
|
||||
|
||||
impl StatefulTask {
|
||||
fn new(value: i32) -> Self {
|
||||
Self { value }
|
||||
}
|
||||
}
|
||||
|
||||
#[crate::task]
|
||||
impl StatefulTask {
|
||||
async fn run(&self) -> i32 {
|
||||
self.value * 2
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Create task with initial state
|
||||
let builder = dag.add_task(StatefulTask::new(21));
|
||||
let builder_id = builder.id;
|
||||
|
||||
// Explicit conversion - ensures From<TaskBuilder> works for stateful tasks
|
||||
let handle: TaskHandle<i32> = builder.into();
|
||||
assert_eq!(handle.id, builder_id);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(dag.get(handle).unwrap(), 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_taskbuilder_conversion_with_complex_output_types() {
|
||||
// Test conversion works with complex output types (Vec, Option, etc.)
|
||||
use crate::runner::DagRunner;
|
||||
|
||||
struct VecTask;
|
||||
#[crate::task]
|
||||
impl VecTask {
|
||||
async fn run(&self) -> Vec<i32> {
|
||||
vec![1, 2, 3]
|
||||
}
|
||||
}
|
||||
|
||||
struct OptionTask;
|
||||
#[crate::task]
|
||||
impl OptionTask {
|
||||
async fn run(&self) -> Option<String> {
|
||||
Some("value".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let vec_builder = dag.add_task(VecTask);
|
||||
let option_builder = dag.add_task(OptionTask);
|
||||
|
||||
let vec_id = vec_builder.id;
|
||||
let option_id = option_builder.id;
|
||||
|
||||
// Convert to handles with complex types
|
||||
let vec_handle: TaskHandle<Vec<i32>> = vec_builder.into();
|
||||
let option_handle: TaskHandle<Option<String>> = option_builder.into();
|
||||
|
||||
assert_eq!(vec_handle.id, vec_id);
|
||||
assert_eq!(option_handle.id, option_id);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(dag.get(vec_handle).unwrap(), vec![1, 2, 3]);
|
||||
assert_eq!(dag.get(option_handle).unwrap(), Some("value".to_string()));
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
|
||||
[test_config]
|
||||
manifest-path = "./Cargo.toml"
|
||||
skip-clean = true
|
||||
engine = "Llvm"
|
||||
# cargo-tarpaulin already reports tracing lines incorrectly when the feature is turned off,
|
||||
# and this is exacerbated when building with all-features.
|
||||
all-features = false
|
||||
run-types = ["Tests"]
|
||||
command = "Test"
|
||||
locked = true
|
||||
frozen = true
|
||||
fail-under = 80.0
|
||||
-1381
File diff suppressed because it is too large.
Load diff
+43
-167
@@ -1,73 +1,11 @@
|
||||
//! Tests for extremely deep dependency chains
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::{DagResult, DagRunner};
|
||||
use dagx_test::task_fn;
|
||||
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_1000_level_deep_chain() -> DagResult<()> {
|
||||
// Test a linear chain of 1000 tasks
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let first = dag.add_task(task_fn(|_: ()| async { 0 }));
|
||||
let mut current = (&first).into(); // Convert to TaskHandle
|
||||
|
||||
for _i in 1..1000 {
|
||||
current = dag
|
||||
.add_task(task_fn(move |prev: i32| async move { prev + 1 }))
|
||||
.depends_on(current);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(current)?, 999);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deep_chain_with_branches() -> DagResult<()> {
|
||||
// Test a deep chain where each node has a side branch
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let first = dag.add_task(task_fn(|_: ()| async { 0 }));
|
||||
let mut main_chain: dagx::TaskHandle<i32> = (&first).into();
|
||||
let mut branches = Vec::new();
|
||||
|
||||
for i in 1..500 {
|
||||
// Main chain continues
|
||||
main_chain = dag
|
||||
.add_task(task_fn(move |prev: i32| async move { prev + 1 }))
|
||||
.depends_on(main_chain);
|
||||
|
||||
// Create a branch at every 10th node
|
||||
if i % 10 == 0 {
|
||||
let branch = dag
|
||||
.add_task(task_fn(move |main_val: i32| async move { main_val * 2 }))
|
||||
.depends_on(main_chain);
|
||||
branches.push(branch);
|
||||
}
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(main_chain)?, 499);
|
||||
|
||||
// Check some branches
|
||||
if !branches.is_empty() {
|
||||
assert_eq!(dag.get(branches[0])?, 20); // 10 * 2
|
||||
assert_eq!(dag.get(branches[4])?, 100); // 50 * 2
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_parallel_deep_chains() -> DagResult<()> {
|
||||
// Test 10 parallel chains of depth 500 each
|
||||
@@ -76,12 +14,12 @@ async fn test_multiple_parallel_deep_chains() -> DagResult<()> {
|
||||
let mut chain_ends = Vec::new();
|
||||
|
||||
for chain_id in 0..8 {
|
||||
let first = dag.add_task(task_fn(move |_: ()| async move { chain_id * 1000 }));
|
||||
let mut current: dagx::TaskHandle<i32> = (&first).into();
|
||||
let first = dag.add_task(task_fn::<(), _, _>(move |_: ()| chain_id * 1000));
|
||||
let mut current: dagx::TaskHandle<i32> = first.into();
|
||||
|
||||
for _ in 0..500 {
|
||||
current = dag
|
||||
.add_task(task_fn(|prev: i32| async move { prev + 1 }))
|
||||
.add_task(task_fn::<i32, _, _>(|&prev: &i32| prev + 1))
|
||||
.depends_on(current);
|
||||
}
|
||||
|
||||
@@ -90,8 +28,8 @@ async fn test_multiple_parallel_deep_chains() -> DagResult<()> {
|
||||
|
||||
// Add a final task that depends on all chains
|
||||
let final_task = dag
|
||||
.add_task(task_fn(
|
||||
|inputs: (i32, i32, i32, i32, i32, i32, i32, i32)| async move {
|
||||
.add_task(task_fn::<(i32, i32, i32, i32, i32, i32, i32, i32), _, _>(
|
||||
|inputs: (&i32, &i32, &i32, &i32, &i32, &i32, &i32, &i32)| {
|
||||
inputs.0
|
||||
+ inputs.1
|
||||
+ inputs.2
|
||||
@@ -113,10 +51,8 @@ async fn test_multiple_parallel_deep_chains() -> DagResult<()> {
|
||||
&chain_ends[7],
|
||||
));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
// Each chain: start + 500
|
||||
// Chain 0: 0 + 500 = 500
|
||||
@@ -134,37 +70,31 @@ async fn test_deep_chain_execution_order() -> DagResult<()> {
|
||||
let dag = DagRunner::new();
|
||||
let order = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let first = dag.add_task(task_fn({
|
||||
let first = dag.add_task(task_fn::<(), _, _>({
|
||||
let order = order.clone();
|
||||
move |_: ()| {
|
||||
let order = order.clone();
|
||||
async move {
|
||||
assert_eq!(order.fetch_add(1, Ordering::SeqCst), 0);
|
||||
0
|
||||
}
|
||||
assert_eq!(order.fetch_add(1, Ordering::SeqCst), 0);
|
||||
0
|
||||
}
|
||||
}));
|
||||
let mut current: dagx::TaskHandle<i32> = (&first).into();
|
||||
let mut current: dagx::TaskHandle<i32> = first.into();
|
||||
|
||||
for expected in 1..100 {
|
||||
current = dag
|
||||
.add_task(task_fn({
|
||||
.add_task(task_fn::<i32, _, _>({
|
||||
let order = order.clone();
|
||||
move |prev: i32| {
|
||||
move |&prev: &i32| {
|
||||
let order = order.clone();
|
||||
async move {
|
||||
assert_eq!(order.fetch_add(1, Ordering::SeqCst), expected);
|
||||
prev + 1
|
||||
}
|
||||
assert_eq!(order.fetch_add(1, Ordering::SeqCst), expected);
|
||||
prev + 1
|
||||
}
|
||||
}))
|
||||
.depends_on(current);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
assert_eq!(order.load(Ordering::SeqCst), 100);
|
||||
assert_eq!(dag.get(current)?, 99);
|
||||
@@ -179,23 +109,23 @@ async fn test_deep_binary_tree() -> DagResult<()> {
|
||||
|
||||
fn build_tree(dag: &DagRunner, depth: usize, value: i32) -> dagx::TaskHandle<i32> {
|
||||
if depth == 0 {
|
||||
let task = dag.add_task(task_fn(move |_: ()| async move { value }));
|
||||
(&task).into() // Convert TaskBuilder to TaskHandle
|
||||
let task = dag.add_task(task_fn::<(), _, _>(move |_: ()| value));
|
||||
task.into() // Convert TaskBuilder to TaskHandle
|
||||
} else {
|
||||
let left = build_tree(dag, depth - 1, value * 2);
|
||||
let right = build_tree(dag, depth - 1, value * 2 + 1);
|
||||
|
||||
dag.add_task(task_fn(move |(l, r): (i32, i32)| async move { l + r }))
|
||||
.depends_on((&left, &right))
|
||||
dag.add_task(task_fn::<(i32, i32), _, _>(move |(l, r): (&i32, &i32)| {
|
||||
l + r
|
||||
}))
|
||||
.depends_on((&left, &right))
|
||||
}
|
||||
}
|
||||
|
||||
let root = build_tree(&dag, 10, 1);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
// Sum of all values in a complete binary tree with values 1024..2047
|
||||
// This is sum from 2^10 to 2^11-1 = 1024 * 1023 / 2 + 1024 * 512 = 1047552
|
||||
@@ -210,25 +140,23 @@ async fn test_fibonacci_chain() -> DagResult<()> {
|
||||
// Test a Fibonacci-like dependency pattern
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let f0_builder = dag.add_task(task_fn(|_: ()| async { 0i64 }));
|
||||
let f1_builder = dag.add_task(task_fn(|_: ()| async { 1i64 }));
|
||||
let f0_builder = dag.add_task(task_fn::<(), _, _>(|_: ()| 0i64));
|
||||
let f1_builder = dag.add_task(task_fn::<(), _, _>(|_: ()| 1i64));
|
||||
|
||||
let mut prev2: dagx::TaskHandle<i64> = (&f0_builder).into();
|
||||
let mut prev1: dagx::TaskHandle<i64> = (&f1_builder).into();
|
||||
let mut prev2: dagx::TaskHandle<i64> = f0_builder.into();
|
||||
let mut prev1: dagx::TaskHandle<i64> = f1_builder.into();
|
||||
|
||||
for _ in 2..50 {
|
||||
let next = dag
|
||||
.add_task(task_fn(|(a, b): (i64, i64)| async move { a + b }))
|
||||
.add_task(task_fn::<(i64, i64), _, _>(|(a, b): (&i64, &i64)| a + b))
|
||||
.depends_on((&prev2, &prev1));
|
||||
|
||||
prev2 = prev1;
|
||||
prev1 = next;
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
// 50th Fibonacci number (0-indexed) = 7778742049
|
||||
assert_eq!(dag.get(prev1)?, 7778742049);
|
||||
@@ -236,93 +164,41 @@ async fn test_fibonacci_chain() -> DagResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deep_chain_with_accumulator() -> DagResult<()> {
|
||||
// Test a deep chain where each node accumulates a value
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Use a simpler accumulation pattern - just sum
|
||||
let first = dag.add_task(task_fn(|_: ()| async { 0i64 }));
|
||||
let mut current: dagx::TaskHandle<i64> = (&first).into();
|
||||
|
||||
for i in 1..=200 {
|
||||
current = dag
|
||||
.add_task(task_fn(move |sum: i64| async move { sum + i as i64 }))
|
||||
.depends_on(current);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
let result = dag.get(current)?;
|
||||
assert_eq!(result, 20100); // sum of 1..=200
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_zigzag_dependencies() -> DagResult<()> {
|
||||
// Test a zigzag pattern: two chains that cross-reference each other
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let a_first = dag.add_task(task_fn(|_: ()| async { 1 }));
|
||||
let b_first = dag.add_task(task_fn(|_: ()| async { 2 }));
|
||||
let a_first = dag.add_task(task_fn::<(), _, _>(|_: ()| 1));
|
||||
let b_first = dag.add_task(task_fn::<(), _, _>(|_: ()| 2));
|
||||
|
||||
let mut chain_a: dagx::TaskHandle<i32> = (&a_first).into();
|
||||
let mut chain_b: dagx::TaskHandle<i32> = (&b_first).into();
|
||||
let mut chain_a: dagx::TaskHandle<i32> = a_first.into();
|
||||
let mut chain_b: dagx::TaskHandle<i32> = b_first.into();
|
||||
|
||||
for i in 0..100 {
|
||||
if i % 2 == 0 {
|
||||
// A depends on B
|
||||
chain_a = dag
|
||||
.add_task(task_fn(move |b_val: i32| async move { b_val + 1 }))
|
||||
.add_task(task_fn::<i32, _, _>(move |b_val: &i32| b_val + 1))
|
||||
.depends_on(chain_b);
|
||||
} else {
|
||||
// B depends on A
|
||||
chain_b = dag
|
||||
.add_task(task_fn(move |a_val: i32| async move { a_val + 2 }))
|
||||
.add_task(task_fn::<i32, _, _>(move |a_val: &i32| a_val + 2))
|
||||
.depends_on(chain_a);
|
||||
}
|
||||
}
|
||||
|
||||
// Final task depends on both chains
|
||||
let final_task = dag
|
||||
.add_task(task_fn(|(a, b): (i32, i32)| async move { a + b }))
|
||||
.add_task(task_fn::<(i32, i32), _, _>(|(a, b): (&i32, &i32)| a + b))
|
||||
.depends_on((&chain_a, &chain_b));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
let result = dag.get(final_task)?;
|
||||
assert!(result > 100);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[cfg_attr(tarpaulin, ignore)] // This test is extremely resource intensive
|
||||
async fn test_10000_level_chain_stress() -> DagResult<()> {
|
||||
// Ultimate deep chain test
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let first = dag.add_task(task_fn(|_: ()| async { 0u64 }));
|
||||
let mut current: dagx::TaskHandle<u64> = (&first).into();
|
||||
|
||||
for _ in 1..10_000 {
|
||||
current = dag
|
||||
.add_task(task_fn(|prev: u64| async move { prev + 1 }))
|
||||
.depends_on(current);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(current)?, 9_999);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,249 +0,0 @@
|
||||
//! Large scale stress tests with thousands of nodes and edges
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::{DagResult, DagRunner};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_10000_independent_tasks() -> DagResult<()> {
|
||||
// Test with 10,000 independent tasks
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let counter = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let tasks: Vec<_> = (0..10_000)
|
||||
.map(|i| {
|
||||
let counter = counter.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let counter = counter.clone();
|
||||
async move {
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
i
|
||||
}
|
||||
}))
|
||||
})
|
||||
.collect();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Verify all tasks completed
|
||||
assert_eq!(counter.load(Ordering::Relaxed), 10_000);
|
||||
|
||||
// Spot check some results
|
||||
assert_eq!(dag.get(&tasks[0])?, 0);
|
||||
assert_eq!(dag.get(&tasks[9_999])?, 9_999);
|
||||
assert_eq!(dag.get(&tasks[5_000])?, 5_000);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_5000_node_pyramid() -> DagResult<()> {
|
||||
// Create a pyramid structure with ~5000 nodes
|
||||
// Layer sizes: 2048 -> 512 -> 128 -> 32 -> 8 -> 2 -> 1
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Bottom layer: 2048 nodes - convert to TaskHandles
|
||||
let mut current_layer: Vec<dagx::TaskHandle<i32>> = (0..2048)
|
||||
.map(|i| {
|
||||
let task = dag.add_task(task_fn(move |_: ()| async move {
|
||||
if i < 1024 {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}));
|
||||
(&task).into()
|
||||
})
|
||||
.collect();
|
||||
|
||||
let layer_sizes = vec![512, 128, 32, 8, 2];
|
||||
|
||||
for layer_size in layer_sizes {
|
||||
let mut next_layer = Vec::with_capacity(layer_size);
|
||||
let group_size = current_layer.len() / layer_size;
|
||||
|
||||
for i in 0..layer_size {
|
||||
let start = i * group_size;
|
||||
let deps: Vec<_> = current_layer[start..start + group_size.min(4)]
|
||||
.iter()
|
||||
.take(4)
|
||||
.collect();
|
||||
|
||||
let task = match deps.len() {
|
||||
1 => dag
|
||||
.add_task(task_fn(|a: i32| async move { a }))
|
||||
.depends_on(deps[0]),
|
||||
2 => dag
|
||||
.add_task(task_fn(|(a, b): (i32, i32)| async move { a + b }))
|
||||
.depends_on((deps[0], deps[1])),
|
||||
3 => dag
|
||||
.add_task(task_fn(
|
||||
|(a, b, c): (i32, i32, i32)| async move { a + b + c },
|
||||
))
|
||||
.depends_on((deps[0], deps[1], deps[2])),
|
||||
4 => dag
|
||||
.add_task(task_fn(|(a, b, c, d): (i32, i32, i32, i32)| async move {
|
||||
a + b + c + d
|
||||
}))
|
||||
.depends_on((deps[0], deps[1], deps[2], deps[3])),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
next_layer.push(task);
|
||||
}
|
||||
current_layer = next_layer;
|
||||
}
|
||||
|
||||
// Final layer combines the last 2
|
||||
let root = dag
|
||||
.add_task(task_fn(|(a, b): (i32, i32)| async move { a + b }))
|
||||
.depends_on((¤t_layer[0], ¤t_layer[1]));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// We created 1024 ones and 1024 zeros at the bottom
|
||||
// The sum should propagate up to 1024
|
||||
let result = dag.get(root)?;
|
||||
assert!(result > 0, "Expected positive result, got {}", result);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_wide_dag_1000_sources_1000_sinks() -> DagResult<()> {
|
||||
// Test a very wide DAG with 1000 sources and 1000 sinks
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Create 1000 source tasks
|
||||
let sources: Vec<_> = (0..1000)
|
||||
.map(|i| dag.add_task(task_fn(move |_: ()| async move { i % 10 })))
|
||||
.collect();
|
||||
|
||||
// Create 1000 sink tasks, each depending on 3 random sources
|
||||
let sinks: Vec<_> = (0..1000)
|
||||
.map(|i| {
|
||||
let idx1 = (i * 7) % 1000;
|
||||
let idx2 = (i * 13) % 1000;
|
||||
let idx3 = (i * 17) % 1000;
|
||||
|
||||
dag.add_task(task_fn(move |(a, b, c): (i32, i32, i32)| async move {
|
||||
a + b + c + i as i32
|
||||
}))
|
||||
.depends_on((&sources[idx1], &sources[idx2], &sources[idx3]))
|
||||
})
|
||||
.collect();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Verify some sinks completed correctly
|
||||
let result0 = dag.get(sinks[0])?;
|
||||
let result500 = dag.get(sinks[500])?;
|
||||
let result999 = dag.get(sinks[999])?;
|
||||
|
||||
assert!(result0 >= 0);
|
||||
assert!(result500 >= 500);
|
||||
assert!(result999 >= 999);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_10000_node_linear_chain_segments() -> DagResult<()> {
|
||||
// Test 10000 nodes arranged as 100 parallel chains of 100 nodes each
|
||||
let dag = DagRunner::new();
|
||||
let counter = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let mut chain_ends = Vec::new();
|
||||
|
||||
for chain_id in 0..100 {
|
||||
let first = dag.add_task(task_fn({
|
||||
let counter = counter.clone();
|
||||
move |_: ()| {
|
||||
let counter = counter.clone();
|
||||
async move {
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
chain_id * 100
|
||||
}
|
||||
}
|
||||
}));
|
||||
let mut current: dagx::TaskHandle<i32> = (&first).into();
|
||||
|
||||
for _i in 1..100 {
|
||||
current = dag
|
||||
.add_task(task_fn({
|
||||
let counter = counter.clone();
|
||||
move |prev: i32| {
|
||||
let counter = counter.clone();
|
||||
async move {
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
prev + 1
|
||||
}
|
||||
}
|
||||
}))
|
||||
.depends_on(current);
|
||||
}
|
||||
|
||||
chain_ends.push(current);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// All 10,000 tasks should have executed
|
||||
assert_eq!(counter.load(Ordering::Relaxed), 10_000);
|
||||
|
||||
// Each chain should end with start + 99
|
||||
for (chain_id, &chain_end) in chain_ends.iter().enumerate() {
|
||||
let expected = chain_id as i32 * 100 + 99;
|
||||
assert_eq!(dag.get(chain_end)?, expected);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[cfg_attr(tarpaulin, ignore)] // This test is very resource intensive
|
||||
async fn test_100000_nodes_stress() -> DagResult<()> {
|
||||
// Ultimate stress test: 100,000 nodes
|
||||
let dag = DagRunner::new();
|
||||
let completed = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
// Create 100,000 tasks in batches
|
||||
let batch_size = 1000;
|
||||
let num_batches = 100;
|
||||
|
||||
for batch in 0..num_batches {
|
||||
let _: Vec<_> = (0..batch_size)
|
||||
.map(|i| {
|
||||
let completed = completed.clone();
|
||||
let task_id = batch * batch_size + i;
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let completed = completed.clone();
|
||||
async move {
|
||||
completed.fetch_add(1, Ordering::Relaxed);
|
||||
task_id
|
||||
}
|
||||
}))
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(completed.load(Ordering::Relaxed), 100_000);
|
||||
Ok(())
|
||||
}
|
||||
@@ -4,6 +4,5 @@
|
||||
//! by normal usage patterns.
|
||||
|
||||
mod deep_chains;
|
||||
mod large_scale;
|
||||
mod type_limits;
|
||||
mod zero_tasks;
|
||||
+32
-235
@@ -1,7 +1,8 @@
|
||||
//! Tests for type system edge cases and limits
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::{DagResult, DagRunner};
|
||||
use dagx::{DagResult, DagRunner, TaskHandle};
|
||||
use dagx_test::task_fn;
|
||||
|
||||
use std::marker::PhantomData;
|
||||
|
||||
#[tokio::test]
|
||||
@@ -18,30 +19,26 @@ async fn test_zero_sized_types_throughout() -> DagResult<()> {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// Task producing ZST
|
||||
let t1 = dag.add_task(task_fn(|_: ()| async { Empty }));
|
||||
let t1 = dag.add_task(task_fn::<(), _, _>(|_: ()| Empty));
|
||||
|
||||
// Task that also produces ZST (independent since we can't extract custom types)
|
||||
let t2 = dag.add_task(task_fn(|_: ()| async { 42 }));
|
||||
let t2: TaskHandle<_> = dag.add_task(task_fn::<(), _, _>(|_: ()| 42)).into();
|
||||
|
||||
// Task with PhantomData
|
||||
let _t3 = dag.add_task(task_fn(|_: ()| async {
|
||||
PhantomWrapper::<i32> {
|
||||
_phantom: PhantomData,
|
||||
}
|
||||
let _t3 = dag.add_task(task_fn::<(), _, _>(|_: ()| PhantomWrapper::<i32> {
|
||||
_phantom: PhantomData,
|
||||
}));
|
||||
|
||||
// Task that depends on t2 (uses primitive type)
|
||||
let t4 = dag
|
||||
.add_task(task_fn(|val: i32| async move {
|
||||
.add_task(task_fn::<i32, _, _>(|&val: &i32| {
|
||||
assert_eq!(val, 42);
|
||||
"success"
|
||||
}))
|
||||
.depends_on(&t2);
|
||||
.depends_on(t2);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(t1)?, Empty);
|
||||
assert_eq!(dag.get(t2)?, 42);
|
||||
@@ -61,7 +58,7 @@ async fn test_large_value_types() -> DagResult<()> {
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let t1 = dag.add_task(task_fn(|_: ()| async {
|
||||
let t1 = dag.add_task(task_fn::<(), _, _>(|_: ()| {
|
||||
let large = LargeStruct {
|
||||
data: [42; 8192],
|
||||
more_data: vec![1; 1000],
|
||||
@@ -71,15 +68,13 @@ async fn test_large_value_types() -> DagResult<()> {
|
||||
}));
|
||||
|
||||
let t2 = dag
|
||||
.add_task(task_fn(|val: usize| async move {
|
||||
.add_task(task_fn::<usize, _, _>(|&val: &usize| {
|
||||
val // Just pass through to verify
|
||||
}))
|
||||
.depends_on(&t1);
|
||||
.depends_on(t1);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(t2)?, 1042);
|
||||
|
||||
@@ -91,17 +86,19 @@ async fn test_unit_type_chains() -> DagResult<()> {
|
||||
// Test chains of unit type operations
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let t1 = dag.add_task(task_fn(|_: ()| async {}));
|
||||
let t2 = dag.add_task(task_fn(|_: ()| async {})).depends_on(&t1);
|
||||
let t3 = dag.add_task(task_fn(|_: ()| async {})).depends_on(t2);
|
||||
let t1: TaskHandle<_> = dag.add_task(task_fn::<(), _, _>(|_: ()| {})).into();
|
||||
let t2 = dag
|
||||
.add_task(task_fn::<(), _, _>(|_: ()| {}))
|
||||
.depends_on((t1,));
|
||||
let t3 = dag
|
||||
.add_task(task_fn::<(), _, _>(|_: ()| {}))
|
||||
.depends_on((t2,));
|
||||
let t4 = dag
|
||||
.add_task(task_fn(|_: ()| async { "done" }))
|
||||
.depends_on(&t3);
|
||||
.add_task(task_fn::<(), _, _>(|_: ()| "done"))
|
||||
.depends_on((t3,));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
dag.get(t1)?; // Verify no error
|
||||
dag.get(t2)?;
|
||||
@@ -111,134 +108,6 @@ async fn test_unit_type_chains() -> DagResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nested_option_result_types() -> DagResult<()> {
|
||||
// Test deeply nested generic types
|
||||
type ComplexType = Result<Option<Result<Vec<Option<i32>>, String>>, &'static str>;
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let t1 = dag.add_task(task_fn(|_: ()| async {
|
||||
let val: ComplexType = Ok(Some(Ok(vec![Some(42), None, Some(13)])));
|
||||
val
|
||||
}));
|
||||
|
||||
let t2 = dag
|
||||
.add_task(task_fn(|complex: ComplexType| async move {
|
||||
match complex {
|
||||
Ok(Some(Ok(vec))) => vec.iter().filter_map(|x| *x).sum::<i32>(),
|
||||
_ => 0,
|
||||
}
|
||||
}))
|
||||
.depends_on(&t1);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(t2)?, 55);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tuple_of_tuples() -> DagResult<()> {
|
||||
// Test nested tuple types
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let t1 = dag.add_task(task_fn(|_: ()| async { ((1, 2), (3, 4)) }));
|
||||
let t2 = dag.add_task(task_fn(|_: ()| async { ((5, 6), (7, 8)) }));
|
||||
|
||||
let t3 = dag
|
||||
.add_task(task_fn(
|
||||
|inputs: (((i32, i32), (i32, i32)), ((i32, i32), (i32, i32)))| async move {
|
||||
let (((a, b), (c, d)), ((e, f), (g, h))) = inputs;
|
||||
a + b + c + d + e + f + g + h
|
||||
},
|
||||
))
|
||||
.depends_on((&t1, &t2));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(t3)?, 36);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_string_types_variety() -> DagResult<()> {
|
||||
// Test various string types
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let t1 = dag.add_task(task_fn(|_: ()| async { "static str" }));
|
||||
let t2 = dag.add_task(task_fn(|_: ()| async { String::from("owned string") }));
|
||||
let t3 = dag.add_task(task_fn(|_: ()| async {
|
||||
std::borrow::Cow::Borrowed("cow str")
|
||||
}));
|
||||
let t4 = dag.add_task(task_fn(|_: ()| async { Box::new("boxed str") }));
|
||||
let t5 = dag.add_task(task_fn(|_: ()| async { std::sync::Arc::new("arc str 2") }));
|
||||
let t6 = dag.add_task(task_fn(|_: ()| async { std::sync::Arc::new("arc str") }));
|
||||
|
||||
let combined = dag
|
||||
.add_task(task_fn(
|
||||
|(s1, s2, s3, s4, s5, s6): (
|
||||
&str,
|
||||
String,
|
||||
std::borrow::Cow<str>,
|
||||
Box<&str>,
|
||||
std::sync::Arc<&str>,
|
||||
std::sync::Arc<&str>,
|
||||
)| async move { format!("{} {} {} {} {} {}", s1, s2, s3, s4, s5, s6) },
|
||||
))
|
||||
.depends_on((&t1, &t2, &t3, &t4, &t5, &t6));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
let result = dag.get(combined)?;
|
||||
assert!(result.contains("static str"));
|
||||
assert!(result.contains("owned string"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_generic_type_constraints() -> DagResult<()> {
|
||||
// Test with types that have trait bounds
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
struct Wrapper<T: Clone + Send + 'static>(T);
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let t1 = dag.add_task(task_fn(|_: ()| async { Wrapper(42) }));
|
||||
let t2 = dag.add_task(task_fn(|_: ()| async { Wrapper("hello") }));
|
||||
let t3 = dag.add_task(task_fn(|_: ()| async { Wrapper(vec![1, 2, 3]) }));
|
||||
|
||||
let combined = dag
|
||||
.add_task(task_fn(
|
||||
|(w1, w2, w3): (Wrapper<i32>, Wrapper<&str>, Wrapper<Vec<i32>>)| async move {
|
||||
format!("{:?} {:?} {:?}", w1, w2, w3)
|
||||
},
|
||||
))
|
||||
.depends_on((&t1, &t2, &t3));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
let result = dag.get(combined)?;
|
||||
assert!(result.contains("Wrapper(42)"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_reference_wrapper_types() -> DagResult<()> {
|
||||
// Test types that wrap references
|
||||
@@ -246,7 +115,7 @@ async fn test_reference_wrapper_types() -> DagResult<()> {
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let t1 = dag.add_task(task_fn(|_: ()| async {
|
||||
let t1 = dag.add_task(task_fn::<(), _, _>(|_: ()| {
|
||||
let arc = Arc::new(Mutex::new(100));
|
||||
// Extract value instead of passing Arc
|
||||
let guard = arc.lock().unwrap();
|
||||
@@ -254,85 +123,13 @@ async fn test_reference_wrapper_types() -> DagResult<()> {
|
||||
}));
|
||||
|
||||
let t2 = dag
|
||||
.add_task(task_fn(|val: i32| async move { val * 2 }))
|
||||
.depends_on(&t1);
|
||||
.add_task(task_fn::<i32, _, _>(|&val: &i32| val * 2))
|
||||
.depends_on(t1);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(t2)?, 200);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_array_types() -> DagResult<()> {
|
||||
// Test with fixed-size arrays
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let t1 = dag.add_task(task_fn(|_: ()| async { [1, 2, 3, 4, 5] }));
|
||||
let t2 = dag.add_task(task_fn(|_: ()| async { [6, 7, 8, 9, 10] }));
|
||||
|
||||
let t3 = dag
|
||||
.add_task(task_fn(|(a1, a2): ([i32; 5], [i32; 5])| async move {
|
||||
let mut sum = 0;
|
||||
for i in 0..5 {
|
||||
sum += a1[i] + a2[i];
|
||||
}
|
||||
sum
|
||||
}))
|
||||
.depends_on((&t1, &t2));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(t3)?, 55);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_enum_variants() -> DagResult<()> {
|
||||
// Test with various enum types
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
enum MyEnum {
|
||||
Unit,
|
||||
Tuple(i32, String),
|
||||
Struct { x: i32, y: i32 },
|
||||
LargeVariant([u8; 1024]),
|
||||
}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let t1 = dag.add_task(task_fn(|_: ()| async { MyEnum::Unit }));
|
||||
let t2 = dag.add_task(task_fn(|_: ()| async {
|
||||
MyEnum::Tuple(42, "test".to_string())
|
||||
}));
|
||||
let t3 = dag.add_task(task_fn(|_: ()| async { MyEnum::Struct { x: 10, y: 20 } }));
|
||||
let t4 = dag.add_task(task_fn(|_: ()| async { MyEnum::LargeVariant([0; 1024]) }));
|
||||
|
||||
let combined = dag
|
||||
.add_task(task_fn(
|
||||
|(e1, e2, e3, e4): (MyEnum, MyEnum, MyEnum, MyEnum)| async move {
|
||||
matches!(e1, MyEnum::Unit)
|
||||
&& matches!(e2, MyEnum::Tuple(42, _))
|
||||
&& matches!(e3, MyEnum::Struct { x: 10, y: 20 })
|
||||
&& matches!(e4, MyEnum::LargeVariant(_))
|
||||
},
|
||||
))
|
||||
.depends_on((&t1, &t2, &t3, &t4));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert!(dag.get(combined)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
+21
-157
@@ -1,98 +1,9 @@
|
||||
//! Tests for empty DAGs and minimal task configurations
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::{task, DagResult, DagRunner};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use dagx_test::task_fn;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_empty_dag_execution() -> DagResult<()> {
|
||||
// Test that an empty DAG can be created and run without issues
|
||||
let dag = DagRunner::new();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Should complete successfully with no tasks
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_single_task_no_deps() -> DagResult<()> {
|
||||
// Test the absolute minimum: one task with no dependencies
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let task = dag.add_task(task_fn(|_: ()| async { 42 }));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(task)?, 42);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_single_task_unit_input_output() -> DagResult<()> {
|
||||
// Test a task that takes unit and returns unit
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let executed = Arc::new(AtomicUsize::new(0));
|
||||
let exec_clone = executed.clone();
|
||||
|
||||
let task = dag.add_task(task_fn(move |_: ()| {
|
||||
let exec = exec_clone.clone();
|
||||
async move {
|
||||
exec.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
dag.get(task)?; // Just verify it doesn't error
|
||||
assert_eq!(executed.load(Ordering::SeqCst), 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_zero_sized_type_task() -> DagResult<()> {
|
||||
// Test with zero-sized types
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
struct ZeroSized;
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let task = dag.add_task(task_fn(|_: ()| async { ZeroSized }));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(&task)?, ZeroSized);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_empty_dag_multiple_runs() -> DagResult<()> {
|
||||
// Test that an empty DAG can be run multiple times
|
||||
let dag = DagRunner::new();
|
||||
|
||||
for _ in 0..10 {
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
use tokio::task::yield_now;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_dag_with_only_source_tasks() -> DagResult<()> {
|
||||
@@ -100,15 +11,13 @@ async fn test_dag_with_only_source_tasks() -> DagResult<()> {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let tasks: Vec<_> = (0..100)
|
||||
.map(|i| dag.add_task(task_fn(move |_: ()| async move { i })))
|
||||
.map(|i| dag.add_task(task_fn::<(), _, _>(move |_: ()| i)))
|
||||
.collect();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
for (i, task) in tasks.iter().enumerate() {
|
||||
for (i, task) in tasks.into_iter().enumerate() {
|
||||
assert_eq!(dag.get(task)?, i);
|
||||
}
|
||||
|
||||
@@ -132,77 +41,32 @@ async fn test_single_self_contained_stateful_task() -> DagResult<()> {
|
||||
let dag = DagRunner::new();
|
||||
let task = dag.add_task(Counter { value: 21 });
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(task)?, 42);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_alternating_empty_and_filled_dag_runs() -> DagResult<()> {
|
||||
// Test alternating between empty DAG and DAG with tasks
|
||||
for i in 0..5 {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
if i % 2 == 0 {
|
||||
// Empty DAG
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
} else {
|
||||
// DAG with a task
|
||||
let task = dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(dag.get(task)?, i);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_dag_with_never_type_simulation() -> DagResult<()> {
|
||||
// Test handling of tasks that conceptually "never" complete
|
||||
// (simulated by immediate completion with special marker)
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum Never {}
|
||||
|
||||
let dag = DagRunner::new();
|
||||
|
||||
// This task completes immediately but returns a Result containing "never" type
|
||||
let task = dag.add_task(task_fn(|_: ()| async { Result::<i32, Never>::Ok(42) }));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(&task)?, Ok(42));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimal_task_with_minimal_async_work() -> DagResult<()> {
|
||||
// Test a task that does the absolute minimum async work
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let task = dag.add_task(task_fn(|_: ()| async {
|
||||
// Just yield once to the runtime
|
||||
tokio::task::yield_now().await;
|
||||
1337
|
||||
}));
|
||||
struct YieldTask;
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
#[task]
|
||||
impl YieldTask {
|
||||
async fn run() -> i32 {
|
||||
yield_now().await;
|
||||
1337
|
||||
}
|
||||
}
|
||||
|
||||
let task = dag.add_task(YieldTask);
|
||||
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(task)?, 1337);
|
||||
Ok(())
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
// Common test utilities and helpers for the test suite
|
||||
|
||||
// Re-export task_fn for internal testing
|
||||
// Note: task_fn is #[doc(hidden)] and not part of the documented public API
|
||||
pub use dagx::task_fn;
|
||||
|
||||
// Re-export commonly used test structs
|
||||
#[allow(dead_code)]
|
||||
pub mod tasks {
|
||||
use dagx::task;
|
||||
|
||||
// Simple seed task with mutable state
|
||||
pub struct Seed {
|
||||
pub value: i32,
|
||||
}
|
||||
|
||||
impl Seed {
|
||||
pub fn new(v: i32) -> Self {
|
||||
Self { value: v }
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl Seed {
|
||||
async fn run(&mut self) -> i32 {
|
||||
self.value
|
||||
}
|
||||
}
|
||||
|
||||
// Stateless increment task
|
||||
pub struct Inc;
|
||||
|
||||
impl Inc {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl Inc {
|
||||
async fn run(x: &i32) -> i32 {
|
||||
x + 1
|
||||
}
|
||||
}
|
||||
|
||||
// Stateless add task
|
||||
pub struct Add;
|
||||
|
||||
impl Add {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl Add {
|
||||
async fn run(a: &i32, b: &i32) -> i32 {
|
||||
a + b
|
||||
}
|
||||
}
|
||||
|
||||
// Stateless multiply task
|
||||
pub struct Mul;
|
||||
|
||||
impl Mul {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl Mul {
|
||||
async fn run(a: &i32, b: &i32) -> i32 {
|
||||
a * b
|
||||
}
|
||||
}
|
||||
|
||||
// Stateful counter task
|
||||
pub struct Counter {
|
||||
pub count: i32,
|
||||
}
|
||||
|
||||
impl Counter {
|
||||
pub fn new(initial: i32) -> Self {
|
||||
Self { count: initial }
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl Counter {
|
||||
async fn run(&mut self, input: &i32) -> i32 {
|
||||
self.count += input;
|
||||
self.count
|
||||
}
|
||||
}
|
||||
|
||||
// Task that outputs multiple types combined as string
|
||||
pub struct OutputResult;
|
||||
|
||||
impl OutputResult {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl OutputResult {
|
||||
async fn run(s: &String, n: &usize, f: &bool) -> String {
|
||||
format!("{s}:{n}:{f}")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
// Test that custom types work with the #[task] macro
|
||||
use dagx::{task, DagRunner, TaskHandle};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
struct Person {
|
||||
name: String,
|
||||
age: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
struct Company {
|
||||
name: String,
|
||||
employees: Vec<Person>,
|
||||
}
|
||||
|
||||
// Source task returning custom type
|
||||
struct FetchPerson;
|
||||
|
||||
#[task]
|
||||
impl FetchPerson {
|
||||
async fn run(&self) -> Person {
|
||||
Person {
|
||||
name: "Alice".to_string(),
|
||||
age: 30,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Task consuming custom type
|
||||
struct ProcessPerson;
|
||||
|
||||
#[task]
|
||||
impl ProcessPerson {
|
||||
async fn run(person: &Person) -> String {
|
||||
format!("{} is {} years old", person.name, person.age)
|
||||
}
|
||||
}
|
||||
|
||||
// Task transforming custom type to another custom type
|
||||
struct PersonToCompany;
|
||||
|
||||
#[task]
|
||||
impl PersonToCompany {
|
||||
async fn run(person: &Person) -> Company {
|
||||
Company {
|
||||
name: format!("{}'s Company", person.name),
|
||||
employees: vec![person.clone()],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_type_single_dependency() {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let fetch = dag.add_task(FetchPerson);
|
||||
let process = dag.add_task(ProcessPerson).depends_on(fetch);
|
||||
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = dag.get(process).unwrap();
|
||||
assert_eq!(result, "Alice is 30 years old");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_type_passthrough() {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let fetch: TaskHandle<_> = dag.add_task(FetchPerson).into();
|
||||
let transform = dag.add_task(PersonToCompany).depends_on(fetch);
|
||||
let process = dag.add_task(ProcessPerson).depends_on(fetch);
|
||||
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let person_result = dag.get(process).unwrap();
|
||||
let company_result = dag.get(transform).unwrap();
|
||||
|
||||
assert_eq!(person_result.as_str(), "Alice is 30 years old");
|
||||
assert_eq!(company_result.name, "Alice's Company");
|
||||
assert_eq!(company_result.employees.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_type_fan_out() {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let fetch: TaskHandle<_> = dag.add_task(FetchPerson).into();
|
||||
let process1 = dag.add_task(ProcessPerson).depends_on(fetch);
|
||||
let process2 = dag.add_task(ProcessPerson).depends_on(fetch);
|
||||
let process3 = dag.add_task(ProcessPerson).depends_on(fetch);
|
||||
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result1 = dag.get(process1).unwrap();
|
||||
let result2 = dag.get(process2).unwrap();
|
||||
let result3 = dag.get(process3).unwrap();
|
||||
|
||||
assert_eq!(result1, "Alice is 30 years old");
|
||||
assert_eq!(result2, "Alice is 30 years old");
|
||||
assert_eq!(result3, "Alice is 30 years old");
|
||||
}
|
||||
+19
-21
@@ -30,11 +30,11 @@ fn test_builder_consumed_by_depends_on() {
|
||||
let consumer_builder = dag.add_task(Consumer);
|
||||
|
||||
// This consumes consumer_builder and returns a TaskHandle
|
||||
let _consumer_handle = consumer_builder.depends_on(&source);
|
||||
let _consumer_handle = consumer_builder.depends_on(source);
|
||||
|
||||
// We CANNOT use consumer_builder again because it was moved
|
||||
// This would fail to compile:
|
||||
// let _consumer2 = consumer_builder.depends_on(&source); // ERROR: use of moved value
|
||||
// let _consumer2 = consumer_builder.depends_on(source); // ERROR: use of moved value
|
||||
}
|
||||
|
||||
/// Demonstrates that you can only get a TaskHandle by finalizing a builder,
|
||||
@@ -53,7 +53,7 @@ fn test_handle_is_immutable_reference() {
|
||||
let builder = dag.add_task(NoInput);
|
||||
|
||||
// Convert to handle (only works because Input = ())
|
||||
let handle: TaskHandle<i32> = builder.into();
|
||||
let handle: TaskHandle<_> = builder.into();
|
||||
|
||||
// TaskHandle is Copy/Clone - just an ID wrapper
|
||||
let _copy1 = handle;
|
||||
@@ -63,7 +63,7 @@ fn test_handle_is_immutable_reference() {
|
||||
// But TaskHandle has NO methods to modify the DAG
|
||||
// There's no way to add dependencies to it
|
||||
// This would fail to compile:
|
||||
// handle.depends_on(&something); // ERROR: no method named `depends_on`
|
||||
// handle.depends_on(something); // ERROR: no method named `depends_on`
|
||||
}
|
||||
|
||||
/// Documents the ordering constraint that prevents cycles
|
||||
@@ -107,9 +107,9 @@ fn test_ordering_prevents_cycles() {
|
||||
// So we can only create DAGs, never cycles!
|
||||
|
||||
// Let's create a valid DAG instead: A→B (B depends on A)
|
||||
let a_handle: TaskHandle<i32> = a_builder.into(); // A has no deps (unit input)
|
||||
let a_handle: TaskHandle<_> = a_builder.into(); // A has no deps (unit input)
|
||||
let _b_handle = b_builder.depends_on(a_handle);
|
||||
// Now we could try: a_handle.depends_on(&b_handle)?
|
||||
// Now we could try: a_handle.depends_on(b_handle)?
|
||||
// But TaskHandle has no such method!
|
||||
}
|
||||
|
||||
@@ -137,8 +137,8 @@ fn test_unit_tasks_cannot_create_cycles() {
|
||||
let task2 = dag.add_task(Task2);
|
||||
|
||||
// Both have unit input, so we can convert to handles
|
||||
let _handle1: TaskHandle<i32> = task1.into();
|
||||
let _handle2: TaskHandle<i32> = task2.into();
|
||||
let _handle1: TaskHandle<_> = task1.into();
|
||||
let _handle2: TaskHandle<_> = task2.into();
|
||||
|
||||
// Now we have two handles, but...
|
||||
// TaskHandle has no methods to add dependencies!
|
||||
@@ -146,7 +146,7 @@ fn test_unit_tasks_cannot_create_cycles() {
|
||||
// And once you've converted to TaskHandle, you can't go back
|
||||
|
||||
// This would fail to compile:
|
||||
// _handle1.depends_on(&_handle2); // ERROR: no method named `depends_on`
|
||||
// _handle1.depends_on(_handle2); // ERROR: no method named `depends_on`
|
||||
}
|
||||
|
||||
/// Demonstrates the catch-22 of three-way cycles
|
||||
@@ -182,9 +182,9 @@ fn test_three_way_cycle_impossible() {
|
||||
let _c_builder = dag.add_task(C);
|
||||
|
||||
// To create A→C→B→A:
|
||||
// 1. a_builder.depends_on(&c_handle) // Need c_handle
|
||||
// 2. c_builder.depends_on(&b_handle) // Need b_handle
|
||||
// 3. b_builder.depends_on(&a_handle) // Need a_handle (from step 1!)
|
||||
// 1. a_builder.depends_on(c_handle) // Need c_handle
|
||||
// 2. c_builder.depends_on(b_handle) // Need b_handle
|
||||
// 3. b_builder.depends_on(a_handle) // Need a_handle (from step 1!)
|
||||
//
|
||||
// But step 3 needs a_handle which doesn't exist until step 1 completes.
|
||||
// And step 1 needs c_handle which doesn't exist until step 2 completes.
|
||||
@@ -209,7 +209,7 @@ fn test_self_loop_prevented() {
|
||||
let _builder = dag.add_task(SelfReferential);
|
||||
|
||||
// To create A→A (self-loop):
|
||||
// builder.depends_on(&builder)
|
||||
// builder.depends_on(builder)
|
||||
//
|
||||
// But this would fail because:
|
||||
// 1. depends_on() takes `self` by value (moves builder)
|
||||
@@ -217,7 +217,7 @@ fn test_self_loop_prevented() {
|
||||
// 3. We can't also use &builder as an argument because it's being moved
|
||||
//
|
||||
// This would fail to compile:
|
||||
// let _handle = builder.depends_on(&builder); // ERROR: use of moved value
|
||||
// let _handle = builder.depends_on(builder); // ERROR: use of moved value
|
||||
}
|
||||
|
||||
/// Comprehensive test showing all the guardrails
|
||||
@@ -263,18 +263,16 @@ async fn test_type_safety_prevents_invalid_graphs() {
|
||||
let s2 = dag.add_task(Source);
|
||||
|
||||
// Transform depends on sources
|
||||
let t1 = dag.add_task(Transform).depends_on(&s1);
|
||||
let t2 = dag.add_task(Transform).depends_on(&s2);
|
||||
let t1 = dag.add_task(Transform).depends_on(s1);
|
||||
let t2 = dag.add_task(Transform).depends_on(s2);
|
||||
|
||||
// Aggregate depends on transforms
|
||||
let result = dag.add_task(Aggregate).depends_on((&t1, &t2));
|
||||
|
||||
// Execute and verify
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(dag.get(result).unwrap(), 40); // (10*2) + (10*2)
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
//! Dependency tests module
|
||||
|
||||
pub mod resolution;
|
||||
pub mod single;
|
||||
pub mod tuples;
|
||||
@@ -1,7 +1,8 @@
|
||||
//! Tests for dependency resolution order and transitive dependencies
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::{DagResult, DagRunner};
|
||||
use dagx::{task, DagResult, DagRunner};
|
||||
use dagx_test::task_fn;
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[tokio::test]
|
||||
@@ -11,80 +12,75 @@ async fn test_dependency_resolution_order() -> DagResult<()> {
|
||||
|
||||
let execution_log = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
struct ATask {
|
||||
log: Arc<Mutex<Vec<(&'static str, i32)>>>,
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl ATask {
|
||||
async fn run(&self) -> i32 {
|
||||
self.log.lock().unwrap().push(("A_start", 0));
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;
|
||||
self.log.lock().unwrap().push(("A_end", 1));
|
||||
1
|
||||
}
|
||||
}
|
||||
|
||||
// Create a chain: A -> B -> C -> D -> E
|
||||
let a = {
|
||||
let log = execution_log.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let log = log.clone();
|
||||
async move {
|
||||
log.lock().unwrap().push(("A_start", 0));
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;
|
||||
log.lock().unwrap().push(("A_end", 1));
|
||||
1
|
||||
}
|
||||
}))
|
||||
};
|
||||
let a = dag.add_task(ATask {
|
||||
log: execution_log.clone(),
|
||||
});
|
||||
|
||||
let b = {
|
||||
let log = execution_log.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
dag.add_task(task_fn::<i32, _, _>(move |&x: &i32| {
|
||||
let log = log.clone();
|
||||
async move {
|
||||
log.lock().unwrap().push(("B_start", x));
|
||||
assert_eq!(x, 1, "B should receive 1 from A");
|
||||
log.lock().unwrap().push(("B_end", x + 1));
|
||||
x + 1
|
||||
}
|
||||
log.lock().unwrap().push(("B_start", x));
|
||||
assert_eq!(x, 1, "B should receive 1 from A");
|
||||
log.lock().unwrap().push(("B_end", x + 1));
|
||||
x + 1
|
||||
}))
|
||||
.depends_on(&a)
|
||||
.depends_on(a)
|
||||
};
|
||||
|
||||
let c = {
|
||||
let log = execution_log.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
dag.add_task(task_fn::<i32, _, _>(move |&x: &i32| {
|
||||
let log = log.clone();
|
||||
async move {
|
||||
log.lock().unwrap().push(("C_start", x));
|
||||
assert_eq!(x, 2, "C should receive 2 from B");
|
||||
log.lock().unwrap().push(("C_end", x + 1));
|
||||
x + 1
|
||||
}
|
||||
log.lock().unwrap().push(("C_start", x));
|
||||
assert_eq!(x, 2, "C should receive 2 from B");
|
||||
log.lock().unwrap().push(("C_end", x + 1));
|
||||
x + 1
|
||||
}))
|
||||
.depends_on(b)
|
||||
};
|
||||
|
||||
let d = {
|
||||
let log = execution_log.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
dag.add_task(task_fn::<i32, _, _>(move |&x: &i32| {
|
||||
let log = log.clone();
|
||||
async move {
|
||||
log.lock().unwrap().push(("D_start", x));
|
||||
assert_eq!(x, 3, "D should receive 3 from C");
|
||||
log.lock().unwrap().push(("D_end", x + 1));
|
||||
x + 1
|
||||
}
|
||||
log.lock().unwrap().push(("D_start", x));
|
||||
assert_eq!(x, 3, "D should receive 3 from C");
|
||||
log.lock().unwrap().push(("D_end", x + 1));
|
||||
x + 1
|
||||
}))
|
||||
.depends_on(c)
|
||||
};
|
||||
|
||||
let e = {
|
||||
let log = execution_log.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
dag.add_task(task_fn::<i32, _, _>(move |&x: &i32| {
|
||||
let log = log.clone();
|
||||
async move {
|
||||
log.lock().unwrap().push(("E_start", x));
|
||||
assert_eq!(x, 4, "E should receive 4 from D");
|
||||
log.lock().unwrap().push(("E_end", x + 1));
|
||||
x + 1
|
||||
}
|
||||
log.lock().unwrap().push(("E_start", x));
|
||||
assert_eq!(x, 4, "E should receive 4 from D");
|
||||
log.lock().unwrap().push(("E_end", x + 1));
|
||||
x + 1
|
||||
}))
|
||||
.depends_on(d)
|
||||
};
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(e)?, 5);
|
||||
|
||||
@@ -103,74 +99,3 @@ async fn test_dependency_resolution_order() -> DagResult<()> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_transitive_dependencies() -> DagResult<()> {
|
||||
// Test that transitive dependencies work correctly
|
||||
// A -> B -> C (creates transitive A -> C relationship)
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let a = dag.add_task(task_fn(|_: ()| async { 10 }));
|
||||
let b = dag
|
||||
.add_task(task_fn(|x: i32| async move { x * 2 }))
|
||||
.depends_on(&a);
|
||||
let c = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 5 }))
|
||||
.depends_on(b);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(&a)?, 10);
|
||||
assert_eq!(dag.get(b)?, 20);
|
||||
assert_eq!(dag.get(c)?, 25); // 20 + 5
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deep_chain_50_levels() -> DagResult<()> {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let first = dag.add_task(task_fn(|_: ()| async { 0 }));
|
||||
let mut handles = vec![(&first).into()];
|
||||
|
||||
for _i in 1..=50 {
|
||||
let next = dag
|
||||
.add_task(task_fn(move |x: i32| async move { x + 1 }))
|
||||
.depends_on(handles.last().unwrap());
|
||||
handles.push(next);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(handles.last().unwrap())?, 50);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deep_chain_200_levels() -> DagResult<()> {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let first = dag.add_task(task_fn(|_: ()| async { 1 }));
|
||||
let mut current = (&first).into();
|
||||
|
||||
for _ in 0..200 {
|
||||
current = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(current);
|
||||
}
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(current)?, 201);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
//! Tests for single dependencies and basic dependency data flow
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::{DagResult, DagRunner};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_single_dependency() -> DagResult<()> {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let source = dag.add_task(task_fn(|_: ()| async { 42 }));
|
||||
let dependent = dag
|
||||
.add_task(task_fn(|x: i32| async move { x + 1 }))
|
||||
.depends_on(&source);
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(dependent)?, 43);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_dependency_data_flow() -> DagResult<()> {
|
||||
// Test that data flows correctly through dependencies
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let source = dag.add_task(task_fn(|_: ()| async { vec![1, 2, 3, 4, 5] }));
|
||||
|
||||
let sum = dag
|
||||
.add_task(task_fn(|v: Vec<i32>| async move { v.iter().sum::<i32>() }))
|
||||
.depends_on(&source);
|
||||
|
||||
let product = dag
|
||||
.add_task(task_fn(
|
||||
|v: Vec<i32>| async move { v.iter().product::<i32>() },
|
||||
))
|
||||
.depends_on(&source);
|
||||
|
||||
let final_result = dag
|
||||
.add_task(task_fn(|(s, p): (i32, i32)| async move { s + p }))
|
||||
.depends_on((&sum, &product));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Note: sum and product are not sinks (final_result depends on them)
|
||||
// Only final_result is a sink and can be retrieved
|
||||
assert_eq!(dag.get(final_result)?, 135); // 15+120
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_dependencies_with_different_types() -> DagResult<()> {
|
||||
// Test that dependencies with different types work correctly
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let int_source = dag.add_task(task_fn(|_: ()| async { 42 }));
|
||||
let string_source = dag.add_task(task_fn(|_: ()| async { "hello".to_string() }));
|
||||
let bool_source = dag.add_task(task_fn(|_: ()| async { true }));
|
||||
|
||||
let combined = dag
|
||||
.add_task(task_fn(|(i, s, b): (i32, String, bool)| async move {
|
||||
format!("{}: {} ({})", s, i, b)
|
||||
}))
|
||||
.depends_on((&int_source, &string_source, &bool_source));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(combined)?, "hello: 42 (true)");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_shared_dependencies() -> DagResult<()> {
|
||||
// Test that a single node can be a dependency for multiple downstream nodes
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let shared = dag.add_task(task_fn(|_: ()| async { 42 }));
|
||||
|
||||
// Create 10 tasks that all depend on the shared node
|
||||
let dependents: Vec<_> = (0..10)
|
||||
.map(|i| {
|
||||
dag.add_task(task_fn(move |x: i32| async move { x * (i + 1) }))
|
||||
.depends_on(&shared)
|
||||
})
|
||||
.collect();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Verify all dependents got the correct value from shared
|
||||
for (i, handle) in dependents.iter().enumerate() {
|
||||
assert_eq!(dag.get(handle)?, 42 * (i as i32 + 1));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_source_nodes() -> DagResult<()> {
|
||||
// Test DAG with multiple source nodes (no dependencies)
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let sources: Vec<_> = (0..20)
|
||||
.map(|i| dag.add_task(task_fn(move |_: ()| async move { i * 10 })))
|
||||
.collect();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
for (i, source) in sources.iter().enumerate() {
|
||||
assert_eq!(dag.get(source)?, i as i32 * 10);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Loaded 100 of 144 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user