feat: dagrunner, initial version
This commit is contained in:
122 files changed
+23101
No files matched your search
@@ -0,0 +1,432 @@
|
||||
//! Concurrent access tests
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::DagRunner;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_concurrent_run_calls() {
|
||||
// Test that multiple run() calls are handled correctly
|
||||
let dag = Arc::new(DagRunner::new());
|
||||
|
||||
// Add some tasks
|
||||
let tasks: Vec<_> = (0..10)
|
||||
.map(|i| dag.add_task(task_fn(move |_: ()| async move { i })))
|
||||
.collect();
|
||||
|
||||
// Try to run the DAG multiple times concurrently
|
||||
let mut handles = JoinSet::new();
|
||||
|
||||
for _ in 0..5 {
|
||||
let dag = Arc::clone(&dag);
|
||||
handles.spawn(async move {
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
});
|
||||
}
|
||||
|
||||
// Only one run should succeed, others should fail with concurrent execution error
|
||||
let mut success_count = 0;
|
||||
let mut error_count = 0;
|
||||
|
||||
while let Some(result) = handles.join_next().await {
|
||||
match result.unwrap() {
|
||||
Ok(_) => success_count += 1,
|
||||
Err(_) => error_count += 1,
|
||||
}
|
||||
}
|
||||
|
||||
// Exactly one should succeed, the rest should fail
|
||||
assert_eq!(success_count, 1);
|
||||
assert_eq!(error_count, 4);
|
||||
|
||||
// Verify results are consistent from the successful run
|
||||
for (i, task) in tasks.iter().enumerate() {
|
||||
assert_eq!(dag.get(task).unwrap(), i);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_concurrent_execution_and_building() {
|
||||
// Test that we can build and execute concurrently
|
||||
let dag = Arc::new(DagRunner::new());
|
||||
|
||||
// Start with some initial tasks
|
||||
let initial: Vec<_> = (0..10)
|
||||
.map(|i| dag.add_task(task_fn(move |_: ()| async move { i })))
|
||||
.collect();
|
||||
|
||||
// Execute the initial DAG
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Now try to add more tasks while also reading results
|
||||
let mut handles = JoinSet::new();
|
||||
|
||||
// Reader tasks
|
||||
for task in &initial {
|
||||
let dag = Arc::clone(&dag);
|
||||
let task_handle: dagx::TaskHandle<i32> = task.into();
|
||||
handles.spawn(async move { dag.get(task_handle).unwrap() });
|
||||
}
|
||||
|
||||
// Builder tasks (adding new nodes)
|
||||
for i in 10..20 {
|
||||
let dag = Arc::clone(&dag);
|
||||
handles.spawn(async move {
|
||||
dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
i
|
||||
});
|
||||
}
|
||||
|
||||
// Wait for all operations
|
||||
while let Some(result) = handles.join_next().await {
|
||||
result.unwrap();
|
||||
}
|
||||
|
||||
// Execute again with the new tasks
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_dag_builder_thread_safety() {
|
||||
// Test that the DAG builder is thread-safe
|
||||
let dag = Arc::new(DagRunner::new());
|
||||
let counter = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let mut handles = JoinSet::new();
|
||||
|
||||
// Spawn tasks that build complex sub-graphs concurrently
|
||||
for thread_id in 0..10 {
|
||||
let dag = Arc::clone(&dag);
|
||||
let counter = Arc::clone(&counter);
|
||||
|
||||
handles.spawn(async move {
|
||||
// Each thread creates a small pipeline
|
||||
let source = dag.add_task(task_fn(move |_: ()| async move { thread_id }));
|
||||
|
||||
let transform1 = dag
|
||||
.add_task(task_fn(move |x: i32| async move { x * 10 }))
|
||||
.depends_on(&source);
|
||||
|
||||
let transform2 = dag
|
||||
.add_task(task_fn(move |x: i32| async move { x + thread_id }))
|
||||
.depends_on(transform1);
|
||||
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
transform2
|
||||
});
|
||||
}
|
||||
|
||||
let mut final_tasks = vec![];
|
||||
while let Some(result) = handles.join_next().await {
|
||||
final_tasks.push(result.unwrap());
|
||||
}
|
||||
|
||||
// Verify all threads completed
|
||||
assert_eq!(counter.load(Ordering::SeqCst), 10);
|
||||
|
||||
// Execute the combined DAG
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Verify each pipeline computed correctly
|
||||
for task in final_tasks.iter() {
|
||||
let result = dag.get(task).unwrap();
|
||||
// Each pipeline: thread_id * 10 + thread_id
|
||||
assert!((0..100).contains(&result));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_concurrent_get_operations() {
|
||||
// Test concurrent get() operations
|
||||
let dag = Arc::new(DagRunner::new());
|
||||
|
||||
// Create and execute tasks
|
||||
let tasks: Vec<_> = (0..100)
|
||||
.map(|i| dag.add_task(task_fn(move |_: ()| async move { i })))
|
||||
.collect();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Now access results concurrently
|
||||
let mut handles = JoinSet::new();
|
||||
|
||||
for task in tasks.iter() {
|
||||
let dag = Arc::clone(&dag);
|
||||
let task_handle: dagx::TaskHandle<i32> = task.into();
|
||||
handles.spawn(async move { dag.get(task_handle).unwrap() });
|
||||
}
|
||||
|
||||
// Collect and verify results
|
||||
let mut results = vec![];
|
||||
while let Some(result) = handles.join_next().await {
|
||||
results.push(result.unwrap());
|
||||
}
|
||||
|
||||
// Results should contain all values 0..100
|
||||
results.sort();
|
||||
for (i, val) in results.iter().enumerate() {
|
||||
assert_eq!(*val, i as i32);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_no_race_in_dependency_tracking() {
|
||||
// Test that concurrent dependency modifications don't cause races
|
||||
let dag = Arc::new(DagRunner::new());
|
||||
let source = dag.add_task(task_fn(|_: ()| async { 42 }));
|
||||
|
||||
let mut handles = JoinSet::new();
|
||||
|
||||
// Multiple threads create tasks depending on the same source
|
||||
let source_handle: dagx::TaskHandle<i32> = (&source).into();
|
||||
for i in 0..50 {
|
||||
let dag = Arc::clone(&dag);
|
||||
let source_clone = source_handle;
|
||||
|
||||
handles.spawn(async move {
|
||||
dag.add_task(task_fn(move |x: i32| async move { x + i }))
|
||||
.depends_on(source_clone)
|
||||
});
|
||||
}
|
||||
|
||||
let mut dependents = vec![];
|
||||
while let Some(result) = handles.join_next().await {
|
||||
dependents.push(result.unwrap());
|
||||
}
|
||||
|
||||
// Execute
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// All dependents should have correct values
|
||||
for dep in dependents.iter() {
|
||||
let result = dag.get(dep).unwrap();
|
||||
assert!((42..92).contains(&result)); // 42 + (0..50)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_high_concurrency_stress() {
|
||||
// Stress test with high concurrency
|
||||
let dag = Arc::new(DagRunner::new());
|
||||
let completed = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let mut handles = JoinSet::new();
|
||||
|
||||
// 1000 concurrent operations
|
||||
for i in 0..1000 {
|
||||
let dag = Arc::clone(&dag);
|
||||
let completed = Arc::clone(&completed);
|
||||
|
||||
handles.spawn(async move {
|
||||
// Mix of operations
|
||||
match i % 3 {
|
||||
0 => {
|
||||
// Add a source task
|
||||
dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
}
|
||||
1 => {
|
||||
// Add a task with a simple dependency
|
||||
let source = dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
dag.add_task(task_fn(move |x: i32| async move { x * 2 }))
|
||||
.depends_on(&source);
|
||||
}
|
||||
2 => {
|
||||
// Add a more complex subgraph
|
||||
let a = dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
let b = dag.add_task(task_fn(move |_: ()| async move { i + 1 }));
|
||||
dag.add_task(task_fn(move |(x, y): (i32, i32)| async move { x + y }))
|
||||
.depends_on((&a, &b));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
completed.fetch_add(1, Ordering::SeqCst);
|
||||
});
|
||||
}
|
||||
|
||||
// Wait for all operations to complete
|
||||
while let Some(result) = handles.join_next().await {
|
||||
result.unwrap();
|
||||
}
|
||||
|
||||
assert_eq!(completed.load(Ordering::SeqCst), 1000);
|
||||
|
||||
// The DAG should still be executable
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_lock_contention_measurement() {
|
||||
// Test to measure lock contention under concurrent load
|
||||
let dag = Arc::new(DagRunner::new());
|
||||
let start = tokio::time::Instant::now();
|
||||
let operations = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let mut handles = JoinSet::new();
|
||||
|
||||
// Create many concurrent operations that will contend for locks
|
||||
for i in 0..100 {
|
||||
let dag = Arc::clone(&dag);
|
||||
let ops = Arc::clone(&operations);
|
||||
|
||||
handles.spawn(async move {
|
||||
for j in 0..10 {
|
||||
dag.add_task(task_fn(move |_: ()| async move { i * 10 + j }));
|
||||
ops.fetch_add(1, Ordering::SeqCst);
|
||||
|
||||
// Small yield to increase contention likelihood
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Wait for completion
|
||||
while let Some(result) = handles.join_next().await {
|
||||
result.unwrap();
|
||||
}
|
||||
|
||||
let duration = start.elapsed();
|
||||
let total_ops = operations.load(Ordering::SeqCst);
|
||||
|
||||
println!(
|
||||
"Completed {} operations in {:?} ({:.0} ops/sec)",
|
||||
total_ops,
|
||||
duration,
|
||||
total_ops as f64 / duration.as_secs_f64()
|
||||
);
|
||||
|
||||
// Should complete reasonably quickly even with contention
|
||||
assert!(
|
||||
duration < Duration::from_secs(5),
|
||||
"Too much lock contention"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_atomic_dag_operations() {
|
||||
// Test that DAG operations are atomic (all-or-nothing)
|
||||
let dag = Arc::new(DagRunner::new());
|
||||
|
||||
// Create a base task
|
||||
let base = dag.add_task(task_fn(|_: ()| async { 100 }));
|
||||
|
||||
// Try to create dependent tasks concurrently
|
||||
let mut handles = JoinSet::new();
|
||||
let success_count = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let base_handle: dagx::TaskHandle<i32> = (&base).into();
|
||||
for i in 0..50 {
|
||||
let dag = Arc::clone(&dag);
|
||||
let base_clone = base_handle;
|
||||
let success = Arc::clone(&success_count);
|
||||
|
||||
handles.spawn(async move {
|
||||
let dep = dag
|
||||
.add_task(task_fn(move |x: i32| async move { x + i }))
|
||||
.depends_on(base_clone);
|
||||
|
||||
success.fetch_add(1, Ordering::SeqCst);
|
||||
dep
|
||||
});
|
||||
}
|
||||
|
||||
let mut deps = vec![];
|
||||
while let Some(result) = handles.join_next().await {
|
||||
deps.push(result.unwrap());
|
||||
}
|
||||
|
||||
// All operations should have succeeded
|
||||
assert_eq!(success_count.load(Ordering::SeqCst), 50);
|
||||
assert_eq!(deps.len(), 50);
|
||||
|
||||
// Execute and verify
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for dep in deps.iter() {
|
||||
let result = dag.get(dep).unwrap();
|
||||
assert!((100..150).contains(&result));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_no_deadlock_on_circular_waiting() {
|
||||
// Test that we don't deadlock even with complex concurrent operations
|
||||
let dag = Arc::new(DagRunner::new());
|
||||
|
||||
// Create multiple threads that might cause circular waiting patterns
|
||||
let mut handles = JoinSet::new();
|
||||
|
||||
for i in 0..10 {
|
||||
let dag = Arc::clone(&dag);
|
||||
|
||||
handles.spawn(async move {
|
||||
// Each thread creates a chain of tasks
|
||||
let first = dag.add_task(task_fn(move |_: ()| async move { i }));
|
||||
let mut chain = vec![(&first).into()];
|
||||
|
||||
for j in 0..5 {
|
||||
let next = dag
|
||||
.add_task(task_fn(move |x: i32| async move { x + j }))
|
||||
.depends_on(chain.last().unwrap());
|
||||
chain.push(next);
|
||||
|
||||
// Yield to potentially cause interleaving
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
*chain.last().unwrap()
|
||||
});
|
||||
}
|
||||
|
||||
// Set a timeout to detect deadlocks
|
||||
let result = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
let mut finals = vec![];
|
||||
while let Some(result) = handles.join_next().await {
|
||||
finals.push(result.unwrap());
|
||||
}
|
||||
finals
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok(), "Deadlock detected - operation timed out");
|
||||
|
||||
// Should be able to execute the DAG
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
//! Tests for execution correctness and parallelism
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::{DagResult, DagRunner};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_layers_execute_in_parallel() -> DagResult<()> {
|
||||
// Test that tasks in the same topological layer execute in parallel
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let layer_timing = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
// Layer 0: 3 independent sources
|
||||
let sources: Vec<_> = (0..3)
|
||||
.map(|i| {
|
||||
let timing = layer_timing.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let timing = timing.clone();
|
||||
async move {
|
||||
let start = Instant::now();
|
||||
timing.lock().unwrap().push((format!("L0_{}", i), start));
|
||||
sleep(Duration::from_millis(20)).await;
|
||||
i * 10
|
||||
}
|
||||
}))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Layer 1: Tasks depending on layer 0
|
||||
let layer1: Vec<_> = sources
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, source)| {
|
||||
let timing = layer_timing.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let timing = timing.clone();
|
||||
async move {
|
||||
let start = Instant::now();
|
||||
timing.lock().unwrap().push((format!("L1_{}", i), start));
|
||||
sleep(Duration::from_millis(20)).await;
|
||||
x + 1
|
||||
}
|
||||
}))
|
||||
.depends_on(source)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Layer 2: Final aggregation
|
||||
let final_task = {
|
||||
let timing = layer_timing.clone();
|
||||
dag.add_task(task_fn(move |(a, b, c): (i32, i32, i32)| {
|
||||
let timing = timing.clone();
|
||||
async move {
|
||||
let start = Instant::now();
|
||||
timing.lock().unwrap().push(("L2_final".to_string(), start));
|
||||
a + b + c
|
||||
}
|
||||
}))
|
||||
.depends_on((&layer1[0], &layer1[1], &layer1[2]))
|
||||
};
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(final_task)?, 33); // (0+1) + (10+1) + (20+1)
|
||||
|
||||
// Analyze timing
|
||||
let timing = layer_timing.lock().unwrap();
|
||||
|
||||
// Get layer 0 start times
|
||||
let l0_times: Vec<Instant> = (0..3)
|
||||
.map(|i| {
|
||||
timing
|
||||
.iter()
|
||||
.find(|(name, _)| name == &format!("L0_{}", i))
|
||||
.map(|(_, t)| *t)
|
||||
.unwrap()
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Get layer 1 start times
|
||||
let l1_times: Vec<Instant> = (0..3)
|
||||
.map(|i| {
|
||||
timing
|
||||
.iter()
|
||||
.find(|(name, _)| name == &format!("L1_{}", i))
|
||||
.map(|(_, t)| *t)
|
||||
.unwrap()
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Layer 0 tasks should all start nearly simultaneously
|
||||
let l0_spread = l0_times
|
||||
.iter()
|
||||
.max()
|
||||
.unwrap()
|
||||
.duration_since(*l0_times.iter().min().unwrap());
|
||||
assert!(
|
||||
l0_spread < Duration::from_millis(5),
|
||||
"Layer 0 tasks should start together, spread was {:?}",
|
||||
l0_spread
|
||||
);
|
||||
|
||||
// Layer 1 tasks should also start nearly simultaneously (after layer 0 completes)
|
||||
let l1_spread = l1_times
|
||||
.iter()
|
||||
.max()
|
||||
.unwrap()
|
||||
.duration_since(*l1_times.iter().min().unwrap());
|
||||
assert!(
|
||||
l1_spread < Duration::from_millis(5),
|
||||
"Layer 1 tasks should start together, spread was {:?}",
|
||||
l1_spread
|
||||
);
|
||||
|
||||
// Layer 1 should start after layer 0
|
||||
let earliest_l1 = *l1_times.iter().min().unwrap();
|
||||
let earliest_l0 = *l0_times.iter().min().unwrap();
|
||||
assert!(
|
||||
earliest_l1 >= earliest_l0 + Duration::from_millis(15),
|
||||
"Layer 1 should start after layer 0 completes"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_completion_order_matches_dependencies() -> DagResult<()> {
|
||||
// Verify that tasks complete in dependency order
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let completion_order = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
// Create a complex DAG with clear ordering requirements
|
||||
let a = {
|
||||
let order = completion_order.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let order = order.clone();
|
||||
async move {
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
order.lock().unwrap().push("A");
|
||||
1
|
||||
}
|
||||
}))
|
||||
};
|
||||
|
||||
let b = {
|
||||
let order = completion_order.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let order = order.clone();
|
||||
async move {
|
||||
sleep(Duration::from_millis(5)).await;
|
||||
order.lock().unwrap().push("B");
|
||||
2
|
||||
}
|
||||
}))
|
||||
};
|
||||
|
||||
let c = {
|
||||
let order = completion_order.clone();
|
||||
dag.add_task(task_fn(move |(x, y): (i32, i32)| {
|
||||
let order = order.clone();
|
||||
async move {
|
||||
order.lock().unwrap().push("C");
|
||||
x + y
|
||||
}
|
||||
}))
|
||||
.depends_on((&a, &b))
|
||||
};
|
||||
|
||||
let d = {
|
||||
let order = completion_order.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let order = order.clone();
|
||||
async move {
|
||||
order.lock().unwrap().push("D");
|
||||
x * 2
|
||||
}
|
||||
}))
|
||||
.depends_on(c)
|
||||
};
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(d)?, 6); // (1 + 2) * 2
|
||||
|
||||
// Check completion order
|
||||
let order = completion_order.lock().unwrap();
|
||||
|
||||
// A and B can complete in any order (parallel)
|
||||
assert!(order.contains(&"A"));
|
||||
assert!(order.contains(&"B"));
|
||||
|
||||
// C must come after both A and B
|
||||
let c_pos = order.iter().position(|name| *name == "C").unwrap();
|
||||
let a_pos = order.iter().position(|name| *name == "A").unwrap();
|
||||
let b_pos = order.iter().position(|name| *name == "B").unwrap();
|
||||
assert!(c_pos > a_pos, "C should complete after A");
|
||||
assert!(c_pos > b_pos, "C should complete after B");
|
||||
|
||||
// D must come after C
|
||||
let d_pos = order.iter().position(|name| *name == "D").unwrap();
|
||||
assert!(d_pos > c_pos, "D should complete after C");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_dependent_tasks_respect_ordering() -> DagResult<()> {
|
||||
// Prove that dependent tasks wait for their dependencies
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let execution_order = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
// Create a chain: A -> B -> C -> D
|
||||
let a = {
|
||||
let order = execution_order.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let order = order.clone();
|
||||
async move {
|
||||
sleep(Duration::from_millis(20)).await;
|
||||
order.lock().unwrap().push("A");
|
||||
1
|
||||
}
|
||||
}))
|
||||
};
|
||||
|
||||
let b = {
|
||||
let order = execution_order.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let order = order.clone();
|
||||
async move {
|
||||
order.lock().unwrap().push("B");
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
x + 1
|
||||
}
|
||||
}))
|
||||
.depends_on(&a)
|
||||
};
|
||||
|
||||
let c = {
|
||||
let order = execution_order.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let order = order.clone();
|
||||
async move {
|
||||
order.lock().unwrap().push("C");
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
x + 1
|
||||
}
|
||||
}))
|
||||
.depends_on(b)
|
||||
};
|
||||
|
||||
let d = {
|
||||
let order = execution_order.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let order = order.clone();
|
||||
async move {
|
||||
order.lock().unwrap().push("D");
|
||||
x + 1
|
||||
}
|
||||
}))
|
||||
.depends_on(c)
|
||||
};
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Verify results
|
||||
assert_eq!(dag.get(a)?, 1);
|
||||
assert_eq!(dag.get(b)?, 2);
|
||||
assert_eq!(dag.get(c)?, 3);
|
||||
assert_eq!(dag.get(d)?, 4);
|
||||
|
||||
// Verify execution order
|
||||
let order = execution_order.lock().unwrap();
|
||||
let expected = vec!["A", "B", "C", "D"];
|
||||
assert_eq!(order.as_slice(), expected.as_slice());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_independent_tasks_dont_wait() -> DagResult<()> {
|
||||
// Prove that independent tasks don't wait for each other
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let task_started = Arc::new(Mutex::new(Vec::new()));
|
||||
let task_finished = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
// Create tasks with different durations
|
||||
let fast1 = {
|
||||
let started = task_started.clone();
|
||||
let finished = task_finished.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let started = started.clone();
|
||||
let finished = finished.clone();
|
||||
async move {
|
||||
started.lock().unwrap().push(("fast1", Instant::now()));
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
finished.lock().unwrap().push(("fast1", Instant::now()));
|
||||
"fast1"
|
||||
}
|
||||
}))
|
||||
};
|
||||
|
||||
let slow = {
|
||||
let started = task_started.clone();
|
||||
let finished = task_finished.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let started = started.clone();
|
||||
let finished = finished.clone();
|
||||
async move {
|
||||
started.lock().unwrap().push(("slow", Instant::now()));
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
finished.lock().unwrap().push(("slow", Instant::now()));
|
||||
"slow"
|
||||
}
|
||||
}))
|
||||
};
|
||||
|
||||
let fast2 = {
|
||||
let started = task_started.clone();
|
||||
let finished = task_finished.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let started = started.clone();
|
||||
let finished = finished.clone();
|
||||
async move {
|
||||
started.lock().unwrap().push(("fast2", Instant::now()));
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
finished.lock().unwrap().push(("fast2", Instant::now()));
|
||||
"fast2"
|
||||
}
|
||||
}))
|
||||
};
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Check results
|
||||
assert_eq!(dag.get(fast1)?, "fast1");
|
||||
assert_eq!(dag.get(slow)?, "slow");
|
||||
assert_eq!(dag.get(fast2)?, "fast2");
|
||||
|
||||
// Verify that fast tasks finished before slow task
|
||||
let finished = task_finished.lock().unwrap();
|
||||
let fast1_finish = finished.iter().find(|x| x.0 == "fast1").unwrap().1;
|
||||
let fast2_finish = finished.iter().find(|x| x.0 == "fast2").unwrap().1;
|
||||
let slow_finish = finished.iter().find(|x| x.0 == "slow").unwrap().1;
|
||||
|
||||
assert!(
|
||||
fast1_finish < slow_finish,
|
||||
"fast1 should finish before slow"
|
||||
);
|
||||
assert!(
|
||||
fast2_finish < slow_finish,
|
||||
"fast2 should finish before slow"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
//! Parallelism tests module
|
||||
|
||||
pub mod concurrent;
|
||||
pub mod execution;
|
||||
pub mod proof;
|
||||
pub mod spawning;
|
||||
pub mod threads;
|
||||
pub mod timing;
|
||||
@@ -0,0 +1,65 @@
|
||||
//! Parallelism proof tests
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::{DagResult, DagRunner};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parallelism_proof_1000_tasks() -> DagResult<()> {
|
||||
// Proof that 1000 tasks sleeping 100ms each complete in ~100ms (not 100 seconds)
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let sleep_duration = Duration::from_millis(100);
|
||||
let num_tasks = 1000u32;
|
||||
|
||||
// Create 1000 independent tasks that each sleep for 100ms
|
||||
let tasks: Vec<_> = (0..num_tasks)
|
||||
.map(|i| {
|
||||
dag.add_task(task_fn(move |_: ()| async move {
|
||||
sleep(sleep_duration).await;
|
||||
i
|
||||
}))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let start = Instant::now();
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Verify all tasks completed
|
||||
for (i, task) in tasks.iter().enumerate() {
|
||||
assert_eq!(dag.get(task)?, i as u32);
|
||||
}
|
||||
|
||||
// Sequential execution would take: 1000 * 100ms = 100 seconds
|
||||
let sequential_time = sleep_duration * num_tasks;
|
||||
|
||||
println!(
|
||||
"Parallelism proof: {} tasks × {}ms completed in {:?} (sequential would be {:?})",
|
||||
num_tasks,
|
||||
sleep_duration.as_millis(),
|
||||
elapsed,
|
||||
sequential_time
|
||||
);
|
||||
|
||||
// Allow generous overhead: should complete in less than 500ms (not 100 seconds!)
|
||||
assert!(
|
||||
elapsed < Duration::from_millis(500),
|
||||
"Tasks appear to be running sequentially! Took {:?} instead of ~100ms",
|
||||
elapsed
|
||||
);
|
||||
|
||||
// Verify we achieved massive speedup
|
||||
let speedup = sequential_time.as_secs_f64() / elapsed.as_secs_f64();
|
||||
assert!(
|
||||
speedup > 200.0,
|
||||
"Speedup too low: {:.1}x (expected > 200x)",
|
||||
speedup
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
//! Tests for spawner functionality
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::{DagResult, DagRunner};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_spawner_actually_spawns_tasks() -> DagResult<()> {
|
||||
// Verify that the spawner function is actually called for each task
|
||||
let dag = DagRunner::new();
|
||||
let spawn_count = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
// Create 10 independent tasks
|
||||
let tasks: Vec<_> = (0..10)
|
||||
.map(|i| dag.add_task(task_fn(move |_: ()| async move { i })))
|
||||
.collect();
|
||||
|
||||
// Custom spawner that counts invocations
|
||||
let counter = spawn_count.clone();
|
||||
dag.run(move |fut| {
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Verify all tasks were spawned
|
||||
for (i, task) in tasks.iter().enumerate() {
|
||||
assert_eq!(dag.get(task)?, i as i32);
|
||||
}
|
||||
|
||||
// The spawner should have been called once for each task
|
||||
let spawns = spawn_count.load(Ordering::SeqCst);
|
||||
assert_eq!(
|
||||
spawns, 10,
|
||||
"Expected spawner to be called 10 times, but was called {} times",
|
||||
spawns
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tasks_not_running_inline() -> DagResult<()> {
|
||||
// Verify that tasks are spawned and CAN run on different threads than the caller
|
||||
// Note: Small/fast tasks may run on the same thread if the scheduler chooses
|
||||
use std::thread;
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let calling_thread = thread::current().id();
|
||||
let task_threads = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
// Create tasks with some work to encourage distribution
|
||||
let tasks: Vec<_> = (0..20)
|
||||
.map(|i| {
|
||||
let threads = task_threads.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let threads = threads.clone();
|
||||
async move {
|
||||
threads.lock().unwrap().push(thread::current().id());
|
||||
|
||||
// Add some async work
|
||||
sleep(Duration::from_millis(2)).await;
|
||||
|
||||
i
|
||||
}
|
||||
}))
|
||||
})
|
||||
.collect();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Verify results
|
||||
for (i, task) in tasks.iter().enumerate() {
|
||||
assert_eq!(dag.get(task)?, i as i32);
|
||||
}
|
||||
|
||||
// Check if tasks ran on different threads than the caller
|
||||
// This is a capability test - the scheduler MAY choose to use the same thread
|
||||
let task_thread_ids = task_threads.lock().unwrap();
|
||||
let different_threads = task_thread_ids
|
||||
.iter()
|
||||
.filter(|&&tid| tid != calling_thread)
|
||||
.count();
|
||||
|
||||
if different_threads == 0 {
|
||||
eprintln!(
|
||||
"Warning: All tasks ran on the calling thread. This may indicate tasks aren't \
|
||||
being spawned, or the scheduler chose not to distribute (valid for very fast tasks)"
|
||||
);
|
||||
}
|
||||
|
||||
// The key thing is that tasks completed successfully via spawner
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,477 @@
|
||||
//! Tests for multi-threading and concurrent execution
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::{DagResult, DagRunner};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tasks_run_on_different_threads() -> DagResult<()> {
|
||||
// Prove that tasks CAN run on multiple different threads (true parallelism capability)
|
||||
// Note: Tokio may choose to run fast tasks on the same thread for efficiency
|
||||
use std::thread;
|
||||
|
||||
let dag = DagRunner::new();
|
||||
let thread_ids = Arc::new(Mutex::new(std::collections::HashSet::new()));
|
||||
|
||||
// Create 50 tasks with actual work to encourage multi-threading
|
||||
let tasks: Vec<_> = (0..50)
|
||||
.map(|i| {
|
||||
let ids = thread_ids.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let ids = ids.clone();
|
||||
async move {
|
||||
let tid = thread::current().id();
|
||||
ids.lock().unwrap().insert(tid);
|
||||
|
||||
// Do some actual async work to give tokio reason to distribute
|
||||
sleep(Duration::from_millis(5)).await;
|
||||
|
||||
// Some CPU work too
|
||||
let mut sum = 0;
|
||||
for j in 0..1000 {
|
||||
sum += j;
|
||||
}
|
||||
i + sum
|
||||
}
|
||||
}))
|
||||
})
|
||||
.collect();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Verify all tasks completed
|
||||
for task in tasks.iter() {
|
||||
let _ = dag.get(task)?;
|
||||
}
|
||||
|
||||
// With a multi-threaded runtime and sufficient work, we typically see multiple threads
|
||||
// But we can't guarantee it (scheduler's choice), so we just verify the capability exists
|
||||
let unique_threads = thread_ids.lock().unwrap().len();
|
||||
|
||||
// On a multi-core system with sufficient work, tokio should use multiple threads
|
||||
// This is a soft assertion - if it fails, it means tasks CAN'T use multiple threads
|
||||
if unique_threads == 1 {
|
||||
eprintln!(
|
||||
"Warning: All tasks ran on 1 thread. This may indicate parallelism isn't working, \
|
||||
or tokio chose not to distribute work (valid for small/fast tasks)"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_concurrent_execution_with_atomic_counter() -> DagResult<()> {
|
||||
// Prove concurrent execution using atomic counters to track simultaneous tasks
|
||||
let dag = DagRunner::new();
|
||||
let concurrent_count = Arc::new(AtomicUsize::new(0));
|
||||
let max_concurrent = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
// Create 20 tasks that track concurrent execution
|
||||
let tasks: Vec<_> = (0..20)
|
||||
.map(|i| {
|
||||
let count = concurrent_count.clone();
|
||||
let max = max_concurrent.clone();
|
||||
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let count = count.clone();
|
||||
let max = max.clone();
|
||||
async move {
|
||||
// Increment concurrent counter
|
||||
let current = count.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
|
||||
// Track maximum concurrency
|
||||
let mut prev_max = max.load(Ordering::SeqCst);
|
||||
while current > prev_max {
|
||||
match max.compare_exchange_weak(
|
||||
prev_max,
|
||||
current,
|
||||
Ordering::SeqCst,
|
||||
Ordering::SeqCst,
|
||||
) {
|
||||
Ok(_) => break,
|
||||
Err(x) => prev_max = x,
|
||||
}
|
||||
}
|
||||
|
||||
// Simulate work
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
|
||||
// Decrement counter
|
||||
count.fetch_sub(1, Ordering::SeqCst);
|
||||
|
||||
i
|
||||
}
|
||||
}))
|
||||
})
|
||||
.collect();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Verify all tasks completed
|
||||
for (i, task) in tasks.iter().enumerate() {
|
||||
assert_eq!(dag.get(task)?, i);
|
||||
}
|
||||
|
||||
// Check that we achieved real parallelism
|
||||
let max_seen = max_concurrent.load(Ordering::SeqCst);
|
||||
println!("Maximum concurrent tasks: {}", max_seen);
|
||||
|
||||
// Should see multiple tasks running concurrently
|
||||
assert!(
|
||||
max_seen > 1,
|
||||
"Expected concurrent execution, max was {}",
|
||||
max_seen
|
||||
);
|
||||
|
||||
// Final count should be 0
|
||||
assert_eq!(concurrent_count.load(Ordering::SeqCst), 0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_diamond_parallel_execution() -> DagResult<()> {
|
||||
// Test diamond pattern: source -> (parallel1, parallel2) -> sink
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let concurrent_middle = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let source = dag.add_task(task_fn(|_: ()| async { 100 }));
|
||||
|
||||
// These should run in parallel
|
||||
let parallel1 = {
|
||||
let counter = concurrent_middle.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let counter = counter.clone();
|
||||
async move {
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
sleep(Duration::from_millis(20)).await;
|
||||
let count = counter.load(Ordering::SeqCst);
|
||||
counter.fetch_sub(1, Ordering::SeqCst);
|
||||
assert!(count > 0, "Should see concurrent execution");
|
||||
x * 2
|
||||
}
|
||||
}))
|
||||
.depends_on(&source)
|
||||
};
|
||||
|
||||
let parallel2 = {
|
||||
let counter = concurrent_middle.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let counter = counter.clone();
|
||||
async move {
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
sleep(Duration::from_millis(20)).await;
|
||||
let count = counter.load(Ordering::SeqCst);
|
||||
counter.fetch_sub(1, Ordering::SeqCst);
|
||||
// Both should be running
|
||||
assert!(count > 0, "Should see concurrent execution");
|
||||
x * 3
|
||||
}
|
||||
}))
|
||||
.depends_on(&source)
|
||||
};
|
||||
|
||||
let sink = dag
|
||||
.add_task(task_fn(|(a, b): (i32, i32)| async move { a + b }))
|
||||
.depends_on((¶llel1, ¶llel2));
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
assert_eq!(dag.get(sink)?, 500); // (100 * 2) + (100 * 3)
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parallel_branches_with_dependencies() -> DagResult<()> {
|
||||
// Test that parallel branches execute independently but respect their own dependencies
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let branch_progress = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
// Source node
|
||||
let source = dag.add_task(task_fn(|_: ()| async { 10 }));
|
||||
|
||||
// Branch 1: source -> b1_1 -> b1_2
|
||||
let b1_1 = {
|
||||
let progress = branch_progress.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let progress = progress.clone();
|
||||
async move {
|
||||
progress
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(("b1_1_start", Instant::now()));
|
||||
sleep(Duration::from_millis(30)).await;
|
||||
progress.lock().unwrap().push(("b1_1_end", Instant::now()));
|
||||
x * 2
|
||||
}
|
||||
}))
|
||||
.depends_on(&source)
|
||||
};
|
||||
|
||||
let b1_2 = {
|
||||
let progress = branch_progress.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let progress = progress.clone();
|
||||
async move {
|
||||
progress
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(("b1_2_start", Instant::now()));
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
progress.lock().unwrap().push(("b1_2_end", Instant::now()));
|
||||
x + 1
|
||||
}
|
||||
}))
|
||||
.depends_on(b1_1)
|
||||
};
|
||||
|
||||
// Branch 2: source -> b2_1 -> b2_2
|
||||
let b2_1 = {
|
||||
let progress = branch_progress.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let progress = progress.clone();
|
||||
async move {
|
||||
progress
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(("b2_1_start", Instant::now()));
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
progress.lock().unwrap().push(("b2_1_end", Instant::now()));
|
||||
x * 3
|
||||
}
|
||||
}))
|
||||
.depends_on(&source)
|
||||
};
|
||||
|
||||
let b2_2 = {
|
||||
let progress = branch_progress.clone();
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let progress = progress.clone();
|
||||
async move {
|
||||
progress
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(("b2_2_start", Instant::now()));
|
||||
sleep(Duration::from_millis(30)).await;
|
||||
progress.lock().unwrap().push(("b2_2_end", Instant::now()));
|
||||
x + 2
|
||||
}
|
||||
}))
|
||||
.depends_on(b2_1)
|
||||
};
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Verify results
|
||||
assert_eq!(dag.get(b1_2)?, 21); // (10 * 2) + 1
|
||||
assert_eq!(dag.get(b2_2)?, 32); // (10 * 3) + 2
|
||||
|
||||
// Analyze timing to prove parallel execution
|
||||
let progress = branch_progress.lock().unwrap();
|
||||
|
||||
// Find timestamps
|
||||
let find_time = |name: &str| -> Instant {
|
||||
progress
|
||||
.iter()
|
||||
.find(|(n, _)| *n == name)
|
||||
.map(|(_, t)| *t)
|
||||
.unwrap_or_else(|| panic!("Couldn't find {}", name))
|
||||
};
|
||||
|
||||
let b1_1_start = find_time("b1_1_start");
|
||||
let b1_1_end = find_time("b1_1_end");
|
||||
let b1_2_start = find_time("b1_2_start");
|
||||
let b2_1_start = find_time("b2_1_start");
|
||||
|
||||
// Branch 1 and Branch 2 first tasks should start nearly simultaneously
|
||||
let branch_start_diff = if b1_1_start > b2_1_start {
|
||||
b1_1_start - b2_1_start
|
||||
} else {
|
||||
b2_1_start - b1_1_start
|
||||
};
|
||||
|
||||
assert!(
|
||||
branch_start_diff < Duration::from_millis(10),
|
||||
"Branches should start in parallel, diff was {:?}",
|
||||
branch_start_diff
|
||||
);
|
||||
|
||||
// b1_2 should only start after b1_1 ends
|
||||
assert!(b1_2_start >= b1_1_end, "b1_2 started before b1_1 finished");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_wide_fanout_parallel_execution() -> DagResult<()> {
|
||||
// Test that a single source can fan out to many parallel tasks
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let max_concurrent = Arc::new(AtomicUsize::new(0));
|
||||
let current_concurrent = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let source = dag.add_task(task_fn(|_: ()| async { 42 }));
|
||||
|
||||
// Create 50 tasks that all depend on the source
|
||||
let dependents: Vec<_> = (0..50)
|
||||
.map(|i| {
|
||||
let max_c = max_concurrent.clone();
|
||||
let curr_c = current_concurrent.clone();
|
||||
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let max_c = max_c.clone();
|
||||
let curr_c = curr_c.clone();
|
||||
async move {
|
||||
// Track concurrency
|
||||
let current = curr_c.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
|
||||
// Update max
|
||||
let mut prev_max = max_c.load(Ordering::SeqCst);
|
||||
while current > prev_max {
|
||||
match max_c.compare_exchange_weak(
|
||||
prev_max,
|
||||
current,
|
||||
Ordering::SeqCst,
|
||||
Ordering::SeqCst,
|
||||
) {
|
||||
Ok(_) => break,
|
||||
Err(v) => prev_max = v,
|
||||
}
|
||||
}
|
||||
|
||||
// Do work
|
||||
sleep(Duration::from_millis(5)).await;
|
||||
|
||||
curr_c.fetch_sub(1, Ordering::SeqCst);
|
||||
x + i
|
||||
}
|
||||
}))
|
||||
.depends_on(&source)
|
||||
})
|
||||
.collect();
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Verify results
|
||||
for (i, task) in dependents.iter().enumerate() {
|
||||
assert_eq!(dag.get(task)?, 42 + i as i32);
|
||||
}
|
||||
|
||||
// Check that we saw significant parallelism
|
||||
let max_seen = max_concurrent.load(Ordering::SeqCst);
|
||||
println!("Max concurrent in fanout: {}", max_seen);
|
||||
assert!(
|
||||
max_seen > 5,
|
||||
"Expected significant parallelism in fanout, got {}",
|
||||
max_seen
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_massive_parallel_fanout() -> DagResult<()> {
|
||||
// Test extreme parallelism with 1000 parallel tasks
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let completed = Arc::new(AtomicUsize::new(0));
|
||||
let max_concurrent = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let source = dag.add_task(task_fn(|_: ()| async { 1 }));
|
||||
|
||||
// Create 1000 tasks all depending on the same source
|
||||
let tasks: Vec<_> = (0..1000)
|
||||
.map(|i| {
|
||||
let comp = completed.clone();
|
||||
let max = max_concurrent.clone();
|
||||
let curr = current.clone();
|
||||
|
||||
dag.add_task(task_fn(move |x: i32| {
|
||||
let comp = comp.clone();
|
||||
let max = max.clone();
|
||||
let curr = curr.clone();
|
||||
async move {
|
||||
// Track concurrency
|
||||
let c = curr.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
|
||||
let mut prev_max = max.load(Ordering::SeqCst);
|
||||
while c > prev_max {
|
||||
match max.compare_exchange_weak(
|
||||
prev_max,
|
||||
c,
|
||||
Ordering::SeqCst,
|
||||
Ordering::SeqCst,
|
||||
) {
|
||||
Ok(_) => break,
|
||||
Err(v) => prev_max = v,
|
||||
}
|
||||
}
|
||||
|
||||
// Quick work
|
||||
sleep(Duration::from_micros(100)).await;
|
||||
|
||||
curr.fetch_sub(1, Ordering::SeqCst);
|
||||
comp.fetch_add(1, Ordering::SeqCst);
|
||||
x + i
|
||||
}
|
||||
}))
|
||||
.depends_on(&source)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let start = Instant::now();
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// All should complete
|
||||
assert_eq!(completed.load(Ordering::SeqCst), 1000);
|
||||
|
||||
// Check some results
|
||||
assert_eq!(dag.get(tasks[0])?, 1);
|
||||
assert_eq!(dag.get(tasks[500])?, 501);
|
||||
assert_eq!(dag.get(tasks[999])?, 1000);
|
||||
|
||||
let max_seen = max_concurrent.load(Ordering::SeqCst);
|
||||
println!(
|
||||
"Massive fanout: {} concurrent tasks, completed in {:?}",
|
||||
max_seen, elapsed
|
||||
);
|
||||
|
||||
// Should see significant parallelism
|
||||
assert!(max_seen > 10, "Expected high parallelism, got {}", max_seen);
|
||||
|
||||
// Should complete quickly (not sequentially)
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"1000 tasks took too long: {:?}",
|
||||
elapsed
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
//! Tests for timing and speedup verification
|
||||
|
||||
use crate::common::task_fn;
|
||||
use dagx::{DagResult, DagRunner, Task};
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parallel_execution_speedup() -> DagResult<()> {
|
||||
// Prove parallel execution by comparing timing
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let work_duration = Duration::from_millis(50);
|
||||
let num_tasks = 10;
|
||||
|
||||
// Create independent tasks that each take 50ms
|
||||
let tasks: Vec<_> = (0..num_tasks)
|
||||
.map(|i| {
|
||||
dag.add_task(task_fn(move |_: ()| async move {
|
||||
sleep(work_duration).await;
|
||||
i
|
||||
}))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let start = Instant::now();
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Verify all tasks completed
|
||||
for (i, task) in tasks.iter().enumerate() {
|
||||
assert_eq!(dag.get(task)?, i);
|
||||
}
|
||||
|
||||
// If tasks ran sequentially: 10 * 50ms = 500ms
|
||||
// If tasks ran in parallel: ~50ms (plus overhead)
|
||||
let sequential_time = work_duration * num_tasks as u32;
|
||||
|
||||
println!(
|
||||
"Parallel execution took {:?}, sequential would be {:?}",
|
||||
elapsed, sequential_time
|
||||
);
|
||||
|
||||
// Parallel execution should be significantly faster
|
||||
assert!(
|
||||
elapsed < sequential_time / 2,
|
||||
"Parallel execution too slow: {:?} vs sequential {:?}",
|
||||
elapsed,
|
||||
sequential_time
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct SyncTask;
|
||||
#[dagx::task]
|
||||
impl SyncTask {
|
||||
fn run(&self, x: &i32) -> i32 {
|
||||
// Synchronous task - no async/await
|
||||
std::thread::sleep(Duration::from_millis(1)); // Blocking sleep
|
||||
x * 2
|
||||
}
|
||||
}
|
||||
|
||||
struct AsyncTask;
|
||||
#[dagx::task]
|
||||
impl AsyncTask {
|
||||
async fn run(&self, x: &i32) -> i32 {
|
||||
// Async task
|
||||
sleep(Duration::from_millis(1)).await; // Non-blocking sleep
|
||||
x * 3
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_sync_tasks_dont_block_async_runtime() -> DagResult<()> {
|
||||
// Test that sync tasks don't block the async runtime
|
||||
let dag = DagRunner::new();
|
||||
|
||||
let start_times = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
// Mix of sync and async tasks
|
||||
let source = dag.add_task(task_fn(|_: ()| async { 10 }));
|
||||
|
||||
// Sync tasks
|
||||
let sync1 = dag.add_task(SyncTask).depends_on(&source);
|
||||
let sync2 = dag.add_task(SyncTask).depends_on(&source);
|
||||
|
||||
// Async tasks
|
||||
let async1 = dag.add_task(AsyncTask).depends_on(&source);
|
||||
let async2 = dag.add_task(AsyncTask).depends_on(&source);
|
||||
|
||||
// Track timing
|
||||
let timing_task = {
|
||||
let times = start_times.clone();
|
||||
dag.add_task(task_fn(move |_: ()| {
|
||||
let times = times.clone();
|
||||
async move {
|
||||
for i in 0..5 {
|
||||
times
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((format!("tick_{}", i), Instant::now()));
|
||||
sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
"timing"
|
||||
}
|
||||
}))
|
||||
};
|
||||
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Verify results
|
||||
assert_eq!(dag.get(sync1)?, 20);
|
||||
assert_eq!(dag.get(sync2)?, 20);
|
||||
assert_eq!(dag.get(async1)?, 30);
|
||||
assert_eq!(dag.get(async2)?, 30);
|
||||
assert_eq!(dag.get(timing_task)?, "timing");
|
||||
|
||||
// Check that timing task made progress (wasn't blocked by sync tasks)
|
||||
let times = start_times.lock().unwrap();
|
||||
assert_eq!(
|
||||
times.len(),
|
||||
5,
|
||||
"Timing task should have completed all ticks"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in new issue
Block a user