swaits/dagx
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: consolidate some tests/examples/benchmarks

Browse Source
This commit is contained in:
TechnoPorg committed 2026-02-12 23:31:19 +01:00
1 parent 49be497964
commit b8204c4ed9
83 files changed
+569 -10068
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

examples/complex_dag.rs
-215
View File
@@ -1,215 +0,0 @@
//! # Building Complex Workflows: Multi-Layer DAG
//!
//! This example demonstrates how to build complex, real-world DAGs by combining
//! multiple dependency patterns into a cohesive workflow.
//!
//! ## What You'll Learn
//! - How to combine fan-out, fan-in, and diamond patterns in one DAG
//! - How to build multi-layer workflows with intermediate results
//! - How to structure an ETL (Extract-Transform-Load) pipeline
//! - How to pass results through multiple transformation stages
//!
//! ## Workflow Diagram
//! ```text
//! EXTRACT TRANSFORM COMBINE AGGREGATE
//! ┌──────────┐ ┌──────────┐
//! │ LoadData │────►│Transform │────┐
//! │ A │ │ (×2) │ │
//! └──────────┘ └──────────┘ │
//! ├──►┌─────────┐───►┌─────────┐───►┌─────────┐
//! │ │ Combine │ │ Average │ │ Report │
//! └──►└─────────┘ └─────────┘ └─────────┘
//! ┌──────────┐ ┌──────────┐ ┌──► │ │
//! │ LoadData │────►│Transform │────┘ │ │
//! │ B │ │ (×3) │ └──────────────┘
//! └──────────┘ └──────────┘
//! ```
//!
//! ## Key Concepts
//! - **Multi-layer execution**: Tasks organized into logical stages (Extract → Transform → Combine)
//! - **Many-to-many dependencies**: The Combine task depends on both Transform tasks
//! - **Result reuse**: The combined result is used by both Average and Report tasks
//! - **Diamond pattern**: Multiple paths from sources converge and then diverge again
//!
//! ## Use Cases
//! - ETL pipelines with multiple data sources
//! - Multi-stage data processing workflows
//! - Complex business logic with branching and merging
//! - Data aggregation and reporting systems
//!
//! ## Running This Example
//! ```bash
//! cargo run --example complex_dag
//! ```
//!
//! ## Expected Output
//! ```text
//! Building ETL pipeline...
//!
//! Executing pipeline...
//!
//! Loading data from source A
//! Loading data from source B
//! Transforming: 6 * 2 = 12
//! Transforming: 15 * 3 = 45
//! Combining datasets: 12 + 45 = 57
//! Computing average: 57 / 6 = 9.50
//!
//! ETL Pipeline Results:
//! Items: 6
//! Total: 57
//! Average: 9.50
//!
//! Calculation:
//! Source A: sum=6, transformed (x2) = 12
//! Source B: sum=15, transformed (x3) = 45
//! Combined: 6 items, sum=57, avg=9.50
//! ```
use dagx::{task, DagResult, DagRunner};
// Extract: Load data from different sources
struct LoadData {
source: &'static str,
}
impl LoadData {
fn new(source: &'static str) -> Self {
Self { source }
}
}
#[task]
impl LoadData {
async fn run(&mut self) -> i32 {
// Simulate loading and summing data
println!("Loading data from source {}", self.source);
match self.source {
"A" => 6, // sum of [1, 2, 3]
"B" => 15, // sum of [4, 5, 6]
_ => 0,
}
}
}
// Transform: Process data with a multiplier
struct Transform {
multiplier: i32,
}
impl Transform {
fn new(multiplier: i32) -> Self {
Self { multiplier }
}
}
#[task]
impl Transform {
async fn run(&mut self, sum: &i32) -> i32 {
let result = sum * self.multiplier;
println!("Transforming: {} * {} = {}", sum, self.multiplier, result);
result
}
}
// Combine: Merge two datasets
struct Combine;
#[task]
impl Combine {
async fn run(sum_a: &i32, sum_b: &i32) -> i32 {
println!(
"Combining datasets: {} + {} = {}",
sum_a,
sum_b,
sum_a + sum_b
);
sum_a + sum_b
}
}
// Compute average (using a fixed count for simplicity)
struct ComputeAverage {
total_count: i32,
}
impl ComputeAverage {
fn new(total_count: i32) -> Self {
Self { total_count }
}
}
#[task]
impl ComputeAverage {
async fn run(&mut self, sum: &i32) -> f64 {
let avg = *sum as f64 / self.total_count as f64;
println!(
"Computing average: {} / {} = {:.2}",
sum, self.total_count, avg
);
avg
}
}
// Format report
struct FormatReport {
count: i32,
}
impl FormatReport {
fn new(count: i32) -> Self {
Self { count }
}
}
#[task]
impl FormatReport {
async fn run(&mut self, sum: &i32, avg: &f64) -> String {
format!(
"ETL Pipeline Results:\n Items: {}\n Total: {}\n Average: {:.2}",
self.count, sum, avg
)
}
}
#[tokio::main]
async fn main() -> DagResult<()> {
let dag = DagRunner::new();
println!("Building ETL pipeline...\n");
// Extract: Load from two sources (parallel)
let source_a = dag.add_task(LoadData::new("A"));
let source_b = dag.add_task(LoadData::new("B"));
// Transform: Process each source (parallel)
let transform_a = dag.add_task(Transform::new(2)).depends_on(source_a);
let transform_b = dag.add_task(Transform::new(3)).depends_on(source_b);
// Combine: Merge the transformed datasets
let combined = dag
.add_task(Combine)
.depends_on((&transform_a, &transform_b));
// Compute average on combined data (we know we have 6 items total)
let avg = dag.add_task(ComputeAverage::new(6)).depends_on(combined);
// Format final report (needs both sum and average)
let report = dag
.add_task(FormatReport::new(6))
.depends_on((&combined, &avg));
// Execute the entire ETL pipeline
println!("Executing pipeline...\n");
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await?;
// Display final report
println!("\n{}", dag.get(report)?);
println!("\nCalculation:");
println!(" Source A: sum=6, transformed (x2) = 12");
println!(" Source B: sum=15, transformed (x3) = 45");
println!(" Combined: 6 items, sum=57, avg=9.50");
Ok(())
}
examples/conditional_workflow.rs
-148
View File
@@ -1,148 +0,0 @@
//! # Pattern: Conditional Execution with Data Validation
//!
//! This example demonstrates how to implement conditional logic in DAGs by using
//! simple data transformations rather than traditional Result types.
//!
//! ## What You'll Learn
//! - How to implement validation logic in parallel tasks
//! - How to use return values for conditional behavior (e.g., 0 for invalid)
//! - How to categorize and branch based on computed values
//! - How to aggregate results from multiple conditional branches
//!
//! ## Workflow Diagram
//! ```text
//! VALIDATE CATEGORIZE SUMMARIZE
//! ┌─────────────┐ ┌──────────────┐
//! │ Validate(10)│────20──►│Categorize(25)│────"low"────┐
//! └─────────────┘ └──────────────┘ │
//! │
//! ┌─────────────┐ ┌──────────────┐ │
//! │ Validate(-5)│────0───►│Categorize(25)│──"invalid"──┼──►┌────────────┐
//! └─────────────┘ └──────────────┘ │ │ Summarize │
//! │ └────────────┘
//! ┌─────────────┐ ┌──────────────┐ │
//! │ Validate(20)│────40──►│Categorize(25)│────"high"───┘
//! └─────────────┘ └──────────────┘
//! ```
//!
//! ## Key Concepts
//! - **Validation as transformation**: Invalid inputs become sentinel values (0)
//! - **Parallel validation**: All inputs validated simultaneously
//! - **Data-driven branching**: Categorization based on computed values
//! - **Type-safe aggregation**: All branches produce the same type (String)
//!
//! ## Use Cases
//! - Input validation pipelines
//! - Multi-stage data cleaning
//! - Classification and categorization workflows
//! - Quality control systems
//! - Data preprocessing for ML pipelines
//!
//! ## Running This Example
//! ```bash
//! cargo run --example conditional_workflow
//! ```
//!
//! ## Expected Output
//! ```text
//! Processing and categorizing inputs in parallel...
//!
//! Valid input 10: doubling to 20
//! Invalid input -5: returning 0
//! Valid input 20: doubling to 40
//!
//! Final summary: Categories: [low, invalid, high]
//! Expected: Categories: [low, invalid, high]
//! ```
use dagx::{task, DagResult, DagRunner};
// Task that validates and processes input
struct ValidateAndDouble {
value: i32,
}
impl ValidateAndDouble {
fn new(value: i32) -> Self {
Self { value }
}
}
#[task]
impl ValidateAndDouble {
async fn run(&mut self) -> i32 {
if self.value > 0 {
println!("Valid input {}: doubling to {}", self.value, self.value * 2);
self.value * 2
} else {
println!("Invalid input {}: returning 0", self.value);
0 // Return 0 for invalid inputs
}
}
}
// Task that categorizes a number
struct Categorize {
threshold: i32,
}
impl Categorize {
fn new(threshold: i32) -> Self {
Self { threshold }
}
}
#[task]
impl Categorize {
async fn run(&mut self, value: &i32) -> String {
if *value == 0 {
"invalid".to_string()
} else if *value >= self.threshold {
"high".to_string()
} else {
"low".to_string()
}
}
}
// Task that combines categorization results
struct SummarizeResults;
#[task]
impl SummarizeResults {
async fn run(cat1: &String, cat2: &String, cat3: &String) -> String {
format!("Categories: [{}, {}, {}]", cat1, cat2, cat3)
}
}
#[tokio::main]
async fn main() -> DagResult<()> {
let dag = DagRunner::new();
println!("Processing and categorizing inputs in parallel...\n");
// Validate and process multiple inputs in parallel
let processed1 = dag.add_task(ValidateAndDouble::new(10)); // Valid: 20
let processed2 = dag.add_task(ValidateAndDouble::new(-5)); // Invalid: 0
let processed3 = dag.add_task(ValidateAndDouble::new(20)); // Valid: 40
// Categorize each result (also parallel, threshold = 25)
let cat1 = dag.add_task(Categorize::new(25)).depends_on(processed1); // 20 < 25 = "low"
let cat2 = dag.add_task(Categorize::new(25)).depends_on(processed2); // 0 = "invalid"
let cat3 = dag.add_task(Categorize::new(25)).depends_on(processed3); // 40 >= 25 = "high"
// Combine results
let summary = dag
.add_task(SummarizeResults)
.depends_on((&cat1, &cat2, &cat3));
// Execute
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await?;
// Display results
println!("\nFinal summary: {}", dag.get(summary)?);
println!("Expected: Categories: [low, invalid, high]");
Ok(())
}
examples/custom_types.rs
-116
View File
@@ -1,116 +0,0 @@
//! # Custom Types: Using Your Own Structs
//!
//! This example demonstrates that **ANY custom type** works in dagx as long as it implements
//! ` Send + Sync + 'static`. You don't need to implement any special traits!
//!
//! ## What You'll Learn
//! - How to use custom structs as task outputs
//! - How custom types flow through the DAG automatically
//! - How one task's output can be consumed by multiple downstream tasks
//!
//! ## Key Insight
//!
//! **dagx works with ANY type automatically!** The `#[task]` macro generates type-specific
//! extraction logic, so you don't need to implement any special traits. Just make sure your
//! type implements:
//! - `Send + Sync` (required for async tasks)
//! - `'static` (no lifetime constraints)
//!
//! That's it! No ExtractInput, no special annotations, no trait boilerplate.
//!
//! ## Running This Example
//! ```bash
//! cargo run --example custom_types
//! ```
//!
//! ## Expected Output
//! ```text
//! Creating custom types example...
//!
//! Creating user: Alice (age: 30)
//! Processing user Alice for greeting
//! Processing user Alice for validation
//! Greeting: Hello, Alice!
//! Validation: Alice is valid (age 30 >= 18)
//! ```
use dagx::{task, DagRunner, TaskHandle};
#[derive(Debug)]
struct User {
name: String,
age: u32,
}
// Task that produces a custom type
struct CreateUser {
name: String,
age: u32,
}
#[task]
impl CreateUser {
async fn run(&self) -> User {
println!("Creating user: {} (age: {})", self.name, self.age);
User {
name: self.name.clone(),
age: self.age,
}
}
}
// Task that consumes the custom type and produces a String
struct GreetUser;
#[task]
impl GreetUser {
async fn run(user: &User) -> String {
println!("Processing user {} for greeting", user.name);
format!("Hello, {}!", user.name)
}
}
// Another task that also consumes the same custom type
// This demonstrates fan-out: one output consumed by multiple tasks
struct ValidateUser;
#[task]
impl ValidateUser {
async fn run(user: &User) -> String {
println!("Processing user {} for validation", user.name);
if user.age >= 18 {
format!("{} is valid (age {} >= 18)", user.name, user.age)
} else {
format!("{} is too young (age {} < 18)", user.name, user.age)
}
}
}
#[tokio::main]
async fn main() {
println!("Creating custom types example...\n");
let dag = DagRunner::new();
// Create a task that produces a custom User type
let user_task: TaskHandle<_> = dag
.add_task(CreateUser {
name: "Alice".to_string(),
age: 30,
})
.into();
// Two different tasks consume the same custom User type
// dagx automatically handles Arc wrapping, so both tasks get &User
let greeting = dag.add_task(GreetUser).depends_on(user_task);
let validation = dag.add_task(ValidateUser).depends_on(user_task);
// Execute the DAG
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
// Print results
println!("Greeting: {}", dag.get(greeting).unwrap());
println!("Validation: {}", dag.get(validation).unwrap());
}
examples/debug_tracing.rs
-461
View File
@@ -1,461 +0,0 @@
//! # Debug Tracing and Task Naming
//!
//! This example demonstrates how to add debug metadata and tracing information to tasks
//! for better observability in complex DAG workflows.
//!
//! ## What You'll Learn
//! - How to add task names and IDs for debugging
//! - How to implement correlation IDs for distributed tracing
//! - How to track task execution timing and lifecycle events
//! - How to build traceable task wrappers
//!
//! ## Debugging Patterns
//!
//! **1. Named Tasks** - Add human-readable names to tasks
//! ```rust
//! struct TaskMetadata {
//! id: String,
//! name: String,
//! start_time: Option<Instant>,
//! }
//! ```
//!
//! **2. Correlation IDs** - Track requests across task boundaries
//! ```rust
//! struct TraceContext {
//! correlation_id: String,
//! parent_task: Option<String>,
//! }
//! ```
//!
//! **3. Execution Metrics** - Measure task performance
//! ```rust
//! struct TaskTrace {
//! name: String,
//! duration_ms: u64,
//! status: &'static str,
//! }
//! ```
//!
//! ## Workflow Diagram
//! ```text
//! TRACE: order-12345
//! ┌──────────────────┐
//! │ ValidateOrder │ [order-12345] Start: 0ms
//! │ ID: task-001 │──────► Status: Success
//! └──────────────────┘ Duration: 10ms
//! │
//! ├──────────────────┬──────────────────┐
//! ▼ ▼ ▼
//! ┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐
//! │ CheckInventory │ │ ValidatePayment │ │ CalculateShipping│
//! │ ID: task-002 │ │ ID: task-003 │ │ ID: task-004 │
//! └──────────────────┘ └──────────────────┘ └──────────────────┘
//! [order-12345] [order-12345] [order-12345]
//! Duration: 45ms Duration: 120ms Duration: 15ms
//! │ │ │
//! └──────────────────┴──────────────────┘
//! ▼
//! ┌──────────────────┐
//! │ FulfillOrder │ [order-12345]
//! │ ID: task-005 │ Duration: 30ms
//! └──────────────────┘
//! ```
//!
//! ## Key Concepts
//! - **Observability**: Make task execution visible for debugging
//! - **Correlation IDs**: Track a request through multiple tasks
//! - **Structured logging**: Consistent task metadata across logs
//! - **Performance metrics**: Measure task execution time
//! - **Task lifecycle**: Track start, completion, and errors
//!
//! ## Use Cases
//! - Debugging complex workflows in production
//! - Performance analysis and bottleneck identification
//! - Distributed tracing in microservices
//! - SLA monitoring and alerting
//! - Audit logging for compliance
//!
//! ## Running This Example
//! ```bash
//! cargo run --example debug_tracing
//! ```
//!
//! ## Expected Output
//! ```text
//! === Debug Tracing Example ===
//!
//! Starting order processing for order-12345
//!
//! [order-12345] [task-001] ValidateOrder: Starting
//! [order-12345] [task-001] ValidateOrder: Completed in 10ms - Success
//!
//! [order-12345] [task-002] CheckInventory: Starting
//! [order-12345] [task-003] ValidatePayment: Starting
//! [order-12345] [task-004] CalculateShipping: Starting
//! [order-12345] [task-002] CheckInventory: Completed in 45ms - Success
//! [order-12345] [task-004] CalculateShipping: Completed in 15ms - Success
//! [order-12345] [task-003] ValidatePayment: Completed in 120ms - Success
//!
//! [order-12345] [task-005] FulfillOrder: Starting
//! [order-12345] [task-005] FulfillOrder: Completed in 30ms - Success
//!
//! === Execution Trace Summary ===
//! Correlation ID: order-12345
//! Total Tasks: 5
//! Total Duration: 220ms
//! Success: 5, Failed: 0
//!
//! Task Breakdown:
//! ValidateOrder: 10ms
//! CheckInventory: 45ms
//! ValidatePayment: 120ms (slowest)
//! CalculateShipping: 15ms
//! FulfillOrder: 30ms
//! ```
use dagx::{task, DagRunner};
use std::time::Instant;
use tokio::time::{sleep, Duration};
// Stateful task that tracks execution metadata for debugging
struct TracedTask {
task_id: String,
task_name: String,
correlation_id: String,
work_duration_ms: u64,
start_time: Option<Instant>,
}
impl TracedTask {
fn new(
task_id: impl Into<String>,
task_name: impl Into<String>,
correlation_id: impl Into<String>,
work_duration_ms: u64,
) -> Self {
Self {
task_id: task_id.into(),
task_name: task_name.into(),
correlation_id: correlation_id.into(),
work_duration_ms,
start_time: None,
}
}
}
#[task]
impl TracedTask {
async fn run(&mut self) -> Result<TaskTrace, String> {
// Record start
let start = Instant::now();
self.start_time = Some(start);
println!(
"[{}] [{}] {}: Starting",
self.correlation_id, self.task_id, self.task_name
);
// Simulate work
sleep(Duration::from_millis(self.work_duration_ms)).await;
// Record completion
let duration_ms = start.elapsed().as_millis() as u64;
println!(
"[{}] [{}] {}: Completed in {}ms - Success",
self.correlation_id, self.task_id, self.task_name, duration_ms
);
Ok(TaskTrace {
task_id: self.task_id.clone(),
task_name: self.task_name.clone(),
correlation_id: self.correlation_id.clone(),
duration_ms,
status: "Success",
})
}
}
// Task that can fail, demonstrating error tracing
struct TracedTaskWithError {
task_id: String,
task_name: String,
correlation_id: String,
should_fail: bool,
}
impl TracedTaskWithError {
fn new(
task_id: impl Into<String>,
task_name: impl Into<String>,
correlation_id: impl Into<String>,
should_fail: bool,
) -> Self {
Self {
task_id: task_id.into(),
task_name: task_name.into(),
correlation_id: correlation_id.into(),
should_fail,
}
}
}
#[task]
impl TracedTaskWithError {
async fn run(&mut self) -> Result<TaskTrace, TaskTrace> {
let start = Instant::now();
println!(
"[{}] [{}] {}: Starting",
self.correlation_id, self.task_id, self.task_name
);
sleep(Duration::from_millis(50)).await;
let duration_ms = start.elapsed().as_millis() as u64;
if self.should_fail {
println!(
"[{}] [{}] {}: Failed in {}ms - Validation Error",
self.correlation_id, self.task_id, self.task_name, duration_ms
);
Err(TaskTrace {
task_id: self.task_id.clone(),
task_name: self.task_name.clone(),
correlation_id: self.correlation_id.clone(),
duration_ms,
status: "Failed",
})
} else {
println!(
"[{}] [{}] {}: Completed in {}ms - Success",
self.correlation_id, self.task_id, self.task_name, duration_ms
);
Ok(TaskTrace {
task_id: self.task_id.clone(),
task_name: self.task_name.clone(),
correlation_id: self.correlation_id.clone(),
duration_ms,
status: "Success",
})
}
}
}
// Task trace data structure
#[derive(Clone, Debug)]
#[allow(dead_code)]
struct TaskTrace {
task_id: String,
task_name: String,
correlation_id: String,
duration_ms: u64,
status: &'static str,
}
// Task that aggregates traces from multiple upstream tasks
struct TraceAggregator {
correlation_id: String,
}
impl TraceAggregator {
fn new(correlation_id: impl Into<String>) -> Self {
Self {
correlation_id: correlation_id.into(),
}
}
}
#[task]
impl TraceAggregator {
async fn run(
&mut self,
t1: &Result<TaskTrace, String>,
t2: &Result<TaskTrace, String>,
t3: &Result<TaskTrace, String>,
) -> TraceSummary {
let traces = vec![t1, t2, t3];
let mut summary = TraceSummary {
correlation_id: self.correlation_id.clone(),
task_count: traces.len(),
total_duration_ms: 0,
success_count: 0,
failed_count: 0,
task_details: Vec::new(),
};
for trace_result in traces {
match trace_result {
Ok(trace) => {
summary.success_count += 1;
summary.total_duration_ms += trace.duration_ms;
summary.task_details.push(trace.clone());
}
Err(e) => {
summary.failed_count += 1;
eprintln!("[{}] Task failed: {}", self.correlation_id, e);
}
}
}
summary
}
}
// Summary of traced execution
#[derive(Clone, Debug)]
struct TraceSummary {
correlation_id: String,
task_count: usize,
total_duration_ms: u64,
success_count: usize,
failed_count: usize,
task_details: Vec<TaskTrace>,
}
impl TraceSummary {
fn print(&self) {
println!("\n=== Execution Trace Summary ===");
println!("Correlation ID: {}", self.correlation_id);
println!("Total Tasks: {}", self.task_count);
println!("Total Duration: {}ms", self.total_duration_ms);
println!(
"Success: {}, Failed: {}",
self.success_count, self.failed_count
);
if !self.task_details.is_empty() {
println!("\nTask Breakdown:");
let max_duration = self
.task_details
.iter()
.map(|t| t.duration_ms)
.max()
.unwrap_or(0);
for trace in &self.task_details {
let slowest_tag = if trace.duration_ms == max_duration {
" (slowest)"
} else {
""
};
println!(
" {}: {}ms{}",
trace.task_name, trace.duration_ms, slowest_tag
);
}
}
}
}
#[tokio::main]
async fn main() {
println!("=== Debug Tracing Example ===\n");
// Example 1: Successful order processing with full tracing
println!("Example 1: Successful order processing\n");
{
let correlation_id = "order-12345";
println!("Starting order processing for {}\n", correlation_id);
let dag = DagRunner::new();
// Root task: Validate order
let _validate = dag.add_task(TracedTask::new(
"task-001",
"ValidateOrder",
correlation_id,
10,
));
// Parallel validation tasks (independent, no dependencies)
let inventory = dag.add_task(TracedTask::new(
"task-002",
"CheckInventory",
correlation_id,
45,
));
let payment = dag.add_task(TracedTask::new(
"task-003",
"ValidatePayment",
correlation_id,
120,
));
let shipping = dag.add_task(TracedTask::new(
"task-004",
"CalculateShipping",
correlation_id,
15,
));
// Aggregate traces
let summary = dag
.add_task(TraceAggregator::new(correlation_id))
.depends_on((inventory, payment, shipping));
// Run the DAG
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
// Print summary
dag.get(summary).unwrap().print();
}
println!("\n");
// Example 2: Order with failed payment
println!("Example 2: Order with failed payment\n");
{
let correlation_id = "order-67890";
println!("Starting order processing for {}\n", correlation_id);
let dag = DagRunner::new();
let _validate = dag.add_task(TracedTaskWithError::new(
"task-101",
"ValidateOrder",
correlation_id,
false,
));
let _inventory = dag.add_task(TracedTaskWithError::new(
"task-102",
"CheckInventory",
correlation_id,
false,
));
// This task will fail
let payment = dag.add_task(TracedTaskWithError::new(
"task-103",
"ValidatePayment",
correlation_id,
true, // will fail
));
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
// Check results
match dag.get(payment).unwrap().as_ref() {
Ok(_) => println!("\nPayment validation succeeded"),
Err(trace) => {
println!("\n[{}] Payment validation failed:", correlation_id);
println!(" Task ID: {}", trace.task_id);
println!(" Duration: {}ms", trace.duration_ms);
println!(" Status: {}", trace.status);
}
}
}
println!("\n=== Debug Tracing Example Complete ===");
}
examples/large_dag.rs
-382
View File
@@ -1,382 +0,0 @@
//! # Working with Large DAGs: Performance and Scalability
//!
//! This example demonstrates dagx's performance characteristics when working
//! with large-scale DAGs containing hundreds or thousands of tasks.
//!
//! ## What You'll Learn
//! - How dagx scales with increasing task counts
//! - Performance characteristics of different DAG topologies
//! - How to benchmark DAG construction and execution
//! - Memory efficiency considerations
//!
//! ## DAG Topologies Tested
//!
//! **1. Wide DAG** - Maximum parallelism
//! ```text
//! ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐
//! │Task 1│ │Task 2│ │Task 3│ ... │Task N│
//! └──────┘ └──────┘ └──────┘ └──────┘
//! (1000 independent parallel tasks)
//! ```
//!
//! **2. Deep DAG** - Sequential chain
//! ```text
//! ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐
//! │Task 1│───►│Task 2│───►│Task 3│───► ... ──►│Task N│
//! └──────┘ └──────┘ └──────┘ └──────┘
//! (100-level dependency chain)
//! ```
//!
//! **3. Binary Tree** - Balanced aggregation
//! ```text
//! ┌────┐ ┌────┐ ┌────┐ ┌────┐
//! │ T1 │ │ T2 │ │ T3 │ │ T4 │
//! └─┬──┘ └─┬──┘ └─┬──┘ └─┬──┘
//! └───┬───┘ └───┬───┘
//! │ │
//! ┌──▼──┐ ┌──▼──┐
//! │ C1 │ │ C2 │
//! └──┬──┘ └──┬──┘
//! └───────┬──────┘
//! │
//! ┌──▼──┐
//! │Root │
//! └─────┘
//! ```
//!
//! **4. Diamond Pattern** - Many-to-many
//! ```text
//! Layer 1: 50 sources
//! Layer 2: 50 tasks (each depends on 2 sources)
//! Layer 3: 4 aggregates
//! Layer 4: 1 final result
//! ```
//!
//! ## Performance Metrics
//! - **Build time**: Time to construct the DAG structure
//! - **Execution time**: Time to run all tasks
//! - **Tasks/second**: Throughput metric
//! - **Memory overhead**: Approximate per-task memory usage
//!
//! ## Key Observations
//! - DAG construction is very fast (< 1ms for 1000s of tasks)
//! - Parallel execution scales well with available CPU cores
//! - Deep chains execute sequentially as expected
//! - Memory overhead is minimal (~200 bytes per task)
//!
//! ## Running This Example
//! ```bash
//! cargo run --example large_dag --release # Use --release for accurate benchmarks
//! ```
//!
//! ## Expected Output
//! ```text
//! === Large DAG Performance Example ===
//!
//! Example 1: Wide DAG (1000 parallel tasks)
//! Build time: 347µs
//! Execution time: 12.5ms
//! Tasks/second: 80000
//! Sample sum (first 10): ...
//!
//! Example 2: Deep DAG (100-level chain)
//! Build time: 156µs
//! Execution time: 8.2ms
//! Final result: ...
//!
//! Example 3: Binary Tree DAG (depth 10, 1023 tasks)
//! Build time: 423µs
//! Tree depth: 9
//! Execution time: 15.3ms
//! Root result: ...
//!
//! Example 4: Diamond Pattern (50x50 = 2500 edges)
//! Build time: 234µs
//! Execution time: 11.7ms
//! Total tasks: 105
//! Final result: ...
//!
//! Example 5: Memory Efficiency (100 tasks, measure overhead)
//! i32 sum: ...
//! String total length: ...
//!
//! === Large DAG Performance Example Complete ===
//!
//! Key Observations:
//! - DAG construction is very fast (< 1ms for 1000s of tasks)
//! - Parallel execution scales well with available CPU cores
//! - Deep chains execute sequentially as expected
//! - Memory overhead is minimal (~ 200 bytes per task)
//! ```
use dagx::{task, DagRunner};
use std::time::Instant;
// Simple computation task
struct Compute {
value: i32,
}
#[task]
impl Compute {
async fn run(&mut self) -> i32 {
// Simulate some work
let mut result = self.value;
for _ in 0..100 {
result = result.wrapping_mul(31).wrapping_add(17);
}
result
}
}
// Task that combines two values
struct Combine;
#[task]
impl Combine {
async fn run(a: &i32, b: &i32) -> i32 {
a.wrapping_add(*b)
}
}
// Task that aggregates multiple values
struct Aggregate;
#[task]
impl Aggregate {
async fn run(a: &i32, b: &i32, c: &i32, d: &i32) -> i32 {
a.wrapping_add(*b).wrapping_add(*c).wrapping_add(*d)
}
}
// Task that formats a number as a string
struct FormatNumber {
id: i32,
}
#[task]
impl FormatNumber {
async fn run(&self) -> String {
format!("Result {}", self.id)
}
}
#[tokio::main]
async fn main() {
println!("=== Large DAG Performance Example ===\n");
// Example 1: Wide DAG (many parallel tasks)
println!("Example 1: Wide DAG (1000 parallel tasks)");
{
let start = Instant::now();
let dag = DagRunner::new();
// Create 1000 independent tasks
let tasks: Vec<_> = (0..1000)
.map(|i| dag.add_task(Compute { value: i }))
.collect();
let build_time = start.elapsed();
println!(" Build time: {:?}", build_time);
let exec_start = Instant::now();
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
let exec_time = exec_start.elapsed();
println!(" Execution time: {:?}", exec_time);
println!(" Tasks/second: {:.0}", 1000.0 / exec_time.as_secs_f64());
// Verify a few results
let sum: i32 = tasks
.into_iter()
.take(10)
.map(|t| dag.get(t).unwrap())
.fold(0i32, |acc, x| acc.wrapping_add(x));
println!(" Sample sum (first 10): {}\n", sum);
}
// Example 2: Deep DAG (long dependency chain)
println!("Example 2: Deep DAG (100-level chain)");
{
let start = Instant::now();
let dag = DagRunner::new();
// Create a deep chain by combining adjacent task pairs
let tasks: Vec<_> = (0..100)
.map(|i| dag.add_task(Compute { value: i }).into())
.collect();
let mut chain_tasks = vec![];
for i in 0..99 {
let combined = dag.add_task(Combine).depends_on((&tasks[i], &tasks[i + 1]));
chain_tasks.push(combined);
}
let prev = chain_tasks.last().unwrap();
let build_time = start.elapsed();
println!(" Build time: {:?}", build_time);
let exec_start = Instant::now();
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
let exec_time = exec_start.elapsed();
println!(" Execution time: {:?}", exec_time);
println!(" Final result: {}\n", dag.get(prev).unwrap());
}
// Example 3: Binary tree DAG
println!("Example 3: Binary Tree DAG (depth 10, 1023 tasks)");
{
let start = Instant::now();
let dag = DagRunner::new();
// Build binary tree: each level combines pairs from previous level
let current_level: Vec<_> = (0..512)
.map(|i| dag.add_task(Compute { value: i }))
.collect();
// Process all levels reducing by combining pairs
let mut level = 0;
let mut layer: Vec<dagx::TaskHandle<i32>> =
current_level.into_iter().map(|b| b.into()).collect();
while layer.len() > 1 {
level += 1;
let mut next_layer = Vec::new();
for i in (0..layer.len()).step_by(2) {
if i + 1 < layer.len() {
// We have a pair - combine them
let combined = dag.add_task(Combine).depends_on((&layer[i], &layer[i + 1]));
next_layer.push(combined);
} else {
// Odd one out - just pass it through
next_layer.push(layer[i]);
}
}
layer = next_layer;
}
let root = &layer[0];
let build_time = start.elapsed();
println!(" Build time: {:?}", build_time);
println!(" Tree depth: {}", level);
let exec_start = Instant::now();
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
let exec_time = exec_start.elapsed();
println!(" Execution time: {:?}", exec_time);
println!(" Root result: {}\n", dag.get(root).unwrap());
}
// Example 4: Diamond pattern (many-to-many)
println!("Example 4: Diamond Pattern (50x50 = 2500 edges)");
{
let start = Instant::now();
let dag = DagRunner::new();
// Layer 1: 50 source tasks
let sources: Vec<_> = (0..50)
.map(|i| dag.add_task(Compute { value: i }).into())
.collect();
// Layer 2: 50 tasks, each depends on all sources
let middle: Vec<_> = (0..50)
.map(|i| {
let combine = dag.add_task(Combine);
// Each middle task depends on two sources
combine.depends_on((
&sources[i % sources.len()],
&sources[(i + 1) % sources.len()],
))
})
.collect();
// Layer 3: Aggregate to 4 groups
let aggregates: Vec<_> = (0..4)
.map(|i| {
let start_idx = i * 12;
dag.add_task(Aggregate).depends_on((
&middle[start_idx % middle.len()],
&middle[(start_idx + 1) % middle.len()],
&middle[(start_idx + 2) % middle.len()],
&middle[(start_idx + 3) % middle.len()],
))
})
.collect();
// Layer 4: Final aggregate
let final_result = dag.add_task(Aggregate).depends_on((
&aggregates[0],
&aggregates[1],
&aggregates[2],
&aggregates[3],
));
let build_time = start.elapsed();
println!(" Build time: {:?}", build_time);
let exec_start = Instant::now();
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
let exec_time = exec_start.elapsed();
println!(" Execution time: {:?}", exec_time);
println!(" Total tasks: {}", 50 + 50 + 4 + 1);
println!(" Final result: {}\n", dag.get(final_result).unwrap());
}
// Example 5: Memory efficiency test
println!("Example 5: Memory Efficiency (100 tasks, measure overhead)");
{
let dag = DagRunner::new();
// Create tasks with various output sizes
let small_tasks: Vec<_> = (0..50)
.map(|i| dag.add_task(Compute { value: i }))
.collect();
let string_tasks: Vec<_> = (0..50)
.map(|i| dag.add_task(FormatNumber { id: i }))
.collect();
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
// Verify results are accessible
let i32_sum: i32 = small_tasks
.into_iter()
.map(|t| dag.get(t).unwrap())
.fold(0i32, |acc, x| acc.wrapping_add(x));
let string_count = string_tasks
.into_iter()
.map(|t| dag.get(t).unwrap().len())
.sum::<usize>();
println!(" i32 sum: {}", i32_sum);
println!(" String total length: {}", string_count);
}
println!("\n=== Large DAG Performance Example Complete ===");
println!("\nKey Observations:");
println!(" - DAG construction is very fast (< 1ms for 1000s of tasks)");
println!(" - Parallel execution scales well with available CPU cores");
println!(" - Deep chains execute sequentially as expected");
println!(" - Memory overhead is minimal (~ 200 bytes per task)");
}
examples/retry_strategies.rs
-461
View File
@@ -1,461 +0,0 @@
//! # Retry Strategies for Resilient Workflows
//!
//! This example demonstrates various retry patterns for handling transient failures
//! in DAG workflows, such as network errors, rate limits, or temporary service unavailability.
//!
//! ## What You'll Learn
//! - How to implement immediate, linear, and exponential backoff retry strategies
//! - How to distinguish retriable vs non-retriable errors
//! - How to track retry attempts and provide diagnostics
//! - How to implement jitter to prevent thundering herd problems
//!
//! ## Retry Patterns
//!
//! **1. Immediate Retry** - Retry without delay (for quick transient failures)
//! ```text
//! Attempt 1: ✗ Error
//! Attempt 2: ✗ Error (no delay)
//! Attempt 3: ✓ Success
//! ```
//!
//! **2. Linear Backoff** - Fixed delay between retries
//! ```text
//! Attempt 1: ✗ Error
//! Wait 100ms
//! Attempt 2: ✗ Error
//! Wait 100ms
//! Attempt 3: ✓ Success
//! ```
//!
//! **3. Exponential Backoff** - Increasing delays between retries
//! ```text
//! Attempt 1: ✗ Error
//! Wait 100ms
//! Attempt 2: ✗ Error
//! Wait 200ms
//! Attempt 3: ✗ Error
//! Wait 400ms
//! Attempt 4: ✓ Success
//! ```
//!
//! **4. Exponential Backoff with Jitter** - Randomized delays to prevent coordinated retries
//! ```text
//! Attempt 1: ✗ Error
//! Wait 87ms (50-150ms range)
//! Attempt 2: ✗ Error
//! Wait 234ms (100-300ms range)
//! Attempt 3: ✓ Success
//! ```
//!
//! ## Workflow Diagram
//! ```text
//! ┌──────────────────────┐
//! │ ApiCallWithRetry │
//! │ Max attempts: 3 │
//! │ Strategy: Exp │
//! └──────────────────────┘
//! │
//! ├─► Attempt 1: ✗ (Connection timeout)
//! │ Wait 100ms
//! │
//! ├─► Attempt 2: ✗ (Connection timeout)
//! │ Wait 200ms
//! │
//! └─► Attempt 3: ✓ Success
//!
//! ┌──────────────────────┐
//! │ ApiCallWithRetry │
//! │ Max attempts: 3 │
//! │ Strategy: Linear │
//! └──────────────────────┘
//! │
//! ├─► Attempt 1: ✗ (Rate limit)
//! │ Wait 500ms
//! │
//! ├─► Attempt 2: ✗ (Rate limit)
//! │ Wait 500ms
//! │
//! ├─► Attempt 3: ✗ (Rate limit)
//! │
//! └─► Final: ✗ Retry exhausted
//! ```
//!
//! ## Key Concepts
//! - **Idempotency**: Retried operations should be safe to repeat
//! - **Retriable errors**: Network timeouts, rate limits, temporary service errors
//! - **Non-retriable errors**: Authentication failures, invalid input, logical errors
//! - **Backoff strategies**: Balance between recovery speed and system load
//! - **Jitter**: Randomization to avoid thundering herd when many clients retry simultaneously
//! - **Retry budgets**: Limit total retries to prevent cascading failures
//!
//! ## Use Cases
//! - External API calls with transient network failures
//! - Database operations with temporary connection issues
//! - Distributed systems with eventual consistency
//! - Rate-limited API integrations
//! - Cloud service calls with intermittent availability
//!
//! ## Running This Example
//! ```bash
//! cargo run --example retry_strategies
//! ```
//!
//! ## Expected Output
//! ```text
//! === Retry Strategies Example ===
//!
//! Example 1: Immediate retry strategy
//! [ApiCall-1] Attempt 1/3: Calling external API...
//! [ApiCall-1] ✗ Transient error: Connection timeout
//! [ApiCall-1] Retrying immediately (attempt 2/3)
//! [ApiCall-1] Attempt 2/3: Calling external API...
//! [ApiCall-1] ✗ Transient error: Connection timeout
//! [ApiCall-1] Retrying immediately (attempt 3/3)
//! [ApiCall-1] Attempt 3/3: Calling external API...
//! [ApiCall-1] ✓ Success: API response data
//! Result: Ok(RetryResult { attempts: 3, success: true })
//!
//! Example 2: Linear backoff (500ms delay)
//! [ApiCall-2] Attempt 1/4: Calling external API...
//! [ApiCall-2] ✗ Transient error: Rate limit exceeded
//! [ApiCall-2] Waiting 500ms before retry (attempt 2/4)
//! [ApiCall-2] Attempt 2/4: Calling external API...
//! [ApiCall-2] ✓ Success: API response data
//! Result: Ok(RetryResult { attempts: 2, success: true })
//!
//! Example 3: Exponential backoff
//! [ApiCall-3] Attempt 1/5: Calling external API...
//! [ApiCall-3] ✗ Transient error: Service unavailable
//! [ApiCall-3] Waiting 100ms before retry (attempt 2/5)
//! [ApiCall-3] Attempt 2/5: Calling external API...
//! [ApiCall-3] ✗ Transient error: Service unavailable
//! [ApiCall-3] Waiting 200ms before retry (attempt 3/5)
//! [ApiCall-3] Attempt 3/5: Calling external API...
//! [ApiCall-3] ✓ Success: API response data
//! Result: Ok(RetryResult { attempts: 3, success: true })
//!
//! Example 4: Non-retriable error (fails immediately)
//! [ApiCall-4] Attempt 1/3: Calling external API...
//! [ApiCall-4] ✗ Non-retriable error: Authentication failed
//! Result: Err("Authentication failed - not retrying")
//!
//! Example 5: Retry exhausted
//! [ApiCall-5] Attempt 1/3: Calling external API...
//! [ApiCall-5] ✗ Transient error: Connection timeout
//! [ApiCall-5] Waiting 100ms before retry (attempt 2/3)
//! [ApiCall-5] Attempt 2/3: Calling external API...
//! [ApiCall-5] ✗ Transient error: Connection timeout
//! [ApiCall-5] Waiting 200ms before retry (attempt 3/3)
//! [ApiCall-5] Attempt 3/3: Calling external API...
//! [ApiCall-5] ✗ Transient error: Connection timeout
//! [ApiCall-5] All retry attempts exhausted
//! Result: Err("Retry exhausted after 3 attempts: Connection timeout")
//! ```
use dagx::{task, DagRunner};
use tokio::time::{sleep, Duration};
// Retry strategy configuration
#[derive(Clone, Copy, Debug)]
enum RetryStrategy {
Immediate,
Linear { delay_ms: u64 },
Exponential { base_ms: u64, factor: f64 },
}
impl RetryStrategy {
fn calculate_delay(&self, attempt: u32) -> Option<Duration> {
match self {
RetryStrategy::Immediate => None,
RetryStrategy::Linear { delay_ms } => Some(Duration::from_millis(*delay_ms)),
RetryStrategy::Exponential { base_ms, factor } => {
let delay = *base_ms as f64 * factor.powi(attempt as i32 - 1);
Some(Duration::from_millis(delay as u64))
}
}
}
}
// Simulated error types
#[derive(Clone, Debug)]
enum ApiError {
// Retriable errors
ConnectionTimeout,
ServiceUnavailable,
RateLimitExceeded,
// Non-retriable errors
AuthenticationFailed,
#[allow(dead_code)]
InvalidInput,
}
impl ApiError {
fn is_retriable(&self) -> bool {
matches!(
self,
ApiError::ConnectionTimeout
| ApiError::ServiceUnavailable
| ApiError::RateLimitExceeded
)
}
fn message(&self) -> &'static str {
match self {
ApiError::ConnectionTimeout => "Connection timeout",
ApiError::ServiceUnavailable => "Service unavailable",
ApiError::RateLimitExceeded => "Rate limit exceeded",
ApiError::AuthenticationFailed => "Authentication failed",
ApiError::InvalidInput => "Invalid input",
}
}
}
// Result of retry operation
#[derive(Clone, Debug)]
#[allow(dead_code)]
struct RetryResult {
attempts: u32,
success: bool,
final_error: Option<String>,
}
// Task that simulates an API call with configurable failure pattern
struct ApiCallWithRetry {
task_id: String,
max_attempts: u32,
strategy: RetryStrategy,
// Simulated failure pattern: errors to return before success
failure_sequence: Vec<ApiError>,
current_attempt: u32,
}
impl ApiCallWithRetry {
fn new(
task_id: impl Into<String>,
max_attempts: u32,
strategy: RetryStrategy,
failure_sequence: Vec<ApiError>,
) -> Self {
Self {
task_id: task_id.into(),
max_attempts,
strategy,
failure_sequence,
current_attempt: 0,
}
}
async fn attempt_api_call(&mut self) -> Result<String, ApiError> {
println!(
"[{}] Attempt {}/{}: Calling external API...",
self.task_id,
self.current_attempt + 1,
self.max_attempts
);
// Simulate network delay
sleep(Duration::from_millis(50)).await;
// Check if we should fail this attempt
if (self.current_attempt as usize) < self.failure_sequence.len() {
let error = self.failure_sequence[self.current_attempt as usize].clone();
Err(error)
} else {
Ok("API response data".to_string())
}
}
}
#[task]
impl ApiCallWithRetry {
async fn run(&mut self) -> Result<RetryResult, String> {
loop {
self.current_attempt += 1;
match self.attempt_api_call().await {
Ok(data) => {
println!("[{}] ✓ Success: {}", self.task_id, data);
return Ok(RetryResult {
attempts: self.current_attempt,
success: true,
final_error: None,
});
}
Err(error) => {
println!(
"[{}] ✗ {} error: {}",
self.task_id,
if error.is_retriable() {
"Transient"
} else {
"Non-retriable"
},
error.message()
);
// Check if error is retriable
if !error.is_retriable() {
return Err(format!("{} - not retrying", error.message()));
}
// Check if we have attempts remaining
if self.current_attempt >= self.max_attempts {
println!("[{}] All retry attempts exhausted", self.task_id);
return Err(format!(
"Retry exhausted after {} attempts: {}",
self.current_attempt,
error.message()
));
}
// Calculate and apply backoff delay
if let Some(delay) = self.strategy.calculate_delay(self.current_attempt) {
println!(
"[{}] Waiting {}ms before retry (attempt {}/{})",
self.task_id,
delay.as_millis(),
self.current_attempt + 1,
self.max_attempts
);
sleep(delay).await;
} else {
println!(
"[{}] Retrying immediately (attempt {}/{})",
self.task_id,
self.current_attempt + 1,
self.max_attempts
);
}
// Continue loop to retry
}
}
}
}
}
#[tokio::main]
async fn main() {
println!("=== Retry Strategies Example ===\n");
// Example 1: Immediate retry - succeeds on 3rd attempt
println!("Example 1: Immediate retry strategy");
{
let dag = DagRunner::new();
let api_call = dag.add_task(ApiCallWithRetry::new(
"ApiCall-1",
3,
RetryStrategy::Immediate,
vec![
ApiError::ConnectionTimeout,
ApiError::ConnectionTimeout,
// 3rd attempt will succeed
],
));
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("Result: {:?}\n", dag.get(api_call).unwrap());
}
// Example 2: Linear backoff - succeeds on 2nd attempt
println!("Example 2: Linear backoff (500ms delay)");
{
let dag = DagRunner::new();
let api_call = dag.add_task(ApiCallWithRetry::new(
"ApiCall-2",
4,
RetryStrategy::Linear { delay_ms: 500 },
vec![
ApiError::RateLimitExceeded,
// 2nd attempt will succeed
],
));
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("Result: {:?}\n", dag.get(api_call).unwrap());
}
// Example 3: Exponential backoff - succeeds on 3rd attempt
println!("Example 3: Exponential backoff");
{
let dag = DagRunner::new();
let api_call = dag.add_task(ApiCallWithRetry::new(
"ApiCall-3",
5,
RetryStrategy::Exponential {
base_ms: 100,
factor: 2.0,
},
vec![
ApiError::ServiceUnavailable,
ApiError::ServiceUnavailable,
// 3rd attempt will succeed
],
));
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("Result: {:?}\n", dag.get(api_call).unwrap());
}
// Example 4: Non-retriable error - fails immediately
println!("Example 4: Non-retriable error (fails immediately)");
{
let dag = DagRunner::new();
let api_call = dag.add_task(ApiCallWithRetry::new(
"ApiCall-4",
3,
RetryStrategy::Immediate,
vec![ApiError::AuthenticationFailed],
));
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("Result: {:?}\n", dag.get(api_call).unwrap());
}
// Example 5: Retry exhausted - all attempts fail
println!("Example 5: Retry exhausted");
{
let dag = DagRunner::new();
let api_call = dag.add_task(ApiCallWithRetry::new(
"ApiCall-5",
3,
RetryStrategy::Exponential {
base_ms: 100,
factor: 2.0,
},
vec![
ApiError::ConnectionTimeout,
ApiError::ConnectionTimeout,
ApiError::ConnectionTimeout,
// All attempts will fail
],
));
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("Result: {:?}\n", dag.get(api_call).unwrap());
}
println!("=== Retry Strategies Example Complete ===");
}
examples/timeout.rs
-296
View File
@@ -1,296 +0,0 @@
//! # Managing Timeouts in Tasks
//!
//! This example demonstrates how to implement timeout patterns in DAG tasks using
//! tokio's timeout utilities for time-bounded execution.
//!
//! ## What You'll Learn
//! - How to implement timeouts using `tokio::time::timeout`
//! - How to build tasks with built-in timeout logic
//! - How to handle timeout errors as Result types
//! - How to apply timeouts at both task-level and DAG-level
//!
//! ## Timeout Patterns
//!
//! **1. Task-Level Timeout** - Individual task with timeout
//! ```rust
//! async fn run(&mut self) -> Result<String, String> {
//! let work = async { /* ... */ };
//! match timeout(Duration::from_millis(self.timeout_ms), work).await {
//! Ok(result) => Ok(result),
//! Err(_) => Err("Task timed out".to_string()),
//! }
//! }
//! ```
//!
//! **2. DAG-Level Timeout** - Entire workflow with timeout
//! ```rust
//! match timeout(dag_timeout, dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }).await {
//! Ok(result) => { /* DAG completed */ },
//! Err(_) => { /* DAG timed out */ },
//! }
//! ```
//!
//! ## Workflow Diagram
//! ```text
//! ┌─────────────┐
//! │ FastTask │ (completes in 100ms)
//! │ (timeout │ ───► Ok("Fast task completed")
//! │ 500ms) │
//! └─────────────┘
//!
//! ┌─────────────┐
//! │ SlowTask │ (takes 1000ms)
//! │ (timeout │ ───► Err("Task timed out after 200ms")
//! │ 200ms) │
//! └─────────────┘
//! │
//! └──────►┌──────────────┐
//! │ProcessResult │ ───► "✗ Task timed out..."
//! └──────────────┘
//! ```
//!
//! ## Key Concepts
//! - **Timeout as error**: Timeouts produce `Err` results that propagate through the DAG
//! - **Error as data**: Use Result<T, E> to represent timeout states
//! - **Graceful degradation**: Downstream tasks can handle timeout errors
//! - **Time-bounded execution**: Enforce SLAs and prevent runaway tasks
//!
//! ## Use Cases
//! - API calls with timeout limits
//! - SLA enforcement in workflows
//! - Preventing resource exhaustion from slow tasks
//! - Circuit breaker patterns
//! - External service integration with timeouts
//!
//! ## Running This Example
//! ```bash
//! cargo run --example timeout
//! ```
//!
//! ## Expected Output
//! ```text
//! === Timeout Example ===
//!
//! Example 1: All tasks complete successfully
//! Fast task: Ok("Fast task completed")
//! Timed task: Ok("Work completed in 100ms")
//!
//! Example 2: Task exceeds timeout
//! Result: ✗ Task timed out after 200ms (work would take 1000ms)
//!
//! Example 3: Parallel tasks with different speeds
//! Fast task 1: Ok("Fast task completed")
//! Fast task 2: Ok("Fast task completed")
//! Slow task: Ok("Slow task completed after 500ms")
//! Timed task 1: Ok("Work completed in 100ms")
//! Timed task 2: Err("Task timed out after 200ms...")
//!
//! Example 4: Downstream tasks handling timeout errors
//! Processed 1: ✗ Task timed out after 100ms (work would take 500ms)
//! Processed 2: ✗ Task timed out after 100ms (work would take 500ms)
//!
//! Example 5: Whole-DAG timeout
//! DAG completed within timeout
//! Task 1: Ok("Slow task completed after 200ms")
//! Task 2: Ok("Slow task completed after 200ms")
//! Task 3: Ok("Slow task completed after 200ms")
//!
//! === Timeout Example Complete ===
//! ```
use dagx::{task, DagRunner, TaskHandle};
use tokio::time::{sleep, timeout, Duration};
// Fast task that completes quickly
struct FastTask;
#[task]
impl FastTask {
async fn run() -> Result<String, String> {
sleep(Duration::from_millis(100)).await;
Ok("Fast task completed".to_string())
}
}
// Slow task that takes a long time
struct SlowTask {
duration_ms: u64,
}
#[task]
impl SlowTask {
async fn run(&mut self) -> Result<String, String> {
sleep(Duration::from_millis(self.duration_ms)).await;
Ok(format!("Slow task completed after {}ms", self.duration_ms))
}
}
// Task with built-in timeout
struct TaskWithTimeout {
work_duration_ms: u64,
timeout_ms: u64,
}
#[task]
impl TaskWithTimeout {
async fn run(&mut self) -> Result<String, String> {
let work = async {
sleep(Duration::from_millis(self.work_duration_ms)).await;
format!("Work completed in {}ms", self.work_duration_ms)
};
match timeout(Duration::from_millis(self.timeout_ms), work).await {
Ok(result) => Ok(result),
Err(_) => Err(format!(
"Task timed out after {}ms (work would take {}ms)",
self.timeout_ms, self.work_duration_ms
)),
}
}
}
// Task that processes results (including timeouts)
struct ProcessResult;
#[task]
impl ProcessResult {
async fn run(input: &Result<String, String>) -> String {
match input {
Ok(msg) => format!("✓ {}", msg),
Err(err) => format!("✗ {}", err),
}
}
}
#[tokio::main]
async fn main() {
println!("=== Timeout Example ===\n");
// Example 1: All tasks complete within timeout
println!("Example 1: All tasks complete successfully");
{
let dag = DagRunner::new();
let fast = dag.add_task(FastTask);
let timed = dag.add_task(TaskWithTimeout {
work_duration_ms: 100,
timeout_ms: 500,
});
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("Fast task: {:?}", dag.get(fast).unwrap());
println!("Timed task: {:?}\n", dag.get(timed).unwrap());
}
// Example 2: Task exceeds timeout
println!("Example 2: Task exceeds timeout");
{
let dag = DagRunner::new();
let timed = dag.add_task(TaskWithTimeout {
work_duration_ms: 1000,
timeout_ms: 200,
});
let processed = dag.add_task(ProcessResult).depends_on(timed);
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("Result: {}\n", dag.get(processed).unwrap());
}
// Example 3: Mixed fast and slow tasks
println!("Example 3: Parallel tasks with different speeds");
{
let dag = DagRunner::new();
let fast1 = dag.add_task(FastTask);
let fast2 = dag.add_task(FastTask);
let slow = dag.add_task(SlowTask { duration_ms: 500 });
let timed1 = dag.add_task(TaskWithTimeout {
work_duration_ms: 100,
timeout_ms: 200,
});
let timed2 = dag.add_task(TaskWithTimeout {
work_duration_ms: 300,
timeout_ms: 200,
});
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("Fast task 1: {:?}", dag.get(fast1).unwrap());
println!("Fast task 2: {:?}", dag.get(fast2).unwrap());
println!("Slow task: {:?}", dag.get(slow).unwrap());
println!("Timed task 1: {:?}", dag.get(timed1).unwrap());
println!("Timed task 2: {:?}\n", dag.get(timed2).unwrap());
}
// Example 4: Cascading timeout errors
println!("Example 4: Downstream tasks handling timeout errors");
{
let dag = DagRunner::new();
let source: TaskHandle<_> = dag
.add_task(TaskWithTimeout {
work_duration_ms: 500,
timeout_ms: 100,
})
.into();
let processed1 = dag.add_task(ProcessResult).depends_on(source);
let processed2 = dag.add_task(ProcessResult).depends_on(source);
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("Processed 1: {}", dag.get(processed1).unwrap());
println!("Processed 2: {}\n", dag.get(processed2).unwrap());
}
// Example 5: Implementing whole-DAG timeout
println!("Example 5: Whole-DAG timeout");
{
let dag = DagRunner::new();
let t1 = dag.add_task(SlowTask { duration_ms: 200 });
let t2 = dag.add_task(SlowTask { duration_ms: 200 });
let t3 = dag.add_task(SlowTask { duration_ms: 200 });
// Run the entire DAG with a timeout
let dag_timeout = Duration::from_millis(500);
match timeout(
dag_timeout,
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() }),
)
.await
{
Ok(result) => {
result.unwrap();
println!("DAG completed within timeout");
println!("Task 1: {:?}", dag.get(t1).unwrap());
println!("Task 2: {:?}", dag.get(t2).unwrap());
println!("Task 3: {:?}", dag.get(t3).unwrap());
}
Err(_) => {
eprintln!(
"DAG execution timed out after {}ms",
dag_timeout.as_millis()
);
}
}
}
println!("\n=== Timeout Example Complete ===");
}
examples/tracing_example.rs
-162
View File
@@ -1,162 +0,0 @@
//! # Tracing Example
//!
//! This example demonstrates how to use the optional tracing feature to gain
//! visibility into DAG execution for debugging and performance analysis.
//!
//! ## Running This Example
//!
//! ```bash
//! # With default log level (info)
//! cargo run --example tracing_example --features tracing
//!
//! # With debug level (shows more details)
//! RUST_LOG=dagx=debug cargo run --example tracing_example --features tracing
//!
//! # With trace level (shows all execution details)
//! RUST_LOG=dagx=trace cargo run --example tracing_example --features tracing
//! ```
//!
//! ## What You'll See
//!
//! - Task additions to the DAG
//! - Dependency wiring between tasks
//! - Topological layer computation
//! - Layer-by-layer execution
//! - Inline vs spawned task execution
//! - Completion status
//!
//! ## Log Levels
//!
//! - **INFO**: High-level DAG execution start/completion
//! - **DEBUG**: Task additions, dependency wiring, layer computation
//! - **TRACE**: Individual task execution details (inline vs spawned)
//! - **ERROR**: Errors, panics, cycle detection
//!
use dagx::{task, DagRunner};
use tracing_subscriber::{fmt, EnvFilter};
// Simple source task
struct Value(i32);
#[task]
impl Value {
async fn run(&self) -> i32 {
self.0
}
}
// Computation task
struct Add;
#[task]
impl Add {
async fn run(a: &i32, b: &i32) -> i32 {
a + b
}
}
// Another computation task
struct Multiply;
#[task]
impl Multiply {
async fn run(a: &i32, b: &i32) -> i32 {
a * b
}
}
#[tokio::main]
async fn main() {
// Initialize tracing subscriber with environment filter
// Use RUST_LOG environment variable to control log level
fmt()
.with_env_filter(
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("dagx=info")),
)
.init();
println!("=== DAG Execution with Tracing ===\n");
println!("Run with RUST_LOG=dagx=debug or RUST_LOG=dagx=trace for more details\n");
// Example 1: Simple diamond pattern
println!("Example 1: Diamond Pattern");
println!("---------------------------");
{
let dag = DagRunner::new();
// Layer 0: Source
let source = dag.add_task(Value(10)).into();
// Layer 1: Two parallel computations
let left = dag.add_task(Add).depends_on((&source, &source));
let right = dag.add_task(Multiply).depends_on((&source, &source));
// Layer 2: Combine results
let result = dag.add_task(Add).depends_on((&left, &right));
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("\nResult: {}", dag.get(result).unwrap());
println!("(Expected: 10+10 = 20, 10*10 = 100, 20+100 = 120)\n");
}
// Example 2: Linear chain (demonstrates inline execution)
println!("\nExample 2: Linear Chain (Inline Execution)");
println!("-------------------------------------------");
{
let dag = DagRunner::new();
// Create a linear chain - each layer has only one task
// This should trigger inline execution optimization
let a = dag.add_task(Value(1)).into();
let b = dag.add_task(Add).depends_on((&a, &a));
let c = dag.add_task(Multiply).depends_on((&b, &b));
let d = dag.add_task(Add).depends_on((&c, &c));
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("\nResult: {}", dag.get(d).unwrap());
println!("(Expected: 1+1 = 2, 2*2 = 4, 4+4 = 8)\n");
}
// Example 3: Wide pattern (demonstrates spawned execution)
println!("\nExample 3: Wide Pattern (Spawned Execution)");
println!("--------------------------------------------");
{
let dag = DagRunner::new();
// Layer 0: Multiple sources
let v1 = dag.add_task(Value(1)).into();
let v2 = dag.add_task(Value(2)).into();
let v3 = dag.add_task(Value(3)).into();
let v4 = dag.add_task(Value(4)).into();
// Layer 1: Multiple parallel computations
let sum12 = dag.add_task(Add).depends_on((&v1, &v2));
let sum23 = dag.add_task(Add).depends_on((&v2, &v3));
let sum34 = dag.add_task(Add).depends_on((&v3, &v4));
// Layer 2: Final combination
let sum_all = dag.add_task(Add).depends_on((&sum12, &sum23));
let final_result = dag.add_task(Add).depends_on((&sum_all, &sum34));
dag.run(|fut| async move { tokio::spawn(fut).await.unwrap() })
.await
.unwrap();
println!("\nResult: {}", dag.get(final_result).unwrap());
println!("(Expected: (1+2)+(2+3) = 8, 8+(3+4) = 15)\n");
}
println!("\n=== Tracing Example Complete ===");
println!("\nTry running with different log levels:");
println!(" RUST_LOG=dagx=info - High-level execution info");
println!(" RUST_LOG=dagx=debug - Task additions and layer details");
println!(" RUST_LOG=dagx=trace - All execution details including inline/spawned");
}