feat: dagrunner, initial version
This commit is contained in:
122 files changed
+23101
No files matched your search
@@ -0,0 +1,231 @@
|
||||
//! # Real-World: Data Processing Pipeline
|
||||
//!
|
||||
//! This example demonstrates a realistic data processing pipeline with multiple
|
||||
//! stages of transformation and statistical analysis.
|
||||
//!
|
||||
//! ## What You'll Learn
|
||||
//! - How to structure a multi-stage data pipeline
|
||||
//! - How to use stateful tasks with configuration (DataTransformer)
|
||||
//! - How to parallelize statistical computations on the same dataset
|
||||
//! - How to combine results from parallel operations into a final report
|
||||
//!
|
||||
//! ## Pipeline Architecture
|
||||
//! ```text
|
||||
//! STAGE 1: Load STAGE 2: Transform STAGE 3: Analyze STAGE 4: Report
|
||||
//! ┌─────────┐ ┌──────────────┐ ┌────────────┐
|
||||
//! │LoadVal1 │─────►│DataTransform │─────►│ │
|
||||
//! │ (100) │ │ (×1.5) │──┐ │ ComputeSum │──┐
|
||||
//! └─────────┘ └──────────────┘ │ │ │ │
|
||||
//! ├──►│ (525) │ │
|
||||
//! ┌─────────┐ ┌──────────────┐ │ └────────────┘ │
|
||||
//! │LoadVal2 │─────►│DataTransform │──┤ ├──►┌──────────┐
|
||||
//! │ (200) │ │ (×2.0) │ │ ┌────────────┐ │ │ Generate │
|
||||
//! └─────────┘ └──────────────┘ │ │ComputeProd │──┤ │ Report │
|
||||
//! ├──►│ │ │ └──────────┘
|
||||
//! ┌─────────┐ ┌──────────────┐ │ │ (3750000) │ │
|
||||
//! │LoadVal3 │─────►│DataTransform │──┘ └────────────┘ │
|
||||
//! │ (50) │ │ (×0.5) │ │ │
|
||||
//! └─────────┘ └──────────────┘ │ ┌────────────┐ │
|
||||
//! │ │ FindMax │ │
|
||||
//! └──►│ (400) │──┘
|
||||
//! └────────────┘
|
||||
//! ```
|
||||
//!
|
||||
//! ## Key Concepts
|
||||
//! - **ETL Pattern**: Extract (Load) → Transform → Load (Report)
|
||||
//! - **Parallel transformation**: Each data source is transformed independently
|
||||
//! - **Parallel analysis**: Sum, Product, and Max computed simultaneously
|
||||
//! - **Stateful tasks**: DataTransformer uses different multipliers for each instance
|
||||
//!
|
||||
//! ## Use Cases
|
||||
//! - Log processing and analysis
|
||||
//! - Data transformation pipelines
|
||||
//! - Batch job orchestration
|
||||
//! - Statistical analysis on datasets
|
||||
//! - Multi-metric reporting systems
|
||||
//!
|
||||
//! ## Running This Example
|
||||
//! ```bash
|
||||
//! cargo run --example data_pipeline
|
||||
//! ```
|
||||
//!
|
||||
//! ## Expected Output
|
||||
//! ```text
|
||||
//! Building data pipeline DAG...
|
||||
//!
|
||||
//! Executing pipeline...
|
||||
//!
|
||||
//! Stage 1: Loading raw value 1...
|
||||
//! Stage 1: Loading raw value 2...
|
||||
//! Stage 1: Loading raw value 3...
|
||||
//! Transforming data with multiplier 1.5...
|
||||
//! Transforming data with multiplier 2...
|
||||
//! Transforming data with multiplier 0.5...
|
||||
//!
|
||||
//! Stage 2: Computing sum...
|
||||
//! Stage 2: Computing product...
|
||||
//! Stage 2: Finding maximum...
|
||||
//!
|
||||
//! Stage 3: Generating report...
|
||||
//!
|
||||
//! === Data Pipeline Results ===
|
||||
//! Sum of transformed values: 525.00
|
||||
//! Product of transformed values: 3750000.00
|
||||
//! Maximum value: 400.00
|
||||
//! ============================
|
||||
//! ```
|
||||
|
||||
use dagx::{task, DagRunner, Task};
|
||||
|
||||
// Data source tasks
|
||||
struct LoadValue1;
|
||||
|
||||
#[task]
|
||||
impl LoadValue1 {
|
||||
async fn run() -> i32 {
|
||||
println!("Stage 1: Loading raw value 1...");
|
||||
100
|
||||
}
|
||||
}
|
||||
|
||||
struct LoadValue2;
|
||||
|
||||
#[task]
|
||||
impl LoadValue2 {
|
||||
async fn run() -> i32 {
|
||||
println!("Stage 1: Loading raw value 2...");
|
||||
200
|
||||
}
|
||||
}
|
||||
|
||||
struct LoadValue3;
|
||||
|
||||
#[task]
|
||||
impl LoadValue3 {
|
||||
async fn run() -> i32 {
|
||||
println!("Stage 1: Loading raw value 3...");
|
||||
50
|
||||
}
|
||||
}
|
||||
|
||||
// Custom task with state for data transformation
|
||||
struct DataTransformer {
|
||||
multiplier: f64,
|
||||
}
|
||||
|
||||
impl DataTransformer {
|
||||
fn new(multiplier: f64) -> Self {
|
||||
Self { multiplier }
|
||||
}
|
||||
}
|
||||
|
||||
#[task]
|
||||
impl DataTransformer {
|
||||
async fn run(&mut self, input: &i32) -> f64 {
|
||||
println!(" Transforming data with multiplier {}...", self.multiplier);
|
||||
*input as f64 * self.multiplier
|
||||
}
|
||||
}
|
||||
|
||||
// Statistical analysis tasks
|
||||
struct ComputeSum;
|
||||
|
||||
#[task]
|
||||
impl ComputeSum {
|
||||
async fn run(a: &f64, b: &f64, c: &f64) -> f64 {
|
||||
println!("\nStage 2: Computing sum...");
|
||||
a + b + c
|
||||
}
|
||||
}
|
||||
|
||||
struct ComputeProduct;
|
||||
|
||||
#[task]
|
||||
impl ComputeProduct {
|
||||
async fn run(a: &f64, b: &f64, c: &f64) -> f64 {
|
||||
println!("Stage 2: Computing product...");
|
||||
a * b * c
|
||||
}
|
||||
}
|
||||
|
||||
struct FindMax;
|
||||
|
||||
#[task]
|
||||
impl FindMax {
|
||||
async fn run(a: &f64, b: &f64, c: &f64) -> f64 {
|
||||
println!("Stage 2: Finding maximum...");
|
||||
a.max(*b).max(*c)
|
||||
}
|
||||
}
|
||||
|
||||
// Report generation task
|
||||
struct GenerateReport;
|
||||
|
||||
#[task]
|
||||
impl GenerateReport {
|
||||
async fn run(sum: &f64, product: &f64, max: &f64) -> String {
|
||||
println!("\nStage 3: Generating report...");
|
||||
format!(
|
||||
"\n=== Data Pipeline Results ===\n\
|
||||
Sum of transformed values: {:.2}\n\
|
||||
Product of transformed values: {:.2}\n\
|
||||
Maximum value: {:.2}\n\
|
||||
============================",
|
||||
sum, product, max
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let dag = DagRunner::new();
|
||||
|
||||
println!("Building data pipeline DAG...\n");
|
||||
|
||||
// Stage 1: Data sources
|
||||
let raw_value_1 = dag.add_task(LoadValue1);
|
||||
let raw_value_2 = dag.add_task(LoadValue2);
|
||||
let raw_value_3 = dag.add_task(LoadValue3);
|
||||
|
||||
// Stage 2: Data transformation (with stateful tasks)
|
||||
let transformed_1 = dag
|
||||
.add_task(DataTransformer::new(1.5))
|
||||
.depends_on(&raw_value_1);
|
||||
|
||||
let transformed_2 = dag
|
||||
.add_task(DataTransformer::new(2.0))
|
||||
.depends_on(&raw_value_2);
|
||||
|
||||
let transformed_3 = dag
|
||||
.add_task(DataTransformer::new(0.5))
|
||||
.depends_on(&raw_value_3);
|
||||
|
||||
// Stage 3: Statistical analysis (parallel operations on transformed data)
|
||||
let sum = dag
|
||||
.add_task(ComputeSum)
|
||||
.depends_on((&transformed_1, &transformed_2, &transformed_3));
|
||||
|
||||
let product =
|
||||
dag.add_task(ComputeProduct)
|
||||
.depends_on((&transformed_1, &transformed_2, &transformed_3));
|
||||
|
||||
let max = dag
|
||||
.add_task(FindMax)
|
||||
.depends_on((&transformed_1, &transformed_2, &transformed_3));
|
||||
|
||||
// Stage 4: Final report
|
||||
let report = dag
|
||||
.add_task(GenerateReport)
|
||||
.depends_on((&sum, &product, &max));
|
||||
|
||||
// Execute the pipeline
|
||||
println!("Executing pipeline...\n");
|
||||
dag.run(|fut| {
|
||||
tokio::spawn(fut);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Print final report
|
||||
println!("{}", dag.get(report).unwrap());
|
||||
}
|
||||
Reference in new issue
Block a user