Skip to content

Example Pipeline

An example pipeline comprised of multiple stages (or 'solids').

Example Dagster solids and pipeline from the Dagster tutorial.
import csv
from typing import Any, Dict, List

import requests
from dagster import execute_pipeline, pipeline, solid
from dagster.core.execution.context.compute import SolidExecutionContext

def download_data(context: SolidExecutionContext) -> List[Dict[str, Any]]:
    """Download dataset."""
    response = requests.get("")
    lines = response.text.split("\n")
    cereals = [row for row in csv.DictReader(lines)]"Found {len(cereals)} cereals")
    return cereals

def find_max_sugar_cereal(
    context: SolidExecutionContext, cereals: List[Dict[str, Any]]
) -> str:
    """Find the product that has the maximum value for sugar content"""
    sorted_by_sugar = sorted(cereals, key=lambda cereal: cereal["sugars"])
    max_sugar_cereal = sorted_by_sugar[-1]["name"]"{max_sugar_cereal} has the greatest amount of sugar.")
    return max_sugar_cereal

def cereal_data_pipeline() -> str:
    """Compose the end-to-end cereal data pipeline."""
    return find_max_sugar_cereal(download_data())

if __name__ == "__main__":
    result = execute_pipeline(cereal_data_pipeline)