spb/tools-spboucher-ai

1{2  "aliases": [3    "Airflow"4  ],5  "workflows": [6    "Nightly ELT: extract, load to warehouse, run dbt",7    "Scheduling ML training and batch inference jobs",8    "Coordinating cross-system jobs with sensors and triggers"9  ],10  "advantages": [11    "Largest ecosystem of operators and integrations",12    "Managed services from multiple vendors",13    "Mature scheduler and UI"14  ],15  "limitations": [16    "Several components to operate (database, scheduler, workers)",17    "Task-centric model; data passing between tasks is indirect",18    "Upgrades between major versions require migration effort"19  ],20  "prerequisites": [21    "Python environment plus a metadata database (PostgreSQL recommended)"22  ],23  "audiences": [24    "Data engineers",25    "ML platform teams"26  ],27  "alternatives": [28    "dagster",29    "prefect",30    "dbt",31    "github-actions"32  ],33  "tags": [34    "orchestration",35    "python",36    "open-source",37    "self-hosted",38    "scheduling"39  ],40  "use_cases": [41    "orchestrate-pipelines",42    "transform-data"43  ],44  "operating_systems": [45    "linux",46    "macos"47  ],48  "deployment_models": [49    "on-premises",50    "managed-cloud"51  ],52  "integrations": [53    "dbt",54    "Apache Spark",55    "Kubernetes",56    "PostgreSQL",57    "Amazon S3",58    "Google Cloud",59    "Snowflake",60    "Slack"61  ],62  "data_formats": [63    "Python DAG files",64    "JSON (XCom)"65  ],66  "languages": [67    "Python"68  ],69  "self_hostable": true,70  "discovery_method": "editorial-research",71  "depth": "standard",72  "editorial_status": "reviewed",73  "visibility": "public",74  "featured": false,75  "author": "batch-D",76  "slug": "apache-airflow",77  "name": "Apache Airflow",78  "vendor": {79    "name": "Apache Software Foundation",80    "slug": "apache-software-foundation",81    "website": "https://www.apache.org/"82  },83  "homepage_url": "https://airflow.apache.org/",84  "docs_url": "https://airflow.apache.org/docs/",85  "repo_url": "https://github.com/apache/airflow",86  "license_url": "https://github.com/apache/airflow/blob/main/LICENSE",87  "product_type": "platform",88  "product_status": "active",89  "summary": "Python workflow orchestrator for scheduling and monitoring DAGs of tasks, with hundreds of provider packages and managed offerings from several clouds",90  "description": "Apache Airflow is an open-source orchestration platform, originally from Airbnb and now an Apache Software Foundation project under Apache 2.0, for authoring, scheduling and monitoring workflows defined as Python code. Each workflow is a DAG of tasks executed by operators from provider packages that cover databases, cloud services, Spark, dbt, Kubernetes and HTTP APIs; the scheduler handles dependencies, retries, backfills and SLAs, and a web UI shows runs, logs and task state. Airflow 3, released in 2025, introduced a new React UI, a task execution API that isolates workers, DAG versioning, event-driven scheduling and data assets as first-class scheduling triggers.\n\nAirflow is the most widely deployed orchestrator and is available as managed services (Astronomer, Amazon MWAA, Google Cloud Composer). Running it well means operating a metadata database, scheduler, web server and workers (Celery or Kubernetes executors), which is heavier than Dagster or Prefect for small teams. Its task-centric model also handles data-passing between tasks less naturally than asset-oriented tools.",91  "capabilities": [92    "DAGs defined in Python with the TaskFlow API",93    "Scheduler with retries, backfills and dependency handling",94    "Provider packages for clouds, databases, Spark, dbt and Kubernetes",95    "Web UI for monitoring runs and logs",96    "Celery, Kubernetes and local executors",97    "Asset-based and event-driven scheduling (Airflow 3)"98  ],99  "categories": [100    {101      "slug": "etl-elt",102      "primary": true103    }104  ],105  "execution_modes": [106    "self-hosted",107    "cloud"108  ],109  "interfaces": [110    "web",111    "cli",112    "api",113    "library"114  ],115  "source_model": "open-source",116  "license_spdx": "Apache-2.0",117  "pricing_model": [118    "open-source",119    "free"120  ],121  "sources": [122    {123      "url": "https://airflow.apache.org/",124      "type": "official_site",125      "supports": [126        "name",127        "summary"128      ],129      "checked_at": "2026-11-02",130      "title": "Apache Airflow"131    },132    {133      "url": "https://airflow.apache.org/docs/",134      "type": "official_docs",135      "supports": [136        "capabilities",137        "integrations"138      ],139      "checked_at": "2026-11-02",140      "title": "Airflow documentation"141    },142    {143      "url": "https://github.com/apache/airflow/blob/main/LICENSE",144      "type": "license_page",145      "supports": [146        "license_spdx",147        "source_model"148      ],149      "checked_at": "2026-11-02",150      "title": "Airflow LICENSE (Apache 2.0)"151    },152    {153      "url": "https://github.com/apache/airflow",154      "type": "repository",155      "supports": [156        "repo_url"157      ],158      "checked_at": "2026-11-02",159      "title": "apache/airflow"160    }161  ],162  "confidence": "high"163}164