airflow_dag_great_expectations_operator_validation_quickstart.py

1import os
2from pathlib import Path
3from datetime import datetime
4
5from airflow import DAG
6from great_expectations_provider.operators.great_expectations import GreatExpectationsOperator
7
8# Define the base path for your Great Expectations project
9base_path = Path(__file__).parents[2]
10data_context_root_dir = os.path.join(base_path, "great_expectations")
11
12with DAG(
13    dag_id="ge_example_dag",
14    start_date=datetime(2023, 1, 1),
15    catchup=False,
16    schedule_interval=None,
17) as dag:
18
19    # Task to run Great Expectations validation
20    ge_validation = GreatExpectationsOperator(
21        task_id="ge_validation_task",
22        data_context_root_dir=data_context_root_dir,
23        checkpoint_name="my_checkpoint",
24        # You can optionally override parameters:
25        # expectation_suite_name="my_suite",
26        # batch_request={"datasource_name": "my_datasource", "data_asset_name": "my_table"},
27    )
28
29    ge_validation