airflow_livy_operator_spark_job_submission_dag.py

1import datetime
2
3from airflow import DAG
4from airflow.providers.apache.livy.operators.livy import LivyOperator
5
6with DAG(
7    dag_id='example_livy_operator',
8    schedule=None,
9    start_date=datetime.datetime(2021, 1, 1),
10    catchup=False,
11) as dag:
12
13    # Example of submitting a Spark job via Livy
14    submit_livy_job = LivyOperator(
15        task_id='submit_livy_job',
16        file='/path/to/spark/job.py',  # Path to the Spark job file
17        class_name='org.apache.spark.examples.SparkPi', # Required for Java/Scala
18        args=[10],
19        livy_conn_id='livy_default',
20        polling_interval=30
21    )
22
23    submit_livy_job