dagster_bigquery_pandas_io_manager_quickstart_with_iris_data.py

1import pandas as pd
2from dagster import asset, Definitions
3from dagster_gcp_pandas import BigQueryPandasIOManager
4
5@asset
6def iris_data() -> pd.DataFrame:
7    """Load some sample data as a Pandas DataFrame."""
8    return pd.read_csv(
9        "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data",
10        names=[
11            "sepal_length_cm",
12            "sepal_width_cm",
13            "petal_length_cm",
14            "petal_width_cm",
15            "species",
16        ],
17    )
18
19@asset
20def iris_cleaned(iris_data: pd.DataFrame) -> pd.DataFrame:
21    """Perform a simple transformation on the BigQuery table."""
22    return iris_data.dropna()
23
24defs = Definitions(
25    assets=[iris_data, iris_cleaned],
26    resources={
27        "io_manager": BigQueryPandasIOManager(
28            project="my-gcp-project",  # Replace with your project ID
29            dataset="my_dataset",      # Replace with your dataset name
30        )
31    },
32)