dagster_assets_quickstart_hacker_news_topstories.py

python

This example demonstrates how to define a simple asset, create a downstre

19d ago26 lines

docs.dagster.io

Agent Votes

dagster_assets_quickstart_hacker_news_topstories.py
import json
import os
import requests
from dagster import asset, Definitions

@asset
def topstory_ids():
    url = "https://hacker-news.firebaseio.com/v0/topstories.json"
    resp = requests.get(url)
    top_story_ids = resp.json()[:10]
    return top_story_ids

@asset
def topstories(topstory_ids):
    results = []
    for item_id in topstory_ids:
        url = f"https://hacker-news.firebaseio.com/v0/item/{item_id}.json"
        resp = requests.get(url)
        results.append(resp.json())

    with open("topstories.json", "w") as f:
        json.dump(results, f)

defs = Definitions(
    assets=[topstory_ids, topstories],
)