crawlee_beautifulsoup_crawler_scrape_website_titles.py

1import asyncio
2
3from crawlee.browsers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
4
5
6async def main() -> None:
7    # Crawler is the main class that orchestrates the crawling process.
8    # In this example, we use the BeautifulSoupCrawler, which is a crawler
9    # that uses the BeautifulSoup library to parse HTML.
10    crawler = BeautifulSoupCrawler()
11
12    # Define the default request handler, which will be called for every URL.
13    @crawler.router.default_handler
14    async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
15        # Extract the page title using BeautifulSoup.
16        title = context.soup.title.string if context.soup.title else None
17        context.log.info(f'Title of {context.request.url}: {title}')
18
19    # Run the crawler with the initial list of URLs.
20    await crawler.run(['https://crawlee.dev/python'])
21
22
23if __name__ == '__main__':
24    asyncio.run(main())