webhdfs_rest_api_directory_listing_with_requests.py

1import requests
2
3# Set the HDFS endpoint and path
4hdfs_host = "http://localhost:9870"
5path = "/user/hadoop/test_dir"
6
7# The official Hadoop documentation recommends using the WebHDFS REST API for Python
8# This example lists the contents of a directory
9url = f"{hdfs_host}/webhdfs/v1{path}?op=LISTSTATUS"
10
11try:
12    response = requests.get(url)
13    if response.status_code == 200:
14        data = response.json()
15        files = data['FileStatuses']['FileStatus']
16        print(f"Contents of {path}:")
17        for file in files:
18            print(f"- {file['pathSuffix']} ({file['type']})")
19    else:
20        print(f"Error: {response.status_code}")
21        print(response.text)
22except Exception as e:
23    print(f"An error occurred: {e}")