azure_datalake_gen1_filesystem_operations_with_aad_auth.py

1from azure.datalake.store import core, lib, multithread
2import logging, getpass, pprint, uuid, os
3
4# Settings for your Azure Data Lake Storage Gen1 account
5token = lib.auth(tenant_id='FILL-IN-HERE', 
6                 client_secret='FILL-IN-HERE', 
7                 client_id='FILL-IN-HERE')
8adl_name = 'FILL-IN-HERE'
9adlsFileSystemClient = core.AzureDLFileSystem(token, store_name=adl_name)
10
11# Create a directory
12adlsFileSystemClient.mkdir('mysampledirectory')
13
14# List the contents of the root directory
15print(adlsFileSystemClient.ls('/'))
16
17# Create a file and write content
18with adlsFileSystemClient.open('mysampledirectory/mysamplefile.txt', 'wb') as f:
19    f.write('Hello World'.encode())
20
21# Read the file
22with adlsFileSystemClient.open('mysampledirectory/mysamplefile.txt', 'rb') as f:
23    print(f.read())
24
25# Get file status
26print(adlsFileSystemClient.info('mysampledirectory/mysamplefile.txt'))
27
28# Download file
29multithread.ADLDownloader(adlsFileSystemClient, 
30                          lpath='C:\\local\\path\\mysamplefile.txt', 
31                          rpath='mysampledirectory/mysamplefile.txt', 
32                          nthreads=4, 
33                          overwrite=True)