azure_datalake_gen1_service_principal_auth_file_operations.py

1from azure.datalake.store import core, lib, multithread
2
3# Set these parameters for your environment
4token = lib.auth(tenant_id='FILL-IN-HERE',
5                 client_secret='FILL-IN-HERE',
6                 client_id='FILL-IN-HERE')
7adl_name = 'FILL-IN-HERE'
8adlfs = core.AzureDLFileSystem(token, store_name=adl_name)
9
10# List a directory
11print(adlfs.ls(''))
12
13# Create and write to a file
14with adlfs.open('mystore/file.txt', 'wb') as f:
15    f.write(b'Hello World')
16
17# Read from a file
18with adlfs.open('mystore/file.txt', 'rb') as f:
19    print(f.read())
20
21# Upload and Download in parallel
22multithread.ADLUploader(adlfs, lpath='C:\\data\\*', rpath='/data/', nthreads=4)
23multithread.ADLDownloader(adlfs, lpath='C:\\data_from_adl\\', rpath='/data/', nthreads=4)