azure_data_factory_sdk_create_factory_linked_service_dataset.py

1import os
2from azure.identity import DefaultAzureCredential
3from azure.mgmt.datafactory import DataFactoryManagementClient
4from azure.mgmt.datafactory.models import *
5
6# 1. Provide your Azure subscription ID
7subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID", "your-subscription-id")
8
9# 2. Provide the resource group name and data factory name
10resource_group_name = "example-resources"
11factory_name = "example-data-factory"
12location = "East US"
13
14# 3. Authenticate and create the management client
15credential = DefaultAzureCredential()
16adf_client = DataFactoryManagementClient(credential, subscription_id)
17
18# 4. Create a Data Factory
19print(f"Creating Data Factory: {factory_name}...")
20df_resource = Factory(location=location)
21adf_client.factories.create_or_update(resource_group_name, factory_name, df_resource)
22
23# 5. Define an Azure Storage Linked Service
24storage_account_name = "yourstorageaccount"
25storage_account_key = "yourstoragekey"
26connection_string = f"DefaultEndpointsProtocol=https;AccountName={storage_account_name};AccountKey={storage_account_key};EndpointSuffix=core.windows.net"
27
28ls_name = "AzureStorageLinkedService"
29ls_properties = AzureStorageLinkedService(
30    connection_string=connection_string
31)
32
33print(f"Creating Linked Service: {ls_name}...")
34adf_client.linked_services.create_or_update(
35    resource_group_name, factory_name, ls_name, LinkedServiceResource(properties=ls_properties)
36)
37
38# 6. Create a Dataset (delimited text example)
39dataset_name = "ExampleDataset"
40dataset_properties = DelimitedTextDataset(
41    linked_service_name=LinkedServiceReference(reference_name=ls_name),
42    location=AzureBlobStorageLocation(container="input", file_name="sample.csv"),
43    column_delimiter=",",
44    first_row_as_header=True
45)
46
47print(f"Creating Dataset: {dataset_name}...")
48adf_client.datasets.create_or_update(
49    resource_group_name, factory_name, dataset_name, DatasetResource(properties=dataset_properties)
50)
51
52print("Setup complete.")