gcs_transfer_manager_concurrent_directory_upload.py

1from google.cloud.storage import Client
2from google.cloud.storage.transfer_manager import upload_directory
3
4def upload_directory_with_transfer_manager(bucket_name, source_directory):
5    """Upload every file in a directory, including all files in subdirectories.
6
7    Each blob name is derived from the local file path and contains the 
8    source_directory name as a prefix.
9    """
10
11    # The ID of your GCS bucket
12    # bucket_name = "your-bucket-name"
13
14    # The path to the local directory to upload
15    # source_directory = "/path/to/directory/"
16
17    storage_client = Client()
18    bucket = storage_client.bucket(bucket_name)
19
20    results = upload_directory(
21        bucket, source_directory, recursive=True
22    )
23
24    for name, result in zip(source_directory, results):
25        # The results list is either the submitted resource or an exception
26        if isinstance(result, Exception):
27            print(f"Failed to upload {name} due to exception: {result}")
28        else:
29            print(f"Uploaded {name} to {result.name}.")
30
31if __name__ == "__main__":
32    import sys
33    # Example usage: python quickstart.py my-bucket ./my-local-dir
34    upload_directory_with_transfer_manager(sys.argv[1], sys.argv[2])