smart_open_quickstart_streaming_s3_http_compressed_files.py

1from smart_open import open
2
3# stream lines from an S3 bucket
4for line in open('s3://commoncrawl/robots.txt', 'rb'):
5    print(line.decode('utf-8'))
6    break  # just print the first line
7
8# stream from HTTP
9for line in open('http://example.com/index.html'):
10    print(line)
11    break
12
13# write to a local compressed file
14with open('example.txt.gz', 'wb') as fout:
15    fout.write(b'Hello world!')