You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
34 lines
1.0 KiB
34 lines
1.0 KiB
6 years ago
|
#!/usr/bin/env/env python3
|
||
|
|
import boto3
|
||
|
|
|
||
|
|
s3 = boto3.client('s3',
|
||
|
|
endpoint_url='http://localhost:9000',
|
||
|
|
aws_access_key_id='minio',
|
||
|
|
aws_secret_access_key='minio123',
|
||
|
|
region_name='us-east-1')
|
||
|
|
|
||
|
|
r = s3.select_object_content(
|
||
|
|
Bucket='mycsvbucket',
|
||
|
|
Key='sampledata/TotalPopulation.csv.gz',
|
||
|
|
ExpressionType='SQL',
|
||
|
|
Expression="select * from s3object s where s.Location like '%United States%'",
|
||
|
|
InputSerialization={
|
||
|
|
'CSV': {
|
||
|
|
"FileHeaderInfo": "USE",
|
||
|
|
},
|
||
|
|
'CompressionType': 'GZIP',
|
||
|
|
},
|
||
|
|
OutputSerialization={'CSV': {}},
|
||
|
|
)
|
||
|
|
|
||
|
|
for event in r['Payload']:
|
||
|
|
if 'Records' in event:
|
||
|
|
records = event['Records']['Payload'].decode('utf-8')
|
||
|
|
print(records)
|
||
|
|
elif 'Stats' in event:
|
||
|
|
statsDetails = event['Stats']['Details']
|
||
|
|
print("Stats details bytesScanned: ")
|
||
|
|
print(statsDetails['BytesScanned'])
|
||
|
|
print("Stats details bytesProcessed: ")
|
||
|
|
print(statsDetails['BytesProcessed'])
|