-
Notifications
You must be signed in to change notification settings - Fork 1
/
createbucket.py
294 lines (220 loc) · 14.1 KB
/
createbucket.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
#!/usr/bin/python3
# Ivan Gregoretti, PhD. June 2017.
import subprocess
import boto3
import json
import jmespath
import datetime # Usage: execute datetime.datetime.now(datetime.timezone.utc); then instead of tzinfo=tzutc() use tzinfo=datetime.timezone.utc.
import pandas as pd
import os
import time
################################################################################
# Create session and identify user (self)
################################################################################
# Create custom session
se1 = boto3.Session( profile_name='default' ) # profile: default
#se2= boto3.Session( profile_name='ivan' ) # profile: ivan
# Create Identity and Access Managemet resource and client
iamre = se1.resource('iam')
iamcl = se1.client( 'iam')
# Identify my user name
iam_user_name = jmespath.search('Users[*] | [?UserName!=`null`] | [?contains(UserName, `Gregoretti`) || contains(UserName, `gregoretti`)].UserName | [0]', iamcl.list_users())
iam_user_id = jmespath.search('Users[*] | [?UserName!=`null`] | [?contains(UserName, `Gregoretti`) || contains(UserName, `gregoretti`)].UserId | [0]', iamcl.list_users())
################################################################################
# Create Resources and Client for interation with S3 and IAM
################################################################################
# Create an S3 resource and/or client
s3re = se1.resource('s3')
s3cl = se1.client( 's3')
# In case it is needed, this is how to display the canonical user ID of this acccount
s3cl.list_buckets()['Owner']
##############################
# Convenience function to avoid the problem of "datetime ... is not JSON serializable".
##############################
def date_handler(obj):
if hasattr(obj, 'isoformat'):
return obj.isoformat()
else:
raise TypeError
# Usage: json.dumps(s3cl.list_buckets(), default=date_handler)
# or json.dumps(s3cl.list_buckets()['Buckets'], default=date_handler)
# Note: Pretty print like this: print(json.dumps(s3cl.list_buckets()['Buckets'], default=date_handler, indent=4))
################################################################################
# Create bucket and the perception of directories
################################################################################
# Create a bucket in the S3 resource
my_s3bucket = s3re.create_bucket(ACL='private', Bucket='cst-compbio-research-00-buc') # Bug in the S3 API. Do nto specidy region if us-east-1 is desired.
#my_s3bucket = s3re.create_bucket(ACL='private', Bucket='cst-compbio-research-00-buc', CreateBucketConfiguration={'LocationConstraint': 'us-east-1'})
# Tag immediately
s3re.BucketTagging( bucket_name=my_s3bucket.name ).put( Tagging={'TagSet': [
{'Key':'Name', 'Value':'cst-compbio-research-00-buc'},
{'Key':'Owner', 'Value':iam_user_name},
{'Key':'Department', 'Value':'Computational Biology Research'}
]})
# Create directories
my_s3bucket.put_object(Key='home/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'1000', 'gid':'1000'})
my_s3bucket.put_object(Key='snapshots/', ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'1000', 'gid':'1000'})
my_s3bucket.put_object(Key='scratch/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'1000', 'gid':'1000'})
my_s3bucket.put_object(Key='tmp/' , ContentType='directory', Metadata={'mmode':'509' , 'mtime':str(int(time.time())), 'uid':'1000', 'gid':'1000'})
# Create subdirectories
my_s3bucket.put_object(Key='home/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6799', 'gid':'6799'})
my_s3bucket.put_object(Key='home/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6810', 'gid':'6810'})
my_s3bucket.put_object(Key='home/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6812', 'gid':'6812'})
my_s3bucket.put_object(Key='home/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6802', 'gid':'6802'})
my_s3bucket.put_object(Key='home/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6813', 'gid':'6813'})
my_s3bucket.put_object(Key='home/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6816', 'gid':'6816'})
my_s3bucket.put_object(Key='home/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6818', 'gid':'6818'})
my_s3bucket.put_object(Key='snapshots/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6799', 'gid':'6799'})
my_s3bucket.put_object(Key='snapshots/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6810', 'gid':'6810'})
my_s3bucket.put_object(Key='snapshots/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6812', 'gid':'6812'})
my_s3bucket.put_object(Key='snapshots/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6802', 'gid':'6802'})
my_s3bucket.put_object(Key='snapshots/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6813', 'gid':'6813'})
my_s3bucket.put_object(Key='snapshots/[email protected]/', ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6816', 'gid':'6816'})
my_s3bucket.put_object(Key='snapshots/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6818', 'gid':'6818'})
my_s3bucket.put_object(Key='scratch/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6799', 'gid':'6799'})
my_s3bucket.put_object(Key='scratch/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6810', 'gid':'6810'})
my_s3bucket.put_object(Key='scratch/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6812', 'gid':'6812'})
my_s3bucket.put_object(Key='scratch/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6802', 'gid':'6802'})
my_s3bucket.put_object(Key='scratch/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6813', 'gid':'6813'})
my_s3bucket.put_object(Key='scratch/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6816', 'gid':'6816'})
my_s3bucket.put_object(Key='scratch/[email protected]/' , ContentType='directory', Metadata={'mmode':'16877', 'mtime':str(int(time.time())), 'uid':'6818', 'gid':'6818'})
################################################################################
# Create the bucket's access policy
################################################################################
# Note: S3 actions and examples are documented here:
# http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
# https://aws.amazon.com/blogs/security/writing-iam-policies-grant-access-to-user-specific-folders-in-an-amazon-s3-bucket/
# Note: The "Principal":"*" component has been removed from the statement because
# in IAM policies it is implicitly assumed that the Principal is the user to
# whom the policy is attached.
my_policy_json = json.dumps(
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "1AllowSeeingBucketsList",
"Effect": "Allow",
"Action": [ "s3:ListAllMyBuckets", "s3:GetBucketLocation" ],
"Resource": [ "arn:aws:s3:::*" ]
},
{
"Sid": "2AllowRootLevelListingOfThisBucketAndItsMainDirectories",
"Effect": "Allow",
"Action": [ "s3:ListBucket" ],
"Resource": [ "arn:aws:s3:::cst-compbio-research-00-buc" ],
"Condition": {
"StringEquals": {
"s3:prefix": [ "", "home/" ],
"s3:delimiter": [ "/" ]
}
}
},
{
"Sid": "3AllowUserListingOfUsersOwnHomeAndEverybodysSnapshotAndScratchAndTmp",
"Effect": "Allow",
"Action": [ "s3:ListBucket" ],
"Resource": [ "arn:aws:s3:::cst-compbio-research-00-buc" ],
"Condition": {
"StringLike": {
"s3:prefix": [ "home/${aws:username}/*", "snapshots/*", "scratch/*", "tmp/*" ]
}
}
},
{
"Sid": "4AllowGetAndPutAndDelTmp",
"Effect": "Allow",
"Action": [ "s3:GetObject", "s3:GetObjectTagging", "s3:PutObject", "s3:PutObjectTagging", "s3:DeleteObject", "s3:DeleteObjectTagging" ],
"Resource": [ "arn:aws:s3:::cst-compbio-research-00-buc/tmp/*" ]
},
{
"Sid": "5AllowGetSnapshotsAndScratch",
"Effect": "Allow",
"Action": [ "s3:GetObject", "s3:GetObjectTagging", "s3:GetObjectVersion", "s3:GetObjectVersionTagging" ],
"Resource": [ "arn:aws:s3:::cst-compbio-research-00-buc/snapshots/*", "arn:aws:s3:::cst-compbio-research-00-buc/scratch/*" ]
},
{
"Sid": "6AllowEverythingInUsersOwnHomeAndSnapshotsAndScratch",
"Effect": "Allow",
"Action": [ "s3:*" ],
"Resource": [ "arn:aws:s3:::cst-compbio-research-00-buc/home/${aws:username}/*", "arn:aws:s3:::cst-compbio-research-00-buc/snapshots/${aws:username}/*", "arn:aws:s3:::cst-compbio-research-00-buc/scratch/${aws:username}/*" ]
}
]
}
, indent=4)
my_iampolicy = iamre.create_policy(PolicyName='compbio-research-s3-00-pol', PolicyDocument=my_policy_json, Description='Computational Biology Research S3 access control.' )
########################################
# Creating a new version of an existing policy
########################################
# Policy documents are organised and indexed by verion (eg v1, v2, v3, etc).
# One version is considered the default policy version.
# Here it is assumed that a new my_policy_json document has been created.
# Add it as non-default.
my_iampolicy.create_version(PolicyDocument=my_policy_json, SetAsDefault=False)
# Show versions and report whether or not it is the default
for x in my_iampolicy.versions.all(): print(x.version_id, x.is_default_version)
########################################
# Deleting a policy, if needed.
########################################
# When there are multiple versions, all non-default versions of the policy must
# be deleted before the policy in its entirety can be deleted.
####################
# Deleting a policy when there are multiple versions
####################
# Show versions and report whether or not it is the default
for x in my_iampolicy.versions.all(): print(x.version_id, x.is_default_version)
# Display the document of the policy (examples)
# iamre.PolicyVersion(arn=my_iampolicy.arn, version_id='v1').document
# iamre.PolicyVersion(arn=my_iampolicy.arn, version_id='v2').document
# Assuming that v2 is not the default version, delete version v2.
# iamcl.delete_policy_version(PolicyArn=my_iampolicy.arn, VersionId='v2')
####################
# Deleting a policy when there is only one version (the default one)
####################
iamcl.delete_policy(PolicyArn=my_iampolicy.arn)
################################################################################
# Attach Policy to a Group of users
################################################################################
#iamre.Group(name='compbio-research-group-00').attach_policy(PolicyArn=my_iampolicy.arn)
# Note: The command above is not executed at the moment because the members of
# this group already have AmazonS3FullAccess.
### DONE ###
################################################################################
# Appendix 1. Delete a non-empty bucket.
################################################################################
# List content of the bucket
for obj in my_s3bucket.objects.all(): print(obj.key)
# Delete the objects inside the bucket
for obj in my_s3bucket.objects.all(): obj.delete()
# Delete the bucket itself.
my_s3bucket.delete()
# Bonus. This is how to list all objects in all bucket. Warning: big output.
#for bucket in s3re.buckets.all():
# for obj in bucket.objects.all():
# print(obj.key)
################################################################################
# Appendix 2. Pretty print JSON documents at the BASH command line.
################################################################################
# Note: the BASH command line is less flexible than ipython3 command line. BASH accepts only double quotes inside the expression.
# Example
# echo '{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":"*","Action":"s3:*","Resource":["arn:aws:s3:::my-bucket-12345","arn:aws:s3:::my-bucket-12345/*"],"Condition":{"StringNotLike":{"aws:userId":["AIDAJ4ICC4DFD5DCGTM5C"]}}}]}' | python3 -m json.tool
#{
# "Version": "2012-10-17",
# "Statement": [
# {
# "Effect": "Deny",
# "Principal": "*",
# "Action": "s3:*",
# "Resource": [
# "arn:aws:s3:::my-bucket-12345",
# "arn:aws:s3:::my-bucket-12345/*"
# ],
# "Condition": {
# "StringNotLike": {
# "aws:userId": [
# "AIDAJ4ICC4DFD5DCGTM5C"
# ]
# }
# }
# }
# ]
#}