Mercurial > repos > shellac > sam_consensus_v3
comparison env/bin/lss3 @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 #!/Users/cmdms/OneDrive-UOB/Development/Projects/2021/sam-consensus-v3/env/bin/python3 | |
2 import boto | |
3 from boto.exception import S3ResponseError | |
4 from boto.s3.connection import OrdinaryCallingFormat | |
5 | |
6 | |
7 def sizeof_fmt(num): | |
8 for x in ['b ', 'KB', 'MB', 'GB', 'TB', 'XB']: | |
9 if num < 1024.0: | |
10 return "%3.1f %s" % (num, x) | |
11 num /= 1024.0 | |
12 return "%3.1f %s" % (num, x) | |
13 | |
14 | |
15 def list_bucket(b, prefix=None, marker=None): | |
16 """List everything in a bucket""" | |
17 from boto.s3.prefix import Prefix | |
18 from boto.s3.key import Key | |
19 total = 0 | |
20 | |
21 if prefix: | |
22 if not prefix.endswith("/"): | |
23 prefix = prefix + "/" | |
24 query = b.list(prefix=prefix, delimiter="/", marker=marker) | |
25 print("%s" % prefix) | |
26 else: | |
27 query = b.list(delimiter="/", marker=marker) | |
28 | |
29 num = 0 | |
30 for k in query: | |
31 num += 1 | |
32 mode = "-rwx---" | |
33 if isinstance(k, Prefix): | |
34 mode = "drwxr--" | |
35 size = 0 | |
36 else: | |
37 size = k.size | |
38 for g in k.get_acl().acl.grants: | |
39 if g.id == None: | |
40 if g.permission == "READ": | |
41 mode = "-rwxr--" | |
42 elif g.permission == "FULL_CONTROL": | |
43 mode = "-rwxrwx" | |
44 if isinstance(k, Key): | |
45 print("%s\t%s\t%010s\t%s" % (mode, k.last_modified, | |
46 sizeof_fmt(size), k.name)) | |
47 else: | |
48 #If it's not a Key object, it doesn't have a last_modified time, so | |
49 #print nothing instead | |
50 print("%s\t%s\t%010s\t%s" % (mode, ' ' * 24, | |
51 sizeof_fmt(size), k.name)) | |
52 total += size | |
53 print ("=" * 80) | |
54 print ("\t\tTOTAL: \t%010s \t%i Files" % (sizeof_fmt(total), num)) | |
55 | |
56 | |
57 def list_buckets(s3, display_tags=False): | |
58 """List all the buckets""" | |
59 for b in s3.get_all_buckets(): | |
60 print(b.name) | |
61 if display_tags: | |
62 try: | |
63 tags = b.get_tags() | |
64 for tag in tags[0]: | |
65 print(" %s:%s" % (tag.key, tag.value)) | |
66 except S3ResponseError as e: | |
67 if e.status != 404: | |
68 raise | |
69 | |
70 | |
71 def main(): | |
72 import optparse | |
73 import sys | |
74 | |
75 usage = "usage: %prog [options] [BUCKET1] [BUCKET2]" | |
76 description = "List all S3 buckets OR list keys in the named buckets" | |
77 parser = optparse.OptionParser(description=description, usage=usage) | |
78 parser.add_option('-m', '--marker', | |
79 help='The S3 key where the listing starts after it.') | |
80 parser.add_option('-t', '--tags', action='store_true', | |
81 help='Display tags when listing all buckets.') | |
82 options, buckets = parser.parse_args() | |
83 marker = options.marker | |
84 | |
85 if not buckets: | |
86 list_buckets(boto.connect_s3(), options.tags) | |
87 sys.exit(0) | |
88 | |
89 if options.tags: | |
90 print("-t option only works for the overall bucket list") | |
91 sys.exit(1) | |
92 | |
93 pairs = [] | |
94 mixedCase = False | |
95 for name in buckets: | |
96 if "/" in name: | |
97 pairs.append(name.split("/", 1)) | |
98 else: | |
99 pairs.append([name, None]) | |
100 if pairs[-1][0].lower() != pairs[-1][0]: | |
101 mixedCase = True | |
102 | |
103 if mixedCase: | |
104 s3 = boto.connect_s3(calling_format=OrdinaryCallingFormat()) | |
105 else: | |
106 s3 = boto.connect_s3() | |
107 | |
108 for name, prefix in pairs: | |
109 list_bucket(s3.get_bucket(name), prefix, marker=marker) | |
110 | |
111 | |
112 if __name__ == "__main__": | |
113 main() |