annotate directory_copier.py @ 1:2470f3968557 draft default tip

Uploaded
author brenninc
date Tue, 21 Jun 2016 04:42:16 -0400
parents 97a9636974bc
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
1 import gzip
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
2 import optparse # using optparse as hydra still python 2.6
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
3 import os.path
1
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
4 import re
0
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
5 import shutil
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
6 import sys
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
7
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
8 def report_error(*args):
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
9 sys.stderr.write(' '.join(map(str,args)) + '\n')
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
10 sys.stderr.flush()
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
11 sys.exit(1)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
12
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
13
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
14 def check_pattern_get_new_name(a_file, ending, options):
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
15 if options.start:
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
16 if not(a_file.startswith(options.start)):
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
17 return None
1
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
18 name = a_file[:-len(ending)]
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
19 if name.endswith("."):
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
20 name = name[:-1]
0
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
21 if options.last:
1
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
22 if not(name.endswith(last)):
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
23 return None
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
24 if options.regex:
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
25 pattern = re.compile(options.regex)
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
26 if pattern.search(name) is None:
0
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
27 return None
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
28 if options.new_ending:
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
29 if options.new_ending[0] ==".":
1
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
30 return name + options.new_ending
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
31 else:
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
32 return name + "." + options.new_ending
0
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
33 if options.decompress:
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
34 if a_file.endswith(".gz"):
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
35 return a_file[:-3]
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
36 return a_file
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
37
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
38
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
39 def check_and_get_new_name(a_file, options):
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
40 for ending in options.endings:
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
41 if a_file.endswith(ending):
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
42 return check_pattern_get_new_name (a_file, ending, options)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
43 return None
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
44
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
45
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
46 def link(a_file, new_name, path):
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
47 file_path = os.path.join(os.path.realpath(path), a_file)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
48 sym_path = os.path.join(os.path.realpath("output"), new_name)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
49 os.link(file_path, sym_path)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
50
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
51
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
52 def decompress(a_file, new_name, path):
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
53 file_path = os.path.join(os.path.realpath(path), a_file)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
54 target_path = os.path.join(os.path.realpath("output"), new_name)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
55 with gzip.open(file_path, 'rb') as f_in, open(target_path, 'wb') as f_out:
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
56 shutil.copyfileobj(f_in, f_out)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
57
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
58
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
59 def copy_and_link(path, options):
1
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
60 if options.decompress or options.link:
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
61 os.mkdir("output")
0
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
62 with open(options.list, 'w') as list_file:
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
63 files = os.listdir(path)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
64 files.sort()
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
65 for a_file in files:
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
66 new_name = check_and_get_new_name(a_file, options)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
67 if new_name:
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
68 list_file.write(new_name)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
69 list_file.write("\n")
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
70 if options.decompress:
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
71 if a_file.endswith(".gz"):
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
72 decompress(a_file, new_name,path)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
73 else:
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
74 link(a_file, new_name, path)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
75 elif options.link:
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
76 link(a_file, new_name, path)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
77
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
78
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
79 if __name__ == '__main__':
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
80 parser = optparse.OptionParser()
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
81 parser.add_option("--path", action="store", type="string",
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
82 help="Path of directory to check. ")
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
83 parser.add_option("--ending", action="append", type="string", dest="endings",
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
84 help="Ending that can be listed and if requested linked or decompressed. ")
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
85 parser.add_option("--start", action="store", type="string",
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
86 help="String that must be at the start of the file name ")
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
87 parser.add_option("--last", action="store", type="string",
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
88 help="String that must be the last bit of the file name before the endings")
1
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
89 parser.add_option("--regex", action="store", type="string",
2470f3968557 Uploaded
brenninc
parents: 0
diff changeset
90 help="Regex for file names not including the endings")
0
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
91 parser.add_option("--new_ending", action="store", type="string",
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
92 help="New ending to replace any previous ending in list and if required links or decompressions. Note: If not set decompression will auto remove the compressioned part of the ending")
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
93 parser.add_option("--list", action="store", type="string",
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
94 help="Path to where all files should be listed. ")
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
95 parser.add_option("--link", action="store_true", default=False,
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
96 help="If set will cause links to be added in output directory. ")
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
97 parser.add_option("--decompress", action="store_true", default=False,
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
98 help="If set will cause gz files to be decompressed or if not a supported decompression ending linked.")
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
99 (options, args) = parser.parse_args()
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
100
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
101
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
102 path = options.path.strip()
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
103 if path[-1] != '/':
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
104 path = path + "/"
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
105 copy_and_link(path, options)
97a9636974bc Uploaded first version
brenninc
parents:
diff changeset
106