Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/galaxy/tool_util/deps/mulled/mulled_update_singularity_containers.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 #!/usr/bin/env python | |
2 | |
3 import argparse | |
4 import subprocess | |
5 import tempfile | |
6 from glob import glob | |
7 from subprocess import check_output | |
8 | |
9 from galaxy.util import unicodify | |
10 from .get_tests import hashed_test_search, main_test_search | |
11 | |
12 | |
13 def get_list_from_file(filename): | |
14 """ | |
15 Returns a list of containers stored in a file (one on each line) | |
16 """ | |
17 with open(filename) as fh: | |
18 return [_ for _ in fh.read().splitlines() if _] # if blank lines are in the file empty strings must be removed | |
19 | |
20 | |
21 def docker_to_singularity(container, installation, filepath, no_sudo=False): | |
22 """ | |
23 Convert docker to singularity container. | |
24 """ | |
25 cmd = [installation, 'build', '/'.join((filepath, container)), 'docker://quay.io/biocontainers/' + container] | |
26 try: | |
27 if no_sudo: | |
28 check_output(cmd, stderr=subprocess.STDOUT) | |
29 else: | |
30 check_output(cmd.insert(0, 'sudo'), stderr=subprocess.STDOUT) | |
31 check_output(['sudo', 'rm', '-rf', '/root/.singularity/docker/'], stderr=subprocess.STDOUT) | |
32 except subprocess.CalledProcessError as e: | |
33 raise Exception("Docker to Singularity conversion failed.\nOutput was:\n%s" % unicodify(e.output)) | |
34 | |
35 | |
36 def singularity_container_test(tests, installation, filepath): | |
37 """ | |
38 Run tests, record if they pass or fail | |
39 """ | |
40 test_results = {'passed': [], 'failed': [], 'notest': []} | |
41 | |
42 # create a 'sanitised home' directory in which the containers may be mounted - see http://singularity.lbl.gov/faq#solution-1-specify-the-home-to-mount | |
43 with tempfile.TemporaryDirectory() as tmpdirname: | |
44 for container, test in tests.items(): | |
45 if 'commands' not in test and 'imports' not in test: | |
46 test_results['notest'].append(container) | |
47 | |
48 else: | |
49 exec_command = [installation, 'exec', '-H', tmpdirname, '/'.join((filepath, container))] | |
50 test_passed = True | |
51 errors = [] | |
52 if test.get('commands', False): | |
53 for test_command in test['commands']: | |
54 test_command = test_command.replace('$PREFIX', '/usr/local/') | |
55 test_command = test_command.replace('${PREFIX}', '/usr/local/') | |
56 test_command = test_command.replace('$R ', 'Rscript ') | |
57 | |
58 try: | |
59 check_output(exec_command.extend(['bash', '-c', test_command]), stderr=subprocess.STDOUT) | |
60 except subprocess.CalledProcessError: | |
61 try: | |
62 check_output(exec_command.append(test_command), stderr=subprocess.STDOUT) | |
63 except subprocess.CalledProcessError as e: | |
64 errors.append( | |
65 {'command': test_command, 'output': unicodify(e.output)}) | |
66 test_passed = False | |
67 | |
68 if test.get('imports', False): | |
69 for imp in test['imports']: | |
70 try: | |
71 check_output(exec_command.extend([test['import_lang'], 'import ' + imp]), stderr=subprocess.STDOUT) | |
72 except subprocess.CalledProcessError as e: | |
73 errors.append({'import': imp, 'output': unicodify(e.output)}) | |
74 test_passed = False | |
75 | |
76 if test_passed: | |
77 test_results['passed'].append(container) | |
78 else: | |
79 test['errors'] = errors | |
80 test_results['failed'].append(test) | |
81 return test_results | |
82 | |
83 | |
84 def main(): | |
85 parser = argparse.ArgumentParser( | |
86 description='Updates index of singularity containers.') | |
87 parser.add_argument('-c', '--containers', dest='containers', nargs='+', default=None, | |
88 help="Containers to be generated. If the number of containers is large, it may be simpler to use the --containers-list option.") | |
89 parser.add_argument('-l', '--container-list', dest='container_list', default=None, | |
90 help="Name of file containing list of containers to be generated. Alternative to --containers.") | |
91 parser.add_argument('-f', '--filepath', dest='filepath', | |
92 help="File path where newly-built Singularity containers are placed.") | |
93 parser.add_argument('-i', '--installation', dest='installation', | |
94 help="File path of Singularity installation.") | |
95 parser.add_argument('--no-sudo', dest='no_sudo', action='store_true', | |
96 help="Build containers without sudo.") | |
97 parser.add_argument('--testing', '-t', dest='testing', default=None, | |
98 help="Performs testing automatically - a name for the output file should be provided. (Alternatively, testing may be done using the separate testing tool.") | |
99 | |
100 args = parser.parse_args() | |
101 | |
102 if args.containers: | |
103 containers = args.containers | |
104 elif args.container_list: | |
105 containers = get_list_from_file(args.container_list) | |
106 else: | |
107 print("Either --containers or --container-list should be selected.") | |
108 return | |
109 | |
110 for container in containers: | |
111 docker_to_singularity(container, args.installation, | |
112 args.filepath, args.no_sudo) | |
113 | |
114 if args.testing: | |
115 container_testing({'anaconda_channel': 'bioconda', 'installation': args.installation, 'filepath': args.filepath, 'github_repo': 'bioconda/bioconda-recipes', | |
116 'deep_search': False, 'github_local_path': None, 'logfile': args.testing, 'containers': containers}) | |
117 | |
118 | |
119 def container_testing(args=None): | |
120 if not args: # i.e. if testing is called directly from CLI and not via main() | |
121 parser = argparse.ArgumentParser(description='Tests.') | |
122 parser.add_argument('-c', '--containers', dest='containers', nargs='+', default=None, | |
123 help="Containers to be tested. If the number of containers is large, it may be simpler to use the --containers-list option.") | |
124 parser.add_argument('-l', '--container-list', dest='container_list', default=None, | |
125 help="Name of file containing list of containers to be tested. Alternative to --containers.") | |
126 parser.add_argument('-f', '--filepath', dest='filepath', | |
127 help="File path where the containers to be tested are located.") | |
128 parser.add_argument('-o', '--logfile', dest='logfile', default='singularity.log', | |
129 help="Filename for a log to be written to.") | |
130 parser.add_argument('-i', '--installation', dest='installation', | |
131 help="File path of Singularity installation.") | |
132 parser.add_argument('--deep-search', dest='deep_search', action='store_true', | |
133 help="Perform a more extensive, but probably slower, search for tests.") | |
134 parser.add_argument('--anaconda-channel', dest='anaconda_channel', default='bioconda', | |
135 help="Anaconda channel to search for tests (default: bioconda).") | |
136 parser.add_argument('--github-repo', dest='github_repo', default='bioconda/bioconda-recipes', | |
137 help="Github repository to search for tests - only relevant if --deep-search is activated (default: bioconda/bioconda-recipes") | |
138 parser.add_argument('--github-local-path', dest='github_local_path', default=None, | |
139 help="If the bioconda-recipes repository (or other repository containing tests) is available locally, provide the path here. Only relevant if --deep-search is activated.") | |
140 args = vars(parser.parse_args()) | |
141 | |
142 if args['containers']: | |
143 containers = args['containers'] | |
144 elif args['container_list']: | |
145 containers = get_list_from_file(args['container_list']) | |
146 else: # if no containers are specified, test everything in the filepath | |
147 containers = [n.split(args['filepath'])[1] | |
148 for n in glob('%s*' % args['filepath'])] | |
149 | |
150 with open(args['logfile'], 'w') as f: | |
151 f.write("SINGULARITY CONTAINERS GENERATED:") | |
152 tests = {} | |
153 for container in containers: | |
154 if container[0:6] == 'mulled': # if it is a 'hashed container' | |
155 tests[container] = hashed_test_search( | |
156 container, args['github_local_path'], args['deep_search'], args['anaconda_channel'], args['github_repo']) | |
157 else: | |
158 tests[container] = main_test_search( | |
159 container, args['github_local_path'], args['deep_search'], args['anaconda_channel'], args['github_repo']) | |
160 test_results = singularity_container_test( | |
161 tests, args['installation'], args['filepath']) | |
162 | |
163 f.write('\n\tTEST PASSED:') | |
164 for container in test_results['passed']: | |
165 f.write('\n\t\t%s' % container) | |
166 f.write('\n\tTEST FAILED:') | |
167 for container in test_results['failed']: | |
168 f.write('\n\t\t%s' % container['container']) | |
169 for error in container['errors']: | |
170 f.write('\n\t\t\tCOMMAND: {}\n\t\t\t\tERROR:{}'.format(error.get( | |
171 'command', 'import' + error.get('import', 'nothing found')), error['output'])) | |
172 f.write('\n\tNO TEST AVAILABLE:') | |
173 for container in test_results['notest']: | |
174 f.write('\n\t\t%s' % container) | |
175 | |
176 | |
177 if __name__ == "__main__": | |
178 main() |