Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/galaxy/tool_util/deps/mulled/mulled_list.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 #!/usr/bin/env python | |
2 | |
3 import argparse | |
4 import logging | |
5 from glob import glob | |
6 from html.parser import HTMLParser | |
7 | |
8 import requests | |
9 | |
10 QUAY_API_ENDPOINT = 'https://quay.io/api/v1/repository' | |
11 | |
12 | |
13 def get_quay_containers(repository='biocontainers'): | |
14 """ | |
15 Get all quay containers in the biocontainers repo | |
16 """ | |
17 containers = [] | |
18 | |
19 repos_parameters = {'public': 'true', 'namespace': repository} | |
20 repos_headers = {'Accept-encoding': 'gzip', 'Accept': 'application/json'} | |
21 repos_response = requests.get( | |
22 QUAY_API_ENDPOINT, headers=repos_headers, params=repos_parameters, timeout=12) | |
23 | |
24 repos = repos_response.json()['repositories'] | |
25 | |
26 for repo in repos: | |
27 logging.info(repo) | |
28 tags_response = requests.get( | |
29 "{}/{}/{}".format(QUAY_API_ENDPOINT, repository, repo['name'])) | |
30 tags = tags_response.json()['tags'] | |
31 for tag in tags: | |
32 containers.append('{}:{}'.format(repo['name'], tag)) | |
33 | |
34 return containers | |
35 | |
36 | |
37 def get_singularity_containers(): | |
38 """ | |
39 Get all existing singularity containers from "https://depot.galaxyproject.org/singularity/" | |
40 """ | |
41 class GetContainerNames(HTMLParser): # small parser which gets list of containers | |
42 def __init__(self): | |
43 HTMLParser.__init__(self) | |
44 self.containers = [] | |
45 | |
46 def handle_starttag(self, tag, attrs): | |
47 try: | |
48 for attr in attrs: | |
49 if attr[0] == 'href' and attr[1] != '../': | |
50 self.containers.append(attr[1].replace('%3A', ':')) | |
51 except IndexError: | |
52 pass | |
53 | |
54 parser = GetContainerNames() | |
55 index = requests.get("https://depot.galaxyproject.org/singularity/") | |
56 parser.feed(index.text) | |
57 return parser.containers | |
58 | |
59 | |
60 def get_conda_envs(filepath): | |
61 """ | |
62 Get list of already existing envs | |
63 """ | |
64 return [n.split('__')[-1].replace('@', ':') for n in glob('%s/*' % filepath)] | |
65 | |
66 | |
67 def get_missing_containers(quay_list, singularity_list, blocklist_file=None): | |
68 r""" | |
69 Return list of quay containers that do not exist as singularity containers. Files stored in a blocklist will be ignored | |
70 """ | |
71 blocklist = [] | |
72 if blocklist_file: | |
73 with open(blocklist_file) as fh: | |
74 blocklist = fh.read().split('\n') | |
75 return [n for n in quay_list if n not in singularity_list and n not in blocklist] | |
76 | |
77 | |
78 def get_missing_envs(quay_list, conda_list, blocklist_file=None): | |
79 r""" | |
80 Compares list of conda envs and docker containers and returns missing conda envs | |
81 """ | |
82 blocklist = [] | |
83 if blocklist_file: | |
84 with open(blocklist_file) as fh: | |
85 blocklist = fh.read().split('\n') | |
86 | |
87 return [n for n in quay_list if n.split('--')[0] not in conda_list and n.split('--')[0] not in blocklist] | |
88 | |
89 | |
90 def main(): | |
91 parser = argparse.ArgumentParser( | |
92 description='Returns list of Docker containers in the quay.io biocontainers repository.') | |
93 parser.add_argument('--source', '-s', | |
94 help="Docker, Singularity or Conda.") | |
95 parser.add_argument('--not-singularity', dest='not_singularity', action="store_true", | |
96 help="Exclude Docker containers from which Singularity containers have already been built.") | |
97 parser.add_argument('--not-conda', dest='not_conda', action="store_true", | |
98 help="Exclude Docker containers from which Conda environments have already been extracted.") | |
99 parser.add_argument('--conda-filepath', dest='conda_filepath', default=None, | |
100 help="If searching for conda environments or employing the --not-conda option, a filepath where the environments are located.") | |
101 parser.add_argument('-b', '--blocklist', '--blacklist', dest='blocklist', default=None, | |
102 help="Provide a 'blocklist file' containing containers which should be excluded from the list.") | |
103 parser.add_argument('-f', '--file', dest='output', default=None, | |
104 help="File to write list to. If not given output will be returned on the command line.") | |
105 | |
106 args = parser.parse_args() | |
107 | |
108 if args.source == 'docker': | |
109 containers = get_quay_containers() | |
110 if args.not_singularity: | |
111 containers = get_missing_containers( | |
112 containers, get_singularity_containers(), args.blocklist) | |
113 if args.not_conda: | |
114 containers = get_missing_envs(containers, get_conda_envs( | |
115 args.conda_filepath), args.blocklist) | |
116 elif args.source == 'singularity': | |
117 containers = get_singularity_containers() | |
118 elif args.source == 'conda': | |
119 containers = get_conda_envs(args.conda_filepath) | |
120 else: | |
121 print("The 'source' argument was not understood.") | |
122 return | |
123 | |
124 if args.output: | |
125 with open(args.output, 'a') as f: | |
126 for container in containers: | |
127 f.write('%s\n' % container) | |
128 else: | |
129 print(containers) | |
130 | |
131 | |
132 if __name__ == '__main__': | |
133 main() |