Mercurial > repos > jay > gaiac_precision
annotate gaiac_pm_data_pulling/gaiac_pm_data_pulling.py @ 2:69b3190e5e64 draft
planemo upload for repository https://github.com/jaidevjoshi83/gaiac commit c31ef5726cd1d2659da9aeb956c22fb834b177ff
author | jay |
---|---|
date | Fri, 23 May 2025 17:57:30 +0000 |
parents | 8a99d8639aa2 |
children | 1f9e67edde6a |
rev | line source |
---|---|
1
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
1 #!/usr/bin/env python |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
2 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
3 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
4 import argparse |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
5 import logging |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
6 from base64 import urlsafe_b64encode |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
7 from html import escape |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
8 from urllib.parse import urljoin |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
9 import requests |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
10 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
11 log = logging.getLogger("tools.iuc.data_managers.data_manager_refgenie_pull") |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
12 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
13 import urllib.request |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
14 import json |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
15 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
16 def get_file_list(ip): |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
17 |
2
69b3190e5e64
planemo upload for repository https://github.com/jaidevjoshi83/gaiac commit c31ef5726cd1d2659da9aeb956c22fb834b177ff
jay
parents:
1
diff
changeset
|
18 url = "%s/api/list-files"%(ip) |
1
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
19 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
20 with urllib.request.urlopen(url) as response: |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
21 data = response.read() |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
22 file_list = json.loads(data) |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
23 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
24 return file_list |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
25 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
26 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
27 def simulate_dynamic_options(ip): |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
28 try: |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
29 rval = [] |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
30 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
31 api_data = {ip:{ |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
32 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
33 }} |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
34 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
35 for i in get_file_list(ip): |
2
69b3190e5e64
planemo upload for repository https://github.com/jaidevjoshi83/gaiac commit c31ef5726cd1d2659da9aeb956c22fb834b177ff
jay
parents:
1
diff
changeset
|
36 api_data[ip][i] = ip+'/api/download/'+i |
1
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
37 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
38 ul = [] |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
39 for dataset, url in api_data[ip].items(): |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
40 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
41 ul.append({ |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
42 'name': dataset, |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
43 'value': url, |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
44 'options': [], |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
45 'selected': False |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
46 }) |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
47 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
48 return ul |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
49 except Exception as e: |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
50 # Handle any potential errors |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
51 return [{ |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
52 'name': escape(str(e)), |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
53 'value': 'ERROR', |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
54 'options': [], |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
55 'selected': False |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
56 }] |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
57 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
58 if __name__ == '__main__': |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
59 |
2
69b3190e5e64
planemo upload for repository https://github.com/jaidevjoshi83/gaiac commit c31ef5726cd1d2659da9aeb956c22fb834b177ff
jay
parents:
1
diff
changeset
|
60 simulate_dynamic_options('http://iotprojects.ddns.net') |