Mercurial > repos > jay > gaiac_precision
annotate gaiac_pm_data_pulling/gaiac_pm_data_pulling.py @ 3:1f9e67edde6a draft default tip
planemo upload for repository https://github.com/jaidevjoshi83/gaiac commit e9587f93346c7b55e1be00bad5844bf2db3ed03d-dirty
author | jay |
---|---|
date | Thu, 10 Jul 2025 19:42:02 +0000 |
parents | 69b3190e5e64 |
children |
rev | line source |
---|---|
1
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
1 #!/usr/bin/env python |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
2 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
3 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
4 import argparse |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
5 import logging |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
6 from base64 import urlsafe_b64encode |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
7 from html import escape |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
8 from urllib.parse import urljoin |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
9 import requests |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
10 import urllib.request |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
11 import json |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
12 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
13 def get_file_list(ip): |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
14 |
2
69b3190e5e64
planemo upload for repository https://github.com/jaidevjoshi83/gaiac commit c31ef5726cd1d2659da9aeb956c22fb834b177ff
jay
parents:
1
diff
changeset
|
15 url = "%s/api/list-files"%(ip) |
1
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
16 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
17 with urllib.request.urlopen(url) as response: |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
18 data = response.read() |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
19 file_list = json.loads(data) |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
20 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
21 return file_list |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
22 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
23 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
24 def simulate_dynamic_options(ip): |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
25 try: |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
26 rval = [] |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
27 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
28 api_data = {ip:{ |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
29 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
30 }} |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
31 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
32 for i in get_file_list(ip): |
2
69b3190e5e64
planemo upload for repository https://github.com/jaidevjoshi83/gaiac commit c31ef5726cd1d2659da9aeb956c22fb834b177ff
jay
parents:
1
diff
changeset
|
33 api_data[ip][i] = ip+'/api/download/'+i |
1
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
34 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
35 ul = [] |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
36 for dataset, url in api_data[ip].items(): |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
37 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
38 ul.append({ |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
39 'name': dataset, |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
40 'value': url, |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
41 'options': [], |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
42 'selected': False |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
43 }) |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
44 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
45 return ul |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
46 except Exception as e: |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
47 # Handle any potential errors |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
48 return [{ |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
49 'name': escape(str(e)), |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
50 'value': 'ERROR', |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
51 'options': [], |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
52 'selected': False |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
53 }] |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
54 |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
55 if __name__ == '__main__': |
8a99d8639aa2
planemo upload for repository https://github.com/jaidevjoshi83/gaiac.git commit a733fec1915a1f6cf5f07092dc33426f765b6cb3-dirty
jay
parents:
diff
changeset
|
56 |
3
1f9e67edde6a
planemo upload for repository https://github.com/jaidevjoshi83/gaiac commit e9587f93346c7b55e1be00bad5844bf2db3ed03d-dirty
jay
parents:
2
diff
changeset
|
57 print(simulate_dynamic_options('http://iotprojects.ddns.net')) |