annotate nist_wrapper.py @ 2:f6da901ffff8

jinja solution
author pieter.lukasse@wur.nl
date Thu, 22 Jan 2015 16:55:19 +0100
parents cce6989ed423
children c3dc158717fc
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
1 #!/usr/bin/env python
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
2 # encoding: utf-8
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
3 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
4 Module wrapping the NIST MSSEARCH application for matching
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
5 spectra one or more spectra libraries.
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
6 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
7 import csv
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
8 import sys
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
9 import fileinput
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
10 import urllib2
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
11 import time
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
12 import utils
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
13 import uuid
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
14 import os
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
15 import subprocess
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
16 from report_generator import ReportGenerator
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
17
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
18 __author__ = "Pieter Lukasse"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
19 __contact__ = "pieterlukasse@gmail.com,pieter.lukasse@wur.nl"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
20 __copyright__ = "Copyright, 2015"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
21 __license__ = "Apache v2"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
22
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
23
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
24
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
25 def _prepare_NIST(uuid_value, nist_home_dir, nist_ini_file, spectrum_file, is_wine):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
26 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
27 executes the following steps:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
28 - copy nist_home_dir folder to nist_home_dir+_uuid
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
29 - copy spectrum_file.msp to ~/.wine/drive_c/NIST_uid
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
30 - creates nist_home_dir+_uuid/MSSEARCH/AUTOIMP.MSD -> pointing to C:\NIST_uid\MSSEARCH\temp.msd (in case of is_wine) or to nist_home_dir+_uuidM\SSEARCH\temp.msd
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
31 - creates nist_home_dir+_uuid/MSSEARCH/temp.msd -> pointing to C:\NIST_uid\spectrum_file.msp (in case of is_wine) or to nist_home_dir+_uuid\spectrum_file.msp
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
32 and the text "10 724" in the second row
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
33 - copy nistms.INI to nist_home_dir+_uuid/MSSEARCH , overwriting the existing one
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
34 - in case of is_wine:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
35 replace all occurrences of C:\NIST with C:\NIST_uid in this new nistms.INI
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
36 else:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
37 replace all occurrences of C:\NIST with nist_home_dir+_uuid in this new nistms.INI
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
38 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
39 if nist_home_dir.endswith("/") or nist_home_dir.endswith("\\"):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
40 nist_home_dir = nist_home_dir[:-1]
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
41
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
42 # small validation for wine scenario
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
43 if is_wine and not nist_home_dir.endswith("drive_c/NIST"):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
44 raise Exception('Error: invalid NIST home. For wine usage NIST home dir must be in the .wine folder and then in drive_c/NIST')
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
45
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
46 new_nist_home = nist_home_dir+uuid_value
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
47 utils.copy_dir(nist_home_dir, new_nist_home)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
48
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
49 utils.copy_file(spectrum_file, new_nist_home+"/spectrum_file.msp")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
50
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
51 # remove old file:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
52 os.remove(new_nist_home+"/MSSEARCH/AUTOIMP.MSD")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
53 with open(new_nist_home + "/MSSEARCH/AUTOIMP.MSD", "a") as text_file:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
54 if is_wine:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
55 text_file.write("C:\\NIST" + uuid_value + "\\MSSEARCH\\temp.msd")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
56 else:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
57 text_file.write(new_nist_home + "\\MSSEARCH\\temp.msd")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
58
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
59 with open(new_nist_home + "/MSSEARCH/temp.msd", "a") as text_file:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
60 if is_wine:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
61 text_file.write("C:\\NIST" + uuid_value + "\\spectrum_file.msp\n")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
62 else:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
63 text_file.write(new_nist_home + "\\spectrum_file.msp\n")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
64 text_file.write("10 724")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
65
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
66 replacement_text = new_nist_home
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
67 if is_wine:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
68 replacement_text = "C:\\NIST" + uuid_value
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
69
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
70 # remove old file
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
71 os.remove(new_nist_home+"/MSSEARCH/nistms.INI")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
72 # make new one
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
73 o = open(new_nist_home+"/MSSEARCH/nistms.INI","a") #open for append
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
74 # TODO : this loop/replace below is a bit limited to specific variables...either test different NIST versions or make more generic (harder in case of wine, or we need extra "home in .INI file" parameter):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
75 for line in open(nist_ini_file):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
76 if "Library Directory=" in line:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
77 line = "Library Directory="+ new_nist_home + "\\MSSEARCH\\\n"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
78 if "Dir=" in line:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
79 line = "Dir="+ replacement_text + "\\MSSEARCH\\\n"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
80
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
81 o.write(line)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
82 o.close()
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
83
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
84 return new_nist_home
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
85
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
86 def _run_NIST(new_nist_home, output_file, is_wine):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
87 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
88 - run : (wine) new_nist_home/MSSEARCH/nistms$.exe /INSTRUMENT /PAR=2
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
89 - monitor : new_nist_home/MSSEARCH/SRCREADY.TXT for content = "1"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
90 - when ready:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
91 > copy SRCRESLT.TXT to output_file
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
92 > kill nist process
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
93 > (optional)remove ~/.wine/drive_c/NIST_uid/
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
94 > finish
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
95 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
96 # to avoid conflicts in the orphan process killing (see end of this method), we will
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
97 # only run NIST again after previous nistms.exe process has been killed:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
98 # TODO : solution is currently only for wine (in the windows context the solution is not there yet, but parallel calls are not expected as in windows we only run tests one by one for now)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
99 # if is_wine:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
100 # while True:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
101 # # check if process exists. If not, break loop and continue
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
102 # pid = utils.get_process_pid("nistms.exe")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
103 # if pid == -1:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
104 # break
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
105 # time.sleep(2)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
106
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
107
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
108 # remove old file, if it is there:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
109 file_to_monitor = new_nist_home+"/MSSEARCH/SRCREADY.TXT"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
110 if os.path.exists(file_to_monitor):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
111 os.remove(file_to_monitor)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
112
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
113 exec_path = new_nist_home + "/MSSEARCH/nistms$.exe"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
114
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
115 pro = ""
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
116 if is_wine:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
117 print "calling wine with " + exec_path
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
118 cmd = ["wine "+ exec_path + " /INSTRUMENT /PAR=2"]
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
119 # The os.setsid() is passed in the argument preexec_fn so
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
120 # it's run after the fork() and before exec() to run the shell.
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
121 pro = subprocess.Popen(cmd, stdout=subprocess.PIPE,
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
122 shell=True, preexec_fn=os.setsid)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
123 else:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
124 cmd = [
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
125 exec_path,
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
126 "/INSTRUMENT",
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
127 "/PAR=2"]
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
128 subprocess.call(cmd)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
129
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
130
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
131 # monitor process by checking state file:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
132 while True:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
133 # check if SRCREADY.TXT is there already:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
134 if os.path.exists(file_to_monitor):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
135 break
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
136 time.sleep(2)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
137
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
138 # kill process:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
139 #p.terminate() - not needed, nistm$ will terminate...nistms.exe is the one that
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
140 #stays open...and orphan..killing it:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
141
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
142 if is_wine:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
143 # pid = utils.get_process_pid("nistms.exe")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
144 # os.kill(pid, 9)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
145 os.killpg(pro.pid, 9)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
146 else:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
147 # windows case:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
148 proc_name = "nistms.exe"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
149 os.system("taskkill /f /im " + proc_name)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
150
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
151 # copy SRCRESLT.TXT to output_file
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
152 result_file = new_nist_home+"/MSSEARCH/SRCRESLT.TXT"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
153 utils.copy_file(result_file, output_file)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
154
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
155
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
156 def _create_html_report(output_html_report, output_html_report_files_path, hits_dict, spectra_dict):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
157 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
158 This report will contain a page that displays essentially the same list as found in the
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
159 tabular output file (rendered with datatables jquery plugin), with some extra features:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
160 - when user clicks on an entry, it should display the query spectrum and the hit spectrum
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
161 in "head to tail" and "difference" mode (see galaxy/report_example.png)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
162 -> the query spectrum can be generated from the data in the input MSP file
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
163 -> the library "online representative" spectrum can be generated from data returned by http://webbook.nist.gov/cgi/cbook.cgi?JCAMP=C537268&Index=0&Type=Mass ,
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
164 where C537268 in this case is the CAS ID without the '-' separators
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
165 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
166 # step 1 : generate HTML via the jinja template engine
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
167 # step 1.1: make sure to link the query spectrum data to the corresponding html object for quick rendering when needed
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
168
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
169 html_file = open(output_html_report,'w')
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
170 html_render = ReportGenerator(os.path.dirname(__file__), 'templates/main_template.html',hits_dict, spectra_dict)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
171 html_render.render(html_file)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
172 # copy necessary .js files as well:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
173 templates_folder = os.path.dirname(__file__) + '/templates/'
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
174 utils.copy_file(templates_folder + 'spectrum_gen.js', output_html_report_files_path+"/spectrum_gen.js")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
175 utils.copy_dir(templates_folder + 'lib', output_html_report_files_path+'/lib' )
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
176 utils.copy_dir(templates_folder + 'images', output_html_report_files_path+'/images' )
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
177
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
178 html_file.close()
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
179
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
180
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
181
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
182
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
183
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
184 return None
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
185
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
186 def _get_extra_info_and_link_cols(data_found, data_type_found, query_link):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
187 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
188 This method will go over the data found and will return a
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
189 list with the following items:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
190 - Experiment details where hits have been found :
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
191 'organism', 'tissue','experiment_name','user_name','column_type'
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
192 - Link that executes same query
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
193
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
194 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
195 # set() makes a unique list:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
196 organism_set = []
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
197 tissue_set = []
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
198 experiment_name_set = []
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
199 user_name_set = []
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
200 column_type_set = []
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
201 cas_nr_set = []
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
202
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
203 if 'organism' in data_found:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
204 organism_set = set(data_found['organism'])
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
205 if 'tissue' in data_found:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
206 tissue_set = set(data_found['tissue'])
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
207 if 'experiment_name' in data_found:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
208 experiment_name_set = set(data_found['experiment_name'])
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
209 if 'user_name' in data_found:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
210 user_name_set = set(data_found['user_name'])
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
211 if 'column_type' in data_found:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
212 column_type_set = set(data_found['column_type'])
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
213 if 'CAS' in data_found:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
214 cas_nr_set = set(data_found['CAS'])
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
215
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
216
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
217 result = [data_type_found,
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
218
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
219 #To let Excel interpret as link, use e.g. =HYPERLINK("http://stackoverflow.com", "friendly name"):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
220 "=HYPERLINK(\""+ query_link + "\", \"Link to entries found in DB \")"]
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
221 return result
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
222
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
223
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
224
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
225
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
226
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
227 # alternative: ?
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
228 # s = requests.Session()
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
229 # s.verify = False
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
230 # #s.auth = (token01, token02)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
231 # resp = s.get(url, params={'name': 'anonymous'}, stream=True)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
232 # content = resp.content
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
233 # # transform to dictionary:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
234
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
235
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
236
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
237 def _save_data(data_rows, headers, out_csv):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
238 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
239 Writes tab-separated data to file
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
240 @param data_rows: dictionary containing merged/enriched dataset
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
241 @param out_csv: output csv file
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
242 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
243
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
244 # Open output file for writing
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
245 outfile_single_handle = open(out_csv, 'wb')
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
246 output_single_handle = csv.writer(outfile_single_handle, delimiter="\t")
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
247
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
248 # Write headers
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
249 output_single_handle.writerow(headers)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
250
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
251 # Write one line for each row
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
252 for data_row in data_rows:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
253 output_single_handle.writerow(data_row)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
254
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
255 def _get_metexp_URL(metexp_dblink_file):
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
256 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
257 Read out and return the URL stored in the given file.
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
258 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
259 file_input = fileinput.input(metexp_dblink_file)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
260 try:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
261 for line in file_input:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
262 if line[0] != '#':
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
263 # just return the first line that is not a comment line:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
264 return line
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
265 finally:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
266 file_input.close()
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
267
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
268
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
269 def main():
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
270 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
271 Wrapper main function
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
272
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
273 The input expected is:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
274 NIST_HOME dir
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
275 nistms.INI
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
276 spectrum_file.msp
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
277 outputfileName
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
278 (optional) htmlReportFile
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
279 (optional) htmlReportFile.files_path
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
280 '''
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
281 seconds_start = int(round(time.time()))
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
282
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
283 nist_home_dir = sys.argv[1]
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
284 nist_ini_file = sys.argv[2]
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
285 spectrum_file = sys.argv[3]
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
286 nist_output_file = sys.argv[4]
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
287 final_output_file = sys.argv[5]
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
288 # html report pars:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
289 output_html_report = None
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
290 output_html_report_files_path = None
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
291 if len(sys.argv) > 6:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
292 output_html_report = sys.argv[6]
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
293 output_html_report_files_path = sys.argv[7]
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
294
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
295 is_wine = False
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
296 if "wine" in nist_home_dir:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
297 is_wine = True
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
298
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
299 uuid_value = str(uuid.uuid4())
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
300
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
301 # prepare NIST environment for running:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
302 new_nist_home = _prepare_NIST(uuid_value, nist_home_dir, nist_ini_file, spectrum_file, is_wine)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
303
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
304 # run NIST search command:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
305 _run_NIST(new_nist_home, nist_output_file, is_wine)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
306
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
307 # write output tabular:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
308 hits_dict = utils.get_nist_out_as_dict(nist_output_file)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
309 utils.save_dict_as_tsv(hits_dict, final_output_file)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
310
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
311 # create report:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
312 if len(sys.argv) > 6:
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
313 spectra_dict = utils.get_spectra_file_as_dict(spectrum_file)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
314 _create_html_report(output_html_report, output_html_report_files_path, hits_dict, spectra_dict)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
315
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
316
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
317 #_save_data(enriched_data, headers, output_result)
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
318
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
319 seconds_end = int(round(time.time()))
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
320 print "Took " + str(seconds_end - seconds_start) + " seconds"
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
321
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
322
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
323
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
324 if __name__ == '__main__':
cce6989ed423 new NIST wrapper demo tools
pieter.lukasse@wur.nl
parents:
diff changeset
325 main()