|
18
|
1 from commons.core.utils.FileUtils import FileUtils
|
|
|
2 from commons.core.sql.DbFactory import DbFactory
|
|
|
3 from commons.tools.PostAnalyzeTELib import PostAnalyzeTELib
|
|
|
4 import subprocess
|
|
|
5 import unittest
|
|
|
6 import os
|
|
|
7
|
|
|
8
|
|
|
9 class Test_F_PostAnalyzeTELib(unittest.TestCase):
|
|
|
10
|
|
|
11 def setUp(self):
|
|
|
12 self._expStatFileName = "expStats.tab"
|
|
|
13 self._obsStatFileName = ""
|
|
|
14 self._genomeSize = 1281640
|
|
|
15 self._pathTableName = "dummyDmelChr4_chr_allTEs_nr_noSSR_join_path"
|
|
|
16 self._seqTableName = "dummyDmelChr4_denovoLibTEs_seq"
|
|
|
17
|
|
|
18 def tearDown(self):
|
|
|
19 try:
|
|
|
20 os.remove(self._expStatFileName)
|
|
|
21 except: pass
|
|
|
22 try:
|
|
|
23 os.remove(self._obsStatFileName)
|
|
|
24 except: pass
|
|
|
25
|
|
|
26 def test_run_analysis1(self):
|
|
|
27 libFileName = "TElib.fa"
|
|
|
28 self._writeInputFasta_analysis1(libFileName)
|
|
|
29 expClusterFileName = "expClusters.tab"
|
|
|
30 self._writeExpClusterFile_analysis1(expClusterFileName)
|
|
|
31 expGlobalStatFileName = "expGlobalStats.txt"
|
|
|
32 self._writeExpGlobalStats_analysis1(expGlobalStatFileName)
|
|
|
33 self._writeExpStatsFile_analysis1(self._expStatFileName)
|
|
|
34 obsClusterFileName = "TElib.tab"
|
|
|
35 obsGlobalStatFileName = "TElib.globalStatsPerCluster.txt"
|
|
|
36 self._obsStatFileName = "TElib.statsPerCluster.tab"
|
|
|
37
|
|
|
38 iPATEL = PostAnalyzeTELib(analysis=1, fastaFileName=libFileName, doClean=True)
|
|
|
39 iPATEL.run()
|
|
|
40
|
|
|
41 self.assertTrue(FileUtils.are2FilesIdentical(expClusterFileName, obsClusterFileName))
|
|
|
42 self.assertTrue(FileUtils.are2FilesIdentical(expGlobalStatFileName, obsGlobalStatFileName))
|
|
|
43 self.assertTrue(FileUtils.are2FilesIdentical(self._expStatFileName, self._obsStatFileName))
|
|
|
44
|
|
|
45 os.remove(libFileName)
|
|
|
46 os.remove(expClusterFileName)
|
|
|
47 os.remove(expGlobalStatFileName)
|
|
|
48 os.remove(obsClusterFileName)
|
|
|
49 os.remove(obsGlobalStatFileName)
|
|
|
50
|
|
|
51 def test_run_as_script_analysis1(self):
|
|
|
52 libFileName = "TElib.fa"
|
|
|
53 with open(libFileName, "w") as f:
|
|
|
54 f.write(">transib2\n")
|
|
|
55 f.write("GGCCAGTCACAATGGGGGTTTCACTGGTGTGTCATGCACATTTAATAGGGGTAAGACTGA\n")
|
|
|
56 f.write("ATAAAAAATGATTATTTGCATGAAATGGGGATGAGAGAGAAGGAAAGAGTTTCATCCTGG\n")
|
|
|
57 f.write("GATTCGTTTCATTCACCGGATCTCTTGCGTCCGCCTCCGCCGTGCGACCTCCGCATTC\n")
|
|
|
58 f.write(">transib3\n")
|
|
|
59 f.write("ATAAAAAATGATTATTTGCATGAAATGGGGATGAGAGAGAAGGAAAGAGTTTCATCCTGG\n")
|
|
|
60 f.write("TGAAACTCGTCAGCGTCGTTTCCAAGTCCT\n")
|
|
|
61 f.write(">transib4\n")
|
|
|
62 f.write("GGCCAGTCACAATGGGGGTTTCACTGGTGTGTCATGCACATTTAATAGGGGTAAGACTGA\n")
|
|
|
63 f.write("ATAAAAAATGATTATTTGCATGAAATGGGGATGAGAGAGAAGGAAAGAGTTTCATCCTGG\n")
|
|
|
64 f.write("GATTCGTTTCATTCACCGGATCTCTTGCGTCCGCCTCCGCCGTGCGACCTCCGCATTCAT\n")
|
|
|
65 f.write("AAAAAATGATTATTTGCATGAAATGGGGATGAGAGAGAAGGAAAGAGTTTCATCCTGG\n")
|
|
|
66 expClusterFileName = "expClusters.tab"
|
|
|
67 with open(expClusterFileName, "w") as f:
|
|
|
68 f.write("transib4 \n")
|
|
|
69 f.write("transib2 \n")
|
|
|
70 f.write("transib3 \n")
|
|
|
71 with open(self._expStatFileName, "w") as f:
|
|
|
72 f.write("cluster\tsequencesNb\tsizeOfSmallestSeq\tsizeOfLargestSeq\taverageSize\tmedSize\n")
|
|
|
73 f.write("1\t1\t238\t238\t238\t238\n")
|
|
|
74 f.write("2\t1\t178\t178\t178\t178\n")
|
|
|
75 f.write("3\t1\t90\t90\t90\t90\n")
|
|
|
76 expGlobalStatFileName = "expGlobalStats.txt"
|
|
|
77 with open(expGlobalStatFileName, "w") as f:
|
|
|
78 f.write("nb of clusters: 3\n")
|
|
|
79 f.write("nb of clusters with 1 sequence: 3\n")
|
|
|
80 f.write("nb of clusters with 2 sequences: 0\n")
|
|
|
81 f.write("nb of clusters with >2 sequences: 0 (0 sequences)\n")
|
|
|
82 f.write("nb of sequences: 3\n")
|
|
|
83 f.write("nb of sequences in the largest cluster: 1\n")
|
|
|
84 f.write("nb of sequences in the smallest cluster: 1\n")
|
|
|
85 f.write("size of the smallest sequence: 90\n")
|
|
|
86 f.write("size of the largest sequence: 238\n")
|
|
|
87 f.write("average sequences size: 168\n")
|
|
|
88 f.write("median sequences size: 178\n")
|
|
|
89 obsClusterFileName = "TElib.tab"
|
|
|
90 obsGlobalStatFileName = "TElib.globalStatsPerCluster.txt"
|
|
|
91 self._obsStatFileName = "TElib.statsPerCluster.tab"
|
|
|
92
|
|
|
93 cmd = "PostAnalyzeTELib.py -i %s -L 98 -S 95 -b -c -v 3" % libFileName
|
|
|
94 process = subprocess.Popen(cmd, shell = True)
|
|
|
95 process.communicate()
|
|
|
96
|
|
|
97 self.assertTrue(FileUtils.are2FilesIdentical(expClusterFileName, obsClusterFileName))
|
|
|
98 self.assertTrue(FileUtils.are2FilesIdentical(expGlobalStatFileName, obsGlobalStatFileName))
|
|
|
99 self.assertTrue(FileUtils.are2FilesIdentical(self._expStatFileName, self._obsStatFileName))
|
|
|
100
|
|
|
101 os.remove(libFileName)
|
|
|
102 os.remove(expClusterFileName)
|
|
|
103 os.remove(expGlobalStatFileName)
|
|
|
104 os.remove(obsClusterFileName)
|
|
|
105 os.remove(obsGlobalStatFileName)
|
|
|
106
|
|
|
107 def test_run_analysis2(self):
|
|
|
108 clusterFileName = "ConsensusClusters.tab"
|
|
|
109 self._writeClusterFile_analysis2(clusterFileName)
|
|
|
110 self._writeExpStatsFile_analysis2(self._expStatFileName)
|
|
|
111 self._obsStatFileName = "ConsensusClusters.classifStatsPerCluster.tab"
|
|
|
112
|
|
|
113 iPATEL = PostAnalyzeTELib(analysis=2, clusterFileName=clusterFileName, verbosity=3)
|
|
|
114 iPATEL.run()
|
|
|
115 print "exp: %s, obs: %s" % (self._expStatFileName, self._obsStatFileName)
|
|
|
116 self.assertTrue(FileUtils.are2FilesIdentical(self._expStatFileName, self._obsStatFileName))
|
|
|
117
|
|
|
118 os.remove(clusterFileName)
|
|
|
119
|
|
|
120 def test_run_analysis3(self):
|
|
|
121 iDb = DbFactory.createInstance()
|
|
|
122 iDb.createTable(self._pathTableName, "path", "%s/Tools/DmelChr4_chr_allTEs_nr_noSSR_join_path.path" % os.environ['REPET_DATA'], True)
|
|
|
123 iDb.createTable(self._seqTableName, "seq", "%s/TEannot/DmelChr4_denovoLibTEs.fa" % os.environ['REPET_DATA'], True)
|
|
|
124
|
|
|
125 expGlobalStatFileName = "expGlobalStats.txt"
|
|
|
126 self._writeExpGlobalStats_analysis3(expGlobalStatFileName)
|
|
|
127 self._writeExpStatsFile_analysis3(self._expStatFileName)
|
|
|
128 obsGlobalStatFileName = "%s.globalAnnotStatsPerTE.txt" % self._pathTableName
|
|
|
129 self._obsStatFileName = "%s.annotStatsPerTE.tab" % self._pathTableName
|
|
|
130
|
|
|
131 iPATEL = PostAnalyzeTELib(analysis=3, pathTableName=self._pathTableName, seqTableName=self._seqTableName, genomeSize=self._genomeSize)
|
|
|
132 iPATEL.run()
|
|
|
133
|
|
|
134 self.assertTrue(FileUtils.are2FilesIdentical(expGlobalStatFileName, obsGlobalStatFileName))
|
|
|
135 self.assertTrue(FileUtils.are2FilesIdentical(self._expStatFileName, self._obsStatFileName))
|
|
|
136
|
|
|
137 iDb.dropTable(self._pathTableName)
|
|
|
138 iDb.dropTable(self._seqTableName)
|
|
|
139 iDb.close()
|
|
|
140 os.remove(expGlobalStatFileName)
|
|
|
141 os.remove(obsGlobalStatFileName)
|
|
|
142
|
|
|
143 def test_run_analysis4(self):
|
|
|
144 iDb = DbFactory.createInstance()
|
|
|
145 iDb.createTable(self._pathTableName, "path", "%s/Tools/DmelChr4_chr_allTEs_nr_noSSR_join_path.path" % os.environ['REPET_DATA'], True)
|
|
|
146 iDb.createTable(self._seqTableName, "seq", "%s/TEannot/DmelChr4_denovoLibTEs.fa" % os.environ['REPET_DATA'], True)
|
|
|
147
|
|
|
148 clusterFileName = "clusters.tab"
|
|
|
149 self._writeClusterFile_analysis4(clusterFileName)
|
|
|
150 self._writeExpStatsFile_analysis4(self._expStatFileName)
|
|
|
151 self._obsStatFileName = "%s.annotStatsPerCluster.tab" % self._pathTableName
|
|
|
152
|
|
|
153 iPATEL = PostAnalyzeTELib(analysis=4, clusterFileName=clusterFileName, pathTableName=self._pathTableName, seqTableName=self._seqTableName, genomeSize=self._genomeSize)
|
|
|
154 iPATEL.run()
|
|
|
155
|
|
|
156 self.assertTrue(FileUtils.are2FilesIdentical(self._expStatFileName, self._obsStatFileName))
|
|
|
157
|
|
|
158 iDb.dropTable(self._pathTableName)
|
|
|
159 iDb.dropTable(self._seqTableName)
|
|
|
160 iDb.close()
|
|
|
161 os.remove(clusterFileName)
|
|
|
162
|
|
|
163 def _writeInputFasta_analysis1(self, fileName):
|
|
|
164 with open(fileName, "w") as f:
|
|
|
165 f.write(">DTX-incomp_DmelChr4-B-R9-Map3_reversed\n")
|
|
|
166 f.write("CATTAGATTCAAGGCATCATGGATCAGCACATTTACACAGATATCCTGGAAAATGTGATG\n")
|
|
|
167 f.write("CTGCCATATGCCGGGGATGAAATGCCGTTGGTTTGGACATTTCAACAGGATAACGATTCA\n")
|
|
|
168 f.write("AAACACACGAGCAAGAAAGCTTGAAAGTGGTTTGAGCAGAAATCGATCCGAGTAATGAAA\n")
|
|
|
169 f.write("TGGCCTGCTCTGTCATCCGACTTGAATCCAATCGAAAACCTTTGGGCGGACGTGGAAAAA\n")
|
|
|
170 f.write(">DTX-incomp_DmelChr4-B-R10-Map3\n")
|
|
|
171 f.write("CATTAGATTCAAGGCATCATGGATCAGCACATTTACACAGATATCCTGGAAAATGTGATG\n")
|
|
|
172 f.write("CTGCCATATGCCGGGGATGAAATGCCGTTGGTTTGGACATTTCAACAGGATAACGATTCA\n")
|
|
|
173 f.write("AAACACACGAGCAAGAAAGCTTGAAAGTGGTTTGAGCAGAAATCGATCCGAGTAATGAAA\n")
|
|
|
174 f.write("TGGCCTGCTCTGTCATCCGACTTGAATCCAATCGAAAACCTTTGGGCGGACGTGGAAAAA\n")
|
|
|
175 f.write(">PotentialHostGene-chim_DmelChr4-B-R4-Map5_reversed\n")
|
|
|
176 f.write("TACCAAAGACACTAGAATAACAAGATGCGTAACGCCATACGATTTTTTGGCACACGATTT\n")
|
|
|
177 f.write("TTTCGCCGTGGCTCTAGAGGTGGCTCCAGGCTCTCTCGAATTTTTGTTAGAGAGCGAGAG\n")
|
|
|
178 f.write("AGCGGAGAGCGCTACAGCGAACAGCTCTTTTCAACGCATAAAGTGATAGCAGACAACTGT\n")
|
|
|
179
|
|
|
180 def _writeExpClusterFile_analysis1(self, fileName):
|
|
|
181 with open(fileName, "w") as f:
|
|
|
182 f.write("DTX-incomp_DmelChr4-B-R10-Map3 DTX-incomp_DmelChr4-B-R9-Map3_reversed \n")
|
|
|
183 f.write("PotentialHostGene-chim_DmelChr4-B-R4-Map5_reversed \n")
|
|
|
184
|
|
|
185 def _writeExpStatsFile_analysis1(self, fileName):
|
|
|
186 #TODO: header in option ?
|
|
|
187 with open(fileName, "w") as f:
|
|
|
188 f.write("cluster\tsequencesNb\tsizeOfSmallestSeq\tsizeOfLargestSeq\taverageSize\tmedSize\n")
|
|
|
189 f.write("1\t2\t240\t240\t240\t240\n")
|
|
|
190 f.write("2\t1\t180\t180\t180\t180\n")
|
|
|
191
|
|
|
192 def _writeExpGlobalStats_analysis1(self, fileName):
|
|
|
193 #TODO: file or STDOUT ?
|
|
|
194 with open(fileName, "w") as f:
|
|
|
195 f.write("nb of clusters: 2\n")
|
|
|
196 f.write("nb of clusters with 1 sequence: 1\n")
|
|
|
197 f.write("nb of clusters with 2 sequences: 1\n")
|
|
|
198 f.write("nb of clusters with >2 sequences: 0 (0 sequences)\n")
|
|
|
199 f.write("nb of sequences: 3\n")
|
|
|
200 f.write("nb of sequences in the largest cluster: 2\n")
|
|
|
201 f.write("nb of sequences in the smallest cluster: 1\n")
|
|
|
202 f.write("size of the smallest sequence: 180\n")
|
|
|
203 f.write("size of the largest sequence: 240\n")
|
|
|
204 f.write("average sequences size: 220\n")
|
|
|
205 f.write("median sequences size: 240\n")
|
|
|
206
|
|
|
207 def _writeClusterFile_analysis2(self, fileName):
|
|
|
208 with open(fileName, "w") as f:
|
|
|
209 f.write("DTX-incomp_DmelChr4-B-R9-Map3_reversed\tDTX-incomp_DmelChr4-B-R10-Map3_reversed\tPotentialHostGene-chim_DmelChr4-B-R4-Map5_reversed\n")
|
|
|
210 f.write("DTX-incomp_Blc1_DmelChr4-B-R9-Map3_reversed\tDTX-incomp_Blc1_DmelChr4-B-R10-Map3\n")
|
|
|
211 f.write("DXX-comp_DmelChr4-B-R9-Map3\tDTX-comp_DmelChr4-B-R10-Map3_reversed\tDTX-incomp_DmelChr4-B-R10-Map3_reversed\tnoCat_DmelChr4-B-G1-Map3\tnoCat_DmelChr4-B-R1-Map4\n")
|
|
|
212 f.write("RXX-MITE_DmelChr4-B-G7-Map3\tRXX-MITE_DmelChr4-B-G5-Map3\tRXX-MITE_DmelChr4-B-G2-Map3\tRXX-MITE_DmelChr4-B-G23-Map3\tRXX-MITE_DmelChr4-B-G6-Map3\n")
|
|
|
213
|
|
|
214 def _writeExpStatsFile_analysis2(self, fileName):
|
|
|
215 #TODO: header in option ?
|
|
|
216 with open(fileName, "w") as f:
|
|
|
217 f.write("cluster\tnoCat\tPotentialChimeric\tcomp\tincomp\tclassifs (nbTEs)\n")
|
|
|
218 f.write("1\t0\t1\t0\t2\tDTX (2)\tPotentialHostGene (1)\n")
|
|
|
219 f.write("2\t0\t0\t0\t2\tDTX (2)\n")
|
|
|
220 f.write("3\t2\t0\t2\t1\tDTX (2)\tDXX (1)\n")
|
|
|
221 f.write("4\t0\t0\t0\t0\tMITE (5)\n")
|
|
|
222
|
|
|
223 def _writeExpStatsFile_analysis3(self, fileName):
|
|
|
224 #TODO: header in option ?
|
|
|
225 with open(fileName, "w") as f:
|
|
|
226 f.write("TE\tlength\tcovg\tfrags\tfullLgthFrags\tcopies\tfullLgthCopies\tmeanId\tmeanLgth\tmeanLgthPerc\n")
|
|
|
227 f.write("DmelChr4-B-G1-Map3_NoCat\t542\t3701\t12\t4\t10\t4\t95.72\t370.10\t68.28\n")
|
|
|
228 f.write("DmelChr4-B-G11-Map20_classII-TIR-incomp\t1240\t8216\t27\t0\t22\t0\t88.80\t375.00\t30.24\n")
|
|
|
229 f.write("DmelChr4-B-G7-Map3_classII-TIR-incomp\t1944\t15212\t49\t1\t42\t1\t89.44\t382.36\t19.67\n")
|
|
|
230 f.write("DmelChr4-B-G9-Map3_NoCat\t1590\t11564\t24\t0\t21\t1\t92.03\t550.67\t34.63\n")
|
|
|
231 f.write("DmelChr4-B-P0.0-Map3_classII-TIR-incomp\t1042\t4001\t13\t3\t11\t3\t85.11\t366.36\t35.16\n")
|
|
|
232 f.write("DmelChr4-B-R1-Map4_NoCat\t2367\t66031\t484\t0\t361\t0\t77.84\t182.91\t7.73\n")
|
|
|
233 f.write("DmelChr4-B-R12-Map3_NoCat\t2284\t4938\t3\t2\t3\t2\t99.26\t1646.00\t72.07\n")
|
|
|
234 f.write("DmelChr4-B-R19-Map4_NoCat\t705\t3328\t10\t3\t10\t3\t88.51\t332.80\t47.21\n")
|
|
|
235 f.write("DmelChr4-B-R2-Map6_NoCat\t4638\t20539\t34\t2\t29\t3\t80.93\t708.24\t15.27\n")
|
|
|
236 f.write("DmelChr4-B-R4-Map5_NoCat\t1067\t7292\t35\t1\t28\t1\t86.50\t260.54\t24.42\n")
|
|
|
237 f.write("DmelChr4-B-R9-Map3_NoCat\t714\t5453\t19\t2\t16\t2\t81.18\t340.81\t47.73\n")
|
|
|
238
|
|
|
239 def _writeExpGlobalStats_analysis3(self, fileName):
|
|
|
240 with open(fileName, "w") as f:
|
|
|
241 f.write("nb of sequences: 11\n")
|
|
|
242 f.write("nb of matched sequences: 11\n")
|
|
|
243 f.write("cumulative coverage: 150275 bp\n")
|
|
|
244 f.write("coverage percentage: 11.73%\n")
|
|
|
245 f.write("\n")
|
|
|
246 f.write("total nb of TE fragments: 710\n")
|
|
|
247 f.write("total nb full-length fragments: 18 (2.54%)\n")
|
|
|
248 f.write("total nb of TE copies: 553\n")
|
|
|
249 f.write("total nb full-length copies: 20 (3.62%)\n")
|
|
|
250 f.write("families with full-length fragments: 8 (72.73%)\n")
|
|
|
251 f.write(" with only one full-length fragment: 2\n")
|
|
|
252 f.write(" with only two full-length fragments: 3\n")
|
|
|
253 f.write(" with only three full-length fragments: 2\n")
|
|
|
254 f.write(" with more than three full-length fragments: 1\n")
|
|
|
255 f.write("families with full-length copies: 9 (81.82%)\n")
|
|
|
256 f.write(" with only one full-length copy: 3\n")
|
|
|
257 f.write(" with only two full-length copies: 2\n")
|
|
|
258 f.write(" with only three full-length copies: 3\n")
|
|
|
259 f.write(" with more than three full-length copies: 1\n")
|
|
|
260 f.write("mean of median identity of all families: 88.30 +- 8.33\n")
|
|
|
261 f.write("mean of median length percentage of all families: 30.83 +- 32.30\n")
|
|
|
262
|
|
|
263 def _writeClusterFile_analysis4(self, fileName):
|
|
|
264 with open(fileName, "w") as f:
|
|
|
265 f.write("1\tDmelChr4-B-R1-Map4_NoCat\tDmelChr4-B-R2-Map6_NoCat\tDmelChr4-B-R4-Map5_NoCat\n")
|
|
|
266 f.write("2\tDmelChr4-B-G7-Map3_classII-TIR-incomp\tDmelChr4-B-P0.0-Map3_classII-TIR-incomp\n")
|
|
|
267
|
|
|
268 def _writeExpStatsFile_analysis4(self, fileName):
|
|
|
269 with open(fileName, "w") as f:
|
|
|
270 f.write("Cluster\tcovg\tfrags\tcopies\n")
|
|
|
271 f.write("1\t93862\t553\t418\n")
|
|
|
272 f.write("2\t19213\t62\t53\n")
|
|
|
273
|
|
|
274 def _writeConfigFile(self, configFileName):
|
|
|
275 with open(configFileName, "w") as fHandle:
|
|
|
276 fHandle.write("[repet_env]\n")
|
|
|
277 fHandle.write("repet_host: %s\n" % os.environ["REPET_HOST"])
|
|
|
278 fHandle.write("repet_user: %s\n" % os.environ["REPET_USER"])
|
|
|
279 fHandle.write("repet_pw: %s\n" % os.environ["REPET_PW"])
|
|
|
280 fHandle.write("repet_db: %s\n" % os.environ["REPET_DB"])
|
|
|
281 fHandle.write("repet_port: 3306\n")
|
|
|
282 fHandle.write("[analysis1]\n")
|
|
|
283 fHandle.write("fasta_name: %s\n" % self._expStatFileName)
|
|
|
284
|
|
|
285 fHandle.write("[analysis2]\n")
|
|
|
286 fHandle.write("clusterFileName: %s\n" % self._expStatFileName)
|
|
|
287
|
|
|
288 fHandle.write("[analysis3]\n")
|
|
|
289 fHandle.write("pathTableName: %s\n" % self._pathTableName)
|
|
|
290 fHandle.write("seqTableName: %s\n" % self._seqTableName)
|
|
|
291 fHandle.write("genomeSize: %s\n" % self._genomeSize)
|
|
|
292
|
|
|
293 if __name__ == "__main__":
|
|
|
294 unittest.main() |
