annotate SMART/Java/Python/ncList/NCList.py @ 71:d96f6c9a39e0 draft default tip

Removed pyc files.
author m-zytnicki
date Thu, 07 Apr 2016 09:25:18 -0400
parents 169d364ddd91
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
6
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
1 #! /usr/bin/env python
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
2 #
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
3 # Copyright INRA-URGI 2009-2010
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
4 #
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
5 # This software is governed by the CeCILL license under French law and
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
6 # abiding by the rules of distribution of free software. You can use,
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
7 # modify and/ or redistribute the software under the terms of the CeCILL
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
8 # license as circulated by CEA, CNRS and INRIA at the following URL
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
9 # "http://www.cecill.info".
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
10 #
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
11 # As a counterpart to the access to the source code and rights to copy,
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
12 # modify and redistribute granted by the license, users are provided only
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
13 # with a limited warranty and the software's author, the holder of the
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
14 # economic rights, and the successive licensors have only limited
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
15 # liability.
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
16 #
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
17 # In this respect, the user's attention is drawn to the risks associated
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
18 # with loading, using, modifying and/or developing or reproducing the
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
19 # software by the user in light of its specific status of free software,
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
20 # that may mean that it is complicated to manipulate, and that also
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
21 # therefore means that it is reserved for developers and experienced
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
22 # professionals having in-depth computer knowledge. Users are therefore
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
23 # encouraged to load and test the software's suitability as regards their
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
24 # requirements in conditions enabling the security of their systems and/or
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
25 # data to be ensured and, more generally, to use and operate it in the
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
26 # same conditions as regards security.
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
27 #
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
28 # The fact that you are presently reading this means that you have had
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
29 # knowledge of the CeCILL license and that you accept its terms.
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
30 #
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
31 import os, os.path
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
32 import struct
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
33 import shelve
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
34 import sys
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
35 from SMART.Java.Python.ncList.NCListFilePickle import NCListFilePickle, NCListFileUnpickle
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
36 from SMART.Java.Python.ncList.NCIndex import NCIndex
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
37 from SMART.Java.Python.misc.Progress import Progress
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
38
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
39 LONG_SIZE = struct.calcsize('l')
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
40
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
41 H = 0
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
42 L = 1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
43 T = 2
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
44 G = 3
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
45
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
46 H_CELL_SIZE = 2
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
47 L_CELL_SIZE = 5
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
48 T_CELL_SIZE = 6
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
49
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
50 START = 0
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
51 END = 1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
52 ADDRESS = 2
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
53 LIST = 3
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
54 PARENT = 4
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
55 NEW = 5
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
56 LENGTH = 1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
57
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
58 def pack(input):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
59 return struct.pack("l", long(input))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
60 def unpack(input):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
61 return struct.unpack("l", input)[0]
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
62
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
63
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
64 class NCList(object):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
65
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
66 def __init__(self, verbosity):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
67 self._verbosity = verbosity
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
68 self._subPos = 0
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
69 self._parentPos = 0
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
70 self._nbLines = 0
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
71 self._nbLists = 0
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
72 self._chromosome = None
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
73 self._transcriptFileName = None
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
74 self._lHandle = None
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
75 self._hHandle = None
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
76 self._tHandle = None
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
77 self._parser = None
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
78 self._sizeDict = {H: H_CELL_SIZE, L: L_CELL_SIZE, T: T_CELL_SIZE}
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
79 self._offsets = {H: 0, L: 0, G: 0}
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
80 self._fileNameDict = {}
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
81 self._handleDict = {}
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
82 self._createIndex = False
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
83 self._missingValues = dict([table, {}] for table in self._sizeDict)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
84 self._missingValues[T][LIST] = -1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
85 self._missingValues[L][LIST] = 0
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
86 self._missingValues[T][NEW] = -1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
87
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
88 def __del__(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
89 for handle in (self._lHandle, self._hHandle):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
90 if handle != None:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
91 handle.close()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
92
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
93 def createIndex(self, boolean):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
94 self._createIndex = boolean
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
95
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
96 def setChromosome(self, chromosome):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
97 self._chromosome = chromosome
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
98
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
99 def setFileName(self, fileName):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
100 self._transcriptFileName = fileName
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
101 self._parser = NCListFileUnpickle(fileName, self._verbosity)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
102 self._setFileNames(fileName)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
103
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
104 def setNbElements(self, nbElements):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
105 self._nbLines = nbElements
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
106
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
107 def setOffset(self, fileType, offset):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
108 self._offsets[fileType] = offset
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
109
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
110 def _setFileNames(self, fileName):
46
169d364ddd91 Uploaded
m-zytnicki
parents: 6
diff changeset
111 print "Got file name", fileName
6
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
112 if self._chromosome != None and fileName != None:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
113 coreName = os.path.splitext(fileName)[0]
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
114 if "SMARTTMPPATH" in os.environ:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
115 coreName = os.path.join(os.environ["SMARTTMPPATH"], coreName)
46
169d364ddd91 Uploaded
m-zytnicki
parents: 6
diff changeset
116 print "Used core name", coreName
6
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
117 self._hFileName = "%s_H.bin" % (coreName)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
118 self._lFileName = "%s_L.bin" % (coreName)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
119 self._tFileName = "%s_T.bin" % (coreName)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
120 self._fileNameDict = {H: self._hFileName, L: self._lFileName, T: self._tFileName}
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
121
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
122 def getSizeFirstList(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
123 return self._sizeFirstList
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
124
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
125 def _writeSubListIntoH(self, SubListAddr, SubListLength):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
126 self._hHandle.write(pack(SubListAddr))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
127 self._hHandle.write(pack(SubListLength))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
128 self._subPos += H_CELL_SIZE
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
129
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
130 def _writeParentIntoL(self, readAddr, subListAddr, parentAddr, start, end):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
131 self._lHandle.write(pack(start))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
132 self._lHandle.write(pack(end))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
133 self._lHandle.write(pack(readAddr))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
134 self._lHandle.write(pack(subListAddr))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
135 self._lHandle.write(pack(parentAddr))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
136 self._parentPos += L_CELL_SIZE
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
137
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
138 def getLLineElements(self, subListLAddr):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
139 if subListLAddr == -1 or subListLAddr == None:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
140 #print "reading bad from L", subListLAddr
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
141 return -1, -1, -1, -1, -1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
142 else:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
143 self._lHandle.seek(subListLAddr * L_CELL_SIZE * LONG_SIZE + self._offsets[L])
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
144 start = self._lHandle.read(LONG_SIZE)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
145 if len(start) < LONG_SIZE:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
146 #print "reading very bad from L", subListLAddr
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
147 return -1, -1, -1, -1, -1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
148 start = unpack(start)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
149 end = unpack(self._lHandle.read(LONG_SIZE))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
150 gff3Addr = unpack(self._lHandle.read(LONG_SIZE))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
151 subListHAddr = unpack(self._lHandle.read(LONG_SIZE))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
152 parentLAddr = unpack(self._lHandle.read(LONG_SIZE))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
153 #print "reading from L", subListLAddr, "-->", gff3Addr, subListHAddr, parentLAddr, start, end
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
154 return gff3Addr, subListHAddr, parentLAddr, start, end
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
155
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
156 def getHLineElements(self, subListHAddr):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
157 self._hHandle.seek(subListHAddr * H_CELL_SIZE * LONG_SIZE + self._offsets[H])
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
158 subListStartBin = self._hHandle.read(LONG_SIZE)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
159 if len(subListStartBin) < 8 :
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
160 #print "reading bad from H"
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
161 return -1, -1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
162 subListStart = unpack(subListStartBin)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
163 subListElementsNb = unpack(self._hHandle.read(LONG_SIZE))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
164 #print "reading from H", subListHAddr, "-->", subListStart, subListElementsNb
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
165 return subListStart, subListElementsNb
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
166
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
167 def getRefGffAddr(self, currentRefLAddr):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
168 RefGff3Addr, subListHAddr, parentLAddr, start, end = self.getLLineElements(currentRefLAddr)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
169 return RefGff3Addr
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
170
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
171 def getIntervalFromAdress(self, address):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
172 self._parser.gotoAddress(int(address) + self._offsets[G])
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
173 iTranscrit = self._parser.getNextTranscript()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
174 return iTranscrit
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
175
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
176 def removeFiles(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
177 return
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
178
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
179 def buildLists(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
180 if self._createIndex:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
181 self._index = NCIndex(self._verbosity)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
182 self._createTables()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
183 self._labelLists()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
184 self._computeSubStart()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
185 self._computeAbsPosition()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
186 self._cleanFiles()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
187
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
188 def _createTables(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
189 self._initLists()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
190 self._createTable(H, self._nbLists)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
191 self._createTable(T, self._nbLines)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
192 self._createTable(L, self._nbLines)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
193 self._fillTables()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
194
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
195 def _initLists(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
196 previousTranscript = None
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
197 self._nbLists = 1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
198 progress = Progress(self._nbLines, "Initializing lists", self._verbosity-5)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
199 for transcript in self._parser.getIterator():
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
200 if self._isIncluded(transcript, previousTranscript):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
201 self._nbLists += 1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
202 previousTranscript = transcript
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
203 progress.inc()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
204 progress.done()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
205
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
206 def _isIncluded(self, transcript1, transcript2):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
207 return transcript1 != None and transcript2 != None and transcript1.getStart() >= transcript2.getStart() and transcript1.getEnd() <= transcript2.getEnd()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
208
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
209 def _createTable(self, name, size):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
210 handle = open(self._fileNameDict[name], "w+b")
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
211 progress = Progress(self._sizeDict[name] * size, "Initializing table %d" % (name), self._verbosity-5)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
212 for i in xrange(self._sizeDict[name] * size):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
213 handle.write(pack(-1))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
214 progress.inc()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
215 progress.done()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
216 self._handleDict[name] = handle
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
217
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
218 def _fillTables(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
219 progress = Progress(self._nbLines, "Filling table T", self._verbosity-5)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
220 for i, transcript in enumerate(self._parser.getIterator()):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
221 self._writeValue(T, i, START, transcript.getStart())
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
222 self._writeValue(T, i, END, transcript.getEnd())
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
223 self._writeValue(T, i, ADDRESS, self._parser.getCurrentTranscriptAddress())
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
224 self._writeValue(T, i, PARENT, -1)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
225 self._writeValue(T, i, LIST, -1)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
226 progress.inc()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
227 progress.done()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
228 progress = Progress(self._nbLists, "Filling table H", self._verbosity-5)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
229 for i in xrange(self._nbLists):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
230 self._writeValue(H, i, LENGTH, 0)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
231 progress.inc()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
232 progress.done()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
233
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
234 def _labelLists(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
235 progress = Progress(self._nbLines, "Getting table structure", self._verbosity-5)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
236 nextL = 0
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
237 for i in xrange(self._nbLines):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
238 p = i - 1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
239 start = self._readValue(T, i, START)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
240 end = self._readValue(T, i, END)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
241 while p != -1 and (start < self._readValue(T, p, START) or end > self._readValue(T, p, END)):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
242 p = self._readValue(T, p, PARENT)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
243 thisL = self._readValue(T, p, LIST)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
244 if thisL == -1:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
245 #print "entering"
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
246 thisL = nextL
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
247 nextL += 1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
248 length = 0
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
249 self._writeValue(T, p, LIST, thisL)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
250 else:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
251 length = self._readValue(H, thisL, LENGTH)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
252 self._writeValue(T, i, PARENT, p)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
253 self._writeValue(H, thisL, LENGTH, length + 1)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
254 progress.inc()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
255 progress.done()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
256
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
257 def _computeSubStart(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
258 progress = Progress(self._nbLines, "Getting table sub-lists", self._verbosity-5)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
259 total = 0
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
260 for i in xrange(self._nbLists):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
261 self._writeValue(H, i, START, total)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
262 total += self._readValue(H, i, LENGTH)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
263 self._writeValue(H, i, LENGTH, 0)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
264 progress.inc()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
265 progress.done()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
266
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
267 def _computeAbsPosition(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
268 progress = Progress(self._nbLines, "Writing table", self._verbosity-5)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
269 self._sizeFirstList = 0
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
270 for i in xrange(self._nbLines):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
271 s = self._readValue(T, i, START)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
272 e = self._readValue(T, i, END)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
273 a = self._readValue(T, i, ADDRESS)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
274 pt = self._readValue(T, i, PARENT)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
275 h = self._readValue(T, pt, LIST)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
276 pl = self._readValue(T, pt, NEW)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
277 nb = self._readValue(H, h, LENGTH)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
278 l = self._readValue(H, h, START) + nb
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
279 self._writeValue(T, i, NEW, l)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
280 self._writeValue(L, l, START, s)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
281 self._writeValue(L, l, END, e)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
282 self._writeValue(L, l, ADDRESS, a)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
283 self._writeValue(L, l, LIST, -1)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
284 self._writeValue(L, l, PARENT, pl)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
285 self._writeValue(H, h, LENGTH, nb+1)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
286 if nb == 0:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
287 #print "adding it"
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
288 self._writeValue(L, pl, LIST, h)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
289 if pl == -1:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
290 self._sizeFirstList += 1
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
291 if self._createIndex:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
292 self._index.addTranscript(e, l)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
293 progress.inc()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
294 progress.done()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
295
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
296 def closeFiles(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
297 for handle in self._handleDict.values():
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
298 handle.close()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
299 del self._handleDict
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
300 self._lHandle = None
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
301 self._hHandle = None
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
302 self._tHandle = None
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
303 self._parser = None
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
304
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
305 def openFiles(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
306 self._lHandle = open(self._fileNameDict[L], "rb")
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
307 self._hHandle = open(self._fileNameDict[H], "rb")
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
308 self._handleDict = {H: self._hHandle, L: self._lHandle}
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
309 self._parser = NCListFileUnpickle(self._transcriptFileName, self._verbosity)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
310
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
311 def _cleanFiles(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
312 self.closeFiles()
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
313 os.remove(self._fileNameDict[T])
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
314
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
315 def _getPosition(self, table, line, key):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
316 handle = self._handleDict[table]
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
317 handle.seek(self._sizeDict[table] * line * LONG_SIZE + key * LONG_SIZE)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
318 return handle
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
319
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
320 def _writeValue(self, table, line, key, value):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
321 #print "writing", table, line, key, "<-", value
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
322 if line == -1:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
323 self._missingValues[table][key] = value
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
324 return
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
325 handle = self._getPosition(table, line, key)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
326 handle.write(pack(value))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
327
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
328 def _readValue(self, table, line, key):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
329 #print "reading", table, line, key, "->",
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
330 if line == -1:
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
331 #print self._missingValues[table][key]
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
332 return self._missingValues[table][key]
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
333 handle = self._getPosition(table, line, key)
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
334 r = unpack(handle.read(LONG_SIZE))
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
335 #print r
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
336 return r
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
337
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
338 def getIndex(self):
769e306b7933 Change the repository level.
yufei-luo
parents:
diff changeset
339 return self._index