0
|
1 /*
|
|
2 * RAZF : Random Access compressed(Z) File
|
|
3 * Version: 1.0
|
|
4 * Release Date: 2008-10-27
|
|
5 *
|
|
6 * Copyright 2008, Jue Ruan <ruanjue@gmail.com>, Heng Li <lh3@sanger.ac.uk>
|
|
7 *
|
|
8 * All rights reserved.
|
|
9 *
|
|
10 * Redistribution and use in source and binary forms, with or without
|
|
11 * modification, are permitted provided that the following conditions
|
|
12 * are met:
|
|
13 * 1. Redistributions of source code must retain the above copyright
|
|
14 * notice, this list of conditions and the following disclaimer.
|
|
15 * 2. Redistributions in binary form must reproduce the above copyright
|
|
16 * notice, this list of conditions and the following disclaimer in the
|
|
17 * documentation and/or other materials provided with the distribution.
|
|
18 *
|
|
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
29 * SUCH DAMAGE.
|
|
30 */
|
|
31
|
|
32 #ifndef _NO_RAZF
|
|
33
|
|
34 #include <fcntl.h>
|
|
35 #include <stdio.h>
|
|
36 #include <stdlib.h>
|
|
37 #include <string.h>
|
|
38 #include <unistd.h>
|
|
39 #include "razf.h"
|
|
40
|
|
41 #if ZLIB_VERNUM < 0x1221
|
|
42 struct _gz_header_s {
|
|
43 int text;
|
|
44 uLong time;
|
|
45 int xflags;
|
|
46 int os;
|
|
47 Bytef *extra;
|
|
48 uInt extra_len;
|
|
49 uInt extra_max;
|
|
50 Bytef *name;
|
|
51 uInt name_max;
|
|
52 Bytef *comment;
|
|
53 uInt comm_max;
|
|
54 int hcrc;
|
|
55 int done;
|
|
56 };
|
|
57 #warning "zlib < 1.2.2.1; RAZF writing is disabled."
|
|
58 #endif
|
|
59
|
|
60 #define DEF_MEM_LEVEL 8
|
|
61
|
|
62 static inline uint32_t byte_swap_4(uint32_t v){
|
|
63 v = ((v & 0x0000FFFFU) << 16) | (v >> 16);
|
|
64 return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8);
|
|
65 }
|
|
66
|
|
67 static inline uint64_t byte_swap_8(uint64_t v){
|
|
68 v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32);
|
|
69 v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16);
|
|
70 return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8);
|
|
71 }
|
|
72
|
|
73 static inline int is_big_endian(){
|
|
74 int x = 0x01;
|
|
75 char *c = (char*)&x;
|
|
76 return (c[0] != 0x01);
|
|
77 }
|
|
78
|
|
79 #ifndef _RZ_READONLY
|
|
80 static void add_zindex(RAZF *rz, int64_t in, int64_t out){
|
|
81 if(rz->index->size == rz->index->cap){
|
|
82 rz->index->cap = rz->index->cap * 1.5 + 2;
|
|
83 rz->index->cell_offsets = realloc(rz->index->cell_offsets, sizeof(int) * rz->index->cap);
|
|
84 rz->index->bin_offsets = realloc(rz->index->bin_offsets, sizeof(int64_t) * (rz->index->cap/RZ_BIN_SIZE + 1));
|
|
85 }
|
|
86 if(rz->index->size % RZ_BIN_SIZE == 0) rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE] = out;
|
|
87 rz->index->cell_offsets[rz->index->size] = out - rz->index->bin_offsets[rz->index->size / RZ_BIN_SIZE];
|
|
88 rz->index->size ++;
|
|
89 }
|
|
90
|
|
91 static void save_zindex(RAZF *rz, int fd){
|
|
92 int32_t i, v32;
|
|
93 int is_be;
|
|
94 is_be = is_big_endian();
|
|
95 if(is_be) write(fd, &rz->index->size, sizeof(int));
|
|
96 else {
|
|
97 v32 = byte_swap_4((uint32_t)rz->index->size);
|
|
98 write(fd, &v32, sizeof(uint32_t));
|
|
99 }
|
|
100 v32 = rz->index->size / RZ_BIN_SIZE + 1;
|
|
101 if(!is_be){
|
|
102 for(i=0;i<v32;i++) rz->index->bin_offsets[i] = byte_swap_8((uint64_t)rz->index->bin_offsets[i]);
|
|
103 for(i=0;i<rz->index->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]);
|
|
104 }
|
|
105 write(fd, rz->index->bin_offsets, sizeof(int64_t) * v32);
|
|
106 write(fd, rz->index->cell_offsets, sizeof(int32_t) * rz->index->size);
|
|
107 }
|
|
108 #endif
|
|
109
|
|
110 static void load_zindex(RAZF *rz, int fd){
|
|
111 int32_t i, v32;
|
|
112 int is_be;
|
|
113 if(!rz->load_index) return;
|
|
114 if(rz->index == NULL) rz->index = malloc(sizeof(ZBlockIndex));
|
|
115 is_be = is_big_endian();
|
|
116 read(fd, &rz->index->size, sizeof(int));
|
|
117 if(!is_be) rz->index->size = byte_swap_4((uint32_t)rz->index->size);
|
|
118 rz->index->cap = rz->index->size;
|
|
119 v32 = rz->index->size / RZ_BIN_SIZE + 1;
|
|
120 rz->index->bin_offsets = malloc(sizeof(int64_t) * v32);
|
|
121 read(fd, rz->index->bin_offsets, sizeof(int64_t) * v32);
|
|
122 rz->index->cell_offsets = malloc(sizeof(int) * rz->index->size);
|
|
123 read(fd, rz->index->cell_offsets, sizeof(int) * rz->index->size);
|
|
124 if(!is_be){
|
|
125 for(i=0;i<v32;i++) rz->index->bin_offsets[i] = byte_swap_8((uint64_t)rz->index->bin_offsets[i]);
|
|
126 for(i=0;i<rz->index->size;i++) rz->index->cell_offsets[i] = byte_swap_4((uint32_t)rz->index->cell_offsets[i]);
|
|
127 }
|
|
128 }
|
|
129
|
|
130 #ifdef _RZ_READONLY
|
|
131 static RAZF* razf_open_w(int fd)
|
|
132 {
|
|
133 fprintf(stderr, "[razf_open_w] Writing is not available with zlib ver < 1.2.2.1\n");
|
|
134 return 0;
|
|
135 }
|
|
136 #else
|
|
137 static RAZF* razf_open_w(int fd){
|
|
138 RAZF *rz;
|
|
139 #ifdef _WIN32
|
|
140 setmode(fd, O_BINARY);
|
|
141 #endif
|
|
142 rz = calloc(1, sizeof(RAZF));
|
|
143 rz->mode = 'w';
|
|
144 rz->filedes = fd;
|
|
145 rz->stream = calloc(sizeof(z_stream), 1);
|
|
146 rz->inbuf = malloc(RZ_BUFFER_SIZE);
|
|
147 rz->outbuf = malloc(RZ_BUFFER_SIZE);
|
|
148 rz->index = calloc(sizeof(ZBlockIndex), 1);
|
|
149 deflateInit2(rz->stream, RZ_COMPRESS_LEVEL, Z_DEFLATED, WINDOW_BITS + 16, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
|
|
150 rz->stream->avail_out = RZ_BUFFER_SIZE;
|
|
151 rz->stream->next_out = rz->outbuf;
|
|
152 rz->header = calloc(sizeof(gz_header), 1);
|
|
153 rz->header->os = 0x03; //Unix
|
|
154 rz->header->text = 0;
|
|
155 rz->header->time = 0;
|
|
156 rz->header->extra = malloc(7);
|
|
157 strncpy((char*)rz->header->extra, "RAZF", 4);
|
|
158 rz->header->extra[4] = 1; // obsolete field
|
|
159 // block size = RZ_BLOCK_SIZE, Big-Endian
|
|
160 rz->header->extra[5] = RZ_BLOCK_SIZE >> 8;
|
|
161 rz->header->extra[6] = RZ_BLOCK_SIZE & 0xFF;
|
|
162 rz->header->extra_len = 7;
|
|
163 rz->header->name = rz->header->comment = 0;
|
|
164 rz->header->hcrc = 0;
|
|
165 deflateSetHeader(rz->stream, rz->header);
|
|
166 rz->block_pos = rz->block_off = 0;
|
|
167 return rz;
|
|
168 }
|
|
169
|
|
170 static void _razf_write(RAZF* rz, const void *data, int size){
|
|
171 int tout;
|
|
172 rz->stream->avail_in = size;
|
|
173 rz->stream->next_in = (void*)data;
|
|
174 while(1){
|
|
175 tout = rz->stream->avail_out;
|
|
176 deflate(rz->stream, Z_NO_FLUSH);
|
|
177 rz->out += tout - rz->stream->avail_out;
|
|
178 if(rz->stream->avail_out) break;
|
|
179 write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
|
|
180 rz->stream->avail_out = RZ_BUFFER_SIZE;
|
|
181 rz->stream->next_out = rz->outbuf;
|
|
182 if(rz->stream->avail_in == 0) break;
|
|
183 };
|
|
184 rz->in += size - rz->stream->avail_in;
|
|
185 rz->block_off += size - rz->stream->avail_in;
|
|
186 }
|
|
187
|
|
188 static void razf_flush(RAZF *rz){
|
|
189 uint32_t tout;
|
|
190 if(rz->buf_len){
|
|
191 _razf_write(rz, rz->inbuf, rz->buf_len);
|
|
192 rz->buf_off = rz->buf_len = 0;
|
|
193 }
|
|
194 if(rz->stream->avail_out){
|
|
195 write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
|
|
196 rz->stream->avail_out = RZ_BUFFER_SIZE;
|
|
197 rz->stream->next_out = rz->outbuf;
|
|
198 }
|
|
199 while(1){
|
|
200 tout = rz->stream->avail_out;
|
|
201 deflate(rz->stream, Z_FULL_FLUSH);
|
|
202 rz->out += tout - rz->stream->avail_out;
|
|
203 if(rz->stream->avail_out == 0){
|
|
204 write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
|
|
205 rz->stream->avail_out = RZ_BUFFER_SIZE;
|
|
206 rz->stream->next_out = rz->outbuf;
|
|
207 } else break;
|
|
208 }
|
|
209 rz->block_pos = rz->out;
|
|
210 rz->block_off = 0;
|
|
211 }
|
|
212
|
|
213 static void razf_end_flush(RAZF *rz){
|
|
214 uint32_t tout;
|
|
215 if(rz->buf_len){
|
|
216 _razf_write(rz, rz->inbuf, rz->buf_len);
|
|
217 rz->buf_off = rz->buf_len = 0;
|
|
218 }
|
|
219 while(1){
|
|
220 tout = rz->stream->avail_out;
|
|
221 deflate(rz->stream, Z_FINISH);
|
|
222 rz->out += tout - rz->stream->avail_out;
|
|
223 if(rz->stream->avail_out < RZ_BUFFER_SIZE){
|
|
224 write(rz->filedes, rz->outbuf, RZ_BUFFER_SIZE - rz->stream->avail_out);
|
|
225 rz->stream->avail_out = RZ_BUFFER_SIZE;
|
|
226 rz->stream->next_out = rz->outbuf;
|
|
227 } else break;
|
|
228 }
|
|
229 }
|
|
230
|
|
231 static void _razf_buffered_write(RAZF *rz, const void *data, int size){
|
|
232 int i, n;
|
|
233 while(1){
|
|
234 if(rz->buf_len == RZ_BUFFER_SIZE){
|
|
235 _razf_write(rz, rz->inbuf, rz->buf_len);
|
|
236 rz->buf_len = 0;
|
|
237 }
|
|
238 if(size + rz->buf_len < RZ_BUFFER_SIZE){
|
|
239 for(i=0;i<size;i++) ((char*)rz->inbuf + rz->buf_len)[i] = ((char*)data)[i];
|
|
240 rz->buf_len += size;
|
|
241 return;
|
|
242 } else {
|
|
243 n = RZ_BUFFER_SIZE - rz->buf_len;
|
|
244 for(i=0;i<n;i++) ((char*)rz->inbuf + rz->buf_len)[i] = ((char*)data)[i];
|
|
245 size -= n;
|
|
246 data += n;
|
|
247 rz->buf_len += n;
|
|
248 }
|
|
249 }
|
|
250 }
|
|
251
|
|
252 int razf_write(RAZF* rz, const void *data, int size){
|
|
253 int ori_size, n;
|
|
254 int64_t next_block;
|
|
255 ori_size = size;
|
|
256 next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE;
|
|
257 while(rz->in + rz->buf_len + size >= next_block){
|
|
258 n = next_block - rz->in - rz->buf_len;
|
|
259 _razf_buffered_write(rz, data, n);
|
|
260 data += n;
|
|
261 size -= n;
|
|
262 razf_flush(rz);
|
|
263 add_zindex(rz, rz->in, rz->out);
|
|
264 next_block = ((rz->in / RZ_BLOCK_SIZE) + 1) * RZ_BLOCK_SIZE;
|
|
265 }
|
|
266 _razf_buffered_write(rz, data, size);
|
|
267 return ori_size;
|
|
268 }
|
|
269 #endif
|
|
270
|
|
271 /* gzip flag byte */
|
|
272 #define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */
|
|
273 #define HEAD_CRC 0x02 /* bit 1 set: header CRC present */
|
|
274 #define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */
|
|
275 #define ORIG_NAME 0x08 /* bit 3 set: original file name present */
|
|
276 #define COMMENT 0x10 /* bit 4 set: file comment present */
|
|
277 #define RESERVED 0xE0 /* bits 5..7: reserved */
|
|
278
|
|
279 static int _read_gz_header(unsigned char *data, int size, int *extra_off, int *extra_len){
|
|
280 int method, flags, n, len;
|
|
281 if(size < 2) return 0;
|
|
282 if(data[0] != 0x1f || data[1] != 0x8b) return 0;
|
|
283 if(size < 4) return 0;
|
|
284 method = data[2];
|
|
285 flags = data[3];
|
|
286 if(method != Z_DEFLATED || (flags & RESERVED)) return 0;
|
|
287 n = 4 + 6; // Skip 6 bytes
|
|
288 *extra_off = n + 2;
|
|
289 *extra_len = 0;
|
|
290 if(flags & EXTRA_FIELD){
|
|
291 if(size < n + 2) return 0;
|
|
292 len = ((int)data[n + 1] << 8) | data[n];
|
|
293 n += 2;
|
|
294 *extra_off = n;
|
|
295 while(len){
|
|
296 if(n >= size) return 0;
|
|
297 n ++;
|
|
298 len --;
|
|
299 }
|
|
300 *extra_len = n - (*extra_off);
|
|
301 }
|
|
302 if(flags & ORIG_NAME) while(n < size && data[n++]);
|
|
303 if(flags & COMMENT) while(n < size && data[n++]);
|
|
304 if(flags & HEAD_CRC){
|
|
305 if(n + 2 > size) return 0;
|
|
306 n += 2;
|
|
307 }
|
|
308 return n;
|
|
309 }
|
|
310
|
|
311 static RAZF* razf_open_r(int fd, int _load_index){
|
|
312 RAZF *rz;
|
|
313 int ext_off, ext_len;
|
|
314 int n, is_be, ret;
|
|
315 int64_t end;
|
|
316 unsigned char c[] = "RAZF";
|
|
317 #ifdef _WIN32
|
|
318 setmode(fd, O_BINARY);
|
|
319 #endif
|
|
320 rz = calloc(1, sizeof(RAZF));
|
|
321 rz->mode = 'r';
|
|
322 rz->filedes = fd;
|
|
323 rz->stream = calloc(sizeof(z_stream), 1);
|
|
324 rz->inbuf = malloc(RZ_BUFFER_SIZE);
|
|
325 rz->outbuf = malloc(RZ_BUFFER_SIZE);
|
|
326 rz->end = rz->src_end = 0x7FFFFFFFFFFFFFFFLL;
|
|
327 n = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE);
|
|
328 ret = _read_gz_header(rz->inbuf, n, &ext_off, &ext_len);
|
|
329 if(ret == 0){
|
|
330 PLAIN_FILE:
|
|
331 rz->in = n;
|
|
332 rz->file_type = FILE_TYPE_PLAIN;
|
|
333 memcpy(rz->outbuf, rz->inbuf, n);
|
|
334 rz->buf_len = n;
|
|
335 free(rz->stream);
|
|
336 rz->stream = NULL;
|
|
337 return rz;
|
|
338 }
|
|
339 rz->header_size = ret;
|
|
340 ret = inflateInit2(rz->stream, -WINDOW_BITS);
|
|
341 if(ret != Z_OK){ inflateEnd(rz->stream); goto PLAIN_FILE;}
|
|
342 rz->stream->avail_in = n - rz->header_size;
|
|
343 rz->stream->next_in = rz->inbuf + rz->header_size;
|
|
344 rz->stream->avail_out = RZ_BUFFER_SIZE;
|
|
345 rz->stream->next_out = rz->outbuf;
|
|
346 rz->file_type = FILE_TYPE_GZ;
|
|
347 rz->in = rz->header_size;
|
|
348 rz->block_pos = rz->header_size;
|
|
349 rz->next_block_pos = rz->header_size;
|
|
350 rz->block_off = 0;
|
|
351 if(ext_len < 7 || memcmp(rz->inbuf + ext_off, c, 4) != 0) return rz;
|
|
352 if(((((unsigned char*)rz->inbuf)[ext_off + 5] << 8) | ((unsigned char*)rz->inbuf)[ext_off + 6]) != RZ_BLOCK_SIZE){
|
|
353 fprintf(stderr, " -- WARNING: RZ_BLOCK_SIZE is not %d, treat source as gz file. in %s -- %s:%d --\n", RZ_BLOCK_SIZE, __FUNCTION__, __FILE__, __LINE__);
|
|
354 return rz;
|
|
355 }
|
|
356 rz->load_index = _load_index;
|
|
357 rz->file_type = FILE_TYPE_RZ;
|
|
358 if(lseek(fd, -16, SEEK_END) == -1){
|
|
359 UNSEEKABLE:
|
|
360 rz->seekable = 0;
|
|
361 rz->index = NULL;
|
|
362 rz->src_end = rz->end = 0x7FFFFFFFFFFFFFFFLL;
|
|
363 } else {
|
|
364 is_be = is_big_endian();
|
|
365 rz->seekable = 1;
|
|
366 read(fd, &end, sizeof(int64_t));
|
|
367 if(!is_be) rz->src_end = (int64_t)byte_swap_8((uint64_t)end);
|
|
368 else rz->src_end = end;
|
|
369 read(fd, &end, sizeof(int64_t));
|
|
370 if(!is_be) rz->end = (int64_t)byte_swap_8((uint64_t)end);
|
|
371 else rz->end = end;
|
|
372 if(n > rz->end){
|
|
373 rz->stream->avail_in -= n - rz->end;
|
|
374 n = rz->end;
|
|
375 }
|
|
376 if(rz->end > rz->src_end){
|
|
377 lseek(fd, rz->in, SEEK_SET);
|
|
378 goto UNSEEKABLE;
|
|
379 }
|
|
380 if(lseek(fd, rz->end, SEEK_SET) != rz->end){
|
|
381 lseek(fd, rz->in, SEEK_SET);
|
|
382 goto UNSEEKABLE;
|
|
383 }
|
|
384 load_zindex(rz, fd);
|
|
385 lseek(fd, n, SEEK_SET);
|
|
386 }
|
|
387 return rz;
|
|
388 }
|
|
389
|
|
390 RAZF* razf_dopen(int fd, const char *mode){
|
|
391 if(strstr(mode, "r")) return razf_open_r(fd, 1);
|
|
392 else if(strstr(mode, "w")) return razf_open_w(fd);
|
|
393 else return NULL;
|
|
394 }
|
|
395
|
|
396 RAZF* razf_dopen2(int fd, const char *mode)
|
|
397 {
|
|
398 if(strstr(mode, "r")) return razf_open_r(fd, 0);
|
|
399 else if(strstr(mode, "w")) return razf_open_w(fd);
|
|
400 else return NULL;
|
|
401 }
|
|
402
|
|
403 static inline RAZF* _razf_open(const char *filename, const char *mode, int _load_index){
|
|
404 int fd;
|
|
405 RAZF *rz;
|
|
406 if(strstr(mode, "r")){
|
|
407 #ifdef _WIN32
|
|
408 fd = open(filename, O_RDONLY | O_BINARY);
|
|
409 #else
|
|
410 fd = open(filename, O_RDONLY);
|
|
411 #endif
|
|
412 rz = razf_open_r(fd, _load_index);
|
|
413 } else if(strstr(mode, "w")){
|
|
414 #ifdef _WIN32
|
|
415 fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644);
|
|
416 #else
|
|
417 fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
|
418 #endif
|
|
419 rz = razf_open_w(fd);
|
|
420 } else return NULL;
|
|
421 return rz;
|
|
422 }
|
|
423
|
|
424 RAZF* razf_open(const char *filename, const char *mode){
|
|
425 return _razf_open(filename, mode, 1);
|
|
426 }
|
|
427
|
|
428 RAZF* razf_open2(const char *filename, const char *mode){
|
|
429 return _razf_open(filename, mode, 0);
|
|
430 }
|
|
431
|
|
432 int razf_get_data_size(RAZF *rz, int64_t *u_size, int64_t *c_size){
|
|
433 int64_t n;
|
|
434 if(rz->mode != 'r' && rz->mode != 'R') return 0;
|
|
435 switch(rz->file_type){
|
|
436 case FILE_TYPE_PLAIN:
|
|
437 if(rz->end == 0x7fffffffffffffffLL){
|
|
438 if((n = lseek(rz->filedes, 0, SEEK_CUR)) == -1) return 0;
|
|
439 rz->end = lseek(rz->filedes, 0, SEEK_END);
|
|
440 lseek(rz->filedes, n, SEEK_SET);
|
|
441 }
|
|
442 *u_size = *c_size = rz->end;
|
|
443 return 1;
|
|
444 case FILE_TYPE_GZ:
|
|
445 return 0;
|
|
446 case FILE_TYPE_RZ:
|
|
447 if(rz->src_end == rz->end) return 0;
|
|
448 *u_size = rz->src_end;
|
|
449 *c_size = rz->end;
|
|
450 return 1;
|
|
451 default:
|
|
452 return 0;
|
|
453 }
|
|
454 }
|
|
455
|
|
456 static int _razf_read(RAZF* rz, void *data, int size){
|
|
457 int ret, tin;
|
|
458 if(rz->z_eof || rz->z_err) return 0;
|
|
459 if (rz->file_type == FILE_TYPE_PLAIN) {
|
|
460 ret = read(rz->filedes, data, size);
|
|
461 if (ret == 0) rz->z_eof = 1;
|
|
462 return ret;
|
|
463 }
|
|
464 rz->stream->avail_out = size;
|
|
465 rz->stream->next_out = data;
|
|
466 while(rz->stream->avail_out){
|
|
467 if(rz->stream->avail_in == 0){
|
|
468 if(rz->in >= rz->end){ rz->z_eof = 1; break; }
|
|
469 if(rz->end - rz->in < RZ_BUFFER_SIZE){
|
|
470 rz->stream->avail_in = read(rz->filedes, rz->inbuf, rz->end -rz->in);
|
|
471 } else {
|
|
472 rz->stream->avail_in = read(rz->filedes, rz->inbuf, RZ_BUFFER_SIZE);
|
|
473 }
|
|
474 if(rz->stream->avail_in == 0){
|
|
475 rz->z_eof = 1;
|
|
476 break;
|
|
477 }
|
|
478 rz->stream->next_in = rz->inbuf;
|
|
479 }
|
|
480 tin = rz->stream->avail_in;
|
|
481 ret = inflate(rz->stream, Z_BLOCK);
|
|
482 rz->in += tin - rz->stream->avail_in;
|
|
483 if(ret == Z_NEED_DICT || ret == Z_MEM_ERROR || ret == Z_DATA_ERROR){
|
|
484 fprintf(stderr, "[_razf_read] inflate error: %d (at %s:%d)\n", ret, __FILE__, __LINE__);
|
|
485 rz->z_err = 1;
|
|
486 break;
|
|
487 }
|
|
488 if(ret == Z_STREAM_END){
|
|
489 rz->z_eof = 1;
|
|
490 break;
|
|
491 }
|
|
492 if ((rz->stream->data_type&128) && !(rz->stream->data_type&64)){
|
|
493 rz->buf_flush = 1;
|
|
494 rz->next_block_pos = rz->in;
|
|
495 break;
|
|
496 }
|
|
497 }
|
|
498 return size - rz->stream->avail_out;
|
|
499 }
|
|
500
|
|
501 int razf_read(RAZF *rz, void *data, int size){
|
|
502 int ori_size, i;
|
|
503 ori_size = size;
|
|
504 while(size > 0){
|
|
505 if(rz->buf_len){
|
|
506 if(size < rz->buf_len){
|
|
507 for(i=0;i<size;i++) ((char*)data)[i] = ((char*)rz->outbuf + rz->buf_off)[i];
|
|
508 rz->buf_off += size;
|
|
509 rz->buf_len -= size;
|
|
510 data += size;
|
|
511 rz->block_off += size;
|
|
512 size = 0;
|
|
513 break;
|
|
514 } else {
|
|
515 for(i=0;i<rz->buf_len;i++) ((char*)data)[i] = ((char*)rz->outbuf + rz->buf_off)[i];
|
|
516 data += rz->buf_len;
|
|
517 size -= rz->buf_len;
|
|
518 rz->block_off += rz->buf_len;
|
|
519 rz->buf_off = 0;
|
|
520 rz->buf_len = 0;
|
|
521 if(rz->buf_flush){
|
|
522 rz->block_pos = rz->next_block_pos;
|
|
523 rz->block_off = 0;
|
|
524 rz->buf_flush = 0;
|
|
525 }
|
|
526 }
|
|
527 } else if(rz->buf_flush){
|
|
528 rz->block_pos = rz->next_block_pos;
|
|
529 rz->block_off = 0;
|
|
530 rz->buf_flush = 0;
|
|
531 }
|
|
532 if(rz->buf_flush) continue;
|
|
533 rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE);
|
|
534 if(rz->z_eof && rz->buf_len == 0) break;
|
|
535 }
|
|
536 rz->out += ori_size - size;
|
|
537 return ori_size - size;
|
|
538 }
|
|
539
|
|
540 int razf_skip(RAZF* rz, int size){
|
|
541 int ori_size;
|
|
542 ori_size = size;
|
|
543 while(size > 0){
|
|
544 if(rz->buf_len){
|
|
545 if(size < rz->buf_len){
|
|
546 rz->buf_off += size;
|
|
547 rz->buf_len -= size;
|
|
548 rz->block_off += size;
|
|
549 size = 0;
|
|
550 break;
|
|
551 } else {
|
|
552 size -= rz->buf_len;
|
|
553 rz->buf_off = 0;
|
|
554 rz->buf_len = 0;
|
|
555 rz->block_off += rz->buf_len;
|
|
556 if(rz->buf_flush){
|
|
557 rz->block_pos = rz->next_block_pos;
|
|
558 rz->block_off = 0;
|
|
559 rz->buf_flush = 0;
|
|
560 }
|
|
561 }
|
|
562 } else if(rz->buf_flush){
|
|
563 rz->block_pos = rz->next_block_pos;
|
|
564 rz->block_off = 0;
|
|
565 rz->buf_flush = 0;
|
|
566 }
|
|
567 if(rz->buf_flush) continue;
|
|
568 rz->buf_len = _razf_read(rz, rz->outbuf, RZ_BUFFER_SIZE);
|
|
569 if(rz->z_eof) break;
|
|
570 }
|
|
571 rz->out += ori_size - size;
|
|
572 return ori_size - size;
|
|
573 }
|
|
574
|
|
575 static void _razf_reset_read(RAZF *rz, int64_t in, int64_t out){
|
|
576 lseek(rz->filedes, in, SEEK_SET);
|
|
577 rz->in = in;
|
|
578 rz->out = out;
|
|
579 rz->block_pos = in;
|
|
580 rz->next_block_pos = in;
|
|
581 rz->block_off = 0;
|
|
582 rz->buf_flush = 0;
|
|
583 rz->z_eof = rz->z_err = 0;
|
|
584 inflateReset(rz->stream);
|
|
585 rz->stream->avail_in = 0;
|
|
586 rz->buf_off = rz->buf_len = 0;
|
|
587 }
|
|
588
|
|
589 int64_t razf_jump(RAZF *rz, int64_t block_start, int block_offset){
|
|
590 int64_t pos;
|
|
591 rz->z_eof = 0;
|
|
592 if(rz->file_type == FILE_TYPE_PLAIN){
|
|
593 rz->buf_off = rz->buf_len = 0;
|
|
594 pos = block_start + block_offset;
|
|
595 pos = lseek(rz->filedes, pos, SEEK_SET);
|
|
596 rz->out = rz->in = pos;
|
|
597 return pos;
|
|
598 }
|
|
599 if(block_start == rz->block_pos && block_offset >= rz->block_off) {
|
|
600 block_offset -= rz->block_off;
|
|
601 goto SKIP; // Needn't reset inflate
|
|
602 }
|
|
603 if(block_start == 0) block_start = rz->header_size; // Automaticly revist wrong block_start
|
|
604 _razf_reset_read(rz, block_start, 0);
|
|
605 SKIP:
|
|
606 if(block_offset) razf_skip(rz, block_offset);
|
|
607 return rz->block_off;
|
|
608 }
|
|
609
|
|
610 int64_t razf_seek(RAZF* rz, int64_t pos, int where){
|
|
611 int64_t idx;
|
|
612 int64_t seek_pos, new_out;
|
|
613 rz->z_eof = 0;
|
|
614 if (where == SEEK_CUR) pos += rz->out;
|
|
615 else if (where == SEEK_END) pos += rz->src_end;
|
|
616 if(rz->file_type == FILE_TYPE_PLAIN){
|
|
617 seek_pos = lseek(rz->filedes, pos, SEEK_SET);
|
|
618 rz->buf_off = rz->buf_len = 0;
|
|
619 rz->out = rz->in = seek_pos;
|
|
620 return seek_pos;
|
|
621 } else if(rz->file_type == FILE_TYPE_GZ){
|
|
622 if(pos >= rz->out) goto SKIP;
|
|
623 return rz->out;
|
|
624 }
|
|
625 if(pos == rz->out) return pos;
|
|
626 if(pos > rz->src_end) return rz->out;
|
|
627 if(!rz->seekable || !rz->load_index){
|
|
628 if(pos >= rz->out) goto SKIP;
|
|
629 }
|
|
630 idx = pos / RZ_BLOCK_SIZE - 1;
|
|
631 seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]);
|
|
632 new_out = (idx + 1) * RZ_BLOCK_SIZE;
|
|
633 if(pos > rz->out && new_out <= rz->out) goto SKIP;
|
|
634 _razf_reset_read(rz, seek_pos, new_out);
|
|
635 SKIP:
|
|
636 razf_skip(rz, (int)(pos - rz->out));
|
|
637 return rz->out;
|
|
638 }
|
|
639
|
|
640 uint64_t razf_tell2(RAZF *rz)
|
|
641 {
|
|
642 /*
|
|
643 if (rz->load_index) {
|
|
644 int64_t idx, seek_pos;
|
|
645 idx = rz->out / RZ_BLOCK_SIZE - 1;
|
|
646 seek_pos = (idx < 0)? rz->header_size:(rz->index->cell_offsets[idx] + rz->index->bin_offsets[idx / RZ_BIN_SIZE]);
|
|
647 if (seek_pos != rz->block_pos || rz->out%RZ_BLOCK_SIZE != rz->block_off)
|
|
648 fprintf(stderr, "[razf_tell2] inconsistent block offset: (%lld, %lld) != (%lld, %lld)\n",
|
|
649 (long long)seek_pos, (long long)rz->out%RZ_BLOCK_SIZE, (long long)rz->block_pos, (long long) rz->block_off);
|
|
650 }
|
|
651 */
|
|
652 return (uint64_t)rz->block_pos<<16 | (rz->block_off&0xffff);
|
|
653 }
|
|
654
|
|
655 int64_t razf_seek2(RAZF *rz, uint64_t voffset, int where)
|
|
656 {
|
|
657 if (where != SEEK_SET) return -1;
|
|
658 return razf_jump(rz, voffset>>16, voffset&0xffff);
|
|
659 }
|
|
660
|
|
661 void razf_close(RAZF *rz){
|
|
662 if(rz->mode == 'w'){
|
|
663 #ifndef _RZ_READONLY
|
|
664 razf_end_flush(rz);
|
|
665 deflateEnd(rz->stream);
|
|
666 save_zindex(rz, rz->filedes);
|
|
667 if(is_big_endian()){
|
|
668 write(rz->filedes, &rz->in, sizeof(int64_t));
|
|
669 write(rz->filedes, &rz->out, sizeof(int64_t));
|
|
670 } else {
|
|
671 uint64_t v64 = byte_swap_8((uint64_t)rz->in);
|
|
672 write(rz->filedes, &v64, sizeof(int64_t));
|
|
673 v64 = byte_swap_8((uint64_t)rz->out);
|
|
674 write(rz->filedes, &v64, sizeof(int64_t));
|
|
675 }
|
|
676 #endif
|
|
677 } else if(rz->mode == 'r'){
|
|
678 if(rz->stream) inflateEnd(rz->stream);
|
|
679 }
|
|
680 if(rz->inbuf) free(rz->inbuf);
|
|
681 if(rz->outbuf) free(rz->outbuf);
|
|
682 if(rz->header){
|
|
683 free(rz->header->extra);
|
|
684 free(rz->header->name);
|
|
685 free(rz->header->comment);
|
|
686 free(rz->header);
|
|
687 }
|
|
688 if(rz->index){
|
|
689 free(rz->index->bin_offsets);
|
|
690 free(rz->index->cell_offsets);
|
|
691 free(rz->index);
|
|
692 }
|
|
693 free(rz->stream);
|
|
694 close(rz->filedes);
|
|
695 free(rz);
|
|
696 }
|
|
697
|
|
698 #endif
|