Eclipse SUMO - Simulation of Urban MObility
Loading...
Searching...
No Matches
zstr.hpp
Go to the documentation of this file.
1//---------------------------------------------------------
2// Copyright 2015 Ontario Institute for Cancer Research
3// Written by Matei David (matei@cs.toronto.edu)
4//---------------------------------------------------------
5
6// Reference:
7// http://stackoverflow.com/questions/14086417/how-to-write-custom-input-stream-in-c
8
9#pragma once
10
11#include <cassert>
12#include <fstream>
13#include <sstream>
14#include <zlib.h>
15#include <memory>
16#include <iostream>
17#include "strict_fstream.hpp"
18
19#if defined(__GNUC__) && !defined(__clang__)
20#if (__GNUC__ > 5) || (__GNUC__ == 5 && __GNUC_MINOR__>0)
21#define CAN_MOVE_IOSTREAM
22#endif
23#else
24#define CAN_MOVE_IOSTREAM
25#endif
26
27namespace zstr
28{
29
30static const std::size_t default_buff_size = static_cast<std::size_t>(1 << 20);
31
34 : public std::ios_base::failure
35{
36public:
37 static std::string error_to_message(z_stream * zstrm_p, int ret)
38 {
39 std::string msg = "zlib: ";
40 switch (ret)
41 {
42 case Z_STREAM_ERROR:
43 msg += "Z_STREAM_ERROR: ";
44 break;
45 case Z_DATA_ERROR:
46 msg += "Z_DATA_ERROR: ";
47 break;
48 case Z_MEM_ERROR:
49 msg += "Z_MEM_ERROR: ";
50 break;
51 case Z_VERSION_ERROR:
52 msg += "Z_VERSION_ERROR: ";
53 break;
54 case Z_BUF_ERROR:
55 msg += "Z_BUF_ERROR: ";
56 break;
57 default:
58 std::ostringstream oss;
59 oss << ret;
60 msg += "[" + oss.str() + "]: ";
61 break;
62 }
63 if (zstrm_p->msg) {
64 msg += zstrm_p->msg;
65 }
66 msg += " ("
67 "next_in: " +
68 std::to_string(uintptr_t(zstrm_p->next_in)) +
69 ", avail_in: " +
70 std::to_string(uintptr_t(zstrm_p->avail_in)) +
71 ", next_out: " +
72 std::to_string(uintptr_t(zstrm_p->next_out)) +
73 ", avail_out: " +
74 std::to_string(uintptr_t(zstrm_p->avail_out)) +
75 ")";
76 return msg;
77 }
78
79 Exception(z_stream * zstrm_p, int ret)
80 : std::ios_base::failure(error_to_message(zstrm_p, ret))
81 {
82 }
83}; // class Exception
84
85namespace detail
86{
87
89 : public z_stream
90{
91public:
92 z_stream_wrapper(bool _is_input, int _level, int _window_bits)
93 : is_input(_is_input)
94 {
95 this->zalloc = nullptr;//Z_NULL
96 this->zfree = nullptr;//Z_NULL
97 this->opaque = nullptr;//Z_NULL
98 int ret;
99 if (is_input)
100 {
101 this->avail_in = 0;
102 this->next_in = nullptr;//Z_NULL
103 ret = inflateInit2(this, _window_bits ? _window_bits : 15+32);
104 }
105 else
106 {
107 ret = deflateInit2(this, _level, Z_DEFLATED, _window_bits ? _window_bits : 15+16, 8, Z_DEFAULT_STRATEGY);
108 }
109 if (ret != Z_OK) throw Exception(this, ret);
110 }
112 {
113 if (is_input)
114 {
115 inflateEnd(this);
116 }
117 else
118 {
119 deflateEnd(this);
120 }
121 }
122private:
124}; // class z_stream_wrapper
125
126} // namespace detail
127
129 : public std::streambuf
130{
131public:
132 istreambuf(std::streambuf * _sbuf_p,
133 std::size_t _buff_size = default_buff_size, bool _auto_detect = true, int _window_bits = 0)
134 : sbuf_p(_sbuf_p),
135 in_buff(),
136 in_buff_start(nullptr),
137 in_buff_end(nullptr),
138 out_buff(),
139 zstrm_p(nullptr),
140 buff_size(_buff_size),
141 auto_detect(_auto_detect),
142 auto_detect_run(false),
143 is_text(false),
144 window_bits(_window_bits)
145 {
146 assert(sbuf_p);
147 in_buff = std::unique_ptr<char[]>(new char[buff_size]);
148 in_buff_start = in_buff.get();
149 in_buff_end = in_buff.get();
150 out_buff = std::unique_ptr<char[]>(new char[buff_size]);
151 setg(out_buff.get(), out_buff.get(), out_buff.get());
152 }
153
154 istreambuf(const istreambuf &) = delete;
155 istreambuf & operator = (const istreambuf &) = delete;
156
157 pos_type seekoff(off_type off, std::ios_base::seekdir dir,
158 std::ios_base::openmode which) override
159 {
160 if (off != 0 || dir != std::ios_base::cur) {
161 return std::streambuf::seekoff(off, dir, which);
162 }
163
164 if (!zstrm_p) {
165 return 0;
166 }
167
168 return static_cast<long int>(zstrm_p->total_out - static_cast<uLong>(in_avail()));
169 }
170
171 std::streambuf::int_type underflow() override
172 {
173 if (this->gptr() == this->egptr())
174 {
175 // pointers for free region in output buffer
176 char * out_buff_free_start = out_buff.get();
177 int tries = 0;
178 do
179 {
180 if (++tries > 1000) {
181 throw std::ios_base::failure("Failed to fill buffer after 1000 tries");
182 }
183
184 // read more input if none available
186 {
187 // empty input buffer: refill from the start
188 in_buff_start = in_buff.get();
189 std::streamsize sz = sbuf_p->sgetn(in_buff.get(), static_cast<std::streamsize>(buff_size));
191 if (in_buff_end == in_buff_start) break; // end of input
192 }
193 // auto detect if the stream contains text or deflate data
195 {
196 auto_detect_run = true;
197 unsigned char b0 = *reinterpret_cast< unsigned char * >(in_buff_start);
198 unsigned char b1 = *reinterpret_cast< unsigned char * >(in_buff_start + 1);
199 // Ref:
200 // http://en.wikipedia.org/wiki/Gzip
201 // http://stackoverflow.com/questions/9050260/what-does-a-zlib-header-look-like
203 && ((b0 == 0x1F && b1 == 0x8B) // gzip header
204 || (b0 == 0x78 && (b1 == 0x01 // zlib header
205 || b1 == 0x9C
206 || b1 == 0xDA))));
207 }
208 if (is_text)
209 {
210 // simply swap in_buff and out_buff, and adjust pointers
211 assert(in_buff_start == in_buff.get());
213 out_buff_free_start = in_buff_end;
214 in_buff_start = in_buff.get();
215 in_buff_end = in_buff.get();
216 }
217 else
218 {
219 // run inflate() on input
220 if (! zstrm_p) zstrm_p = std::unique_ptr<detail::z_stream_wrapper>(new detail::z_stream_wrapper(true, Z_DEFAULT_COMPRESSION, window_bits));
221 zstrm_p->next_in = reinterpret_cast< decltype(zstrm_p->next_in) >(in_buff_start);
222 zstrm_p->avail_in = uint32_t(in_buff_end - in_buff_start);
223 zstrm_p->next_out = reinterpret_cast< decltype(zstrm_p->next_out) >(out_buff_free_start);
224 zstrm_p->avail_out = uint32_t((out_buff.get() + buff_size) - out_buff_free_start);
225 int ret = inflate(zstrm_p.get(), Z_NO_FLUSH);
226 // process return code
227 if (ret != Z_OK && ret != Z_STREAM_END) throw Exception(zstrm_p.get(), ret);
228 // update in&out pointers following inflate()
229 in_buff_start = reinterpret_cast< decltype(in_buff_start) >(zstrm_p->next_in);
230 in_buff_end = in_buff_start + zstrm_p->avail_in;
231 out_buff_free_start = reinterpret_cast< decltype(out_buff_free_start) >(zstrm_p->next_out);
232 assert(out_buff_free_start + zstrm_p->avail_out == out_buff.get() + buff_size);
233
234 if (ret == Z_STREAM_END) {
235 // if stream ended, deallocate inflator
236 zstrm_p.reset();
237 }
238 }
239 } while (out_buff_free_start == out_buff.get());
240 // 2 exit conditions:
241 // - end of input: there might or might not be output available
242 // - out_buff_free_start != out_buff: output available
243 this->setg(out_buff.get(), out_buff.get(), out_buff_free_start);
244 }
245 return this->gptr() == this->egptr()
246 ? traits_type::eof()
247 : traits_type::to_int_type(*this->gptr());
248 }
249private:
250 std::streambuf * sbuf_p;
251 std::unique_ptr<char[]> in_buff;
254 std::unique_ptr<char[]> out_buff;
255 std::unique_ptr<detail::z_stream_wrapper> zstrm_p;
256 std::size_t buff_size;
261
262}; // class istreambuf
263
265 : public std::streambuf
266{
267public:
268 ostreambuf(std::streambuf * _sbuf_p,
269 std::size_t _buff_size = default_buff_size, int _level = Z_DEFAULT_COMPRESSION, int _window_bits = 0)
270 : sbuf_p(_sbuf_p),
271 in_buff(),
272 out_buff(),
273 zstrm_p(new detail::z_stream_wrapper(false, _level, _window_bits)),
274 buff_size(_buff_size)
275 {
276 assert(sbuf_p);
277 in_buff = std::unique_ptr<char[]>(new char[buff_size]);
278 out_buff = std::unique_ptr<char[]>(new char[buff_size]);
279 setp(in_buff.get(), in_buff.get() + buff_size);
280 }
281
282 ostreambuf(const ostreambuf &) = delete;
283 ostreambuf & operator = (const ostreambuf &) = delete;
284
285 int deflate_loop(int flush)
286 {
287 while (true)
288 {
289 zstrm_p->next_out = reinterpret_cast< decltype(zstrm_p->next_out) >(out_buff.get());
290 zstrm_p->avail_out = uint32_t(buff_size);
291 int ret = deflate(zstrm_p.get(), flush);
292 if (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) {
293 failed = true;
294 throw Exception(zstrm_p.get(), ret);
295 }
296 std::streamsize sz = sbuf_p->sputn(out_buff.get(), reinterpret_cast< decltype(out_buff.get()) >(zstrm_p->next_out) - out_buff.get());
297 if (sz != reinterpret_cast< decltype(out_buff.get()) >(zstrm_p->next_out) - out_buff.get())
298 {
299 // there was an error in the sink stream
300 return -1;
301 }
302 if (ret == Z_STREAM_END || ret == Z_BUF_ERROR || sz == 0)
303 {
304 break;
305 }
306 }
307 return 0;
308 }
309
310 virtual ~ostreambuf()
311 {
312 // flush the zlib stream
313 //
314 // NOTE: Errors here (sync() return value not 0) are ignored, because we
315 // cannot throw in a destructor. This mirrors the behaviour of
316 // std::basic_filebuf::~basic_filebuf(). To see an exception on error,
317 // close the ofstream with an explicit call to close(), and do not rely
318 // on the implicit call in the destructor.
319 //
320 if (!failed) try {
321 sync();
322 } catch (...) {}
323 }
324 std::streambuf::int_type overflow(std::streambuf::int_type c = traits_type::eof()) override
325 {
326 zstrm_p->next_in = reinterpret_cast< decltype(zstrm_p->next_in) >(pbase());
327 zstrm_p->avail_in = uint32_t(pptr() - pbase());
328 while (zstrm_p->avail_in > 0)
329 {
330 int r = deflate_loop(Z_NO_FLUSH);
331 if (r != 0)
332 {
333 setp(nullptr, nullptr);
334 return traits_type::eof();
335 }
336 }
337 setp(in_buff.get(), in_buff.get() + buff_size);
338 return traits_type::eq_int_type(c, traits_type::eof()) ? traits_type::eof() : sputc(char_type(c));
339 }
340 int sync() override
341 {
342 // first, call overflow to clear in_buff
343 overflow();
344 if (! pptr()) return -1;
345 // then, call deflate asking to finish the zlib stream
346 zstrm_p->next_in = nullptr;
347 zstrm_p->avail_in = 0;
348 if (deflate_loop(Z_FINISH) != 0) return -1;
349 deflateReset(zstrm_p.get());
350 return 0;
351 }
352private:
353 std::streambuf * sbuf_p = nullptr;
354 std::unique_ptr<char[]> in_buff;
355 std::unique_ptr<char[]> out_buff;
356 std::unique_ptr<detail::z_stream_wrapper> zstrm_p;
357 std::size_t buff_size;
358 bool failed = false;
359
360}; // class ostreambuf
361
363 : public std::istream
364{
365public:
366 istream(std::istream & is,
367 std::size_t _buff_size = default_buff_size, bool _auto_detect = true, int _window_bits = 0)
368 : std::istream(new istreambuf(is.rdbuf(), _buff_size, _auto_detect, _window_bits))
369 {
370 exceptions(std::ios_base::badbit);
371 }
372 explicit istream(std::streambuf * sbuf_p)
373 : std::istream(new istreambuf(sbuf_p))
374 {
375 exceptions(std::ios_base::badbit);
376 }
377 virtual ~istream()
378 {
379 delete rdbuf();
380 }
381}; // class istream
382
384 : public std::ostream
385{
386public:
387 ostream(std::ostream & os,
388 std::size_t _buff_size = default_buff_size, int _level = Z_DEFAULT_COMPRESSION, int _window_bits = 0)
389 : std::ostream(new ostreambuf(os.rdbuf(), _buff_size, _level, _window_bits))
390 {
391 exceptions(std::ios_base::badbit);
392 }
393 explicit ostream(std::streambuf * sbuf_p)
394 : std::ostream(new ostreambuf(sbuf_p))
395 {
396 exceptions(std::ios_base::badbit);
397 }
398 virtual ~ostream()
399 {
400 delete rdbuf();
401 }
402}; // class ostream
403
404namespace detail
405{
406
407template < typename FStream_Type >
409{
410 strict_fstream_holder(const std::string& filename, std::ios_base::openmode mode = std::ios_base::in)
411 : _fs(filename, mode)
412 {}
414 FStream_Type _fs {};
415}; // class strict_fstream_holder
416
417} // namespace detail
418
420 : private detail::strict_fstream_holder< strict_fstream::ifstream >,
421 public std::istream
422{
423public:
424 explicit ifstream(const std::string filename, std::ios_base::openmode mode = std::ios_base::in, size_t buff_size = default_buff_size)
425 : detail::strict_fstream_holder< strict_fstream::ifstream >(filename, mode),
426 std::istream(new istreambuf(_fs.rdbuf(), buff_size))
427 {
428 exceptions(std::ios_base::badbit);
429 }
430 explicit ifstream(): detail::strict_fstream_holder< strict_fstream::ifstream >(), std::istream(new istreambuf(_fs.rdbuf())){}
431 void close() {
432 _fs.close();
433 }
434 #ifdef CAN_MOVE_IOSTREAM
435 void open(const std::string filename, std::ios_base::openmode mode = std::ios_base::in) {
436 _fs.open(filename, mode);
437 std::istream::operator=(std::istream(new istreambuf(_fs.rdbuf())));
438 }
439 #endif
440 bool is_open() const {
441 return _fs.is_open();
442 }
443 virtual ~ifstream()
444 {
445 if (_fs.is_open()) close();
446 if (rdbuf()) delete rdbuf();
447 }
448
450 std::streampos compressed_tellg()
451 {
452 return _fs.tellg();
453 }
454}; // class ifstream
455
457 : private detail::strict_fstream_holder< strict_fstream::ofstream >,
458 public std::ostream
459{
460public:
461 explicit ofstream(const std::string filename, std::ios_base::openmode mode = std::ios_base::out,
462 int level = Z_DEFAULT_COMPRESSION, size_t buff_size = default_buff_size)
463 : detail::strict_fstream_holder< strict_fstream::ofstream >(filename, mode | std::ios_base::binary),
464 std::ostream(new ostreambuf(_fs.rdbuf(), buff_size, level))
465 {
466 exceptions(std::ios_base::badbit);
467 }
468 explicit ofstream(): detail::strict_fstream_holder< strict_fstream::ofstream >(), std::ostream(new ostreambuf(_fs.rdbuf())){}
469 void close() {
470 std::ostream::flush();
471 _fs.close();
472 }
473 #ifdef CAN_MOVE_IOSTREAM
474 void open(const std::string filename, std::ios_base::openmode mode = std::ios_base::out, int level = Z_DEFAULT_COMPRESSION) {
475 flush();
476 _fs.open(filename, mode | std::ios_base::binary);
477 std::ostream::operator=(std::ostream(new ostreambuf(_fs.rdbuf(), default_buff_size, level)));
478 }
479 #endif
480 bool is_open() const {
481 return _fs.is_open();
482 }
484 std::ostream::flush();
485 _fs.flush();
486 return *this;
487 }
488 virtual ~ofstream()
489 {
490 if (_fs.is_open()) close();
491 if (rdbuf()) delete rdbuf();
492 }
493
494 // Return the position within the compressed file (wrapped filestream)
495 std::streampos compressed_tellp()
496 {
497 return _fs.tellp();
498 }
499}; // class ofstream
500
501} // namespace zstr
void open(const std::string &filename, std::ios_base::openmode mode=std::ios_base::in)
void open(const std::string &filename, std::ios_base::openmode mode=std::ios_base::out)
Exception class thrown by failed zlib operations.
Definition zstr.hpp:35
static std::string error_to_message(z_stream *zstrm_p, int ret)
Definition zstr.hpp:37
Exception(z_stream *zstrm_p, int ret)
Definition zstr.hpp:79
z_stream_wrapper(bool _is_input, int _level, int _window_bits)
Definition zstr.hpp:92
void open(const std::string filename, std::ios_base::openmode mode=std::ios_base::in)
Definition zstr.hpp:435
std::streampos compressed_tellg()
Return the position within the compressed file (wrapped filestream)
Definition zstr.hpp:450
void close()
Definition zstr.hpp:431
ifstream(const std::string filename, std::ios_base::openmode mode=std::ios_base::in, size_t buff_size=default_buff_size)
Definition zstr.hpp:424
bool is_open() const
Definition zstr.hpp:440
virtual ~ifstream()
Definition zstr.hpp:443
virtual ~istream()
Definition zstr.hpp:377
istream(std::streambuf *sbuf_p)
Definition zstr.hpp:372
istream(std::istream &is, std::size_t _buff_size=default_buff_size, bool _auto_detect=true, int _window_bits=0)
Definition zstr.hpp:366
istreambuf(std::streambuf *_sbuf_p, std::size_t _buff_size=default_buff_size, bool _auto_detect=true, int _window_bits=0)
Definition zstr.hpp:132
char * in_buff_end
Definition zstr.hpp:253
std::unique_ptr< detail::z_stream_wrapper > zstrm_p
Definition zstr.hpp:255
std::unique_ptr< char[]> out_buff
Definition zstr.hpp:254
istreambuf(const istreambuf &)=delete
std::unique_ptr< char[]> in_buff
Definition zstr.hpp:251
char * in_buff_start
Definition zstr.hpp:252
std::size_t buff_size
Definition zstr.hpp:256
pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which) override
Definition zstr.hpp:157
bool auto_detect_run
Definition zstr.hpp:258
std::streambuf::int_type underflow() override
Definition zstr.hpp:171
std::streambuf * sbuf_p
Definition zstr.hpp:250
istreambuf & operator=(const istreambuf &)=delete
std::streampos compressed_tellp()
Definition zstr.hpp:495
ofstream & flush()
Definition zstr.hpp:483
void open(const std::string filename, std::ios_base::openmode mode=std::ios_base::out, int level=Z_DEFAULT_COMPRESSION)
Definition zstr.hpp:474
ofstream(const std::string filename, std::ios_base::openmode mode=std::ios_base::out, int level=Z_DEFAULT_COMPRESSION, size_t buff_size=default_buff_size)
Definition zstr.hpp:461
void close()
Definition zstr.hpp:469
bool is_open() const
Definition zstr.hpp:480
virtual ~ofstream()
Definition zstr.hpp:488
ostream(std::ostream &os, std::size_t _buff_size=default_buff_size, int _level=Z_DEFAULT_COMPRESSION, int _window_bits=0)
Definition zstr.hpp:387
ostream(std::streambuf *sbuf_p)
Definition zstr.hpp:393
virtual ~ostream()
Definition zstr.hpp:398
ostreambuf & operator=(const ostreambuf &)=delete
int deflate_loop(int flush)
Definition zstr.hpp:285
std::streambuf * sbuf_p
Definition zstr.hpp:353
std::size_t buff_size
Definition zstr.hpp:357
std::unique_ptr< char[]> in_buff
Definition zstr.hpp:354
std::unique_ptr< char[]> out_buff
Definition zstr.hpp:355
ostreambuf(const ostreambuf &)=delete
ostreambuf(std::streambuf *_sbuf_p, std::size_t _buff_size=default_buff_size, int _level=Z_DEFAULT_COMPRESSION, int _window_bits=0)
Definition zstr.hpp:268
std::streambuf::int_type overflow(std::streambuf::int_type c=traits_type::eof()) override
Definition zstr.hpp:324
int sync() override
Definition zstr.hpp:340
virtual ~ostreambuf()
Definition zstr.hpp:310
std::unique_ptr< detail::z_stream_wrapper > zstrm_p
Definition zstr.hpp:356
Definition json.hpp:4471
NLOHMANN_BASIC_JSON_TPL_DECLARATION void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL &j1, nlohmann::NLOHMANN_BASIC_JSON_TPL &j2) noexcept(//NOLINT(readability-inconsistent-declaration-parameter-name) is_nothrow_move_constructible< nlohmann::NLOHMANN_BASIC_JSON_TPL >::value &&//NOLINT(misc-redundant-expression) is_nothrow_move_assignable< nlohmann::NLOHMANN_BASIC_JSON_TPL >::value)
exchanges the values of two JSON objects
Definition json.hpp:21884
Definition zstr.hpp:28
static const std::size_t default_buff_size
Definition zstr.hpp:30
strict_fstream_holder(const std::string &filename, std::ios_base::openmode mode=std::ios_base::in)
Definition zstr.hpp:410