Libosmium  2.6.1
Fast and flexible C++ library for working with OpenStreetMap data
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstring>
40 #include <functional>
41 #include <iterator>
42 #include <memory>
43 #include <stdexcept>
44 #include <utility>
45 
46 #include <osmium/memory/item.hpp>
48 #include <osmium/osm/entity.hpp>
50 
51 namespace osmium {
52 
58  struct buffer_is_full : public std::runtime_error {
59 
61  std::runtime_error("Osmium buffer is full") {
62  }
63 
64  }; // struct buffer_is_full
65 
69  namespace memory {
70 
97  class Buffer {
98 
99  public:
100 
101  // This is needed so we can call std::back_inserter() on a Buffer.
102  using value_type = Item;
103 
104  enum class auto_grow : bool {
105  yes = true,
106  no = false
107  }; // enum class auto_grow
108 
109  private:
110 
111  std::unique_ptr<unsigned char[]> m_memory;
112  unsigned char* m_data;
113  size_t m_capacity;
114  size_t m_written;
115  size_t m_committed;
116  auto_grow m_auto_grow {auto_grow::no};
117  std::function<void(Buffer&)> m_full;
118 
119  public:
120 
129  Buffer() noexcept :
130  m_memory(),
131  m_data(nullptr),
132  m_capacity(0),
133  m_written(0),
134  m_committed(0) {
135  }
136 
147  explicit Buffer(unsigned char* data, size_t size) :
148  m_memory(),
149  m_data(data),
150  m_capacity(size),
151  m_written(size),
152  m_committed(size) {
153  if (size % align_bytes != 0) {
154  throw std::invalid_argument("buffer size needs to be multiple of alignment");
155  }
156  }
157 
169  explicit Buffer(unsigned char* data, size_t capacity, size_t committed) :
170  m_memory(),
171  m_data(data),
172  m_capacity(capacity),
173  m_written(committed),
174  m_committed(committed) {
175  if (capacity % align_bytes != 0) {
176  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
177  }
178  if (committed % align_bytes != 0) {
179  throw std::invalid_argument("buffer parameter 'committed' needs to be multiple of alignment");
180  }
181  }
182 
197  explicit Buffer(size_t capacity, auto_grow auto_grow = auto_grow::yes) :
198  m_memory(new unsigned char[capacity]),
199  m_data(m_memory.get()),
200  m_capacity(capacity),
201  m_written(0),
202  m_committed(0),
203  m_auto_grow(auto_grow) {
204  if (capacity % align_bytes != 0) {
205  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
206  }
207  }
208 
209  // buffers can not be copied
210  Buffer(const Buffer&) = delete;
211  Buffer& operator=(const Buffer&) = delete;
212 
213  // buffers can be moved
214  Buffer(Buffer&&) = default;
215  Buffer& operator=(Buffer&&) = default;
216 
217  ~Buffer() = default;
218 
224  unsigned char* data() const noexcept {
225  assert(m_data);
226  return m_data;
227  }
228 
233  size_t capacity() const noexcept {
234  return m_capacity;
235  }
236 
241  size_t committed() const noexcept {
242  return m_committed;
243  }
244 
250  size_t written() const noexcept {
251  return m_written;
252  }
253 
260  bool is_aligned() const noexcept {
261  assert(m_data);
262  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
263  }
264 
285  OSMIUM_DEPRECATED void set_full_callback(std::function<void(Buffer&)> full) {
286  assert(m_data);
287  m_full = full;
288  }
289 
307  void grow(size_t size) {
308  assert(m_data);
309  if (!m_memory) {
310  throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
311  }
312  if (m_capacity < size) {
313  if (size % align_bytes != 0) {
314  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
315  }
316  std::unique_ptr<unsigned char[]> memory(new unsigned char[size]);
317  std::copy_n(m_memory.get(), m_capacity, memory.get());
318  using std::swap;
319  swap(m_memory, memory);
320  m_data = m_memory.get();
321  m_capacity = size;
322  }
323  }
324 
335  size_t commit() {
336  assert(m_data);
337  assert(is_aligned());
338 
339  const size_t offset = m_committed;
340  m_committed = m_written;
341  return offset;
342  }
343 
349  void rollback() {
350  assert(m_data);
351  m_written = m_committed;
352  }
353 
361  size_t clear() {
362  const size_t committed = m_committed;
363  m_written = 0;
364  m_committed = 0;
365  return committed;
366  }
367 
378  template <typename T>
379  T& get(const size_t offset) const {
380  assert(m_data);
381  return *reinterpret_cast<T*>(&m_data[offset]);
382  }
383 
417  unsigned char* reserve_space(const size_t size) {
418  assert(m_data);
419  // try to flush the buffer empty first.
420  if (m_written + size > m_capacity && m_full) {
421  m_full(*this);
422  }
423  // if there's still not enough space, then try growing the buffer.
424  if (m_written + size > m_capacity) {
425  if (m_memory && (m_auto_grow == auto_grow::yes)) {
426  // double buffer size until there is enough space
427  size_t new_capacity = m_capacity * 2;
428  while (m_written + size > new_capacity) {
429  new_capacity *= 2;
430  }
431  grow(new_capacity);
432  } else {
433  throw osmium::buffer_is_full();
434  }
435  }
436  unsigned char* data = &m_data[m_written];
437  m_written += size;
438  return data;
439  }
440 
456  template <typename T>
457  T& add_item(const T& item) {
458  assert(m_data);
459  unsigned char* target = reserve_space(item.padded_size());
460  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
461  return *reinterpret_cast<T*>(target);
462  }
463 
474  void add_buffer(const Buffer& buffer) {
475  assert(m_data && buffer);
476  unsigned char* target = reserve_space(buffer.committed());
477  std::copy_n(buffer.data(), buffer.committed(), target);
478  }
479 
488  void push_back(const osmium::memory::Item& item) {
489  assert(m_data);
490  add_item(item);
491  commit();
492  }
493 
498  template <typename T>
500 
505  template <typename T>
507 
513 
519 
528  template <typename T>
530  assert(m_data);
531  return t_iterator<T>(m_data, m_data + m_committed);
532  }
533 
543  assert(m_data);
544  return iterator(m_data, m_data + m_committed);
545  }
546 
556  template <typename T>
557  t_iterator<T> get_iterator(size_t offset) {
558  assert(m_data);
559  return t_iterator<T>(m_data + offset, m_data + m_committed);
560  }
561 
571  iterator get_iterator(size_t offset) {
572  assert(m_data);
573  return iterator(m_data + offset, m_data + m_committed);
574  }
575 
584  template <typename T>
586  assert(m_data);
587  return t_iterator<T>(m_data + m_committed, m_data + m_committed);
588  }
589 
599  assert(m_data);
600  return iterator(m_data + m_committed, m_data + m_committed);
601  }
602 
603  template <typename T>
605  assert(m_data);
606  return t_const_iterator<T>(m_data, m_data + m_committed);
607  }
608 
610  assert(m_data);
611  return const_iterator(m_data, m_data + m_committed);
612  }
613 
614  template <typename T>
615  t_const_iterator<T> get_iterator(size_t offset) const {
616  assert(m_data);
617  return t_const_iterator<T>(m_data + offset, m_data + m_committed);
618  }
619 
620  const_iterator get_iterator(size_t offset) const {
621  assert(m_data);
622  return const_iterator(m_data + offset, m_data + m_committed);
623  }
624 
625  template <typename T>
627  assert(m_data);
628  return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
629  }
630 
632  assert(m_data);
633  return const_iterator(m_data + m_committed, m_data + m_committed);
634  }
635 
636  template <typename T>
638  return cbegin<T>();
639  }
640 
642  return cbegin();
643  }
644 
645  template <typename T>
647  return cend<T>();
648  }
649 
650  const_iterator end() const {
651  return cend();
652  }
653 
657  explicit operator bool() const noexcept {
658  return m_data != nullptr;
659  }
660 
661  void swap(Buffer& other) {
662  using std::swap;
663 
664  swap(m_memory, other.m_memory);
665  swap(m_data, other.m_data);
666  swap(m_capacity, other.m_capacity);
667  swap(m_written, other.m_written);
668  swap(m_committed, other.m_committed);
669  swap(m_auto_grow, other.m_auto_grow);
670  swap(m_full, other.m_full);
671  }
672 
689  template <typename TCallbackClass>
690  void purge_removed(TCallbackClass* callback) {
691  assert(m_data);
692  if (begin() == end()) {
693  return;
694  }
695 
696  iterator it_write = begin();
697 
698  iterator next;
699  for (iterator it_read = begin(); it_read != end(); it_read = next) {
700  next = std::next(it_read);
701  if (!it_read->removed()) {
702  if (it_read != it_write) {
703  assert(it_read.data() >= data());
704  assert(it_write.data() >= data());
705  size_t old_offset = static_cast<size_t>(it_read.data() - data());
706  size_t new_offset = static_cast<size_t>(it_write.data() - data());
707  callback->moving_in_buffer(old_offset, new_offset);
708  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
709  }
710  it_write.advance_once();
711  }
712  }
713 
714  assert(it_write.data() >= data());
715  m_written = static_cast<size_t>(it_write.data() - data());
716  m_committed = m_written;
717  }
718 
719  }; // class Buffer
720 
721  inline void swap(Buffer& lhs, Buffer& rhs) {
722  lhs.swap(rhs);
723  }
724 
732  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
733  if (!lhs || !rhs) {
734  return !lhs && !rhs;
735  }
736  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
737  }
738 
739  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
740  return ! (lhs == rhs);
741  }
742 
743  } // namespace memory
744 
745 } // namespace osmium
746 
747 #endif // OSMIUM_MEMORY_BUFFER_HPP
size_t m_written
Definition: buffer.hpp:114
void swap(Buffer &other)
Definition: buffer.hpp:661
t_const_iterator< T > cend() const
Definition: buffer.hpp:626
const_iterator begin() const
Definition: buffer.hpp:641
size_t clear()
Definition: buffer.hpp:361
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
OSMIUM_DEPRECATED void set_full_callback(std::function< void(Buffer &)> full)
Definition: buffer.hpp:285
iterator get_iterator(size_t offset)
Definition: buffer.hpp:571
void grow(size_t size)
Definition: buffer.hpp:307
const_iterator end() const
Definition: buffer.hpp:650
Definition: item_iterator.hpp:119
constexpr bool operator==(const Box &lhs, const Box &rhs) noexcept
Definition: box.hpp:222
unsigned char * m_data
Definition: buffer.hpp:112
constexpr item_size_type align_bytes
Definition: item.hpp:53
size_t capacity() const noexcept
Definition: buffer.hpp:233
Buffer(unsigned char *data, size_t capacity, size_t committed)
Definition: buffer.hpp:169
Definition: reader_iterator.hpp:39
void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:721
ItemIterator< TMember > & advance_once()
Definition: item_iterator.hpp:168
Buffer(size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:197
unsigned char * data() const
Definition: item_iterator.hpp:189
t_iterator< T > end()
Definition: buffer.hpp:585
Definition: item.hpp:97
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
T & add_item(const T &item)
Definition: buffer.hpp:457
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:690
t_iterator< T > begin()
Definition: buffer.hpp:529
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:474
const_iterator cbegin() const
Definition: buffer.hpp:609
size_t m_committed
Definition: buffer.hpp:115
bool is_aligned() const noexcept
Definition: buffer.hpp:260
Buffer() noexcept
Definition: buffer.hpp:129
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
unsigned char * reserve_space(const size_t size)
Definition: buffer.hpp:417
auto_grow m_auto_grow
Definition: buffer.hpp:116
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:488
iterator end()
Definition: buffer.hpp:598
size_t committed() const noexcept
Definition: buffer.hpp:241
iterator begin()
Definition: buffer.hpp:542
const_iterator cend() const
Definition: buffer.hpp:631
size_t m_capacity
Definition: buffer.hpp:113
t_const_iterator< T > get_iterator(size_t offset) const
Definition: buffer.hpp:615
size_t written() const noexcept
Definition: buffer.hpp:250
Buffer(unsigned char *data, size_t size)
Definition: buffer.hpp:147
t_const_iterator< T > end() const
Definition: buffer.hpp:646
Definition: buffer.hpp:97
unsigned char * data() const noexcept
Definition: buffer.hpp:224
Definition: buffer.hpp:58
const_iterator get_iterator(size_t offset) const
Definition: buffer.hpp:620
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:604
t_const_iterator< T > begin() const
Definition: buffer.hpp:637
auto_grow
Definition: buffer.hpp:104
buffer_is_full()
Definition: buffer.hpp:60
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
void rollback()
Definition: buffer.hpp:349
std::unique_ptr< unsigned char[]> m_memory
Definition: buffer.hpp:111
bool operator!=(const Changeset &lhs, const Changeset &rhs)
Definition: changeset.hpp:431
t_iterator< T > get_iterator(size_t offset)
Definition: buffer.hpp:557
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:117
size_t commit()
Definition: buffer.hpp:335