8 #ifndef INCLUDED_ORCUS_DETAIL_THREAD_PARSER_TOKEN_BUFFER_HPP
9 #define INCLUDED_ORCUS_DETAIL_THREAD_PARSER_TOKEN_BUFFER_HPP
11 #include "orcus/exception.hpp"
14 #include <condition_variable>
16 namespace orcus {
namespace detail {
namespace thread {
22 template<
typename _TokensT>
25 typedef _TokensT tokens_type;
27 mutable std::mutex m_mtx_tokens;
28 std::condition_variable m_cv_tokens_empty;
29 std::condition_variable m_cv_tokens_ready;
33 size_t m_token_size_threshold;
34 const size_t m_max_token_size;
36 bool m_parsing_progress;
38 bool tokens_empty()
const
40 std::lock_guard<std::mutex> lock(m_mtx_tokens);
41 return m_tokens.empty();
50 void wait_until_tokens_empty()
52 std::unique_lock<std::mutex> lock(m_mtx_tokens);
53 while (!m_tokens.empty())
54 m_cv_tokens_empty.wait(lock);
59 m_token_size_threshold(std::max<size_t>(min_token_size, 1)),
60 m_max_token_size(max_token_size),
61 m_parsing_progress(
true)
63 if (m_token_size_threshold > m_max_token_size)
65 "initial token size threshold is already larger than the max token size.");
78 if (parser_tokens.size() < m_token_size_threshold)
84 if (m_token_size_threshold < (m_max_token_size/2))
87 m_token_size_threshold *= 2;
93 wait_until_tokens_empty();
96 std::unique_lock<std::mutex> lock(m_mtx_tokens);
97 m_tokens.swap(parser_tokens);
99 m_cv_tokens_ready.notify_one();
113 wait_until_tokens_empty();
116 std::lock_guard<std::mutex> lock(m_mtx_tokens);
117 m_tokens.swap(parser_tokens);
118 m_parsing_progress =
false;
120 m_cv_tokens_ready.notify_one();
138 std::unique_lock<std::mutex> lock(m_mtx_tokens);
139 while (m_tokens.empty() && m_parsing_progress)
140 m_cv_tokens_ready.wait(lock);
143 tokens.swap(m_tokens);
144 bool parsing_progress = m_parsing_progress;
148 m_cv_tokens_empty.notify_one();
150 return parsing_progress;
161 if (m_parsing_progress)
164 return m_token_size_threshold;
Definition: parser_token_buffer.hpp:23
void check_and_notify(tokens_type &parser_tokens)
Definition: parser_token_buffer.hpp:76
size_t token_size_threshold() const
Definition: parser_token_buffer.hpp:159
Definition: tokens.hpp:21
Definition: exception.hpp:29
void notify_and_finish(tokens_type &parser_tokens)
Definition: parser_token_buffer.hpp:110
bool next_tokens(tokens_type &tokens)
Definition: parser_token_buffer.hpp:133