455 lines
12 KiB
C++
455 lines
12 KiB
C++
#include "stream.h"
|
|
#include <iostream>
|
|
#include "exp.h"
|
|
|
|
#ifndef YAML_PREFETCH_SIZE
|
|
#define YAML_PREFETCH_SIZE 2048
|
|
#endif
|
|
|
|
#define S_ARRAY_SIZE( A ) (sizeof(A)/sizeof(*(A)))
|
|
#define S_ARRAY_END( A ) ((A) + S_ARRAY_SIZE(A))
|
|
|
|
#define CP_REPLACEMENT_CHARACTER (0xFFFD)
|
|
|
|
namespace YAML
|
|
{
|
|
enum UtfIntroState {
|
|
uis_start,
|
|
uis_utfbe_b1,
|
|
uis_utf32be_b2,
|
|
uis_utf32be_bom3,
|
|
uis_utf32be,
|
|
uis_utf16be,
|
|
uis_utf16be_bom1,
|
|
uis_utfle_bom1,
|
|
uis_utf16le_bom2,
|
|
uis_utf32le_bom3,
|
|
uis_utf16le,
|
|
uis_utf32le,
|
|
uis_utf8_imp,
|
|
uis_utf16le_imp,
|
|
uis_utf32le_imp3,
|
|
uis_utf8_bom1,
|
|
uis_utf8_bom2,
|
|
uis_utf8,
|
|
uis_error
|
|
};
|
|
|
|
enum UtfIntroCharType {
|
|
uict00,
|
|
uictBB,
|
|
uictBF,
|
|
uictEF,
|
|
uictFE,
|
|
uictFF,
|
|
uictAscii,
|
|
uictOther,
|
|
uictMax
|
|
};
|
|
|
|
static bool s_introFinalState[] = {
|
|
false, //uis_start
|
|
false, //uis_utfbe_b1
|
|
false, //uis_utf32be_b2
|
|
false, //uis_utf32be_bom3
|
|
true, //uis_utf32be
|
|
true, //uis_utf16be
|
|
false, //uis_utf16be_bom1
|
|
false, //uis_utfle_bom1
|
|
false, //uis_utf16le_bom2
|
|
false, //uis_utf32le_bom3
|
|
true, //uis_utf16le
|
|
true, //uis_utf32le
|
|
false, //uis_utf8_imp
|
|
false, //uis_utf16le_imp
|
|
false, //uis_utf32le_imp3
|
|
false, //uis_utf8_bom1
|
|
false, //uis_utf8_bom2
|
|
true, //uis_utf8
|
|
true, //uis_error
|
|
};
|
|
|
|
static UtfIntroState s_introTransitions[][uictMax] = {
|
|
// uict00, uictBB, uictBF, uictEF, uictFE, uictFF, uictAscii, uictOther
|
|
{uis_utfbe_b1, uis_utf8, uis_utf8, uis_utf8_bom1, uis_utf16be_bom1, uis_utfle_bom1, uis_utf8_imp, uis_utf8},
|
|
{uis_utf32be_b2, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf16be, uis_utf8},
|
|
{uis_utf32be, uis_utf8, uis_utf8, uis_utf8, uis_utf32be_bom3, uis_utf8, uis_utf8, uis_utf8},
|
|
{uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf32be, uis_utf8, uis_utf8},
|
|
{uis_utf32be, uis_utf32be, uis_utf32be, uis_utf32be, uis_utf32be, uis_utf32be, uis_utf32be, uis_utf32be},
|
|
{uis_utf16be, uis_utf16be, uis_utf16be, uis_utf16be, uis_utf16be, uis_utf16be, uis_utf16be, uis_utf16be},
|
|
{uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf16be, uis_utf8, uis_utf8},
|
|
{uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf16le_bom2, uis_utf8, uis_utf8, uis_utf8},
|
|
{uis_utf32le_bom3, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le},
|
|
{uis_utf32le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le},
|
|
{uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le},
|
|
{uis_utf32le, uis_utf32le, uis_utf32le, uis_utf32le, uis_utf32le, uis_utf32le, uis_utf32le, uis_utf32le},
|
|
{uis_utf16le_imp, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8},
|
|
{uis_utf32le_imp3, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le},
|
|
{uis_utf32le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le, uis_utf16le},
|
|
{uis_utf8, uis_utf8_bom2, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8},
|
|
{uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8},
|
|
{uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8, uis_utf8},
|
|
};
|
|
|
|
static char s_introUngetCount[][uictMax] = {
|
|
// uict00, uictBB, uictBF, uictEF, uictFE, uictFF, uictAscii, uictOther
|
|
{0, 1, 1, 0, 0, 0, 0, 1},
|
|
{0, 2, 2, 2, 2, 2, 2, 2},
|
|
{3, 3, 3, 3, 0, 3, 3, 3},
|
|
{4, 4, 4, 4, 4, 0, 4, 4},
|
|
{1, 1, 1, 1, 1, 1, 1, 1},
|
|
{1, 1, 1, 1, 1, 1, 1, 1},
|
|
{2, 2, 2, 2, 2, 0, 2, 2},
|
|
{2, 2, 2, 2, 0, 2, 2, 2},
|
|
{0, 1, 1, 1, 1, 1, 1, 1},
|
|
{0, 2, 2, 2, 2, 2, 2, 2},
|
|
{1, 1, 1, 1, 1, 1, 1, 1},
|
|
{1, 1, 1, 1, 1, 1, 1, 1},
|
|
{0, 2, 2, 2, 2, 2, 2, 2},
|
|
{0, 3, 3, 3, 3, 3, 3, 3},
|
|
{4, 4, 4, 4, 4, 4, 4, 4},
|
|
{2, 0, 2, 2, 2, 2, 2, 2},
|
|
{3, 3, 0, 3, 3, 3, 3, 3},
|
|
{1, 1, 1, 1, 1, 1, 1, 1},
|
|
};
|
|
|
|
inline UtfIntroCharType IntroCharTypeOf(std::istream::int_type ch)
|
|
{
|
|
if (std::istream::traits_type::eof() == ch) {
|
|
return uictOther;
|
|
}
|
|
|
|
switch (ch) {
|
|
case 0: return uict00;
|
|
case 0xBB: return uictBB;
|
|
case 0xBF: return uictBF;
|
|
case 0xEF: return uictEF;
|
|
case 0xFE: return uictFE;
|
|
case 0xFF: return uictFF;
|
|
}
|
|
|
|
if ((ch > 0) && (ch < 0xFF)) {
|
|
return uictAscii;
|
|
}
|
|
|
|
return uictOther;
|
|
}
|
|
|
|
inline char Utf8Adjust(unsigned long ch, unsigned char lead_bits, unsigned char rshift)
|
|
{
|
|
const unsigned char header = ((1 << lead_bits) - 1) << (8 - lead_bits);
|
|
const unsigned char mask = (0xFF >> (lead_bits + 1));
|
|
return static_cast<char>(static_cast<unsigned char>(
|
|
header | ((ch >> rshift) & mask)
|
|
));
|
|
}
|
|
|
|
inline void QueueUnicodeCodepoint(std::deque<char>& q, unsigned long ch)
|
|
{
|
|
// We are not allowed to queue the Stream::eof() codepoint, so
|
|
// replace it with CP_REPLACEMENT_CHARACTER
|
|
if (static_cast<unsigned long>(Stream::eof()) == ch)
|
|
{
|
|
ch = CP_REPLACEMENT_CHARACTER;
|
|
}
|
|
|
|
if (ch < 0x80)
|
|
{
|
|
q.push_back(Utf8Adjust(ch, 0, 0));
|
|
}
|
|
else if (ch < 0x800)
|
|
{
|
|
q.push_back(Utf8Adjust(ch, 2, 6));
|
|
q.push_back(Utf8Adjust(ch, 1, 0));
|
|
}
|
|
else if (ch < 0x10000)
|
|
{
|
|
q.push_back(Utf8Adjust(ch, 3, 12));
|
|
q.push_back(Utf8Adjust(ch, 1, 6));
|
|
q.push_back(Utf8Adjust(ch, 1, 0));
|
|
}
|
|
else
|
|
{
|
|
q.push_back(Utf8Adjust(ch, 4, 18));
|
|
q.push_back(Utf8Adjust(ch, 1, 12));
|
|
q.push_back(Utf8Adjust(ch, 1, 6));
|
|
q.push_back(Utf8Adjust(ch, 1, 0));
|
|
}
|
|
}
|
|
|
|
Stream::Stream(std::istream& input)
|
|
: m_input(input), m_nPushedBack(0),
|
|
m_pPrefetched(new unsigned char[YAML_PREFETCH_SIZE]),
|
|
m_nPrefetchedAvailable(0), m_nPrefetchedUsed(0)
|
|
{
|
|
typedef std::istream::traits_type char_traits;
|
|
|
|
if(!input)
|
|
return;
|
|
|
|
// Determine (or guess) the character-set by reading the BOM, if any. See
|
|
// the YAML specification for the determination algorithm.
|
|
char_traits::int_type intro[4];
|
|
int nIntroUsed = 0;
|
|
UtfIntroState state = uis_start;
|
|
for (; !s_introFinalState[state]; ) {
|
|
std::istream::int_type ch = input.get();
|
|
intro[nIntroUsed++] = ch;
|
|
UtfIntroCharType charType = IntroCharTypeOf(ch);
|
|
UtfIntroState newState = s_introTransitions[state][charType];
|
|
int nUngets = s_introUngetCount[state][charType];
|
|
if (nUngets > 0) {
|
|
for (; nUngets > 0; --nUngets) {
|
|
if (char_traits::eof() != intro[--nIntroUsed]) {
|
|
m_bufPushback[m_nPushedBack++] =
|
|
char_traits::to_char_type(intro[nIntroUsed]);
|
|
}
|
|
}
|
|
}
|
|
state = newState;
|
|
}
|
|
|
|
switch (state) {
|
|
case uis_utf8: m_charSet = utf8; break;
|
|
case uis_utf16le: m_charSet = utf16le; break;
|
|
case uis_utf16be: m_charSet = utf16be; break;
|
|
case uis_utf32le: m_charSet = utf32le; break;
|
|
case uis_utf32be: m_charSet = utf32be; break;
|
|
default: m_charSet = utf8; break;
|
|
}
|
|
|
|
ReadAheadTo(0);
|
|
}
|
|
|
|
Stream::~Stream()
|
|
{
|
|
delete[] m_pPrefetched;
|
|
}
|
|
|
|
char Stream::peek() const
|
|
{
|
|
if (m_readahead.empty())
|
|
{
|
|
return Stream::eof();
|
|
}
|
|
|
|
return m_readahead[0];
|
|
}
|
|
|
|
Stream::operator bool() const
|
|
{
|
|
return m_input.good() || (!m_readahead.empty() && m_readahead[0] != Stream::eof());
|
|
}
|
|
|
|
// get
|
|
// . Extracts a character from the stream and updates our position
|
|
char Stream::get()
|
|
{
|
|
char ch = peek();
|
|
AdvanceCurrent();
|
|
m_mark.column++;
|
|
|
|
if(ch == '\n') {
|
|
m_mark.column = 0;
|
|
m_mark.line++;
|
|
}
|
|
|
|
return ch;
|
|
}
|
|
|
|
// get
|
|
// . Extracts 'n' characters from the stream and updates our position
|
|
std::string Stream::get(int n)
|
|
{
|
|
std::string ret;
|
|
ret.reserve(n);
|
|
for(int i=0;i<n;i++)
|
|
ret += get();
|
|
return ret;
|
|
}
|
|
|
|
// eat
|
|
// . Eats 'n' characters and updates our position.
|
|
void Stream::eat(int n)
|
|
{
|
|
for(int i=0;i<n;i++)
|
|
get();
|
|
}
|
|
|
|
void Stream::AdvanceCurrent()
|
|
{
|
|
if (!m_readahead.empty())
|
|
{
|
|
m_readahead.pop_front();
|
|
m_mark.pos++;
|
|
}
|
|
|
|
ReadAheadTo(0);
|
|
}
|
|
|
|
bool Stream::_ReadAheadTo(size_t i) const
|
|
{
|
|
while (m_input.good() && (m_readahead.size() <= i))
|
|
{
|
|
switch (m_charSet)
|
|
{
|
|
case utf8: StreamInUtf8(); break;
|
|
case utf16le: StreamInUtf16(); break;
|
|
case utf16be: StreamInUtf16(); break;
|
|
case utf32le: StreamInUtf32(); break;
|
|
case utf32be: StreamInUtf32(); break;
|
|
}
|
|
}
|
|
|
|
// signal end of stream
|
|
if(!m_input.good())
|
|
m_readahead.push_back(Stream::eof());
|
|
|
|
return m_readahead.size() > i;
|
|
}
|
|
|
|
void Stream::StreamInUtf8() const
|
|
{
|
|
unsigned char b = GetNextByte();
|
|
if (m_input.good())
|
|
{
|
|
m_readahead.push_back(b);
|
|
}
|
|
}
|
|
|
|
void Stream::StreamInUtf16() const
|
|
{
|
|
unsigned long ch = 0;
|
|
unsigned char bytes[2];
|
|
int nBigEnd = (m_charSet == utf16be) ? 0 : 1;
|
|
|
|
bytes[0] = GetNextByte();
|
|
bytes[1] = GetNextByte();
|
|
if (!m_input.good())
|
|
{
|
|
return;
|
|
}
|
|
ch = (static_cast<unsigned long>(bytes[nBigEnd]) << 8) |
|
|
static_cast<unsigned long>(bytes[1 ^ nBigEnd]);
|
|
|
|
if (ch >= 0xDC00 && ch < 0xE000)
|
|
{
|
|
// Trailing (low) surrogate...ugh, wrong order
|
|
QueueUnicodeCodepoint(m_readahead, CP_REPLACEMENT_CHARACTER);
|
|
return;
|
|
}
|
|
else if (ch >= 0xD800 && ch < 0xDC00)
|
|
{
|
|
// ch is a leading (high) surrogate
|
|
|
|
// Four byte UTF-8 code point
|
|
|
|
// Read the trailing (low) surrogate
|
|
for (;;)
|
|
{
|
|
bytes[0] = GetNextByte();
|
|
bytes[1] = GetNextByte();
|
|
if (!m_input.good())
|
|
{
|
|
QueueUnicodeCodepoint(m_readahead, CP_REPLACEMENT_CHARACTER);
|
|
return;
|
|
}
|
|
unsigned long chLow = (static_cast<unsigned long>(bytes[nBigEnd]) << 8) |
|
|
static_cast<unsigned long>(bytes[1 ^ nBigEnd]);
|
|
if (chLow < 0xDC00 || ch >= 0xE000)
|
|
{
|
|
// Trouble...not a low surrogate. Dump a REPLACEMENT CHARACTER into the stream.
|
|
QueueUnicodeCodepoint(m_readahead, CP_REPLACEMENT_CHARACTER);
|
|
|
|
// Deal with the next UTF-16 unit
|
|
if (chLow < 0xD800 || ch >= 0xE000)
|
|
{
|
|
// Easiest case: queue the codepoint and return
|
|
QueueUnicodeCodepoint(m_readahead, ch);
|
|
return;
|
|
}
|
|
else
|
|
{
|
|
// Start the loop over with the new high surrogate
|
|
ch = chLow;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Select the payload bits from the high surrogate
|
|
ch &= 0x3FF;
|
|
ch <<= 10;
|
|
|
|
// Include bits from low surrogate
|
|
ch |= (chLow & 0x3FF);
|
|
|
|
// Add the surrogacy offset
|
|
ch += 0x10000;
|
|
}
|
|
}
|
|
|
|
QueueUnicodeCodepoint(m_readahead, ch);
|
|
}
|
|
|
|
inline char* ReadBuffer(unsigned char* pBuffer)
|
|
{
|
|
return reinterpret_cast<char*>(pBuffer);
|
|
}
|
|
|
|
unsigned char Stream::GetNextByte() const
|
|
{
|
|
if (m_nPushedBack)
|
|
{
|
|
return m_bufPushback[--m_nPushedBack];
|
|
}
|
|
|
|
if (m_nPrefetchedUsed >= m_nPrefetchedAvailable)
|
|
{
|
|
std::streambuf *pBuf = m_input.rdbuf();
|
|
m_nPrefetchedAvailable = pBuf->sgetn(ReadBuffer(m_pPrefetched),
|
|
YAML_PREFETCH_SIZE);
|
|
m_nPrefetchedUsed = 0;
|
|
if (!m_nPrefetchedAvailable)
|
|
{
|
|
m_input.setstate(std::ios_base::eofbit);
|
|
}
|
|
|
|
if (0 == m_nPrefetchedAvailable)
|
|
{
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
return m_pPrefetched[m_nPrefetchedUsed++];
|
|
}
|
|
|
|
void Stream::StreamInUtf32() const
|
|
{
|
|
static int indexes[2][4] = {
|
|
{3, 2, 1, 0},
|
|
{0, 1, 2, 3}
|
|
};
|
|
|
|
unsigned long ch = 0;
|
|
unsigned char bytes[4];
|
|
int* pIndexes = (m_charSet == utf32be) ? indexes[1] : indexes[0];
|
|
|
|
bytes[0] = GetNextByte();
|
|
bytes[1] = GetNextByte();
|
|
bytes[2] = GetNextByte();
|
|
bytes[3] = GetNextByte();
|
|
if (!m_input.good())
|
|
{
|
|
return;
|
|
}
|
|
|
|
for (int i = 0; i < 4; ++i)
|
|
{
|
|
ch <<= 8;
|
|
ch |= bytes[pIndexes[i]];
|
|
}
|
|
|
|
QueueUnicodeCodepoint(m_readahead, ch);
|
|
}
|
|
}
|