Imported Upstream version 0.26.0

components/misc/utf8stream.hpp

@@ -0,0 +1,116 @@
+#ifndef MISC_UTF8ITER_HPP
+#define MISC_UTF8ITER_HPP
+
+#include <boost/tuple/tuple.hpp>
+
+class Utf8Stream
+{
+public:
+
+    typedef uint32_t UnicodeChar;
+    typedef unsigned char const * Point;
+
+    //static const unicode_char sBadChar = 0xFFFFFFFF; gcc can't handle this
+    static UnicodeChar sBadChar () { return UnicodeChar (0xFFFFFFFF); }
+
+    Utf8Stream (Point begin, Point end) :
+        cur (begin), nxt (begin), end (end)
+    {
+    }
+
+    Utf8Stream (std::pair <Point, Point> range) :
+        cur (range.first), nxt (range.first), end (range.second)
+    {
+    }
+
+    bool eof () const
+    {
+        return cur == end;
+    }
+
+    Point current () const
+    {
+        return cur;
+    }
+
+    UnicodeChar peek ()
+    {
+        if (cur == nxt)
+            next ();
+        return val;
+    }
+
+    UnicodeChar consume ()
+    {
+        if (cur == nxt)
+            next ();
+        cur = nxt;
+        return val;
+    }
+
+    static std::pair <UnicodeChar, Point> decode (Point cur, Point end)
+    {
+        if ((*cur & 0x80) == 0)
+        {
+            UnicodeChar chr = *cur++;
+
+            return std::make_pair (chr, cur);
+        }
+
+        int octets;
+        UnicodeChar chr;
+
+        boost::tie (octets, chr) = octet_count (*cur++);
+
+        if (octets > 5)
+            return std::make_pair (sBadChar(), cur);
+
+        Point eoc = cur + octets;
+
+        if (eoc > end)
+            return std::make_pair (sBadChar(), cur);
+
+        while (cur != eoc)
+        {
+            if ((*cur & 0xC0) != 0x80) // check continuation mark
+                return std::make_pair (sBadChar(), cur);;
+
+            chr = (chr << 6) | UnicodeChar ((*cur++) & 0x3F);
+        }
+
+        return std::make_pair (chr, cur);
+    }
+
+private:
+
+    static std::pair <int, UnicodeChar> octet_count (unsigned char octet)
+    {
+        int octets;
+
+        unsigned char mark = 0xC0;
+        unsigned char mask = 0xE0;
+
+        for (octets = 1; octets <= 5; ++octets)
+        {
+            if ((octet & mask) == mark)
+                break;
+
+            mark = (mark >> 1) | 0x80;
+            mask = (mask >> 1) | 0x80;
+        }
+
+        return std::make_pair (octets, octet & ~mask);
+    }
+
+    void next ()
+    {
+        boost::tie (val, nxt) = decode (nxt, end);
+    }
+
+    Point cur;
+    Point nxt;
+    Point end;
+    UnicodeChar val;
+};
+
+#endif