lean4-htt/src/util/utf8.cpp

/*
Copyright (c) 2014 Microsoft Corporation. All rights reserved.
Released under Apache 2.0 license as described in the file LICENSE.

Author: Leonardo de Moura
*/
#include <cstdlib>
#include <string>
#include "util/debug.h"
#include "util/optional.h"
#include "util/utf8.h"

namespace lean {
bool is_utf8_next(unsigned char c) { return (c & 0xC0) == 0x80; }

unsigned get_utf8_size(unsigned char c) {
    if ((c & 0x80) == 0)
        return 1;
    else if ((c & 0xE0) == 0xC0)
        return 2;
    else if ((c & 0xF0) == 0xE0)
        return 3;
    else if ((c & 0xF8) == 0xF0)
        return 4;
    else if ((c & 0xFC) == 0xF8)
        return 5;
    else if ((c & 0xFE) == 0xFC)
        return 6;
    else if (c == 0xFF)
        return 1;
    else
        return 0;
}

size_t utf8_strlen(char const * str) {
    size_t r = 0;
    while (*str != 0) {
        unsigned sz = get_utf8_size(*str);
        r++;
        str += sz;
    }
    return r;
}

optional<size_t> utf8_char_pos(char const * str, size_t char_idx) {
    size_t r = 0;
    while (*str != 0) {
        if (char_idx == 0)
            return some<size_t>(r);
        char_idx--;
        unsigned sz = get_utf8_size(*str);
        r += sz;
        str += sz;
    }
    return optional<size_t>();
}

char const * get_utf8_last_char(char const * str) {
    char const * r;
    lean_assert(*str != 0);
    do {
        r = str;
        unsigned sz = get_utf8_size(*str);
        str += sz;
    } while (*str != 0);
    return r;
}

std::string utf8_trim(std::string const & s) {
    int start = -1, stop = -1;
    for (unsigned i = 0; i < s.size(); i += get_utf8_size(s[i])) {
        if (s[i] == ' ') {
            if (stop == -1)
                stop = i;
        } else {
            if (start == -1)
                start = i;
            stop = -1;
        }
    }
    if (stop == -1)
        stop = s.size();
    return s.substr(start, stop - start);
}

unsigned utf8_to_unicode(uchar const * begin, uchar const * end) {
    unsigned result = 0;
    if (begin == end)
        return result;
    auto it = begin;
    unsigned c = *it;
    ++it;
    if (c < 128)
        return c;
    unsigned mask     = (1u << 6) -1;
    unsigned hmask    = mask;
    unsigned shift    = 0;
    unsigned num_bits = 0;
    while ((c & 0xC0) == 0xC0) {
        c <<= 1;
        c &= 0xff;
        num_bits += 6;
        hmask >>= 1;
        shift++;
        result <<= 6;
        if (it == end)
            return 0;
        result |= *it & mask;
        ++it;
    }
    result |= ((c >> shift) & hmask) << num_bits;
    return result;
}

void utf8_decode(std::string const & str, buffer<unsigned> & out) {
    unsigned i = 0;
    while (i < str.size()) {
        unsigned c = static_cast<unsigned char>(str[i]);
        /* zero continuation (0 to 127) */
        if ((c & 0x80) == 0) {
            out.push_back(c);
            i++;
            continue;
        }

        /* one continuation (128 to 2047) */
        if ((c & 0xe0) == 0xc0 && i + 1 < str.size()) {
            unsigned c1 = static_cast<unsigned char>(str[i+1]);
            unsigned r = ((c & 0x1f) << 6) | c1;
            if (r >= 128) {
                out.push_back(r);
                i += 2;
                continue;
            }
        }

        /* two continuations (2048 to 55295 and 57344 to 65535) */
        if ((c & 0xf0) == 0xe0 && i + 2 < str.size()) {
            unsigned c1 = static_cast<unsigned char>(str[i+1]);
            unsigned c2 = static_cast<unsigned char>(str[i+2]);
            unsigned r = ((c & 0x0f) << 12) | (c1 << 6) | c2;
            if (r >= 2048 && (r < 55296 || r > 57343)) {
                out.push_back(r);
                i += 3;
                continue;
            }
        }

        /* three continuations (65536 to 1114111) */
        if ((c & 0xf8) == 0xf0 && i + 3 < str.size()) {
            unsigned c1 = static_cast<unsigned char>(str[i+1]);
            unsigned c2 = static_cast<unsigned char>(str[i+2]);
            unsigned c3 = static_cast<unsigned char>(str[i+3]);
            unsigned r  = ((c & 0x07) << 18) | (c1 << 12) | (c2 << 6) | c3;
            if (r >= 65536 && r <= 1114111) {
                out.push_back(r);
                i += 4;
                continue;
            }
        }

        /* invalid UTF-8 encoded string */
        out.push_back(c);
        i++;
    }
}

#define TAG_CONT    static_cast<unsigned char>(0b10000000)
#define TAG_TWO_B   static_cast<unsigned char>(0b11000000)
#define TAG_THREE_B static_cast<unsigned char>(0b11100000)
#define TAG_FOUR_B  static_cast<unsigned char>(0b11110000)
#define MAX_ONE_B   0x80
#define MAX_TWO_B   0x800
#define MAX_THREE_B 0x10000

void push_unicode_scalar(std::string & s, unsigned code) {
    if (code < MAX_ONE_B) {
        s.push_back(static_cast<unsigned char>(code));
    } else if (code < MAX_TWO_B) {
        s.push_back(static_cast<unsigned char>(code >> 6 & 0x1F) | TAG_TWO_B);
        s.push_back(static_cast<unsigned char>(code & 0x3F) | TAG_CONT);
    } else if (code < MAX_THREE_B) {
        s.push_back(static_cast<unsigned char>(code >> 12 & 0x0F) | TAG_THREE_B);
        s.push_back(static_cast<unsigned char>(code >>  6 & 0x3F) | TAG_CONT);
        s.push_back(static_cast<unsigned char>(code & 0x3F) | TAG_CONT);
    } else {
        s.push_back(static_cast<unsigned char>(code >> 18 & 0x07) | TAG_FOUR_B);
        s.push_back(static_cast<unsigned char>(code >> 12 & 0x3F) | TAG_CONT);
        s.push_back(static_cast<unsigned char>(code >>  6 & 0x3F) | TAG_CONT);
        s.push_back(static_cast<unsigned char>(code & 0x3F) | TAG_CONT);
    }
}
}