#!/usr/bin/env python
# -*- coding: utf-8 -*-#
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Bleeding-edge version of Unicode Character Database.

Provides an interface similar to Python's own unicodedata package, but with
the bleeding-edge data. The implementation is not efficient at all, it's
just done this way for the ease of use. The data is coming from bleeding
edge version of the Unicode Standard not yet published, so it is expected to
be unstable and sometimes inconsistent.
"""

__author__ = (
    "roozbeh@google.com (Roozbeh Pournader) and " "cibu@google.com (Cibu Johny)"
)

import codecs
import collections
import os
from os import path
import re

from nototools.py23 import unichr, unicode, basestring

try:
    import unicodedata2 as unicodedata  # Unicode 8 compliant native lib
except ImportError:
    import unicodedata  # Python's internal library

from nototools import tool_utils  # parse_int_ranges

# Update this when we update the base version data we use
UNICODE_VERSION = 14.0

_data_is_loaded = False
_property_value_aliases_data = {}
_character_names_data = {}
_general_category_data = {}
_combining_class_data = {}
_decomposition_data = {}
_bidi_mirroring_characters = set()
_script_data = {}
_script_extensions_data = {}
_block_data = {}
_block_range = {}
_block_names = []
_age_data = {}
_bidi_mirroring_glyph_data = {}
_core_properties_data = {}
_indic_positional_data = {}
_indic_syllabic_data = {}
_defined_characters = set()
_script_code_to_long_name = {}
_folded_script_name_to_code = {}
_lower_to_upper_case = {}

# emoji data
_presentation_default_emoji = None
_presentation_default_text = None
_emoji_modifier_base = None
_emoji = None
_emoji_variants = None
_emoji_variants_proposed = None

# non-emoji variant data
_variant_data = None
_variant_data_cps = None

# proposed emoji
_proposed_emoji_data = None
_proposed_emoji_data_cps = None

# emoji sequences
_emoji_sequence_data = None
_emoji_non_vs_to_canonical = None
_emoji_group_data = None

# nameslist/namealiases
_nameslist_see_also = None
_namealiases_alt_names = None


def load_data():
    """Loads the data files needed for the module.

    Could be used by processes that care about controlling when the data is
    loaded. Otherwise, data will be loaded the first time it's needed.
    """
    global _data_is_loaded

    if not _data_is_loaded:
        _load_property_value_aliases_txt()
        _load_unicode_data_txt()
        _load_scripts_txt()
        _load_script_extensions_txt()
        _load_blocks_txt()
        _load_derived_age_txt()
        _load_derived_core_properties_txt()
        _load_bidi_mirroring_txt()
        _load_indic_data()
        _load_emoji_data()
        _load_emoji_sequence_data()
        _load_unicode_emoji_variants()
        _load_variant_data()
        _load_proposed_emoji_data()
        _load_nameslist_data()
        _load_namealiases_data()

        _data_is_loaded = True


def name(char, *args):
    """Returns the name of a character.

    Raises a ValueError exception if the character is undefined, unless an
    extra argument is given, in which case it will return that argument.
    """
    if isinstance(char, int):
        char = unichr(char)
    # First try and get the name from unidata, which is faster and supports
    # CJK and Hangul automatic names
    try:
        return unicodedata.name(char)
    except ValueError as val_error:
        cp = ord(char)
        load_data()
        if cp in _character_names_data:
            return _character_names_data[cp]
        elif (cp,) in _emoji_sequence_data:
            return _emoji_sequence_data[(cp,)][0]
        elif args:
            return args[0]
        else:
            raise Exception('no name for "%0x"' % ord(char))


def _char_to_int(char):
    """Converts a potential character to its scalar value."""
    if type(char) in [str, type("")]:
        return ord(char)
    else:
        return char


def derived_props():
    load_data()
    return frozenset(_core_properties_data.keys())


def chars_with_property(propname):
    load_data()
    return frozenset(_core_properties_data[propname])


def category(char):
    """Returns the general category of a character."""
    load_data()
    char = _char_to_int(char)
    try:
        return _general_category_data[char]
    except KeyError:
        return "Cn"  # Unassigned


def combining(char):
    """Returns the canonical combining class of a character."""
    load_data()
    char = _char_to_int(char)
    try:
        return _combining_class_data[char]
    except KeyError:
        return 0


def to_upper(char):
    """Returns the upper case for a lower case character.
    This is not full upper casing, but simply reflects the 1-1
    mapping in UnicodeData.txt."""
    load_data()
    cp = _char_to_int(char)
    try:
        if _general_category_data[cp] == "Ll":
            return unichr(_lower_to_upper_case[cp])
    except KeyError:
        pass
    return char


def canonical_decomposition(char):
    """Returns the canonical decomposition of a character as a Unicode string."""
    load_data()
    char = _char_to_int(char)
    try:
        return _decomposition_data[char]
    except KeyError:
        return ""


def script(char):
    """Returns the script property of a character as a four-letter code."""
    load_data()
    char = _char_to_int(char)
    try:
        return _script_data[char]
    except KeyError:
        return "Zzzz"  # Unknown


def script_extensions(char):
    """Returns the script extensions property of a character.

    The return value is a frozenset of four-letter script codes.
    """
    load_data()
    char = _char_to_int(char)
    try:
        return _script_extensions_data[char]
    except KeyError:
        return frozenset([script(char)])


def block(char):
    """Returns the block property of a character."""
    load_data()
    char = _char_to_int(char)
    try:
        return _block_data[char]
    except KeyError:
        return "No_Block"


def block_range(block):
    """Returns a range (first, last) of the named block."""
    load_data()
    return _block_range[block]


def block_chars(block):
    """Returns a frozenset of the cps in the named block."""
    load_data()
    first, last = _block_range[block]
    return frozenset(range(first, last + 1))


def block_names():
    """Returns the names of the blocks in block order."""
    load_data()
    return _block_names[:]


def age(char):
    """Returns the age property of a character as a string.

    Returns None if the character is unassigned."""
    load_data()
    char = _char_to_int(char)
    try:
        return _age_data[char]
    except KeyError:
        return None


# Uniscribe treats these ignorables (Hangul fillers) as spacing.
UNISCRIBE_USED_IGNORABLES = frozenset([0x115F, 0x1160, 0x3164, 0xFFA0])


def is_default_ignorable(char):
    """Returns true if the character has the Default_Ignorable property."""
    load_data()
    if isinstance(char, (str, unicode)):
        char = ord(char)
    return char in _core_properties_data["Default_Ignorable_Code_Point"]


def default_ignorables():
    load_data()
    return frozenset(_core_properties_data["Default_Ignorable_Code_Point"])


def is_defined(char):
    """Returns true if the character is defined in the Unicode Standard."""
    load_data()
    if isinstance(char, (str, unicode)):
        char = ord(char)
    return char in _defined_characters


def is_private_use(char):
    """Returns true if the characters is a private use character."""
    return category(char) == "Co"


def mirrored(char):
    """Returns 1 if the characters is bidi mirroring, 0 otherwise."""
    load_data()
    if isinstance(char, (str, unicode)):
        char = ord(char)
    return int(char in _bidi_mirroring_characters)


def bidi_mirroring_glyph(char):
    """Returns the bidi mirroring glyph property of a character."""
    load_data()
    if isinstance(char, (str, unicode)):
        char = ord(char)
    try:
        return _bidi_mirroring_glyph_data[char]
    except KeyError:
        return None


def mirrored_chars():
    return frozenset(_bidi_mirroring_glyph_data.keys())


def indic_positional_category(char):
    """Returns the Indic positional category of a character."""
    load_data()
    if isinstance(char, (str, unicode)):
        char = ord(char)
    try:
        return _indic_positional_data[char]
    except KeyError:
        return "NA"


def indic_syllabic_category(char):
    """Returns the Indic syllabic category of a character."""
    load_data()
    if isinstance(char, (str, unicode)):
        char = ord(char)
    try:
        return _bidi_syllabic_data[char]
    except KeyError:
        return "Other"


def create_script_to_chars():
    """Returns a mapping from script to defined characters, based on script and
    extensions, for all scripts."""
    load_data()
    result = collections.defaultdict(set)
    for cp in _defined_characters:
        if cp in _script_data:
            result[_script_data[cp]].add(cp)
        if cp in _script_extensions_data:
            for script in _script_extensions_data[cp]:
                result[script].add(cp)
    return result


_DEFINED_CHARACTERS_CACHE = {}


def defined_characters(version=None, scr=None):
    """Returns the set of all defined characters in the Unicode Standard."""
    load_data()
    # handle common error where version is passed as string, the age test
    # will always pass
    if version is not None:
        version = float(version)
    try:
        return _DEFINED_CHARACTERS_CACHE[(version, scr)]
    except KeyError:
        pass
    characters = _defined_characters
    if version is not None:
        characters = {
            char
            for char in characters
            if age(char) is not None and float(age(char)) <= version
        }
    if scr is not None:
        characters = {
            char
            for char in characters
            if script(char) == scr or scr in script_extensions(char)
        }
    characters = frozenset(characters)
    _DEFINED_CHARACTERS_CACHE[(version, scr)] = characters
    return characters


_strip_re = re.compile(r"[-'_ ]+")


def _folded_script_name(script_name):
    """Folds a script name to its bare bones for comparison."""
    # string.translate is changed by codecs, the method no longer takes two
    # parameters and so script_name.translate(None, "'-_ ") fails to compile
    return _strip_re.sub("", script_name).lower()


def script_code(script_name):
    """Returns the four-letter ISO 15924 code of a script from its long name."""
    load_data()
    folded_script_name = _folded_script_name(script_name)
    try:
        return _HARD_CODED_FOLDED_SCRIPT_NAME_TO_CODE[folded_script_name]
    except:
        return _folded_script_name_to_code.get(folded_script_name, "Zzzz")


# We use some standard script codes that are not assigned to a codepoint
# by unicode, e.g. Zsym.  The data based off Scripts.txt doesn't contain
# these so we add them here.  There are also a few names with punctuation
# that we special-case
_HARD_CODED_HUMAN_READABLE_SCRIPT_NAMES = {
    "Aran": "Nastaliq",  # not assigned
    "Nkoo": "N'Ko",
    "Phag": "Phags-pa",
    "Piqd": "Klingon",  # not assigned
    "Zmth": "Math",  # not assigned
    "Zsye": "Emoji",  # not assigned
    "Zsym": "Symbols",  # not assigned
}

_HARD_CODED_FOLDED_SCRIPT_NAME_TO_CODE = {
    _folded_script_name(name): code
    for code, name in _HARD_CODED_HUMAN_READABLE_SCRIPT_NAMES.items()
}


def human_readable_script_name(code):
    """Returns a human-readable name for the script code."""
    try:
        return _HARD_CODED_HUMAN_READABLE_SCRIPT_NAMES[code]
    except KeyError:
        load_data()
        return _script_code_to_long_name[code]


def all_scripts():
    """Return a frozenset of all four-letter script codes."""
    load_data()
    return frozenset(_script_code_to_long_name.keys())


_DATA_DIR_PATH = path.join(
    path.abspath(path.dirname(__file__)), os.pardir, "third_party", "ucd"
)


def open_unicode_data_file(data_file_name):
    """Opens a Unicode data file.

    Args:
      data_file_name: A string containing the filename of the data file.

    Returns:
      A file handle to the data file.
    """
    filename = path.join(_DATA_DIR_PATH, data_file_name)
    return codecs.open(filename, "r", "utf-8")


def _parse_code_ranges(input_data):
    """Reads Unicode code ranges with properties from an input string.

    Reads a Unicode data file already imported into a string. The format is
    the typical Unicode data file format with either one character or a
    range of characters separated by a semicolon with a property value (and
    potentially comments after a number sign, that will be ignored).

    Example source data file:
      http://www.unicode.org/Public/UNIDATA/Scripts.txt

    Example data:
      0000..001F    ; Common # Cc  [32] <control-0000>..<control-001F>
      0020          ; Common # Zs       SPACE

    Args:
      input_data: An input string, containing the data.

    Returns:
      A list of tuples corresponding to the input data, with each tuple
      containing the beginning of the range, the end of the range, and the
      property value for the range. For example:
      [(0, 31, 'Common'), (32, 32, 'Common')]
    """
    ranges = []
    line_regex = re.compile(
        r"^"
        r"([0-9A-F]{4,6})"  # first character code
        r"(?:\.\.([0-9A-F]{4,6}))?"  # optional second character code
        r"\s*;\s*"
        r"([^#]+)"
    )  # the data, up until the potential comment
    for line in input_data.split("\n"):
        match = line_regex.match(line)
        if not match:
            continue

        first, last, data = match.groups()
        if last is None:
            last = first

        first = int(first, 16)
        last = int(last, 16)
        data = data.rstrip()

        ranges.append((first, last, data))

    return ranges


def _parse_semicolon_separated_data(input_data):
    """Reads semicolon-separated Unicode data from an input string.

    Reads a Unicode data file already imported into a string. The format is
    the Unicode data file format with a list of values separated by
    semicolons. The number of the values on different lines may be different
    from another.

    Example source data file:
      http://www.unicode.org/Public/UNIDATA/PropertyValueAliases.txt

    Example data:
      sc;  Cher  ; Cherokee
      sc;  Copt  ; Coptic   ; Qaac

    Args:
      input_data: An input string, containing the data.

    Returns:
      A list of lists corresponding to the input data, with each individual
      list containing the values as strings. For example:
      [['sc', 'Cher', 'Cherokee'], ['sc', 'Copt', 'Coptic', 'Qaac']]
    """
    all_data = []
    for line in input_data.split("\n"):
        line = line.split("#", 1)[0].strip()  # remove the comment
        if not line:
            continue

        fields = line.split(";")
        fields = [field.strip() for field in fields]
        all_data.append(fields)

    return all_data


def _load_unicode_data_txt():
    """Load character data from UnicodeData.txt."""
    global _defined_characters
    global _bidi_mirroring_characters
    if _defined_characters:
        return

    with open_unicode_data_file("UnicodeData.txt") as unicode_data_txt:
        unicode_data = _parse_semicolon_separated_data(unicode_data_txt.read())

    for line in unicode_data:
        code = int(line[0], 16)
        char_name = line[1]
        general_category = line[2]
        combining_class = int(line[3])

        decomposition = line[5]
        if decomposition.startswith("<"):
            # We only care about canonical decompositions
            decomposition = ""
        decomposition = decomposition.split()
        decomposition = [unichr(int(char, 16)) for char in decomposition]
        decomposition = "".join(decomposition)

        bidi_mirroring = line[9] == "Y"
        if general_category == "Ll":
            upcode = line[12]
            if upcode:
                upper_case = int(upcode, 16)
                _lower_to_upper_case[code] = upper_case

        if char_name.endswith("First>"):
            last_range_opener = code
        elif char_name.endswith("Last>"):
            # Ignore surrogates
            if "Surrogate" not in char_name:
                for char in range(last_range_opener, code + 1):
                    _general_category_data[char] = general_category
                    _combining_class_data[char] = combining_class
                    if bidi_mirroring:
                        _bidi_mirroring_characters.add(char)
                    _defined_characters.add(char)
        else:
            _character_names_data[code] = char_name
            _general_category_data[code] = general_category
            _combining_class_data[code] = combining_class
            if bidi_mirroring:
                _bidi_mirroring_characters.add(code)
            _decomposition_data[code] = decomposition
            _defined_characters.add(code)

    _defined_characters = frozenset(_defined_characters)
    _bidi_mirroring_characters = frozenset(_bidi_mirroring_characters)


def _load_scripts_txt():
    """Load script property from Scripts.txt."""
    with open_unicode_data_file("Scripts.txt") as scripts_txt:
        script_ranges = _parse_code_ranges(scripts_txt.read())

    for first, last, script_name in script_ranges:
        folded_script_name = _folded_script_name(script_name)
        script = _folded_script_name_to_code[folded_script_name]
        for char_code in range(first, last + 1):
            _script_data[char_code] = script


def _load_script_extensions_txt():
    """Load script property from ScriptExtensions.txt."""
    with open_unicode_data_file("ScriptExtensions.txt") as se_txt:
        script_extensions_ranges = _parse_code_ranges(se_txt.read())

    for first, last, script_names in script_extensions_ranges:
        script_set = frozenset(script_names.split(" "))
        for character_code in range(first, last + 1):
            _script_extensions_data[character_code] = script_set


def _load_blocks_txt():
    """Load block name from Blocks.txt."""
    with open_unicode_data_file("Blocks.txt") as blocks_txt:
        block_ranges = _parse_code_ranges(blocks_txt.read())

    for first, last, block_name in block_ranges:
        _block_names.append(block_name)
        _block_range[block_name] = (first, last)
        for character_code in range(first, last + 1):
            _block_data[character_code] = block_name


def _load_derived_age_txt():
    """Load age property from DerivedAge.txt."""
    with open_unicode_data_file("DerivedAge.txt") as derived_age_txt:
        age_ranges = _parse_code_ranges(derived_age_txt.read())

    for first, last, char_age in age_ranges:
        for char_code in range(first, last + 1):
            _age_data[char_code] = char_age


def _load_derived_core_properties_txt():
    """Load derived core properties from Blocks.txt."""
    with open_unicode_data_file("DerivedCoreProperties.txt") as dcp_txt:
        dcp_ranges = _parse_code_ranges(dcp_txt.read())

    for first, last, property_name in dcp_ranges:
        for character_code in range(first, last + 1):
            try:
                _core_properties_data[property_name].add(character_code)
            except KeyError:
                _core_properties_data[property_name] = {character_code}


def _load_property_value_aliases_txt():
    """Load property value aliases from PropertyValueAliases.txt."""
    with open_unicode_data_file("PropertyValueAliases.txt") as pva_txt:
        aliases = _parse_semicolon_separated_data(pva_txt.read())

    for data_item in aliases:
        if data_item[0] == "sc":  # Script
            code = data_item[1]
            long_name = data_item[2]
            _script_code_to_long_name[code] = long_name.replace("_", " ")
            folded_name = _folded_script_name(long_name)
            _folded_script_name_to_code[folded_name] = code


def _load_bidi_mirroring_txt():
    """Load bidi mirroring glyphs from BidiMirroring.txt."""

    with open_unicode_data_file("BidiMirroring.txt") as bidi_mirroring_txt:
        bmg_pairs = _parse_semicolon_separated_data(bidi_mirroring_txt.read())

    for char, bmg in bmg_pairs:
        char = int(char, 16)
        bmg = int(bmg, 16)
        _bidi_mirroring_glyph_data[char] = bmg


def _load_indic_data():
    """Load Indic properties from Indic(Positional|Syllabic)Category.txt."""
    with open_unicode_data_file("IndicPositionalCategory.txt") as inpc_txt:
        positional_ranges = _parse_code_ranges(inpc_txt.read())
    for first, last, char_position in positional_ranges:
        for char_code in range(first, last + 1):
            _indic_positional_data[char_code] = char_position

    with open_unicode_data_file("IndicSyllabicCategory.txt") as insc_txt:
        syllabic_ranges = _parse_code_ranges(insc_txt.read())
    for first, last, char_syllabic_category in syllabic_ranges:
        for char_code in range(first, last + 1):
            _indic_syllabic_data[char_code] = char_syllabic_category


def _load_emoji_data():
    """Parse the new draft format of emoji-data.txt"""
    global _presentation_default_emoji, _presentation_default_text
    global _emoji, _emoji_modifier_base

    if _presentation_default_emoji:
        return

    emoji_sets = {
        "Emoji": set(),
        "Emoji_Presentation": set(),
        "Emoji_Modifier": set(),
        "Emoji_Modifier_Base": set(),
        "Extended_Pictographic": set(),
        "Emoji_Component": set(),
    }

    set_names = "|".join(sorted(emoji_sets.keys()))
    line_re = re.compile(
        r"([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))?\s*;\s*" r"(%s)\s*#.*$" % set_names
    )

    with open_unicode_data_file("emoji-data.txt") as f:
        for line in f:
            line = line.strip()
            if not line or line[0] == "#":
                continue
            m = line_re.match(line)
            if not m:
                raise ValueError('Did not match "%s"' % line)
            start = int(m.group(1), 16)
            end = start if not m.group(2) else int(m.group(2), 16)
            emoji_set = emoji_sets.get(m.group(3))
            emoji_set.update(range(start, end + 1))

    # allow our legacy use of handshake and wrestlers with skin tone modifiers
    emoji_sets["Emoji_Modifier_Base"] |= {0x1F91D, 0x1F93C}

    _presentation_default_emoji = frozenset(emoji_sets["Emoji_Presentation"])
    _presentation_default_text = frozenset(
        emoji_sets["Emoji"] - emoji_sets["Emoji_Presentation"]
    )
    _emoji_modifier_base = frozenset(emoji_sets["Emoji_Modifier_Base"])
    _emoji = frozenset(emoji_sets["Emoji"])

    # we have no real use for the 'Emoji_Regional_Indicator' and
    # 'Emoji_Component' sets, and they're not documented, so ignore them.
    # The regional indicator set is just the 26 regional indicator
    # symbols, and the component set is number sign, asterisk, ASCII digits,
    # the regional indicators, and the skin tone modifiers.


PROPOSED_EMOJI_AGE = 1000.0
ZWJ = 0x200D
EMOJI_VS = 0xFE0F
EMOJI_SEQUENCE_TYPES = frozenset(
    [
        "Basic_Emoji",
        "Emoji_Keycap_Sequence",
        "Emoji_Combining_Sequence",
        "Emoji_Flag_Sequence",
        "RGI_Emoji_Flag_Sequence",
        "RGI_Emoji_Tag_Sequence",
        "Emoji_Modifier_Sequence",
        "RGI_Emoji_Modifier_Sequence",
        "RGI_Emoji_ZWJ_Sequence",
        "Emoji_ZWJ_Sequence",
        "Emoji_Single_Sequence",
    ]
)

# Unicode 12 decided to be 'helpful' and included single emoji in the sequence
# data, but unlike all the other data represents these in batches as XXXX..XXXX
# rather than one per line.  We can't get name data for these so we can't
# use that data, but still have to parse the line.
def _read_emoji_data(lines):
    """Parse lines of emoji data and return a map from sequence to tuples of
    name, age, type."""
    line_re = re.compile(
        r"(?:([0-9A-F ]+)|([0-9A-F]+\.\.[0-9A-F]+)\s*);\s*(%s)\s*;\s*([^#]*)\s*#\s*E?(\d+\.\d+).*"
        % "|".join(EMOJI_SEQUENCE_TYPES)
    )
    result = {}
    for line in lines:
        line = line.strip()
        if not line or line[0] == "#":
            continue
        m = line_re.match(line)
        if not m:
            raise ValueError('"%s" Did not match "%s"' % (line_re.pattern, line))

        # group 1 is a sequence, group 2 is a range of single character sequences.
        # we can't process the range because we don't have a name for each character
        # in the range, so skip it and get these emoji and their names from
        # UnicodeData
        if m.group(2):
            continue

        seq_type = m.group(3).strip().encode("ascii")
        seq = tuple(int(s, 16) for s in m.group(1).split())
        name = m.group(4).strip()
        age = float(m.group(5))
        result[seq] = (name, age, seq_type)
    return result


def _read_emoji_data_file(filename):
    with open_unicode_data_file(filename) as f:
        return _read_emoji_data(f.readlines())


_EMOJI_QUAL_TYPES = [
    "component",
    "fully-qualified",
    "minimally-qualified",
    "unqualified",
]


def _read_emoji_test_data(data_string):
    """Parse the emoji-test.txt data.  This has names of proposed emoji that are
    not yet in the full Unicode data file.  Returns a list of tuples of
    sequence, group, subgroup, name.

    The data is a string."""
    line_re = re.compile(
        r"([0-9a-fA-F ]+)\s*;\s*(%s)\s*#\s*(?:[^\s]+)\s+(.*)\s*"
        % "|".join(_EMOJI_QUAL_TYPES)
    )
    result = []
    GROUP_PREFIX = "# group: "
    SUBGROUP_PREFIX = "# subgroup: "
    group = None
    subgroup = None
    for line in data_string.splitlines():
        line = line.strip()
        if not line:
            continue

        if line[0] == "#":
            if line.startswith(GROUP_PREFIX):
                group = line[len(GROUP_PREFIX) :].strip().encode("ascii")
                subgroup = None
            elif line.startswith(SUBGROUP_PREFIX):
                subgroup = line[len(SUBGROUP_PREFIX) :].strip().encode("ascii")
            continue

        m = line_re.match(line)
        if not m:
            raise ValueError('Did not match "%s" in emoji-test.txt' % line)
        if m.group(2) not in ["component", "fully-qualified"]:
            # We only want component and fully-qualified sequences, as those are
            # 'canonical'.  'minimally-qualified' apparently just leave off the
            # FEOF emoji presentation tag, we already assume these.
            # Information for the unqualified sequences should be
            # redundant.  At the moment we don't verify this so if the file
            # changes we won't catch that.
            continue
        seq = tuple(int(s, 16) for s in m.group(1).split())
        name = m.group(3).strip()
        if not (group and subgroup):
            raise Exception(
                "sequence %s missing group or subgroup" % seq_to_string(seq)
            )
        result.append((seq, group, subgroup, name))

    return result


_SUPPLEMENTAL_EMOJI_GROUP_DATA = """
# group: Misc

# subgroup: used with keycaps
0023 fe0f ; fully-qualified # ? number sign
002a fe0f ; fully-qualified # ? asterisk
0030 fe0f ; fully-qualified # ? digit zero
0031 fe0f ; fully-qualified # ? digit one
0032 fe0f ; fully-qualified # ? digit two
0033 fe0f ; fully-qualified # ? digit three
0034 fe0f ; fully-qualified # ? digit four
0035 fe0f ; fully-qualified # ? digit five
0036 fe0f ; fully-qualified # ? digit six
0037 fe0f ; fully-qualified # ? digit seven
0038 fe0f ; fully-qualified # ? digit eight
0039 fe0f ; fully-qualified # ? digit nine
20e3 ; fully-qualified # ? combining enclosing keycap

# As of Unicode 11 these have group data defined.
# subgroup: skin-tone modifiers
#1f3fb ; fully-qualified # ? emoji modifier fitzpatrick type-1-2
#1f3fc ; fully-qualified # ? emoji modifier fitzpatrick type-3
#1f3fd ; fully-qualified # ? emoji modifier fitzpatrick type-4
#1f3fe ; fully-qualified # ? emoji modifier fitzpatrick type-5
#1f3ff ; fully-qualified # ? emoji modifier fitzpatrick type-6

# subgroup: regional indicator symbols
1f1e6 ; fully-qualified # ? regional indicator symbol letter A
1f1e7 ; fully-qualified # ? regional indicator symbol letter B
1f1e8 ; fully-qualified # ? regional indicator symbol letter C
1f1e9 ; fully-qualified # ? regional indicator symbol letter D
1f1ea ; fully-qualified # ? regional indicator symbol letter E
1f1eb ; fully-qualified # ? regional indicator symbol letter F
1f1ec ; fully-qualified # ? regional indicator symbol letter G
1f1ed ; fully-qualified # ? regional indicator symbol letter H
1f1ee ; fully-qualified # ? regional indicator symbol letter I
1f1ef ; fully-qualified # ? regional indicator symbol letter J
1f1f0 ; fully-qualified # ? regional indicator symbol letter K
1f1f1 ; fully-qualified # ? regional indicator symbol letter L
1f1f2 ; fully-qualified # ? regional indicator symbol letter M
1f1f3 ; fully-qualified # ? regional indicator symbol letter N
1f1f4 ; fully-qualified # ? regional indicator symbol letter O
1f1f5 ; fully-qualified # ? regional indicator symbol letter P
1f1f6 ; fully-qualified # ? regional indicator symbol letter Q
1f1f7 ; fully-qualified # ? regional indicator symbol letter R
1f1f8 ; fully-qualified # ? regional indicator symbol letter S
1f1f9 ; fully-qualified # ? regional indicator symbol letter T
1f1fa ; fully-qualified # ? regional indicator symbol letter U
1f1fb ; fully-qualified # ? regional indicator symbol letter V
1f1fc ; fully-qualified # ? regional indicator symbol letter W
1f1fd ; fully-qualified # ? regional indicator symbol letter X
1f1fe ; fully-qualified # ? regional indicator symbol letter Y
1f1ff ; fully-qualified # ? regional indicator symbol letter Z

#subgroup: unknown flag
fe82b ; fully-qualified # ? unknown flag PUA codepoint
"""

# These are skin tone sequences that Unicode decided not to define.  Android
# shipped with them, so we're stuck with them forever regardless of what
# Unicode says.
#
# This data is in the format of emoji-sequences.txt and emoji-zwj-sequences.txt
_LEGACY_ANDROID_SEQUENCES = """
1F93C 1F3FB                ; Emoji_Modifier_Sequence ; people wrestling: light skin tone # 9.0
1F93C 1F3FC                ; Emoji_Modifier_Sequence ; people wrestling: medium-light skin tone # 9.0
1F93C 1F3FD                ; Emoji_Modifier_Sequence ; people wrestling: medium skin tone # 9.0
1F93C 1F3FE                ; Emoji_Modifier_Sequence ; people wrestling: medium-dark skin tone # 9.0
1F93C 1F3FF                ; Emoji_Modifier_Sequence ; people wrestling: dark skin tone # 9.0
1F93C 1F3FB 200D 2642 FE0F ; Emoji_ZWJ_Sequence ; men wrestling: light skin tone # 9.0
1F93C 1F3FC 200D 2642 FE0F ; Emoji_ZWJ_Sequence ; men wrestling: medium-light skin tone # 9.0
1F93C 1F3FD 200D 2642 FE0F ; Emoji_ZWJ_Sequence ; men wrestling: medium skin tone # 9.0
1F93C 1F3FE 200D 2642 FE0F ; Emoji_ZWJ_Sequence ; men wrestling: medium-dark skin tone # 9.0
1F93C 1F3FF 200D 2642 FE0F ; Emoji_ZWJ_Sequence ; men wrestling: dark skin tone # 9.0
1F93C 1F3FB 200D 2640 FE0F ; Emoji_ZWJ_Sequence ; women wrestling: light skin tone # 9.0
1F93C 1F3FC 200D 2640 FE0F ; Emoji_ZWJ_Sequence ; women wrestling: medium-light skin tone # 9.0
1F93C 1F3FD 200D 2640 FE0F ; Emoji_ZWJ_Sequence ; women wrestling: medium skin tone # 9.0
1F93C 1F3FE 200D 2640 FE0F ; Emoji_ZWJ_Sequence ; women wrestling: medium-dark skin tone # 9.0
1F93C 1F3FF 200D 2640 FE0F ; Emoji_ZWJ_Sequence ; women wrestling: dark skin tone # 9.0
"""

# Defines how to insert the new sequences into the standard order data.  Would
# have been nice to merge it into the above legacy data but that would have
# required a format change.
_LEGACY_ANDROID_ORDER = """
-1F93C  # people wrestling
1F93C 1F3FB
1F93C 1F3FC
1F93C 1F3FD
1F93C 1F3FE
1F93C 1F3FF
-1F93C 200D 2642 FE0F  # men wrestling
1F93C 1F3FB 200D 2642 FE0F
1F93C 1F3FC 200D 2642 FE0F
1F93C 1F3FD 200D 2642 FE0F
1F93C 1F3FE 200D 2642 FE0F
1F93C 1F3FF 200D 2642 FE0F
-1F93C 200D 2640 FE0F  # women wrestling
1F93C 1F3FB 200D 2640 FE0F
1F93C 1F3FC 200D 2640 FE0F
1F93C 1F3FD 200D 2640 FE0F
1F93C 1F3FE 200D 2640 FE0F
1F93C 1F3FF 200D 2640 FE0F
"""


def _get_order_patch(order_text, seq_to_name):
    """Create a mapping from a key sequence to a list of sequence, name tuples.
    This will be used to insert additional sequences after the key sequence
    in the order data.  seq_to_name is a mapping from new sequence to name,
    so the names don't have to be duplicated in the order data."""

    patch_map = {}
    patch_key = None
    patch_list = None

    def get_sequence(seqtext):
        return tuple([int(s, 16) for s in seqtext.split()])

    for line in order_text.splitlines():
        ix = line.find("#")
        if ix >= 0:
            line = line[:ix]
        line = line.strip()
        if not line:
            continue
        if line.startswith("-"):
            if patch_list and patch_key:
                patch_map[patch_key] = patch_list
            patch_key = get_sequence(line[1:])
            patch_list = []
        else:
            seq = get_sequence(line)
            name = seq_to_name[seq]  # exception if seq is not in sequence_text
            patch_list.append((seq, name))
    if patch_list and patch_key:
        patch_map[patch_key] = patch_list

    return patch_map


def _get_android_order_patch():
    """Get an order patch using the legacy android data."""

    # maps from sequence to (name, age, type), we only need the name
    seq_data = _read_emoji_data(_LEGACY_ANDROID_SEQUENCES.splitlines())
    seq_to_name = {k: v[0] for k, v in seq_data.items()}
    return _get_order_patch(_LEGACY_ANDROID_ORDER, seq_to_name)


def _apply_order_patch(patch, group_list):
    """patch is a map from a key sequence to list of sequence, name pairs, and
    group_list is an ordered list of sequence, group, subgroup, name tuples.
    Iterate through the group list appending each item to a new list, and
    after appending an item matching a key sequence, also append all of its
    associated sequences in order using the same group and subgroup.
    Return the new list.  If there are any unused patches, raise an exception."""

    result = []
    patched = set()
    for t in group_list:
        result.append(t)
        if t[0] in patch:
            patched.add(t[0])
            _, group, subgroup, _ = t
            for seq, name in patch[t[0]]:
                result.append((seq, group, subgroup, name))

    unused = set(patch.keys()) - patched
    if unused:
        raise Exception(
            "%d unused patch%s\n  %s: "
            % (
                len(unused),
                "" if len(unused) == 1 else "es",
                "\n  ".join(seq_to_string(seq) for seq in sorted(unused)),
            )
        )

    return result


def _load_emoji_group_data():
    global _emoji_group_data
    if _emoji_group_data:
        return

    _emoji_group_data = {}

    with open_unicode_data_file("emoji-test.txt") as f:
        text = f.read()
    group_list = _read_emoji_test_data(text)

    # patch with android items
    patch = _get_android_order_patch()
    group_list = _apply_order_patch(patch, group_list)

    group_list.extend(_read_emoji_test_data(_SUPPLEMENTAL_EMOJI_GROUP_DATA))
    for i, (seq, group, subgroup, name) in enumerate(group_list):
        if seq in _emoji_group_data:
            print(
                "seq %s already in group data as %s"
                % (seq_to_string(seq), _emoji_group_data[seq])
            )
            print("    new value would be %s" % str((i, group, subgroup, name)))
        _emoji_group_data[seq] = (i, group, subgroup, name)

    assert len(group_list) == len(_emoji_group_data)


def get_emoji_group_data(seq):
    """Return group data for the canonical sequence seq, or None.
    Group data is a tuple of index, group, subgroup, and name.  The
    index is a unique global sort index for the sequence among all
    sequences in the group data."""
    _load_emoji_group_data()
    return _emoji_group_data.get(seq, None)


def get_emoji_groups():
    """Return the main emoji groups, in order."""
    _load_emoji_group_data()
    groups = []
    group = None
    for _, g, _, _ in sorted(_emoji_group_data.values()):
        if g != group:
            group = g
            groups.append(group)
    return groups


def get_emoji_subgroups(group):
    """Return the subgroups of this group, in order, or None
    if the group is not recognized."""
    _load_emoji_group_data()
    subgroups = []
    subgroup = None
    for _, g, sg, _ in sorted(_emoji_group_data.values()):
        if g == group:
            if sg != subgroup:
                subgroup = sg
                subgroups.append(subgroup)
    return subgroups if subgroups else None


def get_emoji_in_group(group, subgroup=None):
    """Return the sorted list of the emoji sequences in the group (limiting to
    subgroup if subgroup is not None).  Returns None if group does not
    exist, and an empty list if subgroup does not exist in group."""
    _load_emoji_group_data()
    result = None
    for seq, (index, g, sg, _) in _emoji_group_data.items():
        if g == group:
            if result is None:
                result = []
            if subgroup and sg != subgroup:
                continue
            result.append(seq)
    result.sort(key=lambda s: _emoji_group_data[s][0])
    return result


def get_sorted_emoji_sequences(seqs):
    """Seqs is a collection of canonical emoji sequences.  Returns a list of
    these sequences in the canonical emoji group order.  Sequences that are not
    canonical are placed at the end, in unicode code point order.
    """
    _load_emoji_group_data()
    return sorted(seqs, key=lambda s: (_emoji_group_data.get(s, 100000), s))


def _load_emoji_sequence_data():
    """Ensure the emoji sequence data is initialized."""
    global _emoji_sequence_data, _emoji_non_vs_to_canonical

    if _emoji_sequence_data is not None:
        return

    _emoji_sequence_data = {}
    _emoji_non_vs_to_canonical = {}

    def add_data(data):
        for k, t in data.items():
            if k in _emoji_sequence_data:
                print("already have data for sequence:", seq_to_string(k), t)
            _emoji_sequence_data[k] = t
            if EMOJI_VS in k:
                _emoji_non_vs_to_canonical[strip_emoji_vs(k)] = k

    for datafile in ["emoji-zwj-sequences.txt", "emoji-sequences.txt"]:
        add_data(_read_emoji_data_file(datafile))
    add_data(_read_emoji_data(_LEGACY_ANDROID_SEQUENCES.splitlines()))

    _load_unicode_data_txt()  # ensure character_names_data is populated
    _load_emoji_data()  # ensure presentation_default_text is populated
    _load_emoji_group_data()  # ensure group data is populated

    # Get names for single emoji from the test data. We will prefer these over
    # those in UnicodeData (e.g. prefer "one o'clock" to "clock face one oclock"),
    # and if they're not in UnicodeData these are proposed new emoji.
    for seq, (_, _, _, emoji_name) in _emoji_group_data.items():
        non_vs_seq = strip_emoji_vs(seq)
        if len(non_vs_seq) > 1:
            continue

        cp = non_vs_seq[0]

        # If it's not in character names data, it's a proposed emoji.
        if cp not in _character_names_data:
            # use 'ignore' to strip curly quotes etc if they exist, unicode
            # character names are ASCII, and it's probably best to keep it that way.
            cp_name = emoji_name.encode("ascii", "ignore").upper()
            _character_names_data[cp] = cp_name

        is_default_text_presentation = cp in _presentation_default_text
        if is_default_text_presentation:
            seq = (cp, EMOJI_VS)

        emoji_age = age(cp)
        if emoji_age is not None:
            emoji_age = float(emoji_age)
        emoji_age = PROPOSED_EMOJI_AGE

        current_data = _emoji_sequence_data.get(seq) or (
            emoji_name,
            emoji_age,
            "Emoji_Single_Sequence",
        )

        if is_default_text_presentation:
            emoji_name = "(emoji) " + emoji_name

        _emoji_sequence_data[seq] = (emoji_name, current_data[1], current_data[2])

    # Fill in sequences of single emoji, handling non-canonical to canonical also.
    for k in _emoji:
        non_vs_seq = (k,)

        is_default_text_presentation = k in _presentation_default_text
        if is_default_text_presentation:
            canonical_seq = (k, EMOJI_VS)
            _emoji_non_vs_to_canonical[non_vs_seq] = canonical_seq
        else:
            canonical_seq = non_vs_seq

        if canonical_seq in _emoji_sequence_data:
            # Prefer names we have where they exist
            emoji_name, emoji_age, seq_type = _emoji_sequence_data[canonical_seq]
        else:
            emoji_name = name(k, "unnamed").lower()
            if name == "unnamed":
                continue
            emoji_age = age(k)
            seq_type = "Emoji_Single_Sequence"

        if is_default_text_presentation and not emoji_name.startswith("(emoji) "):
            emoji_name = "(emoji) " + emoji_name
        _emoji_sequence_data[canonical_seq] = (emoji_name, emoji_age, seq_type)


def get_emoji_sequences(age=None, types=None):
    """Return the set of canonical emoji sequences, filtering to those <= age
    if age is not None, and those with type in types (if not a string) or
    type == types (if type is a string) if types is not None.  By default
    all sequences are returned, including those for single emoji."""
    _load_emoji_sequence_data()

    result = _emoji_sequence_data.keys()
    if types is not None:
        if isinstance(types, basestring):
            types = frozenset([types])
        result = [k for k in result if _emoji_sequence_data[k][1] in types]
    if age is not None:
        age = float(age)
        result = [k for k in result if _emoji_sequence_data[k][0] <= age]
    return result


def get_emoji_sequence_data(seq):
    """Return a tuple of the name, age, and type for the (possibly non-canonical)
    sequence, or None if not recognized as a sequence."""
    _load_emoji_sequence_data()

    seq = get_canonical_emoji_sequence(seq)
    if not seq or seq not in _emoji_sequence_data:
        return None
    return _emoji_sequence_data[seq]


def get_emoji_sequence_name(seq):
    """Return the name of the (possibly non-canonical)  sequence, or None if
    not recognized as a sequence."""
    data = get_emoji_sequence_data(seq)
    return None if not data else data[0]


def get_emoji_sequence_age(seq):
    """Return the age of the (possibly non-canonical)  sequence, or None if
    not recognized as a sequence.  Proposed sequences have PROPOSED_EMOJI_AGE
    as the age."""
    # floats are a pain since the actual values are decimal.  maybe use
    # strings to represent age.
    data = get_emoji_sequence_data(seq)
    return None if not data else data[1]


def get_emoji_sequence_type(seq):
    """Return the type of the (possibly non-canonical)  sequence, or None if
    not recognized as a sequence.  Types are in EMOJI_SEQUENCE_TYPES."""
    data = get_emoji_sequence_data(seq)
    return None if not data else data[2]


def is_canonical_emoji_sequence(seq):
    """Return true if this is a canonical emoji sequence (has 'vs' where Unicode
    says it should), and is known."""
    _load_emoji_sequence_data()
    return seq in _emoji_sequence_data


def get_canonical_emoji_sequence(seq):
    """Return the canonical version of this emoji sequence if the sequence is
    known, or None."""
    if is_canonical_emoji_sequence(seq):
        return seq
    seq = strip_emoji_vs(seq)
    return _emoji_non_vs_to_canonical.get(seq, None)


def strip_emoji_vs(seq):
    """Return a version of this emoji sequence with emoji variation selectors
    stripped. This is the 'non-canonical' version used by the color emoji font,
    which doesn't care how the sequence is represented in text."""
    if EMOJI_VS in seq:
        return tuple([cp for cp in seq if cp != EMOJI_VS])
    return seq


def seq_to_string(seq):
    """Return a string representation of the codepoint sequence."""
    return "_".join("%04x" % cp for cp in seq)


def string_to_seq(seq_str):
    """Return a codepoint sequence (tuple) given its string representation."""
    return tuple([int(s, 16) for s in seq_str.split("_")])


def is_cp_seq(seq):
    return all(0 <= n <= 0x10FFFF for n in seq)


_REGIONAL_INDICATOR_START = 0x1F1E6
_REGIONAL_INDICATOR_END = 0x1F1FF


def is_regional_indicator(cp):
    return _REGIONAL_INDICATOR_START <= cp <= _REGIONAL_INDICATOR_END


def is_regional_indicator_seq(cps):
    return len(cps) == 2 and all(is_regional_indicator(cp) for cp in cps)


def regional_indicator_to_ascii(cp):
    assert is_regional_indicator(cp)
    return chr(cp - _REGIONAL_INDICATOR_START + ord("A"))


def ascii_to_regional_indicator(ch):
    assert "A" <= ch <= "Z"
    return ord(ch) - ord("A") + _REGIONAL_INDICATOR_START


def string_to_regional_indicator_seq(s):
    assert len(s) == 2
    return ascii_to_regional_indicator(s[0]), ascii_to_regional_indicator(s[1])


def regional_indicator_seq_to_string(cps):
    assert len(cps) == 2
    return "".join(regional_indicator_to_ascii(cp) for cp in cps)


def is_tag(cp):
    return 0xE0020 < cp < 0xE0080 or cp == 0xE0001


def tag_character_to_ascii(cp):
    assert is_tag(cp)
    if cp == 0xE0001:
        return "[begin]"
    if cp == 0xE007F:
        return "[end]"
    return chr(cp - 0xE0000)


def is_regional_tag_seq(seq):
    return (
        seq[0] == 0x1F3F4
        and seq[-1] == 0xE007F
        and all(0xE0020 < cp < 0xE007E for cp in seq[1:-1])
    )


_FITZ_START = 0x1F3FB
_FITZ_END = 0x1F3FF


def is_skintone_modifier(cp):
    return _FITZ_START <= cp <= _FITZ_END


def get_presentation_default_emoji():
    _load_emoji_data()
    return _presentation_default_emoji


def get_presentation_default_text():
    _load_emoji_data()
    return _presentation_default_text


def get_emoji():
    _load_emoji_data()
    return _emoji


def is_emoji(cp):
    _load_emoji_data()
    return cp in _emoji


def is_emoji_modifier_base(cp):
    _load_emoji_data()
    return cp in _emoji_modifier_base


def _load_unicode_emoji_variants():
    """Parse StandardizedVariants.txt and initialize a set of characters
    that have a defined emoji variant presentation.  All such characters
    also have a text variant presentation so a single set works for both."""

    global _emoji_variants, _emoji_variants_proposed
    if _emoji_variants:
        return

    emoji_variants = set()
    # prior to Unicode 11 emoji variants were part of the standard data.
    # as of Unicode 11 however they're only in a separate emoji data file.
    line_re = re.compile(r"([0-9A-F]{4,6})\s+FE0F\s*;\s*emoji style\s*;")
    with open_unicode_data_file("emoji-variation-sequences.txt") as f:
        for line in f:
            m = line_re.match(line)
            if m:
                emoji_variants.add(int(m.group(1), 16))

    _emoji_variants = frozenset(emoji_variants)

    try:
        read = 0
        skipped = 0
        with open_unicode_data_file("proposed-variants.txt") as f:
            for line in f:
                m = line_re.match(line)
                if m:
                    read += 1
                    cp = int(m.group(1), 16)
                    if cp in emoji_variants:
                        skipped += 1
                    else:
                        emoji_variants.add(cp)

        print(
            "skipped %s %d proposed variants"
            % ("all of" if skipped == read else skipped, read)
        )
    except IOError as e:
        if e.errno != 2:
            raise

    _emoji_variants_proposed = frozenset(emoji_variants)


def get_unicode_emoji_variants(include_proposed="proposed"):
    """Returns the emoji characters that have both emoji and text presentations.
    If include_proposed is 'proposed', include the ones proposed in 2016/08.  If
    include_proposed is 'proposed_extra', also include the emoji Noto proposes
    for text presentation treatment to align related characters.  Else
    include_proposed should resolve to boolean False."""
    _load_unicode_emoji_variants()
    if not include_proposed:
        return _emoji_variants
    elif include_proposed == "proposed":
        return _emoji_variants_proposed
    elif include_proposed == "proposed_extra":
        extra = tool_utils.parse_int_ranges("1f4b9 1f4c8-1f4ca 1f507 1f509-1f50a 1f44c")
        return _emoji_variants_proposed | extra
    else:
        raise Exception(
            "include_proposed is %s which is not in ['proposed', 'proposed_extra']"
            % include_proposed
        )


def _load_variant_data():
    """Parse StandardizedVariants.txt and initialize all non-emoji variant
    data.  The data is a mapping from codepoint to a list of tuples of:
    - variant selector
    - compatibility character (-1 if there is none)
    - shaping context (bitmask, 1 2 4 8 for isolate initial medial final)
    The compatibility character is for cjk mappings that map to 'the same'
    glyph as another CJK character."""

    global _variant_data, _variant_data_cps
    if _variant_data:
        return

    compatibility_re = re.compile(r"\s*CJK COMPATIBILITY IDEOGRAPH-([0-9A-Fa-f]+)")
    variants = collections.defaultdict(list)
    with open_unicode_data_file("StandardizedVariants.txt") as f:
        for line in f:
            x = line.find("#")
            if x >= 0:
                line = line[:x]
            line = line.strip()
            if not line:
                continue

            tokens = line.split(";")
            cp, var = tokens[0].split(" ")
            cp = int(cp, 16)
            varval = int(var, 16)
            if varval in [0xFE0E, 0xFE0F]:
                continue  # ignore emoji variants
            m = compatibility_re.match(tokens[1].strip())
            compat = int(m.group(1), 16) if m else -1
            context = 0
            if tokens[2]:
                ctx = tokens[2]
                if ctx.find("isolate") != -1:
                    context += 1
                if ctx.find("initial") != -1:
                    context += 2
                if ctx.find("medial") != -1:
                    context += 4
                if ctx.find("final") != -1:
                    context += 8
            variants[cp].append((varval, compat, context))

    _variant_data_cps = frozenset(variants.keys())
    _variant_data = variants


def has_variant_data(cp):
    _load_variant_data()
    return cp in _variant_data


def get_variant_data(cp):
    _load_variant_data()
    return _variant_data[cp][:] if cp in _variant_data else None


def variant_data_cps():
    _load_variant_data()
    return _variant_data_cps


# proposed emoji


def _load_proposed_emoji_data():
    """Parse proposed-emoji.txt if it exists to get cps/names of proposed emoji
    (but not approved) for this version of Unicode."""

    global _proposed_emoji_data, _proposed_emoji_data_cps
    if _proposed_emoji_data:
        return

    _proposed_emoji_data = {}
    line_re = re.compile(r"^U\+([a-zA-z0-9]{4,5})\s.*\s\d{4}Q\d\s+(.*)$")
    try:
        with open_unicode_data_file("proposed-emoji.txt") as f:
            for line in f:
                line = line.strip()
                if not line or line[0] == "#" or line.startswith("\u2022"):
                    continue

                m = line_re.match(line)
                if not m:
                    raise ValueError('did not match "%s"' % line)
                cp = int(m.group(1), 16)
                name = m.group(2)
                if cp in _proposed_emoji_data:
                    raise ValueError(
                        "duplicate emoji %x, old name: %s, new name: %s"
                        % (cp, _proposed_emoji_data[cp], name)
                    )

                _proposed_emoji_data[cp] = name
    except IOError as e:
        if e.errno != 2:
            # not file not found, rethrow
            raise

    _proposed_emoji_data_cps = frozenset(_proposed_emoji_data.keys())


def proposed_emoji_name(cp):
    _load_proposed_emoji_data()
    return _proposed_emoji_data.get(cp, "")


def proposed_emoji_cps():
    _load_proposed_emoji_data()
    return _proposed_emoji_data_cps


def is_proposed_emoji(cp):
    _load_proposed_emoji_data()
    return cp in _proposed_emoji_data_cps


def read_codeset(text):
    line_re = re.compile(r"^0x([0-9a-fA-F]{2,6})\s+0x([0-9a-fA-F]{4,6})\s+.*")
    codeset = set()
    for line in text.splitlines():
        m = line_re.match(line)
        if m:
            cp = int(m.group(2), 16)
            codeset.add(cp)
    return codeset


def codeset(cpname):
    """Return a set of the unicode codepoints in the code page named cpname, or
    None."""
    filename = ("%s.txt" % cpname).upper()
    filepath = path.join(
        path.dirname(__file__), os.pardir, "third_party", "unicode", filename
    )
    if not path.isfile(filepath):
        return None
    with open(filepath, "r") as f:
        return read_codeset(f.read())


def _dump_emoji_presentation():
    """Dump presentation info, for testing."""

    text_p = 0
    emoji_p = 0
    for cp in sorted(get_emoji()):
        cp_name = name(cp, "<error>")
        if cp in get_presentation_default_emoji():
            presentation = "emoji"
            emoji_p += 1
        elif cp in get_presentation_default_text():
            presentation = "text"
            text_p += 1
        else:
            presentation = "<error>"
        print(
            "%s%04x %5s %s" % (" " if cp < 0x10000 else "", cp, presentation, cp_name)
        )
    print(
        "%d total emoji, %d text presentation, %d emoji presentation"
        % (len(get_emoji()), text_p, emoji_p)
    )


def _load_nameslist_data():
    global _nameslist_see_also
    if _nameslist_see_also is not None:
        return

    _nameslist_see_also = collections.defaultdict(set)
    cp = None
    line_re = re.compile(r"^(?:(?:([0-9A-F]{4,6})\t.*)|(?:^\s+([x=])\s+(.*)))$")
    see_also_re = re.compile(r"\s*(?:\(.*\s-\s+([0-9A-F]{4,6})\))|([0-9A-F]{4,6})")
    with open_unicode_data_file("NamesList.txt") as f:
        for line in f:
            m = line_re.match(line)
            if not m:
                continue
            if m.group(1):
                cp = int(m.group(1), 16)
            else:
                rel = m.group(2).strip()
                val = m.group(3).strip()
                if rel != "x":
                    continue
                m = see_also_re.match(val)
                if not m:
                    raise Exception(
                        'could not match see also val "%s" in line "%s"' % (val, line)
                    )
                ref_cp = int(m.group(1) or m.group(2), 16)
                _nameslist_see_also[cp].add(ref_cp)


def see_also(cp):
    _load_nameslist_data()
    return frozenset(_nameslist_see_also.get(cp))


def _load_namealiases_data():
    global _namealiases_alt_names
    if _namealiases_alt_names is not None:
        return

    _namealiases_alt_names = collections.defaultdict(list)
    line_re = re.compile(r"([0-9A-F]{4,6});([^;]+);(.*)$")
    with open_unicode_data_file("NameAliases.txt") as f:
        for line in f:
            m = line_re.match(line)
            if not m:
                continue
            cp = int(m.group(1), 16)
            name = m.group(2).strip()
            name_type = m.group(3).strip()
            if not name_type in [
                "correction",
                "control",
                "alternate",
                "figment",
                "abbreviation",
            ]:
                raise Exception('unknown name type in "%s"' % line)
            if name_type == "figment":
                continue
            _namealiases_alt_names[cp].append((name, name_type))


def alt_names(cp):
    """Return list of name, nametype tuples for cp, or None."""
    _load_namealiases_data()
    return tuple(_namealiases_alt_names.get(cp))


if __name__ == "__main__":
    all_sequences = sorted(get_emoji_sequences())
    for k in all_sequences:
        if not get_emoji_group_data(k):
            print("no data:", seq_to_string(k))

    for group in get_emoji_groups():
        print("group:", group)
        for subgroup in get_emoji_subgroups(group):
            print("  subgroup:", subgroup)
            print("    %d items" % len(get_emoji_in_group(group, subgroup)))

    # dump some information for annotations
    for k in get_sorted_emoji_sequences(all_sequences):
        age = get_emoji_sequence_age(k)
        if age == 12:
            print(seq_to_string(k).replace("_", " "), "#", get_emoji_sequence_name(k))
