#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Grab Universal Declaration of Human Rights data from unicode.org/udhr,
extract the text of Article 1 where it seems ok, and generate text files
using the bcp47 name (including script) for the language of that sample."""

import argparse
import codecs
import collections
import datetime
import difflib
import os
import re
import shutil

try:
    from urllib.request import urlretrieve
except:
    from urllib import urlretrieve

import xml.etree.ElementTree as ET

from nototools import generate_website_data
from nototools import tool_utils
from nototools import unicode_data

DIR_URL = "http://unicode.org/udhr/d"
UDHR_XML_ZIP_NAME = "udhr_xml.zip"
UDHR_XML_ZIP_URL = "http://unicode.org/udhr/assemblies/" + UDHR_XML_ZIP_NAME


def fetch_udhr(fetch_dir):
    """Fetch UDHR xml bundle from unicode.org to fetch_dir."""
    fetch_dir = tool_utils.ensure_dir_exists(fetch_dir)
    dstfile = os.path.join(fetch_dir, UDHR_XML_ZIP_NAME)
    result = urlretrieve(UDHR_XML_ZIP_URL, dstfile)
    print("Fetched: " + result[0])


def update_udhr(udhr_dir, fetch_dir, in_repo):
    """Delete udhr_dir and rebuild with files extracted from udhr_xml.zip
    in fetch_dir. Stage if udhr_dir is in the repo."""

    zippath = os.path.join(fetch_dir, UDHR_XML_ZIP_NAME)
    tool_utils.check_file_exists(zippath)

    if in_repo and os.path.isdir(udhr_dir) and not tool_utils.git_is_clean(udhr_dir):
        raise ValueError("Please clean %s." % udhr_dir)

    if os.path.isdir(udhr_dir):
        shutil.rmtree(udhr_dir)
    os.makedirs(udhr_dir)
    tool_utils.zip_extract_with_timestamp(zippath, udhr_dir)

    # dos line endings, sheesh
    tool_utils.dos2unix(udhr_dir, ["*.xml", "*.rnc", "*.rng"])

    if in_repo:
        tool_utils.git_add_all(udhr_dir)

    date = datetime.datetime.now().strftime("%Y-%m-%d")
    dst = "in %s " % udhr_dir if not in_repo else ""
    print("Update UDHR files %sfrom %s as of %s." % (dst, fetch_dir, date))


def parse_index(src_dir):
    """Parse the index.xml file in src_dir and return a map from bcp to a set of
    file codes, and a map from file code to ohchr code.

    Skip files at stages 1 (missing) or 2 (not started). Stage 3 files have
    article 1, which is what we want.  Stage 4 and 5 are ok, the vast majority are
    unreviewed (4).

    In some cases more than one file is mapped to the same bcp47 code, this gets
    dealt with in fix_index."""

    tree = ET.parse(os.path.join(src_dir, "index.xml"))
    bcp_to_codes = collections.defaultdict(set)
    code_to_ohchr = {}

    for e in tree.getroot().iter("udhr"):
        s = int(e.attrib.get("stage"))
        if s < 3:
            continue

        code = e.attrib.get("f")

        bcp = e.attrib.get("bcp47")
        if not bcp:
            # don't know what to do with this, maybe we could supply a mapping.
            print("no bcp for %s" % code)
            continue

        script = e.attrib.get("iso15924")
        if script:
            # the bcp codes are minimized, and include scripts only if they are
            # not the default (for some version of default), however they will
            # include variants.
            # if the script code is not present in the bcp code, patch it in
            # after the lang.
            script = "-" + script
            if bcp.find(script) == -1:
                ix = bcp.find("-")
                if ix == -1:
                    bcp += script
                else:
                    bcp = bcp[:ix] + script + bcp[ix:]

        ohchr = e.attrib.get("ohchr")

        bcp_to_codes[bcp].add(code)

        # we use the ohchr code to identify an attribution
        if ohchr:
            code_to_ohchr[code] = ohchr

    return bcp_to_codes, code_to_ohchr


# These handle cases in which the unicode udhr data has multiple files with the
# same bcp47 code.
#
# In some cases, we pick just the one code to use, more or less arbitrarily.
# In other cases, we use multiple codes by adding a region or variant tag to the
# shared bcp47 code.
#
# This map contains all the bcp47 codes that we expect udhr assigns to multple
# files.  For each file we either assign a code, or None if we don't use that
# file.
#
# Possible errors are:
# 1) udhr has multiple files for a code, but we don't list the code.
# 2) udhr has multiple files for a code, but we have a different set of files
#    (in which case we might have ones they don't have, and they might have ones
#     we don't have)
# 3) udhr has a single file for a code, but we list the code
#
# We also need to make sure we don't assign a new code that udhr already uses.

BCP_FIXES = {
    "acu-Latn": {
        "acu": "acu-Latn",
        "acu_1": None,
    },
    "ak": {
        "ak_asante": "ak",
        "ak_fante": "ak-fante",
    },
    "chr-Cher": {
        "chr_cased": "chr-Cher-cased",
        "chr_uppercase": "chr-Cher-monocase",
    },
    "cjk-Latn": {
        "cjk": "cjk-Latn",
        "cjk_AO": "cjk-Latn-AO",
    },
    "ht-Latn": {
        "hat_popular": "ht-Latn-popular",
        "hat_kreyol": "ht-Latn-kreyol",
    },
    "hus-Latn": {
        "hus": "hus-Latn",
        "hva": None,
        "hsf": None,
    },
    "kg-Latn": {
        "kng": "kg-Latn",
        "kng_AO": "kg-Latn-AO",
    },
    "kmb-Latn": {
        "009": None,
        "kmb": "kmb-Latn",
    },
    "la-Latn": {
        "lat": "la-Latn",
        "lat_1": None,
    },
    "ln-Latn": {
        "lin_tones": "ln-Latn",
        "lin": None,
    },
    "ny-Latn": {
        "nya_chinyanja": "ny-Latn-chinyan",  # max 8 chars in bcp47
        "nya_chechewa": "ny-Latn-chechewa",
    },
    "oc-Latn": {
        "lnc": "oc-Latn",
        "auv": None,
        "oci_1": None,
        "oci_2": None,
        "oci_3": None,
        "oci_4": None,
        "prv": None,
    },
    "pov-Latn": {
        "008": None,
        "pov": "pov-Latn",
    },
    "ro-Latn": {
        "ron_2006": "ro-Latn",
        "ron_1993": None,
        "ron_1953": None,
    },
    "rom-Latn": {
        "rmn": "rom-Latn",
        "rmn_1": None,
    },
    "th-Thai": {
        "tha": "th-Thai",
        "tha2": None,
    },
    "ts-Latn": {
        "tso_MZ": "ts-Latn-MZ",
        "tso_ZW": "ts-Latn-ZW",
    },
    "umb-Latn": {
        "011": None,
        "umb": "umb-Latn",
    },
    "ur-Arab": {
        "urd": "ur-Arab",
        "urd_2": None,
    },
}


def fix_index(bcp_to_codes):
    """Take a mapping from bcp47 to a set of file codes, and
    select the mappings we want using a allowlist.  We return
    a mapping from one bcp47 code to one file code.

    We use this opportunity to validate the allowlist, and if there are
    any errors, we fail once we're finished."""
    errors = []
    used_fixes = set()
    result = {}
    for k in sorted(bcp_to_codes):
        if k == "und" or k.startswith("und-"):
            # we don't handle undefined languages
            continue

        v = bcp_to_codes[k]
        if len(v) == 1:
            udhr_code = next(iter(v))
            if k in result:
                errors.append(
                    "%s: udhr assigns %s but we already applied %"
                    % (k, udhr_code, result[k])
                )
                continue

            if k in BCP_FIXES:
                errors.append(
                    "%s: udhr assigns %s but we have fixes %s"
                    % (k, result[k], ", ".join(BCP_FIXES[k].keys))
                )
                continue

            # normal case here
            result[k] = next(iter(v))
            continue

        if k not in BCP_FIXES:
            errors.append(
                "%s: no fixes for udhr multiple codes %s" % (k, ", ".join(sorted(v)))
            )
            continue

        fixes = BCP_FIXES[k]
        fixes_keys = set(fixes.keys())
        unused_fixes = fixes_keys - v
        unknown_codes = v - fixes_keys
        if unused_fixes:
            errors.append("%s: unused fixes %s" % (k, ", ".join(sorted(unused_fixes))))
        if unknown_codes:
            errors.append(
                "%s: unknown codes %s" % (k, ", ".join(sorted(unknown_codes)))
            )
        if unused_fixes or unknown_codes:
            continue

        # apply all of our fixes
        for code, new_bcp in sorted(fixes.items()):
            if not new_bcp:
                continue

            if new_bcp in result:
                errors.append(
                    "%s: fix defines %s as %s but already assigned %s"
                    % (k, code, new_bcp, result[new_bcp])
                )
                continue

            # normal fix
            result[new_bcp] = code

    if errors:
        print("fix_index had %d errors:" % len(errors))
        for e in errors:
            print(" ", e)
        raise Exception("correct the fixes allowlist")

    return result


# The likely script data doesn't always match the samples, so we override it
# here.  Probably should always get the samples first and apply the script
# later, but for now we just check after the fact.

CODE_TO_BCP = {"evn": "evn-Cyrl", "ojb": "oj-Cans"}


def add_likely_scripts(bcp_to_code):
    """Add script subtags where they are not present in the bcp code.  If
    we don't know the script"""
    result = {}
    for bcp, code in bcp_to_code.items():
        if code in CODE_TO_BCP:
            new_bcp = CODE_TO_BCP[code]
        else:
            new_bcp = bcp
            parts = bcp.split("-")
            try:
                script = generate_website_data.find_likely_script(parts[0])
                if len(parts) == 1:
                    new_bcp = "%s-%s" % (bcp, script)
                elif len(parts[1]) != 4 or parts[1].isdigit():
                    # assume a region or variant.  Some 4-char values are years, e.g. '1996'
                    new_bcp = "%s-%s-%s" % (parts[0], script, "-".join(parts[1:]))
                # otherwise, we assume the 4-char value is a script, and leave it alone.
            except KeyError:
                # if we can't provide a script, it's no use for a script sample, so exclude it
                print("no likely subtag (script) data for %s, excluding" % parts[0])
                continue
        result[new_bcp] = code
    return result


# These have been fixed/changed in the noto repo.  We do not want to replace them
# with the UDHR samples, which (as of now, anyway) do not reflect these
# improvements. Our th-Thai is a more colloquial translation than the formal one
# in the UDHR repo.
EXCLUDE_BCP = frozenset(["fa-Arab", "ar-Arab", "th-Thai"])

# The data for these is bad.  The kwi.xml has no article 1 text (only '[?]')
# and the cbi.xml article 1 text has '. mitya, tsenr)1in ' in it, which just looks
# broken.
EXCLUDE_CODES = frozenset(["kwi", "cbi"])


def filter_bcp_to_code(bcp_to_code):
    """Exclude entries for samples improved in noto/sample_texts and for bad samples."""
    return {
        k: v
        for k, v in bcp_to_code.items()
        if k not in EXCLUDE_BCP and v not in EXCLUDE_CODES
    }


# Pick a default sample to use when only lang and script are provided.
OPTION_MAP = {
    "ak-Latn": "ak-Latn-asante",
    "chr-Cher": "chr-Cher-cased",
    "de-Latn": "de-Latn-1996",
    "el-Grek": "el-Grek-monoton",
    "ha-Latn": "ha-Latn-NG",
    "ht-Latn": "ht-Latn-kreyol",
    "ny-Latn": "ny-Latn-chechewa",
    "pt-Latn": "pt-Latn-BR",
    "ts-Latn": "ts-Latn-MZ",  # arbitrarily select Mozambque
    "tw-Latn": "tw-Latn-akuapem",  # 'prestige' dialect according to wikipedia
}


def add_default_lang_script(bcp_to_code_attrib_sample):
    """When we query this data, typically we have only language and
    script.  Some of the bcp codes have variants or regions as well, and in
    particular sometimes none of these has just a language and script.
    Select one of these to be used for that, and update the map."""

    errors = []
    options = collections.defaultdict(set)
    long_keys = {}
    for key in bcp_to_code_attrib_sample:
        tags = key.split("-")
        if len(tags) > 2:
            long_keys[key] = tags

    for key in sorted(long_keys):
        tags = long_keys[key]
        lang_scr = tags[0] + "-" + tags[1]
        options[lang_scr].add(key)

    for lang_scr in sorted(options):
        if lang_scr in bcp_to_code_attrib_sample:
            print(
                "%s exists with variants %s"
                % (lang_scr, ", ".join(sorted(options[lang_scr])))
            )
            del options[lang_scr]

    for lang_scr in sorted(options):
        print("%s options: %s" % (lang_scr, options[lang_scr]))
        if not lang_scr in OPTION_MAP:
            errors.append("%s missing from option map" % lang_scr)
        elif not OPTION_MAP[lang_scr] in options[lang_scr]:
            errors.append(
                "%s selected option for %s not available"
                % (lang_scr, OPTION_MAP[lang_scr])
            )
        else:
            alias = OPTION_MAP[lang_scr]
            print("adding %s (from %s)" % (lang_scr, alias))
            bcp_to_code_attrib_sample[lang_scr] = bcp_to_code_attrib_sample[alias]

    if errors:
        print("add_default_lang_script encountered %d errors:" % len(errors))
        for e in errors:
            print(" ", e)
        raise Exception("oops")


def get_code_to_attrib(src_dir):
    code_to_attrib = {}
    code_file = "attributions.tsv"
    with open(os.path.join(src_dir, code_file)) as f:
        for line in f.readlines():
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            code, attrib_key, lang, attrib = line.split("\t")
            code_to_attrib[code] = attrib_key
    return code_to_attrib


def get_bcp_to_code_attrib_sample(src_dir, ohchr_dir):
    """Return a mapping from bcp47 to code (for debugging), attribution, and
    sample.  The process is:
    1) parse the index.xml file to determine a mapping from bcp47 to code.
       the bcp47 code has at least lang and script, and perhaps region/variant.
       Multiple codes might share the same bcp47 code.
    2) Use a allowlist to fix cases where a bcp47 code maps to multiple codes,
       either by selecting one code, or assigning a separate bcp47 value
       to other codes.
    3) Load samples for each bcp47 code using article 1 from the file
       identified by the code.  If there is no article 1, skip that bcp47 code.
    4) Do more checking on the samples to make sure they look legit and
       in particular contain only the scripts we expect them to have based
       on the script code in the bcp47 code.
    5) Add an attribution based on the code and the attributions file.
    6) Find cases where all the bcp47's sharing a lang and script have
       regions and/or variants, and select one of these to assign to
       the lang_script bcp47 code."""

    bcp_to_codes, code_to_ohchr = parse_index(src_dir)
    bcp_to_code = fix_index(bcp_to_codes)
    bcp_to_sample = get_bcp_to_sample(src_dir, bcp_to_code)
    check_bcp_to_sample(bcp_to_sample)

    code_to_attrib = get_code_to_attrib(ohchr_dir)

    bcp_to_code_attrib_sample = {}
    for bcp in bcp_to_sample:
        code = bcp_to_code[bcp]
        ohchr = code_to_ohchr.get(code)
        attr = code_to_attrib.get(ohchr)
        if not attr:
            attr = "none"
            print("%s (%s) not in ohchr attribution data" % (code, ohchr))
        sample = bcp_to_sample[bcp]
        bcp_to_code_attrib_sample[bcp] = (code, attr, sample)

    add_default_lang_script(bcp_to_code_attrib_sample)

    return bcp_to_code_attrib_sample


def print_bcp_to_code_attrib_sample(bcp_to_code_attrib_sample):
    print("index size: %s" % len(bcp_to_code_attrib_sample))
    for bcp, (code, attrib, sample) in sorted(bcp_to_code_attrib_sample.items()):
        print('%s: %s, %s\n  "%s"' % (bcp, code, attrib, sample))


def extract_para(src_path):
    """Extract the text of article 1 from the sample, or None if we can't find
    it."""
    tree = ET.parse(src_path)
    root = tree.getroot()
    ns = {"udhr": "http://www.unhchr.ch/udhr"}
    article = root.find('udhr:article[@number="1"]', ns)
    if article is None:
        # file kjh.xml is damaged, arrgh. Cyrillic small 'ie' looks just like 'e', and
        # the 'number' attribute is written with the Cyrillic e!
        article = root.find('udhr:article[@numb\u0435r="1"]', ns)
    if article is not None:
        text = "\n".join([para.text for para in article.findall("udhr:para", ns)])
        return text.strip() + "\n"
    return None


def fix_sample(sample, bcp):
    """Fix samples that have known fixable issues."""
    new_sample = None
    if bcp == "zh-Hans":
        new_sample = sample.replace(",", "\uff0c")
    elif bcp == "hu-Latn":
        new_sample = sample.replace("Minden.", "Minden")

    if not new_sample:
        return sample

    if new_sample == sample:
        print("sample for %s was not changed by fix" % bcp)
    else:
        print("fixed sample for %s" % bcp)
    return new_sample


def get_sample_for_code(udhr_dir, code):
    """Get a sample named for code."""
    src_file = "udhr_%s.xml" % code
    src_path = os.path.join(udhr_dir, src_file)
    sample = extract_para(src_path)
    if not sample:
        print("unable to get sample from %s" % src_file)
        return None
    return sample


def get_bcp_to_sample(src_dir, bcp_to_code):
    """Return a map from bcp to sample, for codes that have a sample."""
    bcp_to_sample = {}
    for bcp in sorted(bcp_to_code):
        code = bcp_to_code[bcp]
        sample = get_sample_for_code(src_dir, code)
        if not sample:
            print("bcp %s: no sample found (code %s)" % (bcp, code))
        else:
            bcp_to_sample[bcp] = sample
    return bcp_to_sample


bcp_script_re = re.compile(r"^(?:[A-Z][a-z]{3}|\d{3})$")


def get_bcp_script(bcp):
    """Return the script portion of the bcp tag if it exists, or None."""
    parts = bcp.split("-")
    if len(parts) == 1:
        return None
    return parts[1] if bcp_script_re.match(parts[1]) else None


def check_bcp_sample(bcp, sample):
    """Return error msg if fails, or 'OK'."""

    def info_str(script_data):
        v = {data[0]: script for script, data in script_data.items()}
        return ", ".join("%2d: %s" % tup for tup in sorted(v.items(), reverse=True))

    def info_data(bad_scripts, script_data):
        errs = []
        for s in sorted(bad_scripts):
            count, chars = script_data[s]
            errs.append(
                "%s (%d) %s"
                % (
                    s,
                    count,
                    ",".join("%04x (%s)" % (ord(cp), cp) for cp in sorted(chars)),
                )
            )
        return "; ".join(errs)

    script_data = get_script_histogram(sample)

    if len(sample) < 5:
        return "bcp %s: sample too short (%d): %s" % (
            bcp,
            len(sample),
            info_str(script_data),
        )

    script = get_bcp_script(bcp)  # can include non-unicode scripts like Jpan
    if not script:
        # formerly we were going to generate a script based on the sample if we
        # didn't alredy have one, but at the moment all the samples include a script
        # in index.xml.
        raise Exception("no script for %s" % bcp)

    required, accepted = accept_scripts(script)  # unicode scripts
    allowed = {"Zyyy"}  # common is ok
    if required:
        allowed |= required
    if accepted:
        allowed |= accepted

    if required:
        if required & frozenset(script_data.keys()) != required:
            return "bcp %s: required script %s but had (%s)" % (
                bcp,
                ", ".join(sorted(required)),
                info_str(script_data),
            )
    elif not accepted:
        # my problem
        raise Exception("Script %s has neither required nor accepted" % script)

    disallowed = set(script_data.keys()) - allowed
    if disallowed:
        return "bcp %s: script(s) {%s} not in allowed {%s}, %s" % (
            bcp,
            ", ".join(sorted(disallowed)),
            ", ".join(sorted(allowed)),
            info_data(disallowed, script_data),
        )

    # ok, the sample passes.
    return "OK"


def check_bcp_to_sample(bcp_to_sample):
    """For each bcp/sample pair, check the sample script histogram.  If
    anything looks funny (mismatching scripts, bcp script doesn't match
    histogram), delete the mapping, and if there are any rejects, at the
    end report them."""

    errors = []
    for bcp in sorted(bcp_to_sample):
        sample = bcp_to_sample[bcp]
        result = check_bcp_sample(bcp, sample)
        if result != "OK":
            errors.append(result)
            del bcp_to_sample[bcp]

    if errors:
        print("found %d errors in samples" % len(errors))
        for e in errors:
            print(" ", e)


def update_samples(sample_dir, udhr_dir, bcp_to_code_attrib_sample, in_repo, no_stage):
    """Create samples in sample_dir based on the bcp to c_a_s map.  Stage
    if sample_dir is in the repo.  If sample_dir is in the repo, don't
    overwrite samples whose most recent log entry does not start with
    'Updated by tool'."""

    tool_utils.check_dir_exists(udhr_dir)

    if (
        in_repo
        and not no_stage
        and os.path.isdir(sample_dir)
        and not tool_utils.git_is_clean(sample_dir)
    ):
        raise ValueError("Please clean %s." % sample_dir)

    if in_repo:
        repo, subdir = os.path.split(sample_dir)
        tool_samples = frozenset(tool_utils.get_tool_generated(repo, subdir))
        print(
            "allowing overwrite of %d files:\n  %s"
            % (len(tool_samples), ", ".join(sorted(tool_samples)))
        )

    comments = [
        "# Attributions for sample excerpts:",
        "#   original - in the public domain, no attribution",
        "#   UN - UN, OHCHR, or affiliate, attribute to UN",
        "#   other - not a UN translation",
        "#   none - not on ohchr, not a UN translation",
    ]
    sample_attrib_list = []
    sample_dir = tool_utils.ensure_dir_exists(sample_dir)
    count = 0
    for bcp, (code, attrib, sample) in bcp_to_code_attrib_sample.items():
        dst_file = "%s_udhr.txt" % bcp
        dst_path = os.path.join(sample_dir, dst_file)
        if in_repo and os.path.isfile(dst_path) and dst_file not in tool_samples:
            print("Not overwriting modified file %s" % dst_file)
        else:
            with codecs.open(dst_path, "w", "utf8") as f:
                f.write(sample)
            count += 1
        sample_attrib_list.append("%s: %s" % (dst_file, attrib))
    print("Created %d samples" % count)

    # Some existing samples that we don't overwrite are not in
    # bcp_to_code_attrib_sample, so they're not listed.  Readers of the
    # attributions.txt file will need to default these to 'none'.
    attrib_data = "\n".join(comments + sorted(sample_attrib_list)) + "\n"
    with open(os.path.join(sample_dir, "attributions.txt"), "w") as f:
        f.write(attrib_data)

    if in_repo and not no_stage:
        tool_utils.git_add_all(sample_dir)

    date = datetime.datetime.now().strftime("%Y-%m-%d")
    dst = "in %s " % sample_dir if not in_repo else ""
    noto_ix = udhr_dir.find("nototools")
    src = udhr_dir if noto_ix == -1 else udhr_dir[noto_ix:]

    # prefix of this sample commit message indicates that these were
    # tool-generated
    print("Updated by tool - sample files %sfrom %s as of %s." % (dst, src, date))


def get_scripts(text):
    """Return the set of scripts in this text.  Excludes
    some common chars."""
    # ignore these chars, we assume they are ok in any script
    exclusions = {0x00, 0x0A, 0x0D, 0x20, 0xA0, 0xFEFF}
    zyyy_chars = set()
    scripts = set()
    ustr = text.decode("utf8")
    for cp in ustr:
        if ord(cp) in exclusions:
            continue
        script = unicode_data.script(cp)
        if script == "Zyyy":  # common/undetermined
            zyyy_chars.add(cp if cp < "\u00fe" else ord(cp))
        elif not script == "Zinh":  # inherited
            scripts.add(script)
    return scripts, zyyy_chars


def get_script_histogram(utext):
    """Return a map from script to character count + chars, excluding some common
    whitespace, and inherited characters.  utext is a unicode string."""
    exclusions = {0x00, 0x0A, 0x0D, 0x20, 0xA0, 0xFEFF}
    result = {}
    for cp in utext:
        if ord(cp) in exclusions:
            continue
        script = unicode_data.script(cp)
        if script == "Zinh":
            continue
        if script not in result:
            result[script] = [1, {cp}]
        else:
            r = result[script]
            r[0] += 1
            r[1].add(cp)
    return result


# required, allowed sets
SCRIPT_MAP = {
    "Kore": (None, frozenset(["Hang"])),
    "Jpan": (None, frozenset(["Hani", "Hira"])),
    "Hant": (None, frozenset(["Hani"])),
    "Hans": (None, frozenset(["Hani"])),
}


def accept_scripts(script):
    if script in SCRIPT_MAP:
        required, allowed = SCRIPT_MAP.get(script)
    else:
        required, allowed = frozenset([script]), None
    return required, allowed


def test_sample_scripts(sample_dir):
    tested = 0
    errors = 0
    for filename in os.listdir(sample_dir):
        filepath = os.path.join(sample_dir, filename)
        if not (os.path.isfile(filepath) and filename.endswith("_udhr.txt")):
            continue
        tested += 1
        with open(filepath, "rb") as f:
            textbytes = f.read()
            scripts, zyyy = get_scripts(textbytes)
            bcp = filename[: -len("_udhr.txt")]  # trim off extension
            expected_script = bcp.split("-")[1]
            required, allowed = accept_scripts(expected_script)
            if required and required - scripts:
                required_name = ", ".join(sorted([s for s in required]))
                scripts_name = ", ".join(sorted([s for s in scripts]))
                print(
                    "%s requires %s but contains only %s"
                    % (filename, required_name, scripts_name)
                )
                errors += 1
            else:
                remainder = scripts
                if allowed:
                    remainder -= allowed
                if required:
                    remainder -= required
                if remainder:
                    allowed_name = (
                        "<none>"
                        if not allowed
                        else ", ".join(sorted([s for s in allowed]))
                    )
                    scripts_name = ", ".join(sorted([s for s in scripts]))
                    print(
                        "%s allows %s but contains %s"
                        % (filename, allowed_name, scripts_name)
                    )
                    errors += 1
    print("Found %d errors in %d files tested." % (errors, tested))


def compare_samples(base_dir, trg_dir, trg_to_base_name=lambda x: x, opts=None):
    """Report on differences between samples in base and target directories.
    The trg_to_base_name fn takes a target file name and returns the source
    file name to use in the comparisons."""

    if not os.path.isdir(base_dir):
        print("Original sample dir '%s' does not exist" % base_dir)
        return
    if not os.path.isdir(trg_dir):
        print("New sample dir '%s' does not exist" % trg_dir)
        return

    print("Base (current) dir: %s" % base_dir)
    print("Target (new) dir: %s" % trg_dir)
    print('[a/b] means "a" in base is replaced with "b" in target')

    show_missing = opts and "missing" in opts
    show_diffs = opts and "diffs" in opts

    for trg_name in os.listdir(trg_dir):
        if trg_name == "attributions.txt":
            continue

        trg_path = os.path.join(trg_dir, trg_name)
        if not (os.path.isfile(trg_path) and trg_name.endswith(".txt")):
            continue

        base_name = trg_to_base_name(trg_name)
        base_path = os.path.join(base_dir, base_name)
        if not os.path.exists(base_path):
            if show_missing:
                print("base does not exist: %s" % base_name)
            continue

        base_text = None
        dst_text = None
        with codecs.open(base_path, "r", "utf8") as f:
            base_text = f.read()
        with codecs.open(trg_path, "r", "utf8") as f:
            trg_text = f.read()
        if not base_text:
            print("base text (%s) is empty" % base_path)
            continue
        if not trg_text:
            print("target text is empty: %s" % trg_path)
            continue
        if base_text.find(trg_text) == -1:
            print("target (%s) text not in base (%s)" % (base_name, trg_name))
            if show_diffs:
                # In scripts that use space for word break it might be better to compare
                # word by word, but this suffices.
                sm = difflib.SequenceMatcher(None, base_text, trg_text, autojunk=False)
                lines = []
                for tag, i1, i2, j1, j2 in sm.get_opcodes():
                    if tag == "delete":
                        lines.append("[%s/]" % base_text[i1:i2])
                    elif tag == "equal":
                        lines.append(base_text[i1:i2])
                    elif tag == "insert":
                        lines.append("[/%s]" % trg_text[j1:j2])
                    else:
                        lines.append("[%s/%s]" % (base_text[i1:i2], trg_text[j1:j2]))
                print("".join(lines))


def update_repo(repo_samples, new_samples):
    # Verify directory is clean.
    if not tool_utils.git_is_clean(new_samples):
        print("Please fix.")
        return

    # Copy samples into git repo
    for filename in os.listdir(new_samples):
        filepath = os.path.join(new_samples, filename)
        if not os.path.isfile(filepath):
            continue
        shutil.copy2(filename, repo_samples)

    # Stage changes.
    tool_utils.git_add_all(new_samples)

    # Sample commit message.
    print("Update UDHR sample data.")


def main():
    fetch = "/tmp/udhr/zip"
    udhr = "[tools]/third_party/udhr"
    samples = "[tools]/sample_texts"

    epilog = """The general flow is as follows:
  1) ensure attributions.tsv is in [tools]/third_party/ohchr, using
     extract_ohchr_attributions.py.
  2) use -uu to fetch and stage changes to [tools]/third_party/udhr
  3) use -us --sample_dir=/tmp/foo to generate samples
  4) use -c --sample_dir=/tmp/foo to compare the staged samples
  5) tweak the mapping, use -m to see that it's doing what we want
  6) use --us to generate the samples and stage them to [tools]/sample_texts

  This will not overwrite samples whose most recent log entry does
  not start with 'Updated by tool'.
  """

    parser = argparse.ArgumentParser(
        epilog=epilog, formatter_class=argparse.RawTextHelpFormatter
    )
    parser.add_argument(
        "--fetch_dir",
        help="directory into which to fetch xml.zip " "(default %s)" % fetch,
        metavar="dir",
        default=fetch,
    )
    parser.add_argument(
        "--udhr_dir",
        help="location into which to unpack udhr files " "(default %s)" % udhr,
        metavar="dir",
        default=udhr,
    )
    parser.add_argument(
        "--sample_dir",
        help="directory into which to extract samples " "(default %s)" % samples,
        metavar="dir",
        default=samples,
    )
    parser.add_argument(
        "-f",
        "--fetch",
        help="fetch files from unicode.org/udhr to fetch dir",
        action="store_true",
    )
    parser.add_argument(
        "-uu",
        "--update_udhr",
        help="unpack from fetched files to clean udhr dir\n"
        "(will stage if in repo and not no_stage)",
        action="store_true",
    )
    parser.add_argument(
        "-us",
        "--update_sample",
        help="extract samples from udhr to sample dir, "
        "using the bcp to code mapping\n(will stage if in repo and not no_stage)",
        action="store_true",
    )
    parser.add_argument(
        "-m",
        "--mapping",
        help="print the bcp to code mapping generated from the " "udhr dir",
        action="store_true",
    )
    parser.add_argument(
        "-c",
        "--compare",
        help="compare sample changes from base dir "
        "(default %s)\nto sample dir" % samples,
        metavar="base_dir",
        nargs="?",
        const=samples,
        dest="base_sample_dir",
    )
    parser.add_argument(
        "-co",
        "--compare_opts",
        help="options for comparison, provide any of:\n"
        "  'missing' to show samples files not in base, and/or\n"
        "  'diffs' to show full diffs of samples",
        metavar="opt",
        nargs="+",
    )
    parser.add_argument(
        "-ts",
        "--test_script",
        help="test script of samples in sample dir",
        action="store_true",
    )
    parser.add_argument(
        "-n", "--no_stage", help="do not stage changes in repo", action="store_true"
    )

    args = parser.parse_args()

    # Check arguments

    if not (
        args.fetch
        or args.update_udhr
        or args.update_sample
        or args.mapping
        or args.base_sample_dir
        or args.test_script
    ):
        print("nothing to do.")
        return

    def fix_noto_prefix(argname):
        newval = tool_utils.resolve_path(getattr(args, argname))
        setattr(args, argname, newval)

    if args.update_udhr or args.update_sample or args.mapping:
        fix_noto_prefix("udhr_dir")

    if args.update_sample or args.base_sample_dir or args.test_script:
        fix_noto_prefix("sample_dir")

    if args.base_sample_dir:
        fix_noto_prefix("base_sample_dir")

    if args.sample_dir == args.base_sample_dir:
        parser.error("Compare is no-op when base and target sample dirs are the same.")

    # Perform operations.  Some might still fail despite arg checks.
    try:
        if args.fetch:
            fetch_udhr(args.fetch_dir)

        if args.update_udhr:
            in_repo = args.udhr_dir == tool_utils.resolve_path(udhr)
            update_udhr(args.udhr_dir, args.fetch_dir, in_repo and not args.no_stage)

        if args.update_sample or args.mapping:
            ohchr_dir = tool_utils.resolve_path("[tools]/third_party/ohchr")
            bcp_to_code_attrib_sample = get_bcp_to_code_attrib_sample(
                args.udhr_dir, ohchr_dir
            )

        if args.update_sample:
            in_repo = args.sample_dir == tool_utils.resolve_path(samples)
            update_samples(
                args.sample_dir,
                args.udhr_dir,
                bcp_to_code_attrib_sample,
                in_repo,
                args.no_stage,
            )

        if args.mapping:
            print_bcp_to_code_attrib_sample(bcp_to_code_attrib_sample)

        if args.base_sample_dir:
            compare_samples(
                args.base_sample_dir, args.sample_dir, opts=args.compare_opts
            )

        if args.test_script:
            test_sample_scripts(args.sample_dir)
    except ValueError as e:
        print("Error:", e)


if __name__ == "__main__":
    main()
