#!/usr/bin/env python3

from __future__ import print_function
from collections import defaultdict
from functools import cmp_to_key
import sys
import argparse

sys.path.append("/usr/share/botch")
from util import write_plain, read_yaml_file, cmp, parse_dose_yaml_mc
from util import read_tag_file
import os


def print_package(arg, bin2src, source=None, color=True):
    pkgname, arch, version, vpkg = arg
    sn, _ = src_of_pkg(arg, bin2src)
    if source:
        highlight = sn == source[0]
    else:
        highlight = False
    if pkgname == "":
        pkgstring = "(*)"
        bugs = False
    elif pkgname.startswith("src:"):
        pkgname = pkgname[4:]
        bugs = srcpkgbugs.get(pkgname)
        if version:
            pkgstring = '<a title="%s" style="">src:%s</a>' % (version, pkgname)
        else:
            pkgstring = "src:%s" % pkgname
    else:
        bugs = binpkgbugs.get(pkgname)
        if version:
            pkgstring = '<a title="%s:%s (= %s)">%s</a>' % (
                pkgname,
                arch,
                version,
                pkgname,
            )
        else:
            pkgstring = "%s:%s" % (pkgname, arch)
    if highlight:
        pkgstring = '<span style="color:#090;font-weight:bold">' + pkgstring
        pkgstring += "</span>"
    if bugs:
        bugstring = " ".join(
            '<a href="http://bugs.debian.org/%s">%s</a>' % (b, b) for b in bugs
        )
        if color:
            pkgstring = '<span style="color:#f00">%s</span> (%s)' % (
                pkgstring,
                bugstring,
            )
        else:
            pkgstring = "%s (%s)" % (pkgstring, bugstring)
    if sn:
        wnpp = wnpp_stats.get(sn)
        if wnpp:
            t, i = wnpp
            pkgstring = '%s (%s: <a href="http://bugs.debian.org/%s">#%s</a>)' % (
                pkgstring,
                t,
                i,
                i,
            )

    if vpkg:
        pkgstring += '<a title="%s"> → </a>' % vpkg
    return pkgstring


def by_srcname_chain(a, b):
    num_src = cmp(len(b[1]), len(a[1]))
    if num_src:
        return num_src
    cmp_src = cmp(min(a[1]), min(b[1]))
    if cmp_src:
        return cmp_src
    return cmp(a[0], b[0])


def by_affected_sources_missing(a, b):
    a1 = sum([len(v) for v in a[1].values()])
    b1 = sum([len(v) for v in b[1].values()])
    num_src = cmp(b1, a1)
    if num_src:
        return num_src
    else:
        return cmp(a[0], b[0])


def by_affected_sources_conflict(a, b):
    a1 = sum([len(v) for v in a[1].values()])
    b1 = sum([len(v) for v in b[1].values()])
    num_src = cmp(b1, a1)
    if num_src:
        return num_src
    else:
        return cmp(a[0], b[0])


def print_header(outfile, description, btsuser, btstag):
    print(
        """<html>
    <head>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
    <style>
    table, th, td
    {
        border: 1px solid black;
    }
    </style>
    </head>
    <body>
    """,
        file=outfile,
    )

    print(description, file=outfile)

    if btsuser and btstag:
        print(
            "<p>Bugs are associated with packages on this page if they "
            + 'carry the usertag "%s" of the user ' % btstag
            + '"%s".</p>' % btsuser,
            file=outfile,
        )
        print(
            "<p>You can get an overview of all bugs tagged like that in "
            + 'the <a href="https://bugs.debian.org/cgi-bin/pkgreport.cgi?t'
            + 'ag=%s;users=%s"' % (btstag, btsuser)
            + ">Debian bts</a></p>",
            file=outfile,
        )
    print(
        "<p>Hover over a package name with your cursor for architecture and "
        "version information. Hovering over the arrows in the depchain "
        "columns will show the dependency that led from one package in the "
        "chain to the next.</p>",
        file=outfile,
    )


def print_missing(outfile, missing, bin2src, source=None):
    print("<h2>missing</h2>", file=outfile)

    print(
        "<p>The packages in the third column cannot satisfy their (possibly "
        "transitive) dependencies because of the unsatisfied dependency in "
        "the last column. This is mostly because the binary package "
        "providing the dependency in the last column is Multi-Arch:no. Some "
        "of these packages need to be Multi-Arch:foreign instead. In some "
        "other cases, Build-Depends can be annotated with :native. The "
        "depchains column shows the dependency chain(s) from the packages "
        "in the third column to the unsatisfied dependency in the last "
        'column. The "(*)" placeholder in the depchains column represents '
        "any package in the third column. Hovering over the arrows in the "
        "depchains column with your cursor will show the dependency that "
        "led from one package in the chain to the next.</p>",
        file=outfile,
    )
    print(
        "<p>The output is first grouped by the shared unsatisfied "
        + "dependency (last column) and then by shared dependency chain "
        + "(fourth column). The groups are sorted by the number of "
        + "packages missing the dependency in the last column. Within each "
        + "group, the output is sorted by the number of packages "
        + "sharing the same dependency chain.</p>",
        file=outfile,
    )
    print(
        "<table><tr><th># of packages per missing</th><th># of packages per"
        " depchain</th><th>packages with missing (possibly transitive) depe"
        "ndencies</th><th>Depchains</th>" + "<th>Unsatisfied dependency</th></tr>",
        file=outfile,
    )

    for missing_vpkg, v in sorted(
        list(missing.items()), key=cmp_to_key(by_affected_sources_missing)
    ):
        rows = len(v)
        for c in list(v.keys()):
            if c:
                rows += len(c) - 1
        print(
            '<tr><td rowspan="%d">%d</td>' % (rows, sum([len(s) for s in v.values()])),
            file=outfile,
        )
        first = True
        for chain, v in sorted(list(v.items()), key=cmp_to_key(by_srcname_chain)):
            if chain is None:
                maxnumchains = 1
            else:
                maxnumchains = len(chain)
            if chain:
                c = "".join([print_package(t, bin2src, source) for t in chain[0]])
            else:
                c = ""
            if not first:
                print("<tr>", file=outfile)
            print(
                '<td rowspan="%d">%d</td>' % (maxnumchains, len(v))
                + '<td rowspan="%d">%s</td><td>%s</td>'
                % (
                    maxnumchains,
                    " ".join([print_package(t, bin2src, source) for t in sorted(v)]),
                    c,
                ),
                file=outfile,
            )
            if first:
                print(
                    '<td rowspan="%d">%s</td></tr>' % (rows, missing_vpkg), file=outfile
                )
            else:
                print("</tr>", file=outfile)
            if chain:
                for i in range(1, len(chain)):
                    print(
                        "<tr><td>%s</td></tr>"
                        % (
                            "".join(
                                [print_package(t, bin2src, source) for t in chain[i]]
                            )
                        ),
                        file=outfile,
                    )
            first = False
    print("</table>", file=outfile)


def print_conflict(outfile, conflict, bin2src, source=None):
    print("<h2>conflict</h2>", file=outfile)
    print(
        "<p>The packages in the third column cannot satisfy their (possibly "
        "transitive) dependencies because the last package(s) in the first "
        "depchain have an unsatisfied conflict which is shown in the last "
        "column. The second depchain column shows the dependency chain(s) "
        "to the package which the last package(s) in the first depchain "
        "conflict with. Sometimes, multiple dependency chains sharing the "
        "same conflict exist. Hovering over the arrows in the depchains "
        "column with your cursor will show the dependency that led from one "
        "package in the chain to the next.</p>",
        file=outfile,
    )
    print(
        "<p>The output is first grouped by the shared conflicting "
        + "dependency (last column) and then by the shared dependency "
        + "chains (fourth and fifth column). The groups are sorted by the "
        + "number of packages sharing the conflict in the last "
        + "column. Within each group, the output is sorted by the number of "
        + "packages sharing the same dependency chains.</p>",
        file=outfile,
    )
    print(
        "<table><tr><th># of packages per conflict</th><th># of packages per d"
        "epchain</th><th>packages with (possibly transitive) conflicting depen"
        "dencies</th><th>Depchain " + "1</th><th>Depchain2</th><th>Conflict</th></tr>",
        file=outfile,
    )

    for conflict_vpkg, v in sorted(
        list(conflict.items()), key=cmp_to_key(by_affected_sources_conflict)
    ):
        rows = len(v)
        for c1, c2 in list(v.keys()):
            if c1 is not None:
                c1 = len(c1)
            else:
                c1 = 0
            if c2 is not None:
                c2 = len(c2)
            else:
                c2 = 0
            if max(c1, c2) > 1:
                rows += max(c1, c2) - 1
        print(
            '<tr><td rowspan="%d">%d</td>' % (rows, sum([len(s) for s in v.values()])),
            file=outfile,
        )
        first = True
        for (chain1, chain2), v in sorted(
            list(v.items()), key=cmp_to_key(by_srcname_chain)
        ):
            if chain1 is None:
                maxnumchains = len(chain2)
            elif chain2 is None:
                maxnumchains = len(chain1)
            else:
                maxnumchains = max(len(chain1), len(chain2))
            if chain1:
                c1 = "".join([print_package(t, bin2src, source) for t in chain1[0]])
            else:
                c1 = ""
            if chain2:
                c2 = "".join([print_package(t, bin2src, source) for t in chain2[0]])
            else:
                c2 = ""
            if not first:
                print("<tr>", file=outfile)
            print(
                '<td rowspan="%d">%d</td>' % (maxnumchains, len(v))
                + '<td rowspan="%d">%s</td><td>%s</td><td>%s</td>'
                % (
                    maxnumchains,
                    " ".join([print_package(t, bin2src, source) for t in sorted(v)]),
                    c1,
                    c2,
                ),
                file=outfile,
            )
            if first:
                print(
                    '<td rowspan="%d">%s</td></tr>' % (rows, conflict_vpkg),
                    file=outfile,
                )
            else:
                print("</tr>", file=outfile)
            for i in range(1, maxnumchains):
                if chain1 is not None and len(chain1) >= i + 1:
                    c1 = "".join([print_package(t, bin2src, source) for t in chain1[i]])
                else:
                    c1 = ""
                if chain2 is not None and len(chain2) >= i + 1:
                    c2 = "".join([print_package(t, bin2src, source) for t in chain2[i]])
                else:
                    c2 = ""
                print("<tr><td>%s</td><td>%s</td></tr>" % (c1, c2), file=outfile)
            first = False
    print("</table>", file=outfile)


def print_footer(outfile, timestamp, srcs=[], srcsdir="", wwwroot=""):
    if srcs:
        print("<h2>Affected source packages:</h2>", file=outfile)
        srcsdir += "/"
        if srcsdir.startswith(wwwroot):
            srcsdir = srcsdir[len(wwwroot) : -1]
        for src in sorted([s for s in srcs if s is not None]):
            print('<a href="%s%s.html">%s</a>' % (srcsdir, src, src), file=outfile)

    url = "https://salsa.debian.org/debian-bootstrap-team/boott"
    footer = """
    <hr />
    <p>The JSON data used to generate these pages was computed using botch, the
    bootstrap/build ordering tool chain. The source code of botch can be
    redistributed under the terms of the LGPL3+ with an OCaml linking
    exception. The source code can be retrieved from <a
    href="https://salsa.debian.org/debian-bootstrap-team/botch">
    https://salsa.debian.org/debian-bootstrap-team/botch</a></p>

    <p>The html pages were generated by code which can be retrieved from <a
    href="%s"> %s</a> and which can be redistributed under the terms of the
    AGPL3+</p>

    <p>For questions and bugreports please contact j [dot] schauer [at] email
    [dot] de.</p>
    """ % (
        url,
        url,
    )

    if timestamp:
        print("<p>generated: %s</p>" % timestamp, file=outfile)
    print("%s</body></html>" % footer, file=outfile)


def src_of_pkg(t, bin2src):
    n, a, v, _ = t
    if n.startswith("src:"):
        return n[4:], v
    if n == "":
        return (None, None)
    res = bin2src.get((n, a, v))
    if res:
        return res
    # this can happen for a dependency on a package that is not in the archive
    print("cannot find source for %s %s" % (n, v), file=sys.stderr)
    return (None, None)


def print_top10summary(outfile, missing, conflict, bin2src, source=None):
    if not missing and not conflict:
        return

    print("<h2>Top 10 summary</h2>", file=outfile)

    print(
        '<p>The following is a summary of the full "missing" and '
        + '"conflict" tables below. It only shows the first and last '
        + "columns of the full tables and only displays the top 10 rows.</p>",
        file=outfile,
    )

    if missing:
        print("<h3>Missing</h3>", file=outfile)

        print(
            "<table><tr><th># of packages per missing</th><th>Unsatisfied "
            + "dependency</th></tr>",
            file=outfile,
        )
        for missing_vpkg, v in sorted(
            list(missing.items()), key=cmp_to_key(by_affected_sources_missing)
        )[:10]:
            rows = len(v)
            for c in list(v.keys()):
                if c:
                    rows += len(c) - 1
            print(
                "<tr><td>%d</td><td>%s</td></tr>"
                % (sum([len(s) for s in v.values()]), missing_vpkg),
                file=outfile,
            )
        print("</table>", file=outfile)

    if conflict:
        print("<h3>Conflict</h3>", file=outfile)
        print(
            "<table><tr><th># of packages per conflict</th>" + "<th>Conflict</th></tr>",
            file=outfile,
        )

        for conflict_vpkg, v in sorted(
            list(conflict.items()), key=cmp_to_key(by_affected_sources_conflict)
        )[:10]:
            print(
                "<tr><td>%d</td><td>%s</td></tr>"
                % (sum([len(s) for s in v.values()]), conflict_vpkg),
                file=outfile,
            )
        print("</table>", file=outfile)


def create_srcpkg_pages(
    missing, conflict, bin2src, srcsdir, description, btsuser, btstag, timestamp
):
    srcsdict = defaultdict(
        lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))
    )

    # now go through all missing and conflict and copy them to the source
    # packages that associate with them
    # it is not possible to find a source package for a vpkg (especially if
    # it's missing)
    for vpkg, v in missing.items():
        for chain, v in v.items():
            for c in chain:
                for p in c:
                    sn, sv = src_of_pkg(p, bin2src)
                    if not sn:
                        continue
                    srcsdict[sn][sv]["missing"][vpkg][chain] = v
            for p in v:
                sn, sv = src_of_pkg(p, bin2src)
                srcsdict[sn][sv]["missing"][vpkg][chain] = v

    for vpkg, v in conflict.items():
        for (chain1, chain2), v in v.items():
            for c in chain1:
                for p in c:
                    sn, sv = src_of_pkg(p, bin2src)
                    if not sn:
                        continue
                    srcsdict[sn][sv]["conflict"][vpkg][(chain1, chain2)] = v
            for c in chain2:
                for p in c:
                    sn, sv = src_of_pkg(p, bin2src)
                    if not sn:
                        continue
                    srcsdict[sn][sv]["conflict"][vpkg][(chain1, chain2)] = v
            for p in v:
                sn, sv = src_of_pkg(p, bin2src)
                srcsdict[sn][sv]["conflict"][vpkg][(chain1, chain2)] = v

    if not os.path.isdir(srcsdir):
        os.mkdir(srcsdir)
    for srcpkg, versions in srcsdict.items():
        if not srcpkg:
            continue
        fname = os.path.join(srcsdir, srcpkg) + ".html"
        with open(fname, "w", encoding="utf8") as srcpkgpage:
            print_header(srcpkgpage, description, btsuser, btstag)
            for ver in versions.keys():
                print(
                    "<h1>%s</h1>"
                    % print_package(
                        ("src:" + srcpkg, None, ver, None), bin2src, color=False
                    ),
                    file=srcpkgpage,
                )
                print_top10summary(
                    srcpkgpage,
                    versions[ver].get("missing"),
                    versions[ver].get("conflict"),
                    bin2src,
                    (srcpkg, ver),
                )
                if versions[ver].get("missing"):
                    print_missing(
                        srcpkgpage, versions[ver]["missing"], bin2src, (srcpkg, ver)
                    )
                if versions[ver].get("conflict"):
                    print_conflict(
                        srcpkgpage, versions[ver]["conflict"], bin2src, (srcpkg, ver)
                    )
            print_footer(srcpkgpage, timestamp)

    return srcsdict.keys()


def dose2html(
    yamlin,
    outfile,
    description="",
    srcpkgbugs=set(),
    binpkgbugs=set(),
    btsuser=None,
    btstag=None,
    srcsdir=None,
    packages=[],
    timestamp=None,
    wwwroot=None,
    verbose=False,
):
    bin2src = dict()

    for pkg in packages:
        name = pkg["Package"]
        arch = pkg["Architecture"]
        ver = pkg["Version"]
        src = pkg.get("Source")
        if src:
            if " " in src:
                srcname, srcver = src.split(" ", 1)
                srcver = srcver[1:-1]
            else:
                srcname = src
                srcver = ver
        else:
            srcname = name
            srcver = ver
        bin2src[(name, arch, ver)] = (srcname, srcver)
        bin2src[(name, arch, None)] = (srcname, None)
        provides = pkg.get("Provides")
        if not provides:
            continue
        for pname in provides.split(","):
            pname = pname.strip()
            bin2src[(pname, arch, ver)] = (srcname, srcver)
            bin2src[(pname, arch, None)] = (srcname, None)

    missing, conflict = parse_dose_yaml_mc(yamlin)

    print_header(outfile, description, btsuser, btstag)
    print_top10summary(outfile, missing, conflict, bin2src)
    print_missing(outfile, missing, bin2src)
    print_conflict(outfile, conflict, bin2src)

    if srcsdir and packages:
        srcpkgs = create_srcpkg_pages(
            missing, conflict, bin2src, srcsdir, description, btsuser, btstag, timestamp
        )
        print_footer(outfile, timestamp, srcpkgs, srcsdir, wwwroot)
    else:
        print_footer(outfile, timestamp)


class _ExtendAction(argparse.Action):
    def __init__(self, option_strings, dest, nargs=None, const=None, **kwargs):
        if nargs == 0:
            raise ValueError("nargs for extend actions must be > 0")
        if const is not None and nargs != argparse.OPTIONAL:
            raise ValueError("nargs must be %r to supply const" % argparse.OPTIONAL)
        super(_ExtendAction, self).__init__(
            option_strings=option_strings, dest=dest, nargs=nargs, const=const, **kwargs
        )

    def __call__(self, parser, namespace, values, option_string=None):
        items = getattr(namespace, self.dest, None)
        items = argparse._copy_items(items)
        items.extend(values)
        setattr(namespace, self.dest, items)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="given a buildcheck result, create a html overview"
    )
    parser.register("action", "extend", _ExtendAction)
    parser.add_argument(
        "yamlin", type=read_yaml_file, help="input in yaml format (dose3 output)"
    )
    parser.add_argument("htmlout", type=write_plain, help="output in html format")
    parser.add_argument("--desc", default="", help="descriptive HTML snippet")
    parser.add_argument("--btsuser", help="bts user to associate packages with")
    parser.add_argument("--btstag", help="bts tag to associate packages with")
    parser.add_argument(
        "--srcsdir",
        help="output directory for " + "individual source package overviews",
    )
    parser.add_argument(
        "--wnpp",
        action="store_true",
        help="retrieve and print wnpp status for packages",
    )
    parser.add_argument(
        "--packages",
        type=read_tag_file,
        action="extend",
        default=[],
        help="Packages file to create a mapping from binary " + "to source packages",
    )
    parser.add_argument(
        "--timestamp",
        help="put a freeform timestamp string at the bottom " + "of the generated HTML",
    )
    parser.add_argument(
        "--wwwroot",
        help="The HTML hyperlink to the source package "
        "overview by default will begin with the string given "
        'in the "--srcsdir" option. The string given by this '
        "option will be removed from the beginning of that "
        "path in the HTML href attribute. If this option ends "
        "with a slash, then the resulting hyperlinks will "
        "become relative.",
    )
    parser.add_argument("-v", "--verbose", action="store_true", help="be verbose")
    args = parser.parse_args()

    srcpkgbugs = defaultdict(list)
    binpkgbugs = defaultdict(list)

    if args.btsuser and args.btstag:
        # this only corks with python2 because SOAPpy is not available for
        # python3
        # import SOAPpy

        # server = SOAPpy.SOAPProxy('http://bugs.debian.org/cgi-bin/soap.cgi',
        #                           'Debbugs/SOAP')
        # bugnrs = server.get_usertag('debian-cross@lists.debian.org',
        #                             'cross-satisfiability')
        # buginfos = server.get_status(bugnrs['cross-satisfiability'])

        # for bug in buginfos['item']:
        #    k = bug['key']
        #    v = bug['value']
        #    if v['package']:
        #        binpkgbugs[v['package']].append(k)
        #    else:
        #        for s in v['source'].split(','):
        #            s = s.strip()
        #            srcpkgbugs[s].append(k)
        import subprocess

        res = subprocess.check_output(
            "bts select users:%s " % args.btsuser
            + "tag:%s status:open | bts status " % args.btstag
            + "fields:source,package,bug_num file:-",
            shell=True,
        )
        for b in res.decode().split("\n\n"):
            if not b:
                continue
            d = {k: v for k, v in [L.split("\t") for L in b.split("\n") if L.strip()]}
            binpkgbugs[d["package"]].append(d["bug_num"])
            for s in d["source"].split(","):
                s = s.strip()
                srcpkgbugs[s].append(d["bug_num"])

    wnpp_stats = dict()
    if args.wnpp and args.packages:
        import urllib.request

        url = "https://qa.debian.org/data/bts/wnpp_rm"
        with urllib.request.urlopen(url) as f:
            # the following snippet is from
            # distro_tracker/vendor/debian/tracker_tasks.py
            for line in f:
                line = line.decode().strip()
                try:
                    package_name, wnpp_type, bug_id = line.split("|")[0].split()
                    bug_id = int(bug_id)
                except ValueError:
                    # Badly formatted
                    continue
                # Strip the colon from the end of the package name
                package_name = package_name[:-1]

                wnpp_stats[package_name] = (wnpp_type, bug_id)

    with args.htmlout as f:
        dose2html(
            args.yamlin,
            f,
            args.desc,
            srcpkgbugs,
            binpkgbugs,
            args.btsuser,
            args.btstag,
            args.srcsdir,
            args.packages,
            args.timestamp,
            args.wwwroot,
            args.verbose,
        )
