comparison planemo/lib/python3.7/site-packages/schema_salad/makedoc.py @ 1:56ad4e20f292 draft

"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"

1:56ad4e20f292

from __future__ import absolute_import

import argparse
import codecs
import copy
import logging
import os
import re
import sys
from codecs import StreamWriter  # pylint: disable=unused-import
from io import TextIOWrapper, open
from typing import (
    IO,
    Any,
    Dict,
    List,
    MutableMapping,
    MutableSequence,
    Optional,
    Set,
    Tuple,
    Union,
    cast,
)

import mistune
import six
from six import StringIO
from six.moves import range, urllib
from typing_extensions import Text  # pylint: disable=unused-import

from . import schema
from .exceptions import SchemaSaladException, ValidationException
from .utils import add_dictlist, aslist

# move to a regular typing import when Python 3.3-3.6 is no longer supported


_logger = logging.getLogger("salad")


def has_types(items):  # type: (Any) -> List[Text]
    r = []  # type: List[Text]
    if isinstance(items, MutableMapping):
        if items["type"] == "https://w3id.org/cwl/salad#record":
            return [items["name"]]
        for n in ("type", "items", "values"):
            if n in items:
                r.extend(has_types(items[n]))
        return r
    if isinstance(items, MutableSequence):
        for i in items:
            r.extend(has_types(i))
        return r
    if isinstance(items, six.string_types):
        return [items]
    return []


def linkto(item):  # type: (Text) -> Text
    _, frg = urllib.parse.urldefrag(item)
    return "[{}](#{})".format(frg, to_id(frg))


class MyRenderer(mistune.Renderer):
    def __init__(self):  # type: () -> None
        super(MyRenderer, self).__init__()
        self.options = {}

    def header(self, text, level, raw=None):  # type: (Text, int, Any) -> Text
        return """<h{} id="{}" class="section">{} <a href="#{}">&sect;</a></h{}>""".format(
            level, to_id(text), text, to_id(text), level
        )

    def table(self, header, body):  # type: (Text, Text) -> Text
        return (
            '<table class="table table-striped">\n<thead>{}</thead>\n'
            "<tbody>\n{}</tbody>\n</table>\n"
        ).format(header, body)


def to_id(text):  # type: (Text) -> Text
    textid = text
    if text[0] in ("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"):
        try:
            textid = text[text.index(" ") + 1 :]
        except ValueError:
            pass
    textid = textid.replace(" ", "_")
    return textid


class ToC(object):
    def __init__(self):  # type: () -> None
        self.first_toc_entry = True
        self.numbering = [0]
        self.toc = ""
        self.start_numbering = True

    def add_entry(self, thisdepth, title):  # type: (int, str) -> str
        depth = len(self.numbering)
        if thisdepth < depth:
            self.toc += "</ol>"
            for _ in range(0, depth - thisdepth):
                self.numbering.pop()
                self.toc += "</li></ol>"
            self.numbering[-1] += 1
        elif thisdepth == depth:
            if not self.first_toc_entry:
                self.toc += "</ol>"
            else:
                self.first_toc_entry = False
            self.numbering[-1] += 1
        elif thisdepth > depth:
            self.numbering.append(1)

        if self.start_numbering:
            num = "{}.{}".format(
                self.numbering[0], ".".join([str(n) for n in self.numbering[1:]])
            )
        else:
            num = ""
        self.toc += """<li><a href="#{}">{} {}</a><ol>\n""".format(
            to_id(title), num, title
        )
        return num

    def contents(self, idn):  # type: (str) -> str
        toc = """<h1 id="{}">Table of contents</h1>
               <nav class="tocnav"><ol>{}""".format(
            idn, self.toc
        )
        toc += "</ol>"
        for _ in range(0, len(self.numbering)):
            toc += "</li></ol>"
        toc += """</nav>"""
        return toc


basicTypes = (
    "https://w3id.org/cwl/salad#null",
    "http://www.w3.org/2001/XMLSchema#boolean",
    "http://www.w3.org/2001/XMLSchema#int",
    "http://www.w3.org/2001/XMLSchema#long",
    "http://www.w3.org/2001/XMLSchema#float",
    "http://www.w3.org/2001/XMLSchema#double",
    "http://www.w3.org/2001/XMLSchema#string",
    "https://w3id.org/cwl/salad#record",
    "https://w3id.org/cwl/salad#enum",
    "https://w3id.org/cwl/salad#array",
)


def number_headings(toc, maindoc):  # type: (ToC, str) -> str
    mdlines = []
    skip = False
    for line in maindoc.splitlines():
        if line.strip() == "# Introduction":
            toc.start_numbering = True
            toc.numbering = [0]

        if "```" in line:
            skip = not skip

        if not skip:
            m = re.match(r"^(#+) (.*)", line)
            if m is not None:
                num = toc.add_entry(len(m.group(1)), m.group(2))
                line = "{} {} {}".format(m.group(1), num, m.group(2))
            line = re.sub(r"^(https?://\S+)", r"[\1](\1)", line)
        mdlines.append(line)

    maindoc = "\n".join(mdlines)
    return maindoc


def fix_doc(doc):  # type: (Union[List[str], str]) -> str
    if isinstance(doc, MutableSequence):
        docstr = "".join(doc)
    else:
        docstr = doc
    return "\n".join(
        [
            re.sub(r"<([^>@]+@[^>]+)>", r"[\1](mailto:\1)", d)
            for d in docstr.splitlines()
        ]
    )


class RenderType(object):
    def __init__(self, toc, j, renderlist, redirects, primitiveType):
        # type: (ToC, List[Dict[Text, Text]], str, Dict[Text, Text], str) -> None
        self.typedoc = StringIO()
        self.toc = toc
        self.subs = {}  # type: Dict[str, str]
        self.docParent = {}  # type: Dict[str, List[Text]]
        self.docAfter = {}  # type: Dict[str, List[Text]]
        self.rendered = set()  # type: Set[str]
        self.redirects = redirects
        self.title = None  # type: Optional[str]
        self.primitiveType = primitiveType

        for t in j:
            if "extends" in t:
                for e in aslist(t["extends"]):
                    add_dictlist(self.subs, e, t["name"])
                    # if "docParent" not in t and "docAfter" not in t:
                    #    add_dictlist(self.docParent, e, t["name"])

            if t.get("docParent"):
                add_dictlist(self.docParent, t["docParent"], t["name"])

            if t.get("docChild"):
                for c in aslist(t["docChild"]):
                    add_dictlist(self.docParent, t["name"], c)

            if t.get("docAfter"):
                add_dictlist(self.docAfter, t["docAfter"], t["name"])

        metaschema_loader = schema.get_metaschema()[2]
        alltypes = schema.extend_and_specialize(j, metaschema_loader)

        self.typemap = {}  # type: Dict[Text, Dict[Text, Text]]
        self.uses = {}  # type: Dict[Text, List[Tuple[Text, Text]]]
        self.record_refs = {}  # type: Dict[Text, List[Text]]
        for entry in alltypes:
            self.typemap[entry["name"]] = entry
            try:
                if entry["type"] == "record":
                    self.record_refs[entry["name"]] = []
                    fields = entry.get(
                        "fields", []
                    )  # type: Union[Text, List[Dict[Text, Text]]]
                    if isinstance(fields, Text):
                        raise KeyError("record fields must be a list of mappings")
                    for f in fields:  # type: Dict[Text, Text]
                        p = has_types(f)
                        for tp in p:
                            if tp not in self.uses:
                                self.uses[tp] = []
                            if (entry["name"], f["name"]) not in self.uses[tp]:
                                _, frg1 = urllib.parse.urldefrag(t["name"])
                                _, frg2 = urllib.parse.urldefrag(f["name"])
                                self.uses[tp].append((frg1, frg2))
                            if (
                                tp not in basicTypes
                                and tp not in self.record_refs[entry["name"]]
                            ):
                                self.record_refs[entry["name"]].append(tp)
            except KeyError:
                _logger.error("Did not find 'type' in %s", t)
                _logger.error("record refs is %s", self.record_refs)
                raise

        for entry in alltypes:
            if entry["name"] in renderlist or (
                (not renderlist)
                and ("extends" not in entry)
                and ("docParent" not in entry)
                and ("docAfter" not in entry)
            ):
                self.render_type(entry, 1)

    def typefmt(
        self,
        tp,  # type: Any
        redirects,  # type: Dict[Text, Text]
        nbsp=False,  # type: bool
        jsonldPredicate=None,  # type: Optional[Dict[str, str]]
    ):
        # type: (...) -> Text
        if isinstance(tp, MutableSequence):
            if nbsp and len(tp) <= 3:
                return "&nbsp;|&nbsp;".join(
                    [
                        self.typefmt(n, redirects, jsonldPredicate=jsonldPredicate)
                        for n in tp
                    ]
                )
            return " | ".join(
                [
                    self.typefmt(n, redirects, jsonldPredicate=jsonldPredicate)
                    for n in tp
                ]
            )
        if isinstance(tp, MutableMapping):
            if tp["type"] == "https://w3id.org/cwl/salad#array":
                ar = "array&lt;{}&gt;".format(
                    self.typefmt(tp["items"], redirects, nbsp=True)
                )
                if jsonldPredicate is not None and "mapSubject" in jsonldPredicate:
                    if "mapPredicate" in jsonldPredicate:
                        ar += " | "
                        if len(ar) > 40:
                            ar += "<br>"

                        ar += (
                            "<a href='#map'>map</a>&lt;<code>{}</code>"
                            ",&nbsp;<code>{}</code> | {}&gt".format(
                                jsonldPredicate["mapSubject"],
                                jsonldPredicate["mapPredicate"],
                                self.typefmt(tp["items"], redirects),
                            )
                        )
                    else:
                        ar += " | "
                        if len(ar) > 40:
                            ar += "<br>"
                        ar += "<a href='#map'>map</a>&lt;<code>{}</code>,&nbsp;{}&gt".format(
                            jsonldPredicate["mapSubject"],
                            self.typefmt(tp["items"], redirects),
                        )
                return ar
            if tp["type"] in (
                "https://w3id.org/cwl/salad#record",
                "https://w3id.org/cwl/salad#enum",
            ):
                frg = schema.avro_name(tp["name"])
                if tp["name"] in redirects:
                    return """<a href="{}">{}</a>""".format(redirects[tp["name"]], frg)
                if tp["name"] in self.typemap:
                    return """<a href="#{}">{}</a>""".format(to_id(frg), frg)
                if (
                    tp["type"] == "https://w3id.org/cwl/salad#enum"
                    and len(tp["symbols"]) == 1
                ):
                    return "constant value <code>{}</code>".format(
                        schema.avro_name(tp["symbols"][0])
                    )
                return frg
            if isinstance(tp["type"], MutableMapping):
                return self.typefmt(tp["type"], redirects)
        else:
            if str(tp) in redirects:
                return """<a href="{}">{}</a>""".format(redirects[tp], redirects[tp])
            if str(tp) in basicTypes:
                return """<a href="{}">{}</a>""".format(
                    self.primitiveType, schema.avro_name(str(tp))
                )
            _, frg = urllib.parse.urldefrag(tp)
            if frg != "":
                tp = frg
            return """<a href="#{}">{}</a>""".format(to_id(tp), tp)
        raise SchemaSaladException("We should not be here!")

    def render_type(self, f, depth):  # type: (Dict[Text, Any], int) -> None
        if f["name"] in self.rendered or f["name"] in self.redirects:
            return
        self.rendered.add(f["name"])

        if f.get("abstract"):
            return

        if "doc" not in f:
            f["doc"] = ""

        f["type"] = copy.deepcopy(f)
        f["doc"] = ""
        f = f["type"]

        if "doc" not in f:
            f["doc"] = ""

        def extendsfrom(item, ex):
            # type: (Dict[Text, Any], List[Dict[Text, Any]]) -> None
            if "extends" in item:
                for e in aslist(item["extends"]):
                    ex.insert(0, self.typemap[e])
                    extendsfrom(self.typemap[e], ex)

        ex = [f]
        extendsfrom(f, ex)

        enumDesc = {}
        if f["type"] == "enum" and isinstance(f["doc"], MutableSequence):
            for e in ex:
                for i in e["doc"]:
                    idx = i.find(":")
                    if idx > -1:
                        enumDesc[i[:idx]] = i[idx + 1 :]
                e["doc"] = [
                    i
                    for i in e["doc"]
                    if i.find(":") == -1 or i.find(" ") < i.find(":")
                ]

        f["doc"] = fix_doc(f["doc"])

        if f["type"] == "record":
            for field in f.get("fields", []):
                if "doc" not in field:
                    field["doc"] = ""

        if f["type"] != "documentation":
            lines = []
            for line in f["doc"].splitlines():
                if len(line) > 0 and line[0] == "#":
                    line = ("#" * depth) + line
                lines.append(line)
            f["doc"] = "\n".join(lines)

            _, frg = urllib.parse.urldefrag(f["name"])
            num = self.toc.add_entry(depth, frg)
            doc = u"{} {} {}\n".format(("#" * depth), num, frg)
        else:
            doc = u""

        if self.title is None and f["doc"]:
            title = f["doc"][0 : f["doc"].index("\n")]
            if title.startswith("# "):
                self.title = title[2:]
            else:
                self.title = title

        if f["type"] == "documentation":
            f["doc"] = number_headings(self.toc, f["doc"])

        doc = doc + "\n\n" + f["doc"]

        doc = mistune.markdown(doc, renderer=MyRenderer())

        if f["type"] == "record":
            doc += "<h3>Fields</h3>"
            doc += """
<div class="responsive-table">
<div class="row responsive-table-header">
<div class="col-xs-3 col-lg-2">field</div>
<div class="col-xs-2 col-lg-1">required</div>
<div class="col-xs-7 col-lg-3">type</div>
<div class="col-xs-12 col-lg-6 description-header">description</div>
</div>"""
            required = []
            optional = []
            for i in f.get("fields", []):
                tp = i["type"]
                if (
                    isinstance(tp, MutableSequence)
                    and tp[0] == "https://w3id.org/cwl/salad#null"
                ):
                    opt = False
                    tp = tp[1:]
                else:
                    opt = True

                desc = i["doc"]

                rfrg = schema.avro_name(i["name"])
                tr = """
<div class="row responsive-table-row">
<div class="col-xs-3 col-lg-2"><code>{}</code></div>
<div class="col-xs-2 col-lg-1">{}</div>
<div class="col-xs-7 col-lg-3">{}</div>
<div class="col-xs-12 col-lg-6 description-col">{}</div>
</div>""".format(
                    rfrg,
                    "required" if opt else "optional",
                    self.typefmt(
                        tp, self.redirects, jsonldPredicate=i.get("jsonldPredicate")
                    ),
                    mistune.markdown(desc),
                )
                if opt:
                    required.append(tr)
                else:
                    optional.append(tr)
            for i in required + optional:
                doc += i
            doc += """</div>"""
        elif f["type"] == "enum":
            doc += "<h3>Symbols</h3>"
            doc += """<table class="table table-striped">"""
            doc += "<tr><th>symbol</th><th>description</th></tr>"
            for e in ex:
                for i in e.get("symbols", []):
                    doc += "<tr>"
                    efrg = schema.avro_name(i)
                    doc += "<td><code>{}</code></td><td>{}</td>".format(
                        efrg, enumDesc.get(efrg, "")
                    )
                    doc += "</tr>"
            doc += """</table>"""
        f["doc"] = doc

        self.typedoc.write(f["doc"])

        subs = self.docParent.get(f["name"], []) + self.record_refs.get(f["name"], [])
        if len(subs) == 1:
            self.render_type(self.typemap[subs[0]], depth)
        else:
            for s in subs:
                self.render_type(self.typemap[s], depth + 1)

        for s in self.docAfter.get(f["name"], []):
            self.render_type(self.typemap[s], depth)


def avrold_doc(
    j,  # type: List[Dict[Text, Any]]
    outdoc,  # type: Union[IO[Any], StreamWriter]
    renderlist,  # type: str
    redirects,  # type: Dict[Text, Text]
    brand,  # type: str
    brandlink,  # type: str
    primtype,  # type: str
    brandstyle=None,  # type: Optional[str]
    brandinverse=False,  # type: bool
):  # type: (...) -> None
    toc = ToC()
    toc.start_numbering = False

    rt = RenderType(toc, j, renderlist, redirects, primtype)
    content = rt.typedoc.getvalue()  # type: Text

    if brandstyle is None:
        bootstrap_url = (
            "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css"
        )
        bootstrap_integrity = (
            "sha384-604wwakM23pEysLJAhja8Lm42IIwYrJ0dEAqzFsj9pJ/P5buiujjywArgPCi8eoz"
        )
        brandstyle_template = (
            '<link rel="stylesheet" href={} integrity={} crossorigin="anonymous">'
        )
        brandstyle = brandstyle_template.format(bootstrap_url, bootstrap_integrity)

    picturefill_url = (
        "https://cdn.rawgit.com/scottjehl/picturefill/3.0.2/dist/picturefill.min.js"
    )
    picturefill_integrity = (
        "sha384-ZJsVW8YHHxQHJ+SJDncpN90d0EfAhPP+yA94n+EhSRzhcxfo84yMnNk+v37RGlWR"
    )
    outdoc.write(
        """
    <!DOCTYPE html>
    <html>
    <head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    {}
    <script>
    // Picture element HTML5 shiv
    document.createElement( "picture" );
    </script>
    <script src="{}"
        integrity="{}"
        crossorigin="anonymous" async></script>
    """.format(
            brandstyle, picturefill_url, picturefill_integrity
        )
    )

    outdoc.write("<title>{}</title>".format(rt.title))

    outdoc.write(
        """
    <style>
    :target {
      padding-top: 61px;
      margin-top: -61px;
    }
    body {
      padding-top: 61px;
    }
    .tocnav ol {
      list-style: none
    }
    pre {
      margin-left: 2em;
      margin-right: 2em;
    }
    .section a {
      visibility: hidden;
    }
    .section:hover a {
      visibility: visible;
      color: rgb(201, 201, 201);
    }
    .responsive-table-header {
      text-align: left;
      padding: 8px;
      vertical-align: top;
      font-weight: bold;
      border-top-color: rgb(221, 221, 221);
      border-top-style: solid;
      border-top-width: 1px;
      background-color: #f9f9f9
    }
    .responsive-table > .responsive-table-row {
      text-align: left;
      padding: 8px;
      vertical-align: top;
      border-top-color: rgb(221, 221, 221);
      border-top-style: solid;
      border-top-width: 1px;
    }
    @media (min-width: 0px), print {
      .description-header {
        display: none;
      }
      .description-col {
        margin-top: 1em;
        margin-left: 1.5em;
      }
    }
    @media (min-width: 1170px) {
      .description-header {
        display: inline;
      }
      .description-col {
        margin-top: 0px;
        margin-left: 0px;
      }
    }
    .responsive-table-row:nth-of-type(odd) {
       background-color: #f9f9f9
    }
    </style>
    </head>
    <body>
    """
    )

    navbar_extraclass = ""
    if brandinverse:
        navbar_extraclass = "navbar-inverse"
    outdoc.write(
        """
      <nav class="navbar navbar-default navbar-fixed-top {}">
        <div class="container">
          <div class="navbar-header">
            <a class="navbar-brand" href="{}">{}</a>
    """.format(
            navbar_extraclass, brandlink, brand
        )
    )

    if u"<!--ToC-->" in content:
        content = content.replace(u"<!--ToC-->", toc.contents("toc"))
        outdoc.write(
            """
                <ul class="nav navbar-nav">
                  <li><a href="#toc">Table of contents</a></li>
                </ul>
        """
        )

    outdoc.write(
        """
          </div>
        </div>
      </nav>
    """
    )

    outdoc.write(
        """
    <div class="container">
    """
    )

    outdoc.write(
        """
    <div class="row">
    """
    )

    outdoc.write(
        """
    <div class="col-md-12" role="main" id="main">"""
    )

    outdoc.write(content)

    outdoc.write("""</div>""")

    outdoc.write(
        """
    </div>
    </div>
    </body>
    </html>"""
    )


def main():  # type: () -> None
    parser = argparse.ArgumentParser()
    parser.add_argument("schema")
    parser.add_argument("--only", action="append")
    parser.add_argument("--redirect", action="append")
    parser.add_argument("--brand")
    parser.add_argument("--brandlink")
    parser.add_argument("--brandstyle")
    parser.add_argument("--brandinverse", default=False, action="store_true")
    parser.add_argument("--primtype", default="#PrimitiveType")

    args = parser.parse_args()

    makedoc(args)


def makedoc(args):  # type: (argparse.Namespace) -> None

    s = []  # type: List[Dict[Text, Any]]
    a = args.schema
    with open(a, encoding="utf-8") as f:
        if a.endswith("md"):
            s.append(
                {
                    "name": os.path.splitext(os.path.basename(a))[0],
                    "type": "documentation",
                    "doc": f.read(),
                }
            )
        else:
            uri = "file://" + os.path.abspath(a)
            metaschema_loader = schema.get_metaschema()[2]
            j, _ = metaschema_loader.resolve_ref(uri, "")
            if isinstance(j, MutableSequence):
                s.extend(j)
            elif isinstance(j, MutableMapping):
                s.append(j)
            else:
                raise ValidationException("Schema must resolve to a list or a dict")
    redirect = {}
    for r in args.redirect or []:
        redirect[r.split("=")[0]] = r.split("=")[1]
    renderlist = args.only if args.only else []
    stdout = cast(TextIOWrapper, sys.stdout)  # type: Union[TextIOWrapper, StreamWriter]
    if sys.stdout.encoding != "UTF-8":
        if sys.version_info >= (3,):
            stdout = TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
        else:
            stdout = codecs.getwriter("utf-8")(sys.stdout)
    avrold_doc(
        s,
        stdout,
        renderlist,
        redirect,
        args.brand,
        args.brandlink,
        args.primtype,
        brandstyle=args.brandstyle,
        brandinverse=args.brandinverse,
    )


if __name__ == "__main__":
    main()