Mercurial > repos > shellac > sam_consensus_v3
diff env/lib/python3.9/site-packages/schema_salad/makedoc.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
| author | shellac |
|---|---|
| date | Mon, 22 Mar 2021 18:12:50 +0000 |
| parents | |
| children |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/env/lib/python3.9/site-packages/schema_salad/makedoc.py Mon Mar 22 18:12:50 2021 +0000 @@ -0,0 +1,744 @@ +import argparse +import copy +import logging +import os +import re +import sys +from codecs import StreamWriter +from io import StringIO, TextIOWrapper +from typing import ( + IO, + Any, + Dict, + List, + MutableMapping, + MutableSequence, + Optional, + Set, + Tuple, + Union, + cast, +) +from urllib.parse import urldefrag + +import mistune + +from . import schema +from .exceptions import SchemaSaladException, ValidationException +from .utils import add_dictlist, aslist + +_logger = logging.getLogger("salad") + + +def has_types(items: Any) -> List[str]: + r = [] # type: List[str] + if isinstance(items, MutableMapping): + if items["type"] == "https://w3id.org/cwl/salad#record": + return [items["name"]] + for n in ("type", "items", "values"): + if n in items: + r.extend(has_types(items[n])) + return r + if isinstance(items, MutableSequence): + for i in items: + r.extend(has_types(i)) + return r + if isinstance(items, str): + return [items] + return [] + + +def linkto(item: str) -> str: + frg = urldefrag(item)[1] + return "[{}](#{})".format(frg, to_id(frg)) + + +class MyRenderer(mistune.Renderer): + def __init__(self) -> None: + super().__init__() + self.options = {} + + def header(self, text: str, level: int, raw: Optional[Any] = None) -> str: + return ( + """<h{} id="{}" class="section">{} <a href="#{}">§</a></h{}>""".format( + level, to_id(text), text, to_id(text), level + ) + ) + + def table(self, header: str, body: str) -> str: + return ( + '<table class="table table-striped">\n<thead>{}</thead>\n' + "<tbody>\n{}</tbody>\n</table>\n" + ).format(header, body) + + +def to_id(text: str) -> str: + textid = text + if text[0] in ("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"): + try: + textid = text[text.index(" ") + 1 :] + except ValueError: + pass + return textid.replace(" ", "_") + + +class ToC: + def __init__(self) -> None: + self.first_toc_entry = True + self.numbering = [0] + self.toc = "" + self.start_numbering = True + + def add_entry(self, thisdepth, title): # type: (int, str) -> str + depth = len(self.numbering) + if thisdepth < depth: + self.toc += "</ol>" + for _ in range(0, depth - thisdepth): + self.numbering.pop() + self.toc += "</li></ol>" + self.numbering[-1] += 1 + elif thisdepth == depth: + if not self.first_toc_entry: + self.toc += "</ol>" + else: + self.first_toc_entry = False + self.numbering[-1] += 1 + elif thisdepth > depth: + self.numbering.append(1) + + num = ( + "{}.{}".format( + self.numbering[0], ".".join([str(n) for n in self.numbering[1:]]) + ) + if self.start_numbering + else "" + ) + self.toc += """<li><a href="#{}">{} {}</a><ol>\n""".format( + to_id(title), num, title + ) + return num + + def contents(self, idn: str) -> str: + toc = """<h1 id="{}">Table of contents</h1> + <nav class="tocnav"><ol>{}""".format( + idn, self.toc + ) + toc += "</ol>" + for _ in range(0, len(self.numbering)): + toc += "</li></ol>" + toc += """</nav>""" + return toc + + +basicTypes = ( + "https://w3id.org/cwl/salad#null", + "http://www.w3.org/2001/XMLSchema#boolean", + "http://www.w3.org/2001/XMLSchema#int", + "http://www.w3.org/2001/XMLSchema#long", + "http://www.w3.org/2001/XMLSchema#float", + "http://www.w3.org/2001/XMLSchema#double", + "http://www.w3.org/2001/XMLSchema#string", + "https://w3id.org/cwl/salad#record", + "https://w3id.org/cwl/salad#enum", + "https://w3id.org/cwl/salad#array", +) + + +def number_headings(toc: ToC, maindoc: str) -> str: + mdlines = [] + skip = False + for line in maindoc.splitlines(): + if line.strip() == "# Introduction": + toc.start_numbering = True + toc.numbering.clear() + toc.numbering.append(0) + + if "```" in line: + skip = not skip + + if not skip: + m = re.match(r"^(#+) (.*)", line) + if m is not None: + group1 = m.group(1) + assert group1 is not None # nosec + group2 = m.group(2) + assert group2 is not None # nosec + num = toc.add_entry(len(group1), group2) + line = f"{group1} {num} {group2}" + line = re.sub(r"^(https?://\S+)", r"[\1](\1)", line) + mdlines.append(line) + + maindoc = "\n".join(mdlines) + return maindoc + + +def fix_doc(doc: Union[List[str], str]) -> str: + docstr = "".join(doc) if isinstance(doc, MutableSequence) else doc + return "\n".join( + [ + re.sub(r"<([^>@]+@[^>]+)>", r"[\1](mailto:\1)", d) + for d in docstr.splitlines() + ] + ) + + +class RenderType: + def __init__( + self, + toc: ToC, + j: List[Dict[str, str]], + renderlist: str, + redirects: Dict[str, str], + primitiveType: str, + ) -> None: + self.typedoc = StringIO() + self.toc = toc + self.subs = {} # type: Dict[str, str] + self.docParent = {} # type: Dict[str, List[str]] + self.docAfter = {} # type: Dict[str, List[str]] + self.rendered = set() # type: Set[str] + self.redirects = redirects + self.title = None # type: Optional[str] + self.primitiveType = primitiveType + + for t in j: + if "extends" in t: + for e in aslist(t["extends"]): + add_dictlist(self.subs, e, t["name"]) + # if "docParent" not in t and "docAfter" not in t: + # add_dictlist(self.docParent, e, t["name"]) + + if t.get("docParent"): + add_dictlist(self.docParent, t["docParent"], t["name"]) + + if t.get("docChild"): + for c in aslist(t["docChild"]): + add_dictlist(self.docParent, t["name"], c) + + if t.get("docAfter"): + add_dictlist(self.docAfter, t["docAfter"], t["name"]) + + metaschema_loader = schema.get_metaschema()[2] + alltypes = schema.extend_and_specialize(j, metaschema_loader) + + self.typemap = {} # type: Dict[str, Dict[str, str]] + self.uses = {} # type: Dict[str, List[Tuple[str, str]]] + self.record_refs = {} # type: Dict[str, List[str]] + for entry in alltypes: + self.typemap[entry["name"]] = entry + try: + if entry["type"] == "record": + self.record_refs[entry["name"]] = [] + fields = entry.get( + "fields", [] + ) # type: Union[str, List[Dict[str, str]]] + if isinstance(fields, str): + raise KeyError("record fields must be a list of mappings") + for f in fields: # type: Dict[str, str] + p = has_types(f) + for tp in p: + if tp not in self.uses: + self.uses[tp] = [] + if (entry["name"], f["name"]) not in self.uses[tp]: + _, frg1 = urldefrag(t["name"]) + _, frg2 = urldefrag(f["name"]) + self.uses[tp].append((frg1, frg2)) + if ( + tp not in basicTypes + and tp not in self.record_refs[entry["name"]] + ): + self.record_refs[entry["name"]].append(tp) + except KeyError: + _logger.error("Did not find 'type' in %s", t) + _logger.error("record refs is %s", self.record_refs) + raise + + for entry in alltypes: + if entry["name"] in renderlist or ( + (not renderlist) + and ("extends" not in entry) + and ("docParent" not in entry) + and ("docAfter" not in entry) + ): + self.render_type(entry, 1) + + def typefmt( + self, + tp: Any, + redirects: Dict[str, str], + nbsp: bool = False, + jsonldPredicate: Optional[Dict[str, str]] = None, + ) -> str: + if isinstance(tp, MutableSequence): + if nbsp and len(tp) <= 3: + return " | ".join( + [ + self.typefmt(n, redirects, jsonldPredicate=jsonldPredicate) + for n in tp + ] + ) + return " | ".join( + [ + self.typefmt(n, redirects, jsonldPredicate=jsonldPredicate) + for n in tp + ] + ) + if isinstance(tp, MutableMapping): + if tp["type"] == "https://w3id.org/cwl/salad#array": + ar = "array<{}>".format( + self.typefmt(tp["items"], redirects, nbsp=True) + ) + if jsonldPredicate is not None and "mapSubject" in jsonldPredicate: + if "mapPredicate" in jsonldPredicate: + ar += " | " + if len(ar) > 40: + ar += "<br>" + + ar += ( + "<a href='#map'>map</a><<code>{}</code>" + ", <code>{}</code> | {}>".format( + jsonldPredicate["mapSubject"], + jsonldPredicate["mapPredicate"], + self.typefmt(tp["items"], redirects), + ) + ) + else: + ar += " | " + if len(ar) > 40: + ar += "<br>" + ar += "<a href='#map'>map</a><<code>{}</code>, {}>".format( + jsonldPredicate["mapSubject"], + self.typefmt(tp["items"], redirects), + ) + return ar + if tp["type"] in ( + "https://w3id.org/cwl/salad#record", + "https://w3id.org/cwl/salad#enum", + ): + frg = schema.avro_name(tp["name"]) + if tp["name"] in redirects: + return """<a href="{}">{}</a>""".format(redirects[tp["name"]], frg) + if tp["name"] in self.typemap: + return """<a href="#{}">{}</a>""".format(to_id(frg), frg) + if ( + tp["type"] == "https://w3id.org/cwl/salad#enum" + and len(tp["symbols"]) == 1 + ): + return "constant value <code>{}</code>".format( + schema.avro_name(tp["symbols"][0]) + ) + return frg + if isinstance(tp["type"], MutableMapping): + return self.typefmt(tp["type"], redirects) + else: + if str(tp) in redirects: + return """<a href="{}">{}</a>""".format(redirects[tp], redirects[tp]) + if str(tp) in basicTypes: + return """<a href="{}">{}</a>""".format( + self.primitiveType, schema.avro_name(str(tp)) + ) + frg2 = urldefrag(tp)[1] + if frg2 != "": + tp = frg2 + return """<a href="#{}">{}</a>""".format(to_id(tp), tp) + raise SchemaSaladException("We should not be here!") + + def render_type(self, f: Dict[str, Any], depth: int) -> None: + if f["name"] in self.rendered or f["name"] in self.redirects: + return + self.rendered.add(f["name"]) + + if f.get("abstract"): + return + + if "doc" not in f: + f["doc"] = "" + + f["type"] = copy.deepcopy(f) + f["doc"] = "" + f = f["type"] + + if "doc" not in f: + f["doc"] = "" + + def extendsfrom(item: Dict[str, Any], ex: List[Dict[str, Any]]) -> None: + if "extends" in item: + for e in aslist(item["extends"]): + ex.insert(0, self.typemap[e]) + extendsfrom(self.typemap[e], ex) + + ex = [f] + extendsfrom(f, ex) + + enumDesc = {} + if f["type"] == "enum" and isinstance(f["doc"], MutableSequence): + for e in ex: + for i in e["doc"]: + idx = i.find(":") + if idx > -1: + enumDesc[i[:idx]] = i[idx + 1 :] + e["doc"] = [ + i + for i in e["doc"] + if i.find(":") == -1 or i.find(" ") < i.find(":") + ] + + f["doc"] = fix_doc(f["doc"]) + + if f["type"] == "record": + for field in f.get("fields", []): + if "doc" not in field: + field["doc"] = "" + + if f["type"] != "documentation": + lines = [] + for line in f["doc"].splitlines(): + if len(line) > 0 and line[0] == "#": + line = ("#" * depth) + line + lines.append(line) + f["doc"] = "\n".join(lines) + + frg = urldefrag(f["name"])[1] + num = self.toc.add_entry(depth, frg) + doc = "{} {} {}\n".format(("#" * depth), num, frg) + else: + doc = "" + + if self.title is None and f["doc"]: + title = f["doc"][0 : f["doc"].index("\n")] + if title.startswith("# "): + self.title = title[2:] + else: + self.title = title + + if f["type"] == "documentation": + f["doc"] = number_headings(self.toc, f["doc"]) + + doc = doc + "\n\n" + f["doc"] + + doc = mistune.markdown(doc, renderer=MyRenderer()) + + if f["type"] == "record": + doc += "<h3>Fields</h3>" + doc += """ +<div class="responsive-table"> +<div class="row responsive-table-header"> +<div class="col-xs-3 col-lg-2">field</div> +<div class="col-xs-2 col-lg-1">required</div> +<div class="col-xs-7 col-lg-3">type</div> +<div class="col-xs-12 col-lg-6 description-header">description</div> +</div>""" + required = [] + optional = [] + for i in f.get("fields", []): + tp = i["type"] + if ( + isinstance(tp, MutableSequence) + and tp[0] == "https://w3id.org/cwl/salad#null" + ): + opt = False + tp = tp[1:] + else: + opt = True + + desc = i["doc"] + + rfrg = schema.avro_name(i["name"]) + tr = """ +<div class="row responsive-table-row"> +<div class="col-xs-3 col-lg-2"><code>{}</code></div> +<div class="col-xs-2 col-lg-1">{}</div> +<div class="col-xs-7 col-lg-3">{}</div> +<div class="col-xs-12 col-lg-6 description-col">{}</div> +</div>""".format( + rfrg, + "required" if opt else "optional", + self.typefmt( + tp, self.redirects, jsonldPredicate=i.get("jsonldPredicate") + ), + mistune.markdown(desc), + ) + if opt: + required.append(tr) + else: + optional.append(tr) + for i in required + optional: + doc += i + doc += """</div>""" + elif f["type"] == "enum": + doc += "<h3>Symbols</h3>" + doc += """<table class="table table-striped">""" + doc += "<tr><th>symbol</th><th>description</th></tr>" + for e in ex: + for i in e.get("symbols", []): + doc += "<tr>" + efrg = schema.avro_name(i) + doc += "<td><code>{}</code></td><td>{}</td>".format( + efrg, enumDesc.get(efrg, "") + ) + doc += "</tr>" + doc += """</table>""" + f["doc"] = doc + + self.typedoc.write(f["doc"]) + + subs = self.docParent.get(f["name"], []) + self.record_refs.get(f["name"], []) + if len(subs) == 1: + self.render_type(self.typemap[subs[0]], depth) + else: + for s in subs: + self.render_type(self.typemap[s], depth + 1) + + for s in self.docAfter.get(f["name"], []): + self.render_type(self.typemap[s], depth) + + +def avrold_doc( + j: List[Dict[str, Any]], + outdoc: Union[IO[Any], StreamWriter], + renderlist: str, + redirects: Dict[str, str], + brand: str, + brandlink: str, + primtype: str, + brandstyle: Optional[str] = None, + brandinverse: Optional[bool] = False, +) -> None: + toc = ToC() + toc.start_numbering = False + + rt = RenderType(toc, j, renderlist, redirects, primtype) + content = rt.typedoc.getvalue() + + if brandstyle is None: + bootstrap_url = ( + "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css" + ) + bootstrap_integrity = ( + "sha384-604wwakM23pEysLJAhja8Lm42IIwYrJ0dEAqzFsj9pJ/P5buiujjywArgPCi8eoz" + ) + brandstyle_template = ( + '<link rel="stylesheet" href={} integrity={} crossorigin="anonymous">' + ) + brandstyle = brandstyle_template.format(bootstrap_url, bootstrap_integrity) + + picturefill_url = ( + "https://cdn.rawgit.com/scottjehl/picturefill/3.0.2/dist/picturefill.min.js" + ) + picturefill_integrity = ( + "sha384-ZJsVW8YHHxQHJ+SJDncpN90d0EfAhPP+yA94n+EhSRzhcxfo84yMnNk+v37RGlWR" + ) + outdoc.write( + """ + <!DOCTYPE html> + <html> + <head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + {} + <script> + // Picture element HTML5 shiv + document.createElement( "picture" ); + </script> + <script src="{}" + integrity="{}" + crossorigin="anonymous" async></script> + """.format( + brandstyle, picturefill_url, picturefill_integrity + ) + ) + + outdoc.write(f"<title>{rt.title}</title>") + + outdoc.write( + """ + <style> + :target { + padding-top: 61px; + margin-top: -61px; + } + body { + padding-top: 61px; + } + .tocnav ol { + list-style: none + } + pre { + margin-left: 2em; + margin-right: 2em; + } + .section a { + visibility: hidden; + } + .section:hover a { + visibility: visible; + color: rgb(201, 201, 201); + } + .responsive-table-header { + text-align: left; + padding: 8px; + vertical-align: top; + font-weight: bold; + border-top-color: rgb(221, 221, 221); + border-top-style: solid; + border-top-width: 1px; + background-color: #f9f9f9 + } + .responsive-table > .responsive-table-row { + text-align: left; + padding: 8px; + vertical-align: top; + border-top-color: rgb(221, 221, 221); + border-top-style: solid; + border-top-width: 1px; + } + @media (min-width: 0px), print { + .description-header { + display: none; + } + .description-col { + margin-top: 1em; + margin-left: 1.5em; + } + } + @media (min-width: 1170px) { + .description-header { + display: inline; + } + .description-col { + margin-top: 0px; + margin-left: 0px; + } + } + .responsive-table-row:nth-of-type(odd) { + background-color: #f9f9f9 + } + </style> + </head> + <body> + """ + ) + + navbar_extraclass = "navbar-inverse" if brandinverse else "" + outdoc.write( + """ + <nav class="navbar navbar-default navbar-fixed-top {}"> + <div class="container"> + <div class="navbar-header"> + <a class="navbar-brand" href="{}">{}</a> + """.format( + navbar_extraclass, brandlink, brand + ) + ) + + if "<!--ToC-->" in content: + content = content.replace("<!--ToC-->", toc.contents("toc")) + outdoc.write( + """ + <ul class="nav navbar-nav"> + <li><a href="#toc">Table of contents</a></li> + </ul> + """ + ) + + outdoc.write( + """ + </div> + </div> + </nav> + """ + ) + + outdoc.write( + """ + <div class="container"> + """ + ) + + outdoc.write( + """ + <div class="row"> + """ + ) + + outdoc.write( + """ + <div class="col-md-12" role="main" id="main">""" + ) + + outdoc.write(content) + + outdoc.write("""</div>""") + + outdoc.write( + """ + </div> + </div> + </body> + </html>""" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("schema") + parser.add_argument("--only", action="append") + parser.add_argument("--redirect", action="append") + parser.add_argument("--brand") + parser.add_argument("--brandlink") + parser.add_argument("--brandstyle") + parser.add_argument("--brandinverse", default=False, action="store_true") + parser.add_argument("--primtype", default="#PrimitiveType") + + args = parser.parse_args() + + makedoc(args) + + +def makedoc(args: argparse.Namespace) -> None: + + s = [] # type: List[Dict[str, Any]] + a = args.schema + with open(a, encoding="utf-8") as f: + if a.endswith("md"): + s.append( + { + "name": os.path.splitext(os.path.basename(a))[0], + "type": "documentation", + "doc": f.read(), + } + ) + else: + uri = "file://" + os.path.abspath(a) + metaschema_loader = schema.get_metaschema()[2] + j = metaschema_loader.resolve_ref(uri, "")[0] + if isinstance(j, MutableSequence): + s.extend(j) + elif isinstance(j, MutableMapping): + s.append(j) + else: + raise ValidationException("Schema must resolve to a list or a dict") + redirect = {} + for r in args.redirect or []: + redirect[r.split("=")[0]] = r.split("=")[1] + renderlist = args.only if args.only else [] + stdout = ( + TextIOWrapper(sys.stdout.buffer, encoding="utf-8") + if sys.stdout.encoding != "UTF-8" + else cast(TextIOWrapper, sys.stdout) + ) # type: Union[TextIOWrapper, StreamWriter] + avrold_doc( + s, + stdout, + renderlist, + redirect, + args.brand, + args.brandlink, + args.primtype, + brandstyle=args.brandstyle, + brandinverse=args.brandinverse, + ) + + +if __name__ == "__main__": + main()