diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/check-compare-generated.sh | 11 | ||||
| -rwxr-xr-x | tools/check-docs.sh | 24 | ||||
| -rwxr-xr-x | tools/generate-docs-nm-property-infos.py | 519 | ||||
| -rwxr-xr-x | tools/generate-docs-nm-settings-docs-gir.py | 45 | ||||
| -rwxr-xr-x | tools/generate-docs-nm-settings-docs-merge.py | 31 |
5 files changed, 485 insertions, 145 deletions
diff --git a/tools/check-compare-generated.sh b/tools/check-compare-generated.sh index 44b70f95..b66c5a10 100755 --- a/tools/check-compare-generated.sh +++ b/tools/check-compare-generated.sh @@ -2,17 +2,16 @@ set -e -srcdir="$1" -builddir="$2" -doc_h="$3" +f_commited="$1" +f_generated="$2" [ -n "$NMTST_NO_CHECK_SETTINGS_DOCS" ] && exit 0 -cmp -s "${srcdir}/${doc_h}.in" "${builddir}/${doc_h}" && exit 0 +cmp -s "$f_commited" "$f_generated" && exit 0 if [ "$NM_TEST_REGENERATE" = 1 ] ; then - cp -f "${builddir}/${doc_h}" "${srcdir}/${doc_h}.in" + cp -f "$f_generated" "$f_commited" else - echo "*** Error: the generated file '${builddir}/${doc_h}' differs from the source file '${srcdir}/${doc_h}.in'. You probably should copy the generated file over to the source file. You can skip this test by setting NMTST_NO_CHECK_SETTINGS_DOCS=yes. You can also automatically copy the file by rerunning the test with NM_TEST_REGENERATE=1" + echo "*** Error: the generated file '$f_generated' differs from the source file '$f_commited'. You probably should copy the generated file over to the source file. You can skip this test by setting NMTST_NO_CHECK_SETTINGS_DOCS=yes. You can also automatically copy the file by rerunning the test with NM_TEST_REGENERATE=1" exit 1 fi diff --git a/tools/check-docs.sh b/tools/check-docs.sh index c2e6761d..2746848c 100755 --- a/tools/check-docs.sh +++ b/tools/check-docs.sh @@ -69,12 +69,36 @@ F2="$(grep -l "$(sed -n 's/^[\t ]*\(.*_get_type\);/\1/p' "$SOURCEDIR/src/libnm-c F2_EXTRA=" annotation-glossary api-index-full +nm-conn-utils nm-dbus-interface nm-errors +nm-ethtool-utils nm-keyfile nm-utils nm-version +nm-version-macros +nm-vpn-dbus-interface " if ! same_lines "$F1"$'\n'"$F1_EXTRA" "$F2"$'\n'"$F2_EXTRA"; then die "*** Error: libnm classes not included in docs/libnm/libnm-docs.xml ***" fi + +F1="$(sed -n 's/^#include "\(nm-.*\).h"$/xml\/\1.xml/p' "$SOURCEDIR/src/libnm-client-public/NetworkManager.h")" +F1_EXTRA=" +xml/annotation-glossary.xml +xml/api-index-full.xml +xml/nm-setting-ovs-external-ids.xml +xml/nm-setting-ovs-other-config.xml +xml/nm-version-macros.xml +xml/nm-secret-agent-old.xml +xml/nm-vpn-plugin-old.xml +" +F2="$(sed -n 's/.*<xi:include href="\(xml\/.*.xml\)".*/\1/p' "$SOURCEDIR/docs/libnm/libnm-docs.xml")" +F2_EXTRA=" +xml/nm-autoptr.xml +xml/nm-core-enum-types.xml +xml/nm-enum-types.xml +" +if ! same_lines "$F1"$'\n'"$F1_EXTRA" "$F2"$'\n'"$F2_EXTRA"; then + die "*** Error: documentation from public headers not included in docs/libnm/libnm-docs.xml ***" +fi diff --git a/tools/generate-docs-nm-property-infos.py b/tools/generate-docs-nm-property-infos.py index 25aa272a..58ad751f 100755 --- a/tools/generate-docs-nm-property-infos.py +++ b/tools/generate-docs-nm-property-infos.py @@ -1,143 +1,436 @@ #!/usr/bin/env python # SPDX-License-Identifier: LGPL-2.1-or-later +import os import re import sys +import collections import xml.etree.ElementTree as ET -def get_setting_name(one_file): - setting_name = "" - assert re.match(r".*/libnm-core-impl/nm-setting-.*\.c$", one_file) - header_path = one_file.replace("libnm-core-impl", "libnm-core-public") - header_path = header_path.replace(".c", ".h") - try: - header_reader = open(header_path, "r") - except OSError: - print("Can not open header file: %s" % (header_path)) - exit(1) +class LineError(Exception): + def __init__(self, line_no, msg): + Exception.__init__(self, msg) + self.line_no = line_no + + +_dbg_level = 0 +try: + _dbg_level = int(os.getenv("NM_DEBUG_GENERATE_DOCS", 0)) +except Exception: + pass + + +def dbg(msg, level=1): + if level <= _dbg_level: + print(msg) + + +def iter_unique(iterable, default=None): + found = False + for i in iterable: + assert not found + found = True + i0 = i + if found: + return i0 + return default + - line = header_reader.readline() - while line != "": - setting_name_found = re.search(r"NM_SETTING_.+SETTING_NAME\s+\"(\S+)\"", line) - if setting_name_found: - setting_name = setting_name_found.group(1) - break - line = header_reader.readline() - header_reader.close() - return setting_name +def xnode_get_or_create(root_node, node_name, name): + # From root_node, get the node "<{node_name} name={name} .../>" + # or create one, if it doesn't exist. + node = iter_unique( + (node for node in root_node.findall(node_name) if node.attrib["name"] == name) + ) + if node is None: + created = True + node = ET.SubElement(root_node, node_name, name=name) + else: + created = False + return node, created + + +def get_setting_names(source_file): + m = re.match(r"^(.*)/libnm-core-impl/(nm-setting-[^/]*)\.c$", source_file) + assert m + + path_prefix, file_base = (m.group(1), m.group(2)) + + if file_base == "nm-setting-ip-config": + # Special case ip-config, which is a base class. + return 0, ("ipv4", "ipv6") + + header_file = "%s/libnm-core-public/%s.h" % (path_prefix, file_base) -def scan_doc_comments(plugin, setting_node, file, start_tag, end_tag): - data = [] - push_flag = 0 try: - file_reader = open(file, "r") + f = open(header_file, "r") except OSError: - print("Can not open file: %s" % (file)) - exit(1) + raise Exception( + 'Can not open header file "%s" for "%s"' % (header_file, source_file) + ) + + with f: + for line in f: + m = re.search(r"^#define +NM_SETTING_.+SETTING_NAME\s+\"(\S+)\"$", line) + if m: + return 1, (m.group(1),) + + raise Exception( + 'Can\'t find setting name in header file "%s" for "%s"' + % (header_file, source_file) + ) - line = file_reader.readline() - while line != "": - if start_tag in line: - push_flag = 1 - elif end_tag in line and push_flag == 1: - push_flag = 0 - parsed_data = process_data(data) - if parsed_data: - write_data(setting_node, parsed_data) - data = [] - elif push_flag == 1: - data.append(line) - line = file_reader.readline() - file_reader.close() - return - - -keywords = [ - "property", - "variable", - "format", - "values", - "default", - "example", - "description", - "description-docbook", -] -kwd_first_line_re = re.compile( - r"^\s*\**\s+({}):\s+(.*?)\s*$".format("|".join(keywords)) + +def get_file_infos(source_files): + # This function parses the source files and detects the + # used setting name. The returned sections are sorted by setting + # name. + # + # The file "nm-setting-ip-config.c" can contain information + # for "ipv4" and "ipv6" settings. Thus, to sort the files + # is a bit more involved. + + # First, get a list of priority and setting-names that belong + # to the source file. Sort by priority,setting-names. It's + # important that "nm-setting-ip-config.c" gets parsed before + # "nm-setting-ip[46]-config.c". + file_infos = [] + for source_file in source_files: + priority, setting_names = get_setting_names(source_file) + file_infos.append((priority, setting_names, source_file)) + file_infos.sort() + + d = {} + for priority, setting_names, source_file in file_infos: + for setting_name in setting_names: + l = d.get(setting_name, None) + if l is None: + l = list() + d[setting_name] = l + l.append(source_file) + for key in sorted(d.keys()): + for f in d[key]: + yield key, f + + +KEYWORD_XML_TYPE_NESTED = "nested" +KEYWORD_XML_TYPE_NODE = "node" +KEYWORD_XML_TYPE_ATTR = "attr" + +keywords = collections.OrderedDict( + [ + ("property", KEYWORD_XML_TYPE_ATTR), + ("variable", KEYWORD_XML_TYPE_ATTR), + ("format", KEYWORD_XML_TYPE_ATTR), + ("values", KEYWORD_XML_TYPE_ATTR), + ("default", KEYWORD_XML_TYPE_ATTR), + ("example", KEYWORD_XML_TYPE_ATTR), + ("description", KEYWORD_XML_TYPE_ATTR), + ("description-docbook", KEYWORD_XML_TYPE_NESTED), + ] ) -kwd_more_line_re = re.compile(r"^\s*\**\s+(.*?)\s*$") -def process_data(data): +def keywords_allowed(tag, keyword): + # certain keywords might not be valid for some tags. + # Currently, all of them are always valid. + assert keyword in keywords + return True + + +def write_data(tag, setting_node, line_no, parsed_data): + + for k in parsed_data.keys(): + assert keywords_allowed(tag, k) + assert k in keywords + + name = parsed_data["property"] + property_node, created = xnode_get_or_create(setting_node, "property", name) + if not created: + raise LineError(line_no, 'Duplicate property <property name="%s"...' % (name,)) + + for k, xmltype in keywords.items(): + if k == "property": + continue + + v = parsed_data.get(k, None) + if v is None: + continue + + if xmltype == KEYWORD_XML_TYPE_NESTED: + # Set as XML nodes. The input data is XML itself. + des = ET.fromstring("<%s>%s</%s>" % (k, v, k)) + property_node.append(des) + elif xmltype == KEYWORD_XML_TYPE_NODE: + node = ET.SubElement(property_node, k) + node.text = v + elif xmltype == KEYWORD_XML_TYPE_ATTR: + property_node.set(k, v) + else: + assert False + + +kwd_first_line_re = re.compile(r"^ *\* ([-a-z0-9]+): (.*)$") +kwd_more_line_re = re.compile(r"^ *\*( *)(.*?)\s*$") + + +def parse_data(tag, line_no, lines): + assert lines parsed_data = {} - if not data: - return parsed_data keyword = "" - for line in data: - kwd_first_line_found = kwd_first_line_re.search(line) - if kwd_first_line_found: - keyword = kwd_first_line_found.group(1) - if keyword == "description-docbook": - value = kwd_first_line_found.group(2) + "\n" + indent = None + for line in lines: + assert "\n" not in line + line_no += 1 + m = re.search(r"^ \*(| .*)$", line) + if not m: + raise LineError(line_no, 'Invalid formatted line "%s"' % (line,)) + content = m.group(1) + + m = re.search("^ ([-a-z0-9]+):(.*)$", content) + text_keyword_started = None + if m: + keyword = m.group(1) + if keyword in parsed_data: + raise LineError(line_no, 'Duplicated keyword "%s"' % (keyword,)) + text = m.group(2) + text_keyword_started = text + if text: + if text[0] != " " or len(text) == 1: + raise LineError(line_no, 'Invalid formatted line "%s"' % (line,)) + text = text[1:] + if not keywords_allowed(tag, keyword): + raise LineError(line_no, 'Invalid key "%s" for %s' % (keyword, tag)) + if parsed_data and keyword == "property": + raise LineError(line_no, 'The "property:" keywork must be first') + parsed_data[keyword] = text + indent = None + else: + if content == "": + text = "" + elif content[0] == " " and len(content) > 1: + text = content[1:] + assert text + if indent is None: + indent = re.search("^( *)", text).group(1) + if not text.startswith(indent): + raise LineError(line_no, 'Unexpected indention in "%s"' % (line,)) + text = text[len(indent) :] else: - value = kwd_first_line_found.group(2) + " " - parsed_data[keyword] = value - continue - kwd_more_line_found = kwd_more_line_re.search(line) - if kwd_more_line_found: + raise LineError(line_no, 'Unexpected line "%s"' % (line,)) if not keyword: - print("Extra mess in a comment: %s" % (line)) - exit(1) - if keyword == "description-docbook": - value = kwd_more_line_found.group(1) + "\n" + raise LineError(line_no, "Expected data in comment: %s" % (line)) + if text and text[0] == "\\": + assert False + text = text[1:] + if separator == " " and text == "": + # No separator to add. This is a blank line + pass else: - value = kwd_more_line_found.group(1) + " " - parsed_data[keyword] += value - for keyword in keywords: - if keyword == "variable" and keyword not in parsed_data: - parsed_data[keyword] = parsed_data["property"] - elif keyword not in parsed_data: - parsed_data[keyword] = "" - for key in parsed_data.keys(): - parsed_data[key] = parsed_data[key].rstrip() + parsed_data[keyword] = parsed_data[keyword] + separator + text + + if keywords[keyword] == KEYWORD_XML_TYPE_NESTED: + # This is plain XML. They lines are joined by newlines. + separator = "\n" + elif text_keyword_started == "": + # If the previous line was just "tag:$", we don't need a separator + # the next time. + separator = "" + elif not text: + # A blank line is used to mark a line break, while otherwise + # lines are joined by space. + separator = "\n" + else: + separator = " " + if "property" not in parsed_data: + raise LineError(line_no, 'Missing "property:" tag') + for keyword in keywords.keys(): + if not keywords_allowed(tag, keyword): + continue + if keyword not in parsed_data: + parsed_data[keyword] = None return parsed_data -def write_data(setting_node, parsed_data): - property_node = ET.SubElement(setting_node, "property") - property_node.set("name", parsed_data["property"]) - property_node.set("variable", parsed_data["variable"]) - property_node.set("format", parsed_data["format"]) - property_node.set("values", parsed_data["values"]) - property_node.set("default", parsed_data["default"]) - property_node.set("example", parsed_data["example"]) - property_node.set("description", parsed_data["description"]) - if parsed_data["description-docbook"]: - des = ET.fromstring( - "<description-docbook>" - + parsed_data["description-docbook"] - + "</description-docbook>" - ) - property_node.append(des) +def process_setting(tag, root_node, source_file, setting_name): + + dbg( + "> > tag:%s, source_file:%s, setting_name:%s" % (tag, source_file, setting_name) + ) + + start_tag = "---" + tag + "---" + end_tag = "---end---" + + setting_node, created = xnode_get_or_create(root_node, "setting", setting_name) + + try: + f = open(source_file, "r") + except OSError: + raise Exception("Can not open file: %s" % (source_file)) + + lines = None + with f: + line_no = 0 + just_had_end_tag = False + line_no_start = None + for line in f: + line_no += 1 + if line and line[-1] == "\n": + line = line[:-1] + if just_had_end_tag: + # After the end-tag, we still expect one particular line. Be strict about + # this. + just_had_end_tag = False + if line != " */": + raise LineError( + line_no, + 'Invalid end tag "%s". Expects literally " */" after end-tag' + % (line,), + ) + elif start_tag in line: + if line != " /* " + start_tag: + raise LineError( + line_no, + 'Invalid start tag "%s". Expects literally " /* %s"' + % (line, start_tag), + ) + if lines is not None: + raise LineError( + line_no, 'Invalid start tag "%s", missing end-tag' % (line,) + ) + lines = [] + line_no_start = line_no + elif end_tag in line and lines is not None: + if line != " * " + end_tag: + raise LineError(line_no, 'Invalid end tag: "%s"' % (line,)) + parsed_data = parse_data(tag, line_no_start, lines) + if not parsed_data: + raise Exception('invalid data: line %s, "%s"' % (line_no, lines)) + dbg("> > > property: %s" % (parsed_data["property"],)) + if _dbg_level > 1: + for keyword in sorted(parsed_data.keys()): + v = parsed_data[keyword] + if v is not None: + v = '"%s"' % (v,) + dbg( + "> > > > [%s] (%s) = %s" % (keyword, keywords[keyword], v), + level=2, + ) + write_data(tag, setting_node, line_no_start, parsed_data) + lines = None + elif lines is not None: + lines.append(line) + if lines is not None or just_had_end_tag: + raise LineError(line_no_start, "Unterminated start tag") + + +def process_settings_docs(tag, output, source_files): + + dbg("> tag:%s, output:%s" % (tag, output)) + + root_node = ET.Element("nm-setting-docs") + + for setting_name, source_file in get_file_infos(source_files): + try: + process_setting(tag, root_node, source_file, setting_name) + except LineError as e: + raise Exception( + "Error parsing %s, line %s (tag:%s, setting_name:%s): %s" + % (source_file, e.line_no, tag, setting_name, str(e)) + ) + except Exception as e: + raise Exception( + "Error parsing %s (tag:%s, setting_name:%s): %s" + % (source_file, tag, setting_name, str(e)) + ) + + ET.ElementTree(root_node).write(output) + + +def main(): + if len(sys.argv) < 4: + print("Usage: %s [tag] [output-xml-file] [srcfiles...]" % (sys.argv[0])) + exit(1) + + process_settings_docs( + tag=sys.argv[1], output=sys.argv[2], source_files=sys.argv[3:] + ) + + +if __name__ == "__main__": + main() + + +############################################################################### +# Tests +############################################################################### + + +def setup_module(): + global pytest + import pytest + + +def t_srcdir(): + return os.path.abspath(os.path.dirname(__file__) + "/..") + + +def t_setting_c(name): + return t_srcdir() + "/src/libnm-core-impl/nm-setting-" + name + ".c" + + +def test_file_location(): + assert t_srcdir() + "/tools/generate-docs-nm-property-infos.py" == os.path.abspath( + __file__ + ) + assert os.path.isfile(t_srcdir() + "/src/libnm-core-impl/nm-setting-connection.c") + + assert os.path.isfile(t_setting_c("ip-config")) + + +def test_get_setting_names(): + assert (1, ("connection",)) == get_setting_names( + t_srcdir() + "/src/libnm-core-impl/nm-setting-connection.c" + ) + assert (1, ("ipv4",)) == get_setting_names( + t_srcdir() + "/src/libnm-core-impl/nm-setting-ip4-config.c" + ) + assert (0, ("ipv4", "ipv6")) == get_setting_names( + t_srcdir() + "/src/libnm-core-impl/nm-setting-ip-config.c" + ) + +def test_get_file_infos(): -if len(sys.argv) < 4: - print("Usage: %s [plugin] [output-xml-file] [srcfiles]" % (sys.argv[0])) - exit(1) + t = ["connection", "ip-config", "ip4-config", "proxy", "wired"] -argv = list(sys.argv[1:]) -plugin, output, source_files = argv[0], argv[1], argv[2:] -start_tag = "---" + plugin + "---" -end_tag = "---end---" -root_node = ET.Element("nm-setting-docs") + assert [ + ( + "802-3-ethernet", + t_setting_c("wired"), + ), + ( + "connection", + t_setting_c("connection"), + ), + ( + "ipv4", + t_setting_c("ip-config"), + ), + ( + "ipv4", + t_setting_c("ip4-config"), + ), + ( + "ipv6", + t_setting_c("ip-config"), + ), + ("proxy", t_setting_c("proxy")), + ] == list(get_file_infos([t_setting_c(x) for x in t])) -for one_file in source_files: - setting_name = get_setting_name(one_file) - if setting_name: - setting_node = ET.SubElement(root_node, "setting", name=setting_name) - setting_node.text = "\n" - scan_doc_comments(plugin, setting_node, one_file, start_tag, end_tag) -ET.ElementTree(root_node).write(output) +def test_process_setting(): + root_node = ET.Element("nm-setting-docs") + process_setting("nmcli", root_node, t_setting_c("connection"), "connection") diff --git a/tools/generate-docs-nm-settings-docs-gir.py b/tools/generate-docs-nm-settings-docs-gir.py index f7a20e2b..88b87a72 100755 --- a/tools/generate-docs-nm-settings-docs-gir.py +++ b/tools/generate-docs-nm-settings-docs-gir.py @@ -120,14 +120,8 @@ def remove_prefix(line, prefix): return line[len(prefix) :] if line.startswith(prefix) else line -def get_docs(propxml): - doc_xml = propxml.find("gi:doc", ns_map) - if doc_xml is None: - return None - +def format_docs(doc_xml): doc = doc_xml.text - if "deprecated" in propxml.attrib: - doc = doc + " Deprecated: " + propxml.attrib["deprecated"] # split docs into lines lines = re.split("\n", doc) @@ -136,11 +130,11 @@ def get_docs(propxml): doc = "" for l in lines: if l: - doc += " " + l + doc += l + " " else: - doc += "\n\n" + doc = doc.strip(" ") + "\n\n" - doc = doc.strip("\n") + doc = doc.strip("\n ") # Expand constants doc = re.sub(r"%([^%]\w*)", lambda match: constants[match.group(1)], doc) @@ -173,6 +167,14 @@ def get_docs(propxml): return doc +def get_docs(propxml): + doc_xml = propxml.find("gi:doc", ns_map) + if doc_xml is None: + return None + else: + return format_docs(doc_xml) + + def get_default_value(setting, pspec, propxml): default_value = setting.get_property(pspec.name.replace("-", "_")) if default_value is None: @@ -274,6 +276,13 @@ def main(gir_path_str, output_path_str): value_desc = get_docs(propxml) default_value = get_default_value(setting, pspec, propxml) + if "deprecated" in propxml.attrib: + deprecated = True + deprecated_since = propxml.attrib["deprecated-version"] + deprecated_desc = format_docs(propxml.find("gi:doc-deprecated", ns_map)) + else: + deprecated = False + prop_upper = prop.upper().replace("-", "_") if value_desc is None: @@ -310,6 +319,22 @@ def main(gir_path_str, output_path_str): create_desc_docbook(description_docbook, value_desc) + if deprecated: + ET.SubElement( + property_element, + "deprecated", + attrib={ + "since": deprecated_since, + }, + ).text = deprecated_desc + + deprecated_docbook = ET.SubElement( + property_element, + "deprecated-docbook", + ) + + create_desc_docbook(deprecated_docbook, deprecated_desc) + docs_gir.write( output_path_str, xml_declaration=True, diff --git a/tools/generate-docs-nm-settings-docs-merge.py b/tools/generate-docs-nm-settings-docs-merge.py index 17c78a24..0131a024 100755 --- a/tools/generate-docs-nm-settings-docs-merge.py +++ b/tools/generate-docs-nm-settings-docs-merge.py @@ -111,11 +111,12 @@ def node_set_attr(dst_node, name, nodes): dst_node.set(name, x) -def find_first_not_none(itr): - for i in itr: - if i is not None: - return i - return None +def find_attr(properties_attrs, name): + for p_attr in properties_attrs: + if p_attr is not None: + p_attr = p_attr.find(name) + if p_attr is not None: + return p_attr ############################################################################### @@ -180,17 +181,10 @@ for setting_name in iter_keys_of_dicts(settings_roots, key_fcn_setting_name): dbg("> > > > property_name: %s" % (property_name)) properties_attrs = list([p.get(property_name) for p in properties]) - description_docbook = find_first_not_none( - p_attr.find("description-docbook") - for p_attr in properties_attrs - if p_attr is not None - ) - - description = find_first_not_none( - p_attr.find("description") - for p_attr in properties_attrs - if p_attr is not None - ) + description_docbook = find_attr(properties_attrs, "description-docbook") + description = find_attr(properties_attrs, "description") + deprecated_docbook = find_attr(properties_attrs, "deprecated-docbook") + deprecated = find_attr(properties_attrs, "deprecated") if gl_only_from_first and properties_attrs[0] is None: dbg("> > > > skip (only-from-first") @@ -221,4 +215,9 @@ for setting_name in iter_keys_of_dicts(settings_roots, key_fcn_setting_name): elif description is not None: property_node.append(description) + if deprecated_docbook is not None: + property_node.insert(0, deprecated_docbook) + if deprecated is not None: + property_node.insert(0, deprecated) + ET.ElementTree(root_node).write(gl_output_xml_file) |