#!/usr/bin/python3 # # Copyright (c) 2015-2019 The Khronos Group Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # checkLinks.py - validate link/reference API constructs in files # # Usage: checkLinks.py [options] files > logfile # # Options: # -follow attempt to follow include:: directives. This script isn't # an # Asciidoctor processor, so only literal relative paths can # be followed. # -info print some internal diagnostics. # -paramcheck attempt to validate param: names against the surrounding # context (the current structure/function being validated, for example). # This generates many false positives, so is not enabled by default. # -fatal unvalidatable links cause immediate error exit from the script. # Otherwise, errors are accumulated and summarized at the end. # # Depends on vkapi.py, which is a Python representation of relevant parts # of the Vulkan API. Only works when vkapi.py is generated for the full # API, e.g. 'makeAllExts checklinks'; otherwise many false-flagged errors # will occur. import copy, os, pdb, re, string, sys from vkapi import * global curFile, curLine, sectionDepth global errCount, warnCount, emittedPrefix, printInfo curFile = '???' curLine = -1 sectionDepth = 0 emittedPrefix = {} printInfo = False # Called before printing a warning or error. Only prints once prior # to output for a given file. def emitPrefix(): global curFile, curLine, emittedPrefix if (curFile not in emittedPrefix.keys()): emittedPrefix[curFile] = None print('Checking file:', curFile) print('-------------------------------') def info(*args, **kwargs): global curFile, curLine, printInfo if (printInfo): emitPrefix() print('INFO: %s line %d:' % (curFile, curLine), ' '.join([str(arg) for arg in args])) # Print a validation warning found in a file def warning(*args, **kwargs): global curFile, curLine, warnCount warnCount = warnCount + 1 emitPrefix() print('WARNING: %s line %d:' % (curFile, curLine), ' '.join([str(arg) for arg in args])) # Print a validation error found in a file def error(*args, **kwargs): global curFile, curLine, errCount errCount = errCount + 1 emitPrefix() print('ERROR: %s line %d:' % (curFile, curLine), ' '.join([str(arg) for arg in args])) # See if a tag value exists in the specified dictionary and # suggest it as an alternative if so. def checkTag(tag, value, dict, dictName, tagName): if (value in dict.keys()): warning(value, 'exists in the API but not as a', tag + ': .', 'Try using the', tagName + ': tag.') # Report an error due to an asciidoc tag which doesn't match # a corresponding API entity. def foundError(errType, tag, value, fatal): global curFile, curLine error('no such', errType, tag + ':' + value) # Try some heuristics to detect likely problems such as missing vk # prefixes or the wrong tag. # Look in all the dictionaries in vkapi.py to see if the tag # is just wrong but the API entity actually exists. checkTag(tag, value, flags, 'flags', 'tlink/tname') checkTag(tag, value, enums, 'enums', 'elink') checkTag(tag, value, structs, 'structs', 'slink/sname') checkTag(tag, value, handles, 'handles', 'slink/sname') checkTag(tag, value, defines, 'defines', 'slink/sname') checkTag(tag, value, consts, 'consts', 'ename') checkTag(tag, value, protos, 'protos', 'flink/fname') checkTag(tag, value, funcpointers, 'funcpointers', 'tlink/tname') # Look for missing vk prefixes (quirky since it's case-dependent) # NOT DONE YET if fatal: print('ERROR: %s line %d:' % (curFile, curLine), ' '.join(['no such', errType, tag + ':' + value]), file=sys.stderr) sys.exit(1) # Look for param in the list of all parameters of the specified functions # Returns True if found, False otherwise def findParam(param, funclist): for f in funclist: if (param in protos[f]): info('parameter:', param, 'found in function:', f) return True return False # Initialize tracking state for checking links/includes def initChecks(): global curFile, curLine, curFuncs, curStruct, accumFunc, sectionDepth global errCount, warnCount global incPat, linkPat, pathPat, sectionPat # Matches asciidoc single-line section tags sectionPat = re.compile('^(=+) ') # Matches any asciidoc include:: directive pathPat = re.compile('^include::([\w./_]+)\[\]') # Matches asciidoc include:: directives used in spec/ref pages (and also # others such as validity). This is specific to the layout of the api/ # includes and allows any path precding 'api/' followed by the category # (protos, structs, enums, etc.) followed by the name of the proto, # struct, etc. file. incPat = re.compile('^.*api/(\w+)/(\w+)\.txt') # Lists of current /protos/ (functions) and /structs/ includes. There # can be several protos contiguously for different forms of a command curFuncs = [] curStruct = None # Tag if we should accumulate funcs or start a new list. Any intervening # pname: tags or struct includes will restart the list. accumFunc = False # Matches all link names in the current spec/man pages. Assumes these # macro names are not trailing subsets of other macros. Used to # precede the regexp with [^A-Za-z], but this didn't catch macros # at start of line. linkPat = re.compile('([efpst](name|link)):(\w*)') # Total error/warning counters errCount = 0 warnCount = 0 # Validate asciidoc internal links in specified file. # infile - filename to validate # follow - if True, recursively follow include:: directives # paramCheck - if True, try to verify pname: refers to valid # parameter/member names. This generates many false flags currently # included - if True, function was called recursively # fatalExit - if True, validation errors cause an error exit immediately # Links checked are: # fname:vkBlah - Vulkan command name (generates internal link) # flink:vkBlah - Vulkan command name # sname:VkBlah - Vulkan struct name (generates internal link) # slink:VkBlah - Vulkan struct name # elink:VkEnumName - Vulkan enumeration ('enum') type name (generates internal link) # ename:VK_BLAH - Vulkan enumerant token name # pname:name - parameter name to a command or a struct member # tlink:name - Other Vulkan type name (generates internal link) # tname:name - Other Vulkan type name def checkLinks(infile, follow = False, paramCheck = True, included = False, fatalExit = False): global curFile, curLine, curFuncs, curStruct, accumFunc, sectionDepth global errCount, warnCount global incPat, linkPat, pathPat, sectionPat # Global state which gets saved and restored by this function oldCurFile = curFile oldCurLine = curLine curFile = infile curLine = 0 # N.b. dirname() returns an empty string for a path with no directories, # unlike the shell dirname(1). if (not os.path.exists(curFile)): error('No such file', curFile, '- skipping check') # Restore global state before exiting the function curFile = oldCurFile curLine = oldCurLine return inPath = os.path.dirname(curFile) fp = open(curFile, 'r', encoding='utf-8') for line in fp: curLine = curLine + 1 # Track changes up and down section headers, and forget # the current functions/structure when popping up a level match = sectionPat.search(line) if (match): info('Match sectionPat for line:', line) depth = len(match.group(1)) if (depth < sectionDepth): info('Resetting current function/structure for section:', line) curFuncs = [] curStruct = None sectionDepth = depth match = pathPat.search(line) if (match): incpath = match.group(1) info('Match pathPat for line:', line) info(' incpath =', incpath) # An include:: directive. First check if it looks like a # function or struct include file, and modify the corresponding # current function or struct state accordingly. match = incPat.search(incpath) if (match): info('Match incPat for line:', line) # For prototypes, if it is preceded by # another include:: directive with no intervening link: tags, # add to the current function list. Otherwise start a new list. # There is only one current structure. category = match.group(1) tag = match.group(2) # @ Validate tag! # @ Arguably, any intervening text should shift to accumFuncs = False, # e.g. only back-to-back includes separated by blank lines would be # accumulated. if (category == 'protos'): if (tag in protos.keys()): if (accumFunc): curFuncs.append(tag) else: curFuncs = [ tag ] # Restart accumulating functions accumFunc = True info('curFuncs =', curFuncs, 'accumFunc =', accumFunc) else: error('include of nonexistent function', tag) elif (category == 'structs'): if (tag in structs.keys()): curStruct = tag # Any /structs/ include means to stop accumulating /protos/ accumFunc = False info('curStruct =', curStruct) else: error('include of nonexistent struct', tag) if (follow): # Actually process the included file now, recursively newpath = os.path.normpath(os.path.join(inPath, incpath)) info(curFile, ': including file:', newpath) checkLinks(newpath, follow, paramCheck, included = True, fatalExit = fatalExit) matches = linkPat.findall(line) for match in matches: # Start actual validation work. Depending on what the # asciidoc tag name is, look up the value in the corresponding # dictionary. tag = match[0] value = match[2] if (tag == 'fname' or tag == 'flink'): if (value not in protos.keys()): foundError('function', tag, value, False) elif (tag == 'sname' or tag == 'slink'): if (value not in structs.keys() and value not in handles.keys()): foundError('aggregate/scalar/handle/define type', tag, value, False) elif (tag == 'ename'): if (value not in consts.keys() and value not in defines.keys()): foundError('enumerant/constant', tag, value, False) elif (tag == 'elink'): if (value not in enums.keys() and value not in flags.keys()): foundError('enum/bitflag type', tag, value, fatalExit) # tname and tlink are the same except if the errors are treated as fatal # They can be recombined once both are error-clean elif (tag == 'tname'): if (value not in funcpointers.keys() and value not in flags.keys()): foundError('function pointer/other type', tag, value, fatalExit) elif (tag == 'tlink'): if (value not in funcpointers.keys() and value not in flags.keys()): foundError('function pointer/other type', tag, value, False) elif (tag == 'pname'): # Any pname: tag means to stop accumulating /protos/ accumFunc = False # See if this parameter is in the current proto(s) and struct foundParam = False if (curStruct and value in structs[curStruct]): info('parameter', value, 'found in struct', curStruct) elif (curFuncs and findParam(value, curFuncs)): True else: if paramCheck: warning('parameter', value, 'not found. curStruct =', curStruct, 'curFuncs =', curFuncs) else: # This is a logic error error('unknown tag', tag + ':' + value) fp.close() if (errCount > 0 or warnCount > 0): if (not included): print('Errors found:', errCount, 'Warnings found:', warnCount) print('') if (included): info('----- returning from:', infile, 'to parent file', '-----') # Don't generate any output for files without errors # else: # print(curFile + ': No errors found') # Restore global state before exiting the function curFile = oldCurFile curLine = oldCurLine if __name__ == '__main__': follow = False paramCheck = False included = False fatalExit = False totalErrCount = 0 totalWarnCount = 0 if (len(sys.argv) > 1): for file in sys.argv[1:]: if (file == '-follow'): follow = True elif (file == '-info'): printInfo = True elif file == '-paramcheck': paramCheck = True elif (file == '-fatal'): fatalExit = True else: initChecks() checkLinks(file, follow, paramCheck = paramCheck, included = included, fatalExit = fatalExit) totalErrCount = totalErrCount + errCount totalWarnCount = totalWarnCount + warnCount else: print('Need arguments: [-follow] [-info] [-paramcheck] [-fatal] infile [infile...]', file=sys.stderr) if (totalErrCount > 0 or totalWarnCount > 0): if (not included): print('TOTAL Errors found:', totalErrCount, 'Warnings found:', totalWarnCount) if totalErrCount > 0: sys.exit(1)