OpenXR-Docs/specification/scripts/extract_code.py

#!/usr/bin/python3
#
# Copyright (c) 2018 Collabora, Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author(s):    Ryan Pavlik <ryan.pavlik@collabora.com>
#
# Purpose:      This script searches for and extracts embedded source code
#               from specification chapters.

import argparse
import errno
import re
from abc import ABC, abstractmethod
from enum import Enum, unique
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent.parent
ALL_DOCS = sorted((ROOT / 'specification/sources/').glob('**/*.adoc'))

CODEDIR = ROOT / 'specification/example-builds'
GENCODEDIR = CODEDIR / 'generated'
# if it already exists, that's OK
try:
    GENCODEDIR.mkdir(parents=True)
except OSError as e:
    if e.errno != errno.EEXIST:
        raise


class LinewiseFileProcessor(ABC):
    def __init__(self):
        self._lines = []
        self._lineNum = 0
        self._line = ''
        self._filename = Path()

    @property
    def filename(self):
        """The Path object of the currently processed file"""
        return self._filename

    @property
    def relativeFilename(self):
        """The current file's Path relative to the current working directory"""
        return self.filename.relative_to(Path('.').resolve())

    @property
    def line(self):
        """The current line, including any trailing whitespace and the line ending."""
        return self._line

    @property
    def lineNumber(self):
        """Get 1-indexed line number."""
        return self._lineNum

    @property
    def rstrippedLine(self):
        """The current line without any trailing whitespace."""
        return self.line.rstrip()

    @property
    def trailingWhitespace(self):
        """The trailing whitespace of the current line that gets removed when accessing rstrippedLine"""
        nonWhitespaceLength = len(self.rstrippedLine)
        return self.line[nonWhitespaceLength:]

    def precedingLine(self, relativeIndex):
        """Retrieve the line at an line number at the given relative index, if one exists. Returns None if there is no line there."""
        if relativeIndex >= 0:
            raise RuntimeError(
                'relativeIndex must be negative, to retrieve a preceding line.')
        if relativeIndex + self.lineNumber <= 0:
            # There is no line at this index
            return None
        return self._lines[self.lineNumber + relativeIndex - 1]

    def getPrecedingLines(self, num):
        """Get *up to* the preceding num lines. Fewer may be returned if the requested number aren't available."""
        return self._lines[- (num + 1):-1]

    @abstractmethod
    def processLine(self, lineNum, line):
        """Implement in your subclass to handle each new line."""
        raise NotImplementedError

    def _internalProcessFile(self, fileHandle):

        for lineNum, line in enumerate(fileHandle, 1):
            self._lineNum = lineNum
            self._line = line
            self._lines.append(line)
            self.processLine(lineNum, line)

    def processFile(self, filename, fileHandle=None):
        """Main entry point - call with a filename and optionally the file handle to read from."""
        if isinstance(filename, str):
            filename = Path(filename).resolve()

        self._filename = filename

        if fileHandle:
            self._internalProcessFile(fileHandle)
        else:
            with self._filename.open('r', encoding='utf-8') as f:
                self._internalProcessFile(f)


@unique
class Language(Enum):
    C = 'C'
    CPP = 'C++'
    XML = 'XML'
    ASCIIDOC = 'asciidoc'
    JSON = 'JSON'
    SH = 'sh'

    UNKNOWN = 'UNKNOWN'

    def __str__(self):
        return self.value

    @property
    def extension(self):
        if self == Language.UNKNOWN:
            raise RuntimeError(
                "Can't get extension for UNKNOWN language")
        if self == Language.ASCIIDOC:
            return 'adoc'
        return str(self).lower().replace('+', 'p')

    @classmethod
    def from_string(self, s):
        s = s.upper()
        for val in Language:
            if s == str(val).upper():
                return val
        return Language.UNKNOWN


class CodeExtractor(LinewiseFileProcessor):
    def __init__(self, outputLineNumbers=False, quiet=False):
        super().__init__()
        self.MIN_LINES = 5

        self.outputLineNumbers = outputLineNumbers
        self.quiet = quiet

        self.nextSnippetId = 0
        self.inCodeBlock = False
        self.blockPattern = re.compile(r'\[source(,?)(?P<tags>.*)\]')
        self.languagesToExtract = set((Language.CPP, Language.C))
        self.codeLines = None

        self.generatedFiles = []

        # list of (generated file path, include path) pairs
        self.deps = []

        # key: generated file path
        # value: line number where a code snippet starts
        self.origins = {}

    def getUniqueId(self):
        ret = self.nextSnippetId
        self.nextSnippetId += 1
        return ret

    def makeNumberedFilename(self, language):
        name = self.filename.with_suffix('.{num}.{ext}'.format(
            num=self.getUniqueId(), ext=language.extension)).name
        return GENCODEDIR / name

    def printMessage(self, s):
        if not self.quiet:
            print('{}:{}: {}'.format(self.filename, self.lineNumber, s))

    def processStartOfCodeBlock(self):
        prevLine = self.precedingLine(-1)
        if not prevLine:
            # No previous line to find language.
            return

        codeBlockTag = self.blockPattern.match(prevLine.rstrip())
        if not codeBlockTag:
            # Not going to handle this.
            return

        tags = set(codeBlockTag.group('tags').upper().split(','))

        self.language = Language.UNKNOWN
        for lang in Language:
            if str(lang).upper() in tags:
                self.language = lang
                break
        if self.language == Language.UNKNOWN:
            self.printMessage('Not extracting code snippet introduced with {} (tags = {})'.format(
                codeBlockTag.group(), tags))
            return

        if self.language not in self.languagesToExtract:
            self.printMessage('Not extracting code snippet identified as {}'.format(
                self.language))
            return
        if 'SUPPRESS-BUILD' in tags:
            self.printMessage(
                'Suppressing extraction of code snippet because we saw "suppress-build"')
            return

        # self.printMessage('Considering code snippet identified as {}'.format(
        #     self.language))
        self.codeLines = []
        self.startOfCodeBlock = self.lineNumber

    def processEndOfCodeBlock(self):
        if self.codeLines is not None:
            codeLines = self.codeLines
            self.codeLines = None

            if len(codeLines) < self.MIN_LINES:
                self.printMessage(
                    'Not extracting code snippet - only {} lines.'.format(len(codeLines)))
                return

            outFilename = self.makeNumberedFilename(self.language)
            self.printMessage('Writing {} extracted lines to file {}\n'.format(
                len(codeLines), outFilename.relative_to(Path('.').resolve())))
            self.generatedFiles.append(outFilename)

            self.origins[outFilename] = self.startOfCodeBlock

            includeFile = CODEDIR / outFilename.with_suffix('.h').name

            with outFilename.open('w', encoding='utf-8') as f:
                f.write('#include "common_include.h"\n')
                if includeFile.exists():
                    f.write('#include "{}"\n\n'.format(includeFile.name))
                    self.deps.append((outFilename, includeFile))
                f.write('void func() {\n')
                f.write(''.join(codeLines))
                f.write('\n}\n')

    def processCodeBlockLine(self):
        if self.codeLines is not None:
            if self.outputLineNumbers:
                self.codeLines.append('# {} "{}"\n'.format(
                    self.lineNumber, self.filename))
            self.codeLines.append(self.line)

    def processLine(self, lineNum, line):
        if line.startswith('---'):
            # Toggle code block status.
            self.inCodeBlock = not self.inCodeBlock

            if self.inCodeBlock:
                # We just started a code block
                self.processStartOfCodeBlock()
            else:
                # We just ended one.
                self.processEndOfCodeBlock()

        elif self.inCodeBlock:
            self.processCodeBlockLine()


class CodeExtractorGroup(object):
    def __init__(self, outputLineNumbers=False, quiet=False):
        self.outputLineNumbers = outputLineNumbers
        self.quiet = quiet

        # key: adoc file path. value: list of generated source files.
        self.generatedFiles = {}

        # all generated sources files
        self.allGenerated = []

        # list of (generated file path, include path) pairs
        self.deps = []

        # key: generated file path
        # value: (adoc file path, line number where a code snippet starts) pair
        self.origins = {}

    def process(self, files):
        for fn in files:
            extractor = CodeExtractor(outputLineNumbers=self.outputLineNumbers,
                                      quiet=self.quiet)
            extractor.processFile(fn)

            if extractor.generatedFiles:
                self.generatedFiles[extractor.filename] = extractor.generatedFiles
                self.allGenerated.extend(extractor.generatedFiles)
                self.deps.extend(extractor.deps)
                self.origins.update({fn: (extractor.filename, lineNum)
                                     for fn, lineNum in extractor.origins.items()})

    def outputMakefile(self, makefile):
        with open(makefile, 'w', encoding='utf-8') as f:

            generated_c_string = ' \\\n'.join(str(fn)
                                              for fn in self.allGenerated if fn.suffix == '.c')
            generated_cpp_string = ' \\\n'.join(str(fn)
                                                for fn in self.allGenerated if fn.suffix == '.cpp')
            deps_string = '\n'.join('{}: {} $(CODEDIR)/common_include.h'.format(fn.with_suffix('.o'), dep)
                                    for fn, dep in self.deps)
            extra_arg = ''
            if self.outputLineNumbers:
                extra_arg = '--linenumbers'
            f.write("""
OUTDIR  ?= $(CURDIR)/{out}
CODEDIR ?= $(CURDIR)/{codedir}
PYTHON   ?= python3
QUIET    ?= @

GENERATED_C := {c}
C_OBJECTS := $(patsubst %.c,%.o,$(GENERATED_C))

GENERATED_CPP := {cpp}
CPP_OBJECTS := $(patsubst %.cpp,%.o,$(GENERATED_CPP))

build-examples: $(C_OBJECTS) $(CPP_OBJECTS)
.PHONY: build-examples

clean-examples:
\trm -f $(C_OBJECTS) $(CPP_OBJECTS)
.PHONY: clean-examples

$(C_OBJECTS) : %.o : %.c $(OUTDIR)/openxr/openxr.h
\t@echo '$(ORIGIN)'
\t$(QUIET)gcc -std=gnu99 -c -I$(OUTDIR) -I$(CODEDIR) $< -o $@

$(CPP_OBJECTS) : %.o : %.cpp $(OUTDIR)/openxr/openxr.h
\t@echo '$(ORIGIN)'
\t$(QUIET)g++ -std=gnu++11 -c -I$(OUTDIR) -I$(CODEDIR) $< -o $@

ifeq ($(strip $(QUIET)),@)
EXTRACT_QUIET := --quiet
endif

$(GENERATED_C) $(GENERATED_CPP) {makefile}: {script} {inputs}
\t$(QUIET)$(PYTHON) $< {extra} --makefile={makefile} $(EXTRACT_QUIET)

gen: {script}
\t$(QUIET)$(PYTHON) $< {extra} --makefile={makefile} $(EXTRACT_QUIET)
.PHONY: gen

{deps}
""".format(out=(ROOT / 'specification/out/prerelease').relative_to(Path('.').resolve()),
                codedir=CODEDIR.relative_to(Path('.').resolve()),
                c=generated_c_string,
                cpp=generated_cpp_string,
                makefile=makefile,
                script=Path(__file__),
                extra=extra_arg,
                inputs=' '.join(str(infile) for infile in self.generatedFiles),
                deps=deps_string))
            for fn, gen in self.generatedFiles.items():
                f.write('{stem}: {files}\n.PHONY: {stem}\n'.format(
                    stem=fn.stem, files=' '.join(str(g.with_suffix('.o')) for g in gen)))
            width = max(len(generated.name) for generated in self.origins)

            for generated, origin in self.origins.items():
                pad = ' ' * (width - len(generated.name))
                originFile, originLine = origin
                if generated.suffix == '.cpp':
                    compiler = '[c++] '
                else:
                    compiler = '[cc]  '
                originStr = '{} {} {} extracted from {}:{}'.format(compiler, generated.name, pad,
                                                                   originFile, originLine)
                f.write('{obj}: ORIGIN := {originstr}\n'.format(
                    obj=generated.with_suffix('.o'), originstr=originStr))


parser = argparse.ArgumentParser()
parser.add_argument("file",
                    help="Only extract from the indicated file(s). By default, all chapters and extensions are examined.",
                    nargs="*")
parser.add_argument("--linenumbers",
                    help='Add lines of the form "# linenumber filename" to the output, for build errors/warnings that point to the adoc files.',
                    action='store_true')
parser.add_argument('--makefile',
                    help='Output a makefile with a build-examples target (and matching clean-examples target).',
                    type=str)
parser.add_argument('--quiet', '-q',
                    help="Don't output debug information about what we are extracting and not extracting.",
                    action='store_true')
# type=argparse.FileType('w', encoding='UTF-8'))
args = parser.parse_args()


if args.file:
    files = [Path(f).resolve() for f in args.file]
else:
    files = ALL_DOCS

extractors = CodeExtractorGroup(outputLineNumbers=args.linenumbers,
                                quiet=args.quiet)
extractors.process(files)

if args.makefile:
    extractors.outputMakefile(args.makefile)