From f929f3005c24ea759c4b818546fc4942b2e1c2c9 Mon Sep 17 00:00:00 2001 From: Surf1nb1rd Date: Thu, 25 Oct 2018 15:33:49 +0300 Subject: [PATCH] Implemented the chronicles_tail tool --- chronicles.nim | 27 +- chronicles.nimble | 5 +- chronicles/bin/chronicles_tail.nim | 516 +++++++++++++++++++++++++ chronicles/bin/chronicles_tail.nim.cfg | 1 + chronicles/log_output.nim | 2 +- chronicles/options.nim | 2 +- chronicles/topics_registry.nim | 23 ++ 7 files changed, 551 insertions(+), 25 deletions(-) create mode 100644 chronicles/bin/chronicles_tail.nim create mode 100644 chronicles/bin/chronicles_tail.nim.cfg diff --git a/chronicles.nim b/chronicles.nim index e75b8fd..aafe1f8 100644 --- a/chronicles.nim +++ b/chronicles.nim @@ -121,32 +121,15 @@ when runtimeFilteringEnabled: # block surrounding the entire log statement. result = newStmtList() var - matchEnabledTopics = genSym(nskVar, "matchEnabledTopics") - requiredTopicsCount = genSym(nskVar, "requiredTopicsCount") - topicChecks = newStmtList() - - result.add quote do: - - var `matchEnabledTopics` = registry.totalEnabledTopics == 0 - var `requiredTopicsCount` = registry.totalRequiredTopics + topicStateIMPL = bindSym("topicStateIMPL") + topicsMatch = bindSym("topicsMatch") + var topicsArray = newTree(nnkBracket) for topic in topics: - result.add quote do: - let t = topicStateIMPL(`topic`) - case t.state - of Normal: discard - of Enabled: `matchEnabledTopics` = true - of Disabled: break chroniclesLogStmt - of Required: dec `requiredTopicsCount` - - if t.logLevel == NONE: - if LogLevel(`logLevel`) < gActiveLogLevel: - break chroniclesLogStmt - elif LogLevel(`logLevel`) < t.logLevel: - break chroniclesLogStmt + topicsArray.add newCall(topicStateIMPL, topic) result.add quote do: - if not `matchEnabledTopics` or `requiredTopicsCount` > 0: + if not `topicsMatch`(`topicsArray`): break chroniclesLogStmt else: template runtimeFilteringDisabledError = diff --git a/chronicles.nimble b/chronicles.nimble index 4121845..1513397 100644 --- a/chronicles.nimble +++ b/chronicles.nimble @@ -6,8 +6,11 @@ author = "Status Research & Development GmbH" description = "A crafty implementation of structured logging for Nim" license = "Apache License 2.0" skipDirs = @["tests"] +bin = @["chronicles/bin/chronicles_tail"] -requires "nim >= 0.18.1" +requires "nim >= 0.18.1", + "compiler", + "https://github.com/surf1nb1rd/nim-prompt" task test, "run CPU tests": cd "tests" diff --git a/chronicles/bin/chronicles_tail.nim b/chronicles/bin/chronicles_tail.nim new file mode 100644 index 0000000..29ac0a3 --- /dev/null +++ b/chronicles/bin/chronicles_tail.nim @@ -0,0 +1,516 @@ +import os, osproc, streams, macros, queues, threadpool, algorithm, terminal +import json, tables, parseopt, chronicles, chronicles/topics_registry +import strutils, sequtils, unicode, re, parseopt +import compiler / [ast, vmdef, vm, nimeval, options, parser, idents, condsyms, + nimconf, extccomp, astalgo, llstream, pathutils] +import prompt + +type + SyntaxError = object of Exception + + Messagekind = enum + Cmd, + Log + + Message = object + content: string + kind: Messagekind + + Command = enum + clearFilter, + clearGrep, + clearTopics, + extract, + filter, + format, + grep, + help, + quit, + topics + + RecordPrinter = proc (j: JsonNode) + +proc printTextBlock(j: JsonNode) +proc printTextLine(j: JsonNode) +proc printJson(j: JsonNode) + +var + optParser = initOptParser() + program = "" + commandLine = "" + channel: Channel[Message] + fltr: PNode + regex = re("") + recordFormat = "" + jTest = %*{"msg": "foo", "lvl": "dbg", "ts": 3.14, "topics": "bar", "thread": 0} + activeRecordPrinter: RecordPrinter = printTextLine + activeFilter = "" + activeGrep = "" + activeTopics = "" + +proc createTopicState(name: string): ptr Topic = + result = getTopicState(name) + if result == nil: + return registerTopic(name, create(Topic)) + +for kind, key, val in optParser.getopt(): + case kind + of cmdArgument: + program = key + commandLine = optParser.cmdLineRest() + break + of cmdLongOption, cmdShortOption: + case key + of "format": + case val + of "json": + activeRecordPrinter = printJson + of "textblocks": + activeRecordPrinter = printTextBlock + of "textlines": + activeRecordPrinter = printTextLine + of "enabled_topics": + var topics = val.split(Whitespace) + for t in topics: + discard createTopicState(t) + let s = setTopicState(t, Enabled) + assert s + activeTopics.add(" +" & t ) + of "disabled_topics": + var topics = val.split(Whitespace) + for t in topics: + discard createTopicState(t) + let s = setTopicState(t, Disabled) + assert s + activeTopics.add(" -" & t ) + of "required_topics": + var topics = val.split(Whitespace) + for t in topics: + discard createTopicState(t) + let s = setTopicState(t, Required) + activeTopics.add(" *" & t ) + assert s + of cmdEnd: discard + +proc printUsage() = + echo """ +Usage: + chronicles-tail [options] program [program-options] + + Topic filtering options: + + --enabled_topics:"topics" + --disabled_topics:"topics" + --required_topics:"topics" + + Output formatting options: + + --format:json or textblocks or textlines +""" + +if program.len == 0: + echo "Please specify a program to run\n" + printUsage() + quit 1 + +proc parse*(s: string): PNode = + var conf = newConfigRef() + conf.verbosity = 0 + var cache = newIdentCache() + condsyms.initDefines(conf.symbols) + conf.projectName = "stdinfile" + conf.projectFull = AbsoluteFile("stdinfile") + let cwd = AbsoluteFile(getCurrentDir()) + conf.projectPath = AbsoluteDir(canonicalizePath(conf, cwd)) + conf.projectIsStdin = true + loadConfigs(DefaultConfig, cache, conf) + extccomp.initVars(conf) + result = parseString(s, cache, conf) + +# Handling keyboard inputs with the edited mofunoise library + +const commands = [ + ("!filter", "clears the active filter"), + ("!grep", "clears the active grep"), + ("!topics", "clears active topics"), + ("extract", "shows only certain properties of the log statement"), + ("filter", + """shows only statements with the specified property. E.g.: lvl == "DEBUG" """), + ("format", + "sets the format of the log outputs - textblocks, textlines or json"), + ("grep", "uses regular expressions to filter the log"), + ("help", "shows this help"), + ("quit", "quits the program"), + ("topics", """filters by topic using the operators +, - and *. Ex.: topics +enabled_topic -disabled_topic *required_topic +another_enabled_topic """) +] + +static: assert commands.len == int(high(Command)) + 1 + +proc provideCompletions*(pline: seq[Rune], cursorPos: int): seq[string] = + result = @[] + var line = "" + for i in 0 ..< pline.len: + add(line, $pline[i]) + var firstWord = line[0..(cursorPos-1)] + var index = lowerBound(commands, (firstWord, "")) + if commands.len < index + 1: + return + if not startsWith(commands[index][0], firstWord): + return + var i = 0 + while index + i < commands.len and + startsWith(commands[index+i][0], firstWord): + result.add commands[index + i][0] + i += 1 + +var p = Prompt.init("chronicles > ", provideCompletions) +p.useHistoryFile() + +proc pSaveHistory(){.noconv.} = + try: + p.saveHistory() + except IOError: + p.writeLine("Error saving history to " & p.historyPath) + +addQuitProc pSaveHistory + +open(channel) + +var pAddr = addr(p) +proc print(s: string) = pAddr[].writeline s + +proc ctrlCHandler() {.noconv.} = quit 0 +setControlCHook ctrlCHandler + +proc printHelp() = + for el in commands: + var spaces = repeat(" ", 18-el[0].len) + if el[1].len > 50: + var line1 = el[1][0 .. 50] + var line2 = el[1][51 .. ^1] + var empty = repeat(" ", 18) + print el[0] & spaces & line1 & "\n\r" & empty & line2 + else: + print el[0] & spaces & el[1] + +proc printRecord(Record: type, j: JsonNode) = + var record: Record + let severity = parseEnum[LogLevel](j["lvl"].str) + let msg = j["msg"].str + let topic = j["topics"].str + + pAddr[].withOutput do (): + initLogRecord(record, severity, topic, msg) + delete(j, "msg") + delete(j, "lvl") + delete(j, "ts") + delete(j, "topics") + var b = true + for field, value in j: + if b: + record.setFirstProperty($field, value) + b = false + else: + case value.kind + of JString: + record.setProperty($field, value.str) + of JInt: + record.setProperty($field, value.num) + of JFloat: + record.setProperty($field, value.fnum) + of JBool: + record.setProperty($field, value.bval) + of JNull: + assert false + of JObject: + record.setProperty($field, $value) + of JArray: + record.setProperty($field, value.elems) + flushRecord(record) + +proc printTextBlock(j: JsonNode) = + printRecord(TextBlockRecord[StdOutOutput, RfcTime, NativeColors], j) + +proc printTextLine(j: JsonNode) = + printRecord(TextLineRecord[StdOutOutput, RfcTime, NativeColors], j) + +proc printJson(j: JsonNode) = + printRecord(JsonRecord[StdOutOutput, RfcTime], j) + +proc compare(r: JsonNode, n: PNode): int = + case r.kind + of JString: + if n.kind == nkStrLit: + return cmpIgnoreCase(r.str, n.strVal) + of JInt: + if n.kind == nkIntLit: + return cmp(r.num, n.intVal) + of JFloat: + if n.kind == nkFloatLit: + return cmp(r.fnum, n.floatVal) + of JBool: + if n.intVal == 0 or n.intVal == 1: + return cmp(BiggestInt(r.bval), n.intVal) + of JNull: + if n.kind == nkNilLit: + return 0 + of JObject: + raise newException(ValueError, "Type object is not supported") + of JArray: + if n.kind == nkBracket: + let minlen = min(n.sons.len, r.elems.len) + for i in 0 ..< minlen: + var res = compare(r.elems[i], n.sons[i]) + if res == 0: + return n.sons.len - r.elems.len + else: + return res + raise newException(ValueError, + "The value in your filter is of a different type than the json value") + +#Proc 'matches' returns true if a record 'r' matches a filtering condition 'n'. +proc matches(n: PNode, r: JsonNode, allowMissingFields: bool): bool = + if n == nil: + return true + else: + case n.kind + of nkStmtList: + return matches(n[0], r, allowMissingFields) + of nkInfix: + assert n[0].kind == nkIdent + case n[0].ident.s + of "or": + return matches(n[1], r, allowMissingFields) or + matches(n[2], r, allowMissingFields) + of "and": + return matches(n[1], r, allowMissingFields) and + matches(n[2], r, allowMissingFields) + of "==": + if n[1].kind != nkIdent: + raise newException(SyntaxError, "Syntax error") + let jsonPropertyName = n[1].ident.s + if r.hasKey(jsonPropertyName): + if compare(r[jsonPropertyName], n[2]) == 0: + return true + return false + return allowMissingFields + of ">": + if n[1].kind != nkIdent: + raise newException(SyntaxError, "Syntax error") + let jsonPropertyName = n[1].ident.s + if r.hasKey(jsonPropertyName): + if compare(r[jsonPropertyName], n[2]) > 0: + return true + return false + return allowMissingFields + of "<": + if n[1].kind != nkIdent: + raise newException(SyntaxError, "Syntax error") + let jsonPropertyName = n[1].ident.s + if r.hasKey(jsonPropertyName): + if compare(r[jsonPropertyName], n[2]) < 0: + return true + return false + return allowMissingFields + of "in": + if n[2].kind != nkBracket: + raise newException(SyntaxError, + "Please, enter a valid nim expression") + let jsonPropertyName = n[1].ident.s + if r.hasKey(jsonPropertyName): + for i in n[2].sons: + if compare(r[jsonPropertyName], i) == 0: + return true + return false + return allowMissingFields + else: + raise newException(SyntaxError, "Unsupported operator") + else: + raise newException(SyntaxError, "Syntax Error") + +proc matchRE(s: string, re: Regex): bool = + if find(s, re) > -1: + return true + return false + +proc handleCommand(line: string) = + var pos = find(line, " ") + var cmdParam = "" + var firstWord: string #'firstWord' is expected to be a command or command shortcut + if pos > 0: + firstWord = line[0..(pos-1)] + cmdParam = line[(pos+1)..line.high] #command parameters + else: + firstWord = line + var index = lowerBound(commands, (firstWord, "")) + var currentCmd: string + var nextCmd: string + if commands.len < index + 1: + print "Wrong command" + return + elif commands.len == index + 1: + currentCmd = commands[index][0] + nextCmd = "" + else: + currentCmd = commands[index][0] + nextCmd = commands[index + 1][0] + + if not startsWith(currentCmd, firstWord): + print "Wrong command" + return + if startsWith(nextCmd, firstWord): + print "Did you mean " & currentCmd & " or " & nextCmd & "?" + return + var command = Command(index) + + case command + of clearFilter: + fltr = nil + activeFilter = "" + print "clearing filter" + of clearGrep: + print "clearing grep" + regex = re("") + activeGrep = "" + of clearTopics: + clearTopicsRegistry() + activeTopics = "" + of extract: + print "extracting" + of filter: + if pos < 0: + print "Please, enter a parameter for filter" + return + try: + let parsed = parse(cmdParam) + discard matches(parsed, jTest, false) + fltr = parsed + activeFilter = cmdParam + except: + print "Error: " & getCurrentExceptionMsg() + of format: + print "formatting" + if cmdParam == "textblocks": + activeRecordPrinter = printTextBlock + elif cmdParam == "textlines": + activeRecordPrinter = printTextLine + elif cmdParam == "json": + activeRecordPrinter = printJson + else: + print "Please, enter a valid format: TextBlockRecord, TextLineRecord or JsonRecord" + of grep: + if pos < 0: + print "Please, enter a parameter for grep" + return + try: + regex = re(cmdParam, {reIgnoreCase, reStudy}) + activeGrep = cmdParam + except: + print "Error: " & getCurrentExceptionMsg() + of help: + printHelp() + of quit: + quit(0) + of topics: + clearTopicsRegistry() + #echo repr(registry.topicStatesTable) + var params = cmdParam.split(Whitespace) + if cmdParam == "": + print "Please, enter a parameter for grep" + for p in params: + if p.len == 0: + continue + var operator = p[0] + var topic = p[1 .. ^1] + if operator notin ['+', '-', '*']: + print "Syntax Error" + return + activeTopics = cmdParam + if operator == '+': + discard createTopicState(topic) + let s = setTopicState(topic, Enabled) + assert s + elif operator == '-': + discard createTopicState(topic) + let s = setTopicState(topic, Disabled) + assert s + elif operator == '*': + discard createTopicState(topic) + let s = setTopicState(topic, Required) + assert s + +proc inputThread {.thread.} = + try: + while true: + var msg: Message + msg.kind = Cmd + msg.content = pAddr[].readLine() + send(channel, msg) + except: + let ex = getCurrentException() + # print ex.getStackTrace + # print ex.msg + quit 1 + +spawn inputThread() + +var process = startProcess(command = program, args = [commandLine], workingDir = getCurrentDir()) + +proc quitProc(){.noconv.} = terminate(process) +system.addQuitProc(quitProc) + +# Transform json input into TextBlockRecord/TextLineRecord or JsonRecord +proc processTailingThread(process: Process) = + var msg: Message + msg.kind = Log + for line in outputStream(process).lines: + msg.content = line + send(channel, msg) + +spawn processTailingThread(process) + +proc checkType(j: JsonNode, key: string, kind: JsonNodeKind): bool = + j.hasKey(key) and j[key].kind == kind + +proc setStatus() = + var statusBar: seq[StatusBarItem] = @[] + if activeTopics != "" : + statusBar.add(("topics", activeTopics)) + if activeFilter != "" : + statusBar.add(("filter", activeFilter)) + if activeGrep != "" : + statusBar.add(("grep", activeGrep)) + p.setStatusBar(statusBar) + +setStatus() + +proc mainLoop() = + var msg: Message + while true: + msg = recv(channel) + if msg.kind == Cmd: + handleCommand(msg.content) + setStatus() + if msg.kind == Log: + if matchRE(msg.content, regex): + var j = parseJson(msg.content) + var t = j["topics"].str + var topics = t.split(Whitespace + {',', ';'}) + var topicStates: seq[ptr Topic] = @[] + for t in topics: + topicStates.add(createTopicState(t)) + #print (t & $(createTopicState(t)[])) + if j.kind == JObject and + j.checkType("ts", JString) and + j.checkType("lvl", JString) and + j.checkType("msg", JString) and + j.checkType("topics", JString) and + topicsMatch(topicStates) and + matches(fltr, j, false): + activeRecordPrinter(j) + else: + discard + else: + discard + +mainLoop() diff --git a/chronicles/bin/chronicles_tail.nim.cfg b/chronicles/bin/chronicles_tail.nim.cfg new file mode 100644 index 0000000..aed303e --- /dev/null +++ b/chronicles/bin/chronicles_tail.nim.cfg @@ -0,0 +1 @@ +--threads:on diff --git a/chronicles/log_output.nim b/chronicles/log_output.nim index 5960357..cedcead 100644 --- a/chronicles/log_output.nim +++ b/chronicles/log_output.nim @@ -375,7 +375,7 @@ template appendHeader(r: var TextLineRecord | var TextBlockRecord, # choose an arbitrary number and use that - should be fine even for # 80-char terminals # XXX: This should be const, but the compiler fails with an ICE - let padding = static(repeat(' ', if pad: 42 - min(42, name.len) else: 0)) + let padding = repeat(' ', if pad: 42 - min(42, name.len) else: 0) append(r.output, " ") applyStyle(r, styleBright) diff --git a/chronicles/options.nim b/chronicles/options.nim index 627fd1e..a80c3fd 100644 --- a/chronicles/options.nim +++ b/chronicles/options.nim @@ -128,7 +128,7 @@ proc handleEnumOption(E: typedesc[enum], template handleEnumOption(E, varName: untyped): auto = handleEnumOption(E, astToStr(varName), varName) -template topicsAsSeq(topics: string): untyped = +template topicsAsSeq*(topics: string): untyped = when topics.len > 0: topics.split({','} + Whitespace) else: diff --git a/chronicles/topics_registry.nim b/chronicles/topics_registry.nim index 002bb6e..9accd55 100644 --- a/chronicles/topics_registry.nim +++ b/chronicles/topics_registry.nim @@ -22,6 +22,12 @@ proc initTopicsRegistry: TopicsRegisty = var registry* = initTopicsRegistry() +proc clearTopicsRegistry* = + registry.totalEnabledTopics = 0 + registry.totalRequiredTopics = 0 + for val in registry.topicStatesTable.values: + val.state = Normal + iterator topicStates*: (string, TopicState) = for name, topic in registry.topicStatesTable: yield (name, topic.state) @@ -52,3 +58,20 @@ proc setTopicState*(name: string, topicPtr.logLevel = logLevel return true + +proc topicsMatch*(topics: openarray[ptr Topic]): bool = + if topics.len == 0: + return true + var matchEnabledTopics = registry.totalEnabledTopics == 0 + var requiredTopicsCount = registry.totalRequiredTopics + for topic in topics: + case topic.state + of Normal: discard + of Enabled: matchEnabledTopics = true + of Disabled: return false + of Required: dec requiredTopicsCount + return matchEnabledTopics and requiredTopicsCount == 0 + +proc getTopicState*(topic: string): ptr Topic = + return registry.topicStatesTable.getOrDefault(topic) +