From 8e1323740c9cc7aaff3a3c5cec0dc254de41ad43 Mon Sep 17 00:00:00 2001 From: George Lemon Date: Sun, 20 Feb 2022 11:42:57 +0200 Subject: [PATCH] Added package toktok (#2159) --- packages.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages.json b/packages.json index 594595e..4c5717b 100644 --- a/packages.json +++ b/packages.json @@ -25312,5 +25312,22 @@ "license": "Apache-2.0", "web": "https://github.com/thexxiv/dogapi-nim", "doc": "https://thexxiv.github.io/dogapi-nim" + }, + { + "name": "toktok", + "url": "https://github.com/openpeep/toktok", + "method": "git", + "tags": [ + "lexer", + "token", + "tokenizer", + "lex", + "toktok", + "lexbase", + "macros" + ], + "description": "Generic tokenizer written in Nim language 👑 Powered by Nim's Macros", + "license": "MIT", + "web": "https://github.com/openpeep/toktok" } ]