Added package toktok (#2159)

This commit is contained in:
George Lemon 2022-02-20 11:42:57 +02:00 committed by GitHub
parent af0c13f503
commit 8e1323740c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -25312,5 +25312,22 @@
"license": "Apache-2.0",
"web": "https://github.com/thexxiv/dogapi-nim",
"doc": "https://thexxiv.github.io/dogapi-nim"
},
{
"name": "toktok",
"url": "https://github.com/openpeep/toktok",
"method": "git",
"tags": [
"lexer",
"token",
"tokenizer",
"lex",
"toktok",
"lexbase",
"macros"
],
"description": "Generic tokenizer written in Nim language 👑 Powered by Nim's Macros",
"license": "MIT",
"web": "https://github.com/openpeep/toktok"
}
]