diff --git a/scrapper/config.mjs b/docusaurus-utils/scrapper/config.mjs similarity index 100% rename from scrapper/config.mjs rename to docusaurus-utils/scrapper/config.mjs diff --git a/scrapper/fetch-content.mjs b/docusaurus-utils/scrapper/fetch-content.mjs similarity index 92% rename from scrapper/fetch-content.mjs rename to docusaurus-utils/scrapper/fetch-content.mjs index 985b142..8cd8609 100644 --- a/scrapper/fetch-content.mjs +++ b/docusaurus-utils/scrapper/fetch-content.mjs @@ -43,7 +43,8 @@ export async function fetchDirectoryContents(dirUrl, basePath, prefixToRemove) { for (const file of files) { const prefixRemovalRegex = new RegExp(`^${prefixToRemove}`) const relativePath = file.path.replace(prefixRemovalRegex, '') - const filePath = path.join(basePath, adjustPathForMarkdown(relativePath)) + // const filePath = path.join(basePath, adjustPathForMarkdown(relativePath)) + const filePath = path.join(basePath, relativePath) if (file.type === 'file') { await downloadAndModifyFile(file.download_url, filePath) diff --git a/scrapper/file.mjs b/docusaurus-utils/scrapper/file.mjs similarity index 100% rename from scrapper/file.mjs rename to docusaurus-utils/scrapper/file.mjs diff --git a/scrapper/github.mjs b/docusaurus-utils/scrapper/github.mjs similarity index 100% rename from scrapper/github.mjs rename to docusaurus-utils/scrapper/github.mjs diff --git a/scrapper/main.mjs b/docusaurus-utils/scrapper/main.mjs similarity index 100% rename from scrapper/main.mjs rename to docusaurus-utils/scrapper/main.mjs diff --git a/scrapper/markdown-convertor.mjs b/docusaurus-utils/scrapper/markdown-convertor.mjs similarity index 85% rename from scrapper/markdown-convertor.mjs rename to docusaurus-utils/scrapper/markdown-convertor.mjs index 31b8c98..e29a495 100644 --- a/scrapper/markdown-convertor.mjs +++ b/docusaurus-utils/scrapper/markdown-convertor.mjs @@ -86,10 +86,16 @@ function unescapeHtmlComments(htmlString) { return htmlString.replace(/\\<\!--/g, '\n\n') } -function updateMarkdownImagePath(content, number) { - const regex = /(!\[.*?\]\(\.\/)images/g +function updateMarkdownLinksToExcludeMD(content) { + function replaceLinks(match, p1, p2, p3) { + let url = p2.replace(/\.md$/, ''); // Remove .md extension from URL + let anchor = p3.replace(/^\//, ''); // Remove preceding '/' from anchor if exists + return `[${p1}](${url}${anchor ? '#' + anchor : ''})`; + } - return content.replace(regex, `$1${number}/images`) + const regex = /\[((?:(?!\]).)+)\]\(([^)]*?\.md)(?:\/#|\/#)?([^)]*)\)/g + + return content.replace(regex, replaceLinks) } export function vacMarkdownToDocusaurusMarkdown(fileContent) { @@ -113,23 +119,9 @@ export function vacMarkdownToDocusaurusMarkdown(fileContent) { convertedContent = unescapeHtmlComments(convertedContent) - // // parse sidebarPosition from the slug in the frontmatter - const sidebarPosition = parseSlugFromFrontmatter(convertedContent) || 1 - convertedContent = enhanceMarkdownWithBulletPointsCorrected(convertedContent) - convertedContent = updateMarkdownImagePath(convertedContent, sidebarPosition) - - // Insert sidebar_position at the end of frontmatter if it doesn't exist - if ( - /^---\s*[\s\S]+?---/.test(convertedContent) && - !/sidebar_position: \d+/.test(convertedContent) - ) { - convertedContent = convertedContent.replace( - /^---\s*([\s\S]+?)---/, - `---\n$1sidebar_position: ${sidebarPosition}\n---`, - ) - } + convertedContent = updateMarkdownLinksToExcludeMD(convertedContent) return convertedContent; } diff --git a/docusaurus-utils/sidebar/generator.js b/docusaurus-utils/sidebar/generator.js new file mode 100644 index 0000000..b487f8c --- /dev/null +++ b/docusaurus-utils/sidebar/generator.js @@ -0,0 +1,28 @@ +const { compose } = require("./utils"); +const { + positionDefaultReadmeToTop, + removeRFCNumberedDirectories, + separateFoldersAndFilesOrder, + orderAlphabeticallyAndByNumber +} = require("./modifiers") + +async function sidebarItemsGenerator({defaultSidebarItemsGenerator, ...args}) { + const defaultSidebarItems = await defaultSidebarItemsGenerator(args); + + /* + We'll have multiple O(N) passes through the items depending on the reducer implementation, + but we'll sacrifice very small performance for sake of easier maintainability + */ + const sidebarModifier = compose( + positionDefaultReadmeToTop, + separateFoldersAndFilesOrder, + removeRFCNumberedDirectories, + orderAlphabeticallyAndByNumber + ) + + return sidebarModifier(defaultSidebarItems) +} + +module.exports = { + sidebarItemsGenerator +} \ No newline at end of file diff --git a/docusaurus-utils/sidebar/helpers.js b/docusaurus-utils/sidebar/helpers.js new file mode 100644 index 0000000..b45c4ca --- /dev/null +++ b/docusaurus-utils/sidebar/helpers.js @@ -0,0 +1,15 @@ +function isIndexDocument(documentId, parentDirectory) { + if (!documentId) { + return false + } + + return ( + documentId.toUpperCase() === "README" || + documentId.toUpperCase() === "INDEX" || + (!!parentDirectory && documentId.toUpperCase() === parentDirectory.toUpperCase()) + ) +} + +module.exports = { + isIndexDocument +} \ No newline at end of file diff --git a/docusaurus-utils/sidebar/modifiers.js b/docusaurus-utils/sidebar/modifiers.js new file mode 100644 index 0000000..bf8d3fd --- /dev/null +++ b/docusaurus-utils/sidebar/modifiers.js @@ -0,0 +1,89 @@ +const { isNumber } = require('./utils') +const { + isIndexDocument +} = require("./helpers") + +function orderAlphabeticallyAndByNumber(sidebarItems) { + let newSidebarItems = [...sidebarItems] + + for (let i = 0; i < newSidebarItems.length; i++) { + const sidebarItem = newSidebarItems[i]; + + if (sidebarItem.type === 'category' && sidebarItem.items && sidebarItem.items.length > 1) { + newSidebarItems[i] = { + ...sidebarItem, + items: orderAlphabeticallyAndByNumber(sidebarItem.items) + } + } + } + + const collator = new Intl.Collator(undefined, {numeric: true, sensitivity: 'base'}); + const compareFunction = (a, b) => { + return collator.compare(a.label, b.label) + } + + return newSidebarItems.sort(compareFunction) +} + +function removeRFCNumberedDirectories(sidebarItems) { + let newSidebarItems = [] + + sidebarItems.forEach(sidebarItem => { + if (sidebarItem.type === 'category') { + const isRFCNumberedCategory = isNumber(sidebarItem.label); + + if (isRFCNumberedCategory) { + newSidebarItems = [...newSidebarItems, ...sidebarItem.items] + } else { + const newSidebarItem = { + ...sidebarItem, + items: removeRFCNumberedDirectories(sidebarItem.items) + } + newSidebarItems = [...newSidebarItems, newSidebarItem] + } + } else { + newSidebarItems = [...newSidebarItems, sidebarItem] + } + }) + + return newSidebarItems +} + +function separateFoldersAndFilesOrder(sidebarItems) { + let categories = []; + let pages = []; + + sidebarItems.forEach(sidebarItem => { + if (sidebarItem.type === 'category') { + categories = [...categories, sidebarItem] + } else { + pages = [...pages, sidebarItem] + } + }) + + return [ + ...categories, + ...pages + ] +} + +function positionDefaultReadmeToTop(sidebarItems) { + let newSidebarItems = [] + + sidebarItems.forEach(sidebarItem => { + if (sidebarItem.type === "doc" && isIndexDocument(sidebarItem.id)) { + newSidebarItems = [sidebarItem, ...newSidebarItems] + } else { + newSidebarItems = [...newSidebarItems, sidebarItem] + } + }) + + return newSidebarItems +} + +module.exports = { + orderAlphabeticallyAndByNumber, + removeRFCNumberedDirectories, + separateFoldersAndFilesOrder, + positionDefaultReadmeToTop, +} \ No newline at end of file diff --git a/docusaurus-utils/sidebar/utils.js b/docusaurus-utils/sidebar/utils.js new file mode 100644 index 0000000..208eabd --- /dev/null +++ b/docusaurus-utils/sidebar/utils.js @@ -0,0 +1,27 @@ +function isNumber(value) { + if (true === Array.isArray(value)) { + return false; + } + return !isNaN(parseInt(value, 10)); +} + +/* + Composes multiple functions with same arguments into a single one + NOTE: Functions are executed from end of array to start (right to left) + */ +function compose(...funcs) { + if (funcs.length === 1) { + return funcs[0] + } + + return funcs.reduce( + (firstFunction, nextFunction) => + (...args) => + firstFunction(nextFunction(...args)) + ) +} + +module.exports = { + isNumber, + compose +} \ No newline at end of file diff --git a/docusaurus.config.js b/docusaurus.config.js index 320fc03..3d77491 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -1,6 +1,7 @@ // @ts-check // Note: type annotations allow type checking and IDEs autocompletion require('dotenv').config() +const { sidebarItemsGenerator } = require("./docusaurus-utils/sidebar/generator") /** @type {import('@docusaurus/types').Config} */ const config = { @@ -29,6 +30,7 @@ const config = { id: 'codex', path: 'codex', routeBasePath: 'codex', + sidebarItemsGenerator, }, ], [ @@ -37,6 +39,7 @@ const config = { id: 'nomos', path: 'nomos', routeBasePath: 'nomos', + sidebarItemsGenerator, }, ], [ @@ -45,6 +48,7 @@ const config = { id: 'status', path: 'status', routeBasePath: 'status', + sidebarItemsGenerator, }, ], [ @@ -53,6 +57,7 @@ const config = { id: 'vac', path: 'vac', routeBasePath: 'vac', + sidebarItemsGenerator, }, ], [ @@ -61,6 +66,7 @@ const config = { id: 'waku', path: 'waku', routeBasePath: 'waku', + sidebarItemsGenerator, }, ], ], diff --git a/package.json b/package.json index fbebaf4..076a5b8 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "write-translations": "docusaurus write-translations", "write-heading-ids": "docusaurus write-heading-ids", "typecheck": "tsc", - "scrape": "node ./scrapper/main.mjs" + "scrape": "node ./docusaurus-utils/scrapper/main.mjs" }, "dependencies": { "@acid-info/logos-docusaurus-preset": "^1.0.0-alpha.14", diff --git a/static/.gitkeep b/static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/static/docs/codex-whitepaper.pdf b/static/docs/codex-whitepaper.pdf deleted file mode 100644 index 8576864..0000000 Binary files a/static/docs/codex-whitepaper.pdf and /dev/null differ