vacp2p/rfc.vac.dev
1
0
Fork 0
mirror of https://github.com/vacp2p/rfc.vac.dev.git synced 2025-02-22 12:28:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: Fresh scraping on each sync (#4)

Browse Source
This commit is contained in:
Filip Pajic 2024-05-09 12:40:06 +02:00 committed by GitHub
parent c51271253a
commit 6a9b7654cc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 24 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
docusaurus-utils/scrapper/file.mjs
View File

@ -36,7 +36,7 @@ export async function writeLargeFile(path, data) {
export function removeDirectory(path) {
return new Promise((resolve, reject) => {
fs.rmdir(path, {recursive: true}, err => {
fs.rm(path, {recursive: true}, err => {
if (err) {
reject(err)
}
@ -46,6 +46,20 @@ export function removeDirectory(path) {
})
}
export function directoryExists(path) {
return new Promise((resolve, _) => {
const accessBitwiseOperator = fs.constants.F_OK | fs.constants.R_OK;
fs.access(path, accessBitwiseOperator, err => {
if (err) {
resolve(false)
} else {
resolve(true)
}
})
})
}
export async function createDirectory(path) {
try {
/*

9
docusaurus-utils/scrapper/main.mjs
View File

@ -1,4 +1,6 @@
import { fetchDirectoryContents } from './fetch-content.mjs'
import path from 'path'
import { directoryExists, removeDirectory } from './file.mjs'
const directoriesToSync = ['codex', 'nomos', 'status', 'vac', 'waku']
@ -10,6 +12,13 @@ async function main() {
const baseSavePath = `./${dirName}/`
const prefixToRemove = dirName + '/'
const directoryPath = path.join(process.cwd(), dirName);
const shouldRemoveOldContent = await directoryExists(directoryPath);
if (shouldRemoveOldContent) {
await removeDirectory(directoryPath)
console.log(`Removed old ${dirName}`)
}
await fetchDirectoryContents(baseUrl, baseSavePath, prefixToRemove)
console.log(`Synced ${dirName}`)