docs.waku.org/fetch-content.js

const https = require('https');
const fs = require('fs');
const path = require('path');

async function fetchFromGitHub(url, callback) {
    https.get(url, { headers: { 'User-Agent': 'Node.js' } }, (res) => {
        let data = '';

        res.on('data', (chunk) => {
            data += chunk;
        });

        res.on('end', () => {
            callback(null, JSON.parse(data));
        });
    }).on('error', (err) => {
        callback(err, null);
    });
}

async function fetchDirectoryContents(dirUrl, basePath, prefixToRemove) {
    fetchFromGitHub(dirUrl, async (err, files) => {
        if (err) {
            console.error('Error fetching files:', err.message);
            return;
        }

        for (const file of files) {
            const relativePath = file.path.replace(new RegExp(`^${prefixToRemove}`), '');
            const filePath = path.join(basePath, relativePath);

            if (file.type === 'file') {
                await downloadAndSaveFile(file.download_url, filePath);
            } else if (file.type === 'dir') {
                await fetchDirectoryContents(file.url, basePath, prefixToRemove); // Recursive call for subdirectories
            }
        }
    });
}

async function downloadAndSaveFile(url, filePath) {
    const fullFilePath = path.join(__dirname, filePath);

    https.get(url, (res) => {
        const directory = path.dirname(fullFilePath);

        // Ensure the directory exists
        fs.mkdirSync(directory, { recursive: true });

        const fileStream = fs.createWriteStream(fullFilePath);
        res.pipe(fileStream);

        fileStream.on('finish', () => {
            fileStream.close();
            console.log('Downloaded and saved:', filePath);
        });
    }).on('error', (err) => {
        console.error('Error downloading file:', err.message);
    });
}

const repositories = [
    {
        baseUrl: 'https://api.github.com/repos/waku-org/nwaku/contents/docs/benchmarks',
        baseSavePath: '/docs/research/benchmarks/',
        prefixToRemove: 'docs/benchmarks/'
    },
    {
        baseUrl: 'https://api.github.com/repos/waku-org/research/contents/docs',
        baseSavePath: '/docs/research/research-and-studies/',
        prefixToRemove: 'docs/'
    }
];

fs.rmSync('docs/research/', { recursive: true, force: true });

repositories.forEach(repo => {
    fetchDirectoryContents(repo.baseUrl, repo.baseSavePath, repo.prefixToRemove);
});