Deploying multi-source sites to Netlify

Deploying one site (from a single source repo) to Netlify ain’t that hard – see my instructions here – but what if your sources are spread out across multiple repos? How do you combine the data without duplicating it into a monorepo?

That’s exactly the problem Spatie was having for their docs.spatie.be: the website holds the documentation for all of their (bigger) open source packages, but the documentation itself is not part of said repo but is stored within each project’s repository.

To solve this they first created an inventory file repositories.json. The file keeps track of each part of the documentation and its location. Using a script named fetch-content.js they aggregate all those documentation parts:

const util = require("util");
const exec = util.promisify(require("child_process").exec);
console.log('Fetching repositories...');
console.time('Fetched repositories');

const repositories = require("./repositories.json");

function transformBranchToFolderName(branch) {
    return branch.startsWith('v') ? branch.substring(1) : branch;
}

(async function () {
    let promises = [];

    for (const repository of repositories) {
        for (const [branch, alias] of Object.entries(repository.branches)) {
            const folder = `content/${repository.name}/${alias}`;
            const url = `https://codeload.github.com/${repository.repository}/tar.gz/${branch}`;

            promises.push(exec(`mkdir -p ${folder} && curl ${url} \
             | tar -xz -C ${folder} --strip=2 ${repository.repository.split('/')[1]}-${transformBranchToFolderName(branch)}/docs \
             && echo "---\ntitle: ${repository.name}\ncategory: ${repository.category}\n---" > content/${repository.name}/_index.md`));
      
        }
    }

    await Promise.all(promises);
    console.timeEnd('Fetched repositories');
})();

To build the site they first launch fetch-content.js and successively run hugo.

[build]
  command = "node fetch-content.js && hugo -b https://docs.spatie.be"
  publish = "public"

By adding webhooks which point to the same endpoint on all of the listed repos, the docs website is built whenever a commit to any of those repos is pushed.

Going serverless with Hugo and Netlify →
docs.spatie.be Source (GitHub) →