website: resolve unlinked content (#10087)

* website: add back unlinked pages to match previous state

* website: add unlinked content check

* website: add hidden nav-data to unlinked content check

.github/workflows/check-unlinked-content.js

@@ -0,0 +1,147 @@
+var fs = require("fs");
+var path = require("path");
+
+const COLOR_RESET = "\x1b[0m";
+const COLOR_GREEN = "\x1b[32m";
+const COLOR_RED = "\x1b[31m";
+
+runCheck([
+  {
+    contentDir: "website/content/docs",
+    navDataFiles: [
+      "website/data/docs-nav-data.json",
+      "website/data/docs-nav-data-hidden.json",
+    ],
+  },
+  {
+    contentDir: "website/content/api-docs",
+    navDataFiles: [
+      "website/data/api-docs-nav-data.json",
+      "website/data/api-docs-nav-data-hidden.json",
+    ],
+  },
+  {
+    contentDir: "website/content/commands",
+    navDataFiles: [
+      "website/data/commands-nav-data.json",
+      "website/data/commands-nav-data-hidden.json",
+    ],
+  },
+]);
+
+async function runCheck(baseRoutes) {
+  const validatedBaseRoutes = await Promise.all(
+    baseRoutes.map(async ({ contentDir, navDataFiles }) => {
+      const missingRoutes = await validateMissingRoutes(
+        contentDir,
+        navDataFiles
+      );
+      return { contentDir, navDataFiles, missingRoutes };
+    })
+  );
+  const allMissingRoutes = validatedBaseRoutes.reduce((acc, baseRoute) => {
+    return acc.concat(baseRoute.missingRoutes);
+  }, []);
+  if (allMissingRoutes.length == 0) {
+    console.log(
+      `\n${COLOR_GREEN}✓ All content files have routes, and are included in navigation data.${COLOR_RESET}\n`
+    );
+  } else {
+    validatedBaseRoutes.forEach(
+      ({ contentDir, navDataFiles, missingRoutes }) => {
+        if (missingRoutes.length == 0) return true;
+        console.log(
+          `\n${COLOR_RED}Error: Missing pages found in the ${contentDir} directory.\n\nPlease add these paths to ${navDataFiles.join(
+            " or "
+          )}, or remove the .mdx files.\n\n${JSON.stringify(
+            missingRoutes,
+            null,
+            2
+          )}${COLOR_RESET}\n\n`
+        );
+      }
+    );
+    process.exit(1);
+  }
+}
+
+async function validateMissingRoutes(contentDir, navDataFiles) {
+  // Read in nav-data.json, and make a flattened array of nodes
+  const navDataFlat = navDataFiles.reduce((acc, navDataFile) => {
+    const navDataPath = path.join(process.cwd(), navDataFile);
+    const navData = JSON.parse(fs.readFileSync(navDataPath));
+    return acc.concat(flattenNodes(navData));
+  }, []);
+  // Read all files in the content directory
+  const files = await walkAsync(contentDir);
+  // Filter out content files that are already
+  // included in nav-data.json
+  const missingPages = files
+    // Ignore non-.mdx files
+    .filter((filePath) => {
+      return path.extname(filePath) == ".mdx";
+    })
+    // Transform the filePath into an expected route
+    .map((filePath) => {
+      // Get the relative filepath, that's what we'll see in the route
+      const contentDirPath = path.join(process.cwd(), contentDir);
+      const relativePath = path.relative(contentDirPath, filePath);
+      // Remove extensions, these will not be in routes
+      const pathNoExt = relativePath.replace(/\.mdx$/, "");
+      // Resolve /index routes, these will not have /index in their path
+      const routePath = pathNoExt.replace(/\/?index$/, "");
+      return routePath;
+    })
+    // Determine if there is a match in nav-data.
+    // If there is no match, then this is an unlinked content file.
+    .filter((pathToMatch) => {
+      // If it's the root path index page, we know
+      // it'll be rendered (hard-coded into docs-page/server.js)
+      const isIndexPage = pathToMatch === "";
+      if (isIndexPage) return false;
+      // Otherwise, needs a path match in nav-data
+      const matches = navDataFlat.filter(({ path }) => path == pathToMatch);
+      return matches.length == 0;
+    });
+  return missingPages;
+}
+
+function flattenNodes(nodes) {
+  return nodes.reduce((acc, n) => {
+    if (!n.routes) return acc.concat(n);
+    return acc.concat(flattenNodes(n.routes));
+  }, []);
+}
+
+function walkAsync(relativeDir) {
+  const dirPath = path.join(process.cwd(), relativeDir);
+  return new Promise((resolve, reject) => {
+    walk(dirPath, function (err, result) {
+      if (err) reject(err);
+      resolve(result);
+    });
+  });
+}
+
+function walk(dir, done) {
+  var results = [];
+  fs.readdir(dir, function (err, list) {
+    if (err) return done(err);
+    var pending = list.length;
+    if (!pending) return done(null, results);
+    list.forEach(function (file) {
+      file = path.resolve(dir, file);
+      fs.stat(file, function (err, stat) {
+        if (stat && stat.isDirectory()) {
+          walk(file, function (err, res) {
+            results = results.concat(res);
+            if (!--pending) done(null, results);
+          });
+        } else {
+          results.push(file);
+          if (!--pending) done(null, results);
+        }
+      });
+    });
+  });
+}

.github/workflows/check-unlinked-content.yml

@@ -0,0 +1,26 @@
+#
+# This GitHub action checks that all .mdx files in the
+# the website/content directory are being published.
+# It fails if any of these files are not included
+# in the expected nav-data.json file.
+#
+# To resolve failed checks, add the listed paths
+# to the corresponding nav-data.json file
+# in website/data.
+
+name: "website: Check unlinked content"
+on:
+  pull_request:
+    paths:
+      - "website/**"
+
+jobs:
+  check-unlinked-content:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Node
+        uses: actions/setup-node@v1
+      - name: Check that all content files are included in navigation
+        run: node .github/workflows/check-unlinked-content.js

website/components/_temp-enable-hidden-pages/index.js

@@ -0,0 +1,85 @@
+// Imports below are used in server-side only
+import fs from 'fs'
+import path from 'path'
+import {
+  generateStaticPaths as docsPageStaticPaths,
+  generateStaticProps as docsPageStaticProps,
+} from '@hashicorp/react-docs-page/server'
+
+/**
+ * DEBT
+ * This is a short term hotfix for "hidden" docs-sidenav items.
+ *
+ * We likely do NOT want to support this in docs-page/server,
+ * instead, a simple "hidden" attribute supported on docs-sidenav
+ * nodes would do the trick, ensuring the "path" is "registered"
+ * in the appropriate nav-data file, and located in the correct spot
+ * in the nav-data tree, while also hiding that item in the sidebar.
+ *
+ * We can remove this hack with once support lands for "hidden" items,
+ * currently this is somewhat blocked by branding rollout:
+ * Asana task that will resolve this debt:
+ * https://app.asana.com/0/1100423001970639/1200197752405255/f
+ * Draft PR to support "hidden" nav items:
+ * https://github.com/hashicorp/react-components/pull/220
+ **/
+
+const DEFAULT_PARAM_ID = 'page'
+
+export async function generateStaticPaths({
+  navDataFile,
+  navDataFileHidden,
+  localContentDir,
+}) {
+  const visiblePaths = await docsPageStaticPaths({
+    navDataFile,
+    localContentDir,
+  })
+  const hiddenPaths = await docsPageStaticPaths({
+    navDataFile: navDataFileHidden,
+    localContentDir,
+  })
+  return visiblePaths.concat(hiddenPaths)
+}
+
+export async function generateStaticProps({
+  navDataFile,
+  navDataFileHidden,
+  localContentDir,
+  product,
+  params,
+  paramId = DEFAULT_PARAM_ID,
+  additionalComponents,
+}) {
+  // Read in the "hidden" nav data, and flatten it
+  const navDataVisible = readNavData(navDataFile)
+  const navDataHidden = readNavData(navDataFileHidden)
+  // Check if this is a "hidden" page, if so, use the navDataHidden
+  // to generate static props.
+  const currentPath = params[paramId] ? params[paramId].join('/') : ''
+  const hiddenPaths = flattenNavData(navDataHidden).map((n) => n.path)
+  const isHiddenPage = hiddenPaths.filter((p) => p == currentPath).length > 0
+  // Return the static props, but always pass the navDataVisible
+  // as the navData to be displayed.
+  const staticProps = await docsPageStaticProps({
+    navDataFile: isHiddenPage ? navDataFileHidden : navDataFile,
+    localContentDir,
+    product,
+    params,
+    paramId,
+    additionalComponents,
+  })
+  return { ...staticProps, navData: navDataVisible }
+}
+
+function readNavData(navDataFile) {
+  const filePath = path.join(process.cwd(), navDataFile)
+  return JSON.parse(fs.readFileSync(filePath))
+}
+
+function flattenNavData(nodes) {
+  return nodes.reduce((acc, n) => {
+    if (!n.routes) return acc.concat(n)
+    return acc.concat(flattenNavData(n.routes))
+  }, [])
+}

website/data/api-docs-nav-data-hidden.json

@@ -0,0 +1,6 @@
+[
+  {
+    "title": "ACL (Legacy)",
+    "path": "acl-legacy"
+  }
+]

website/data/commands-nav-data-hidden.json

@@ -0,0 +1,21 @@
+[
+  {
+    "title": "Connect",
+    "routes": [
+      {
+        "title": "redirect-traffic",
+        "path": "connect/redirect-traffic"
+      }
+    ]
+  },
+
+  {
+    "title": "Intention",
+    "routes": [
+      {
+        "title": "list",
+        "path": "intention/list"
+      }
+    ]
+  }
+]

website/data/docs-nav-data-hidden.json

@@ -0,0 +1,59 @@
+[
+  {
+    "title": "Internals",
+    "routes": [
+      {
+        "title": "Overview",
+        "path": "internals"
+      },
+      {
+        "title": "ACL",
+        "path": "internals/acl"
+      }
+    ]
+  },
+  {
+    "title": "Get Started",
+    "routes": [
+      {
+        "title": "Manual Bootstrap",
+        "path": "install/manual-bootstrap"
+      }
+    ]
+  },
+  {
+    "title": "Connect",
+    "routes": [
+      {
+        "title": "Proxies",
+        "routes": [
+          {
+            "title": "Managed (Deprecated)",
+            "path": "connect/proxies/managed-deprecated"
+          }
+        ]
+      },
+      {
+        "title": "Security",
+        "path": "connect/security"
+      }
+    ]
+  },
+  {
+    "title": "Agent",
+    "routes": [
+      {
+        "title": "RPC",
+        "path": "agent/rpc"
+      },
+      {
+        "title": "Sentinel",
+        "path": "agent/sentinel"
+      }
+    ]
+  },
+  {
+    "title": "Guides",
+    "path": "guides"
+  }
+]

website/pages/api-docs/[[...page]].jsx

@@ -1,14 +1,22 @@
 import { productName, productSlug } from 'data/metadata'
 import DocsPage from '@hashicorp/react-docs-page'
 // Imports below are only used server-side
+/**
+ * DEBT: short term patch for "hidden" docs-sidenav items.
+ * See components/_temp-enable-hidden-pages for details.
+ * Revert to importing from @hashicorp/react-docs-page/server
+ * once https://app.asana.com/0/1100423001970639/1200197752405255/f
+ * is complete.
+ **/
 import {
   generateStaticPaths,
   generateStaticProps,
-} from '@hashicorp/react-docs-page/server'
+} from 'components/_temp-enable-hidden-pages'
 
 //  Configure the docs path
 const baseRoute = 'api-docs'
 const navDataFile = `data/${baseRoute}-nav-data.json`
+const navDataFileHidden = `data/${baseRoute}-nav-data-hidden.json`
 const localContentDir = `content/${baseRoute}`
 const mainBranch = 'master'
 const product = { name: productName, slug: productSlug }
@@ -20,7 +28,11 @@ export default function ApiDocsLayout(props) {
 }
 
 export async function getStaticPaths() {
-  const paths = await generateStaticPaths({ localContentDir, navDataFile })
+  const paths = await generateStaticPaths({
+    localContentDir,
+    navDataFile,
+    navDataFileHidden,
+  })
   return { paths, fallback: false }
 }
 
@@ -29,6 +41,7 @@ export async function getStaticProps({ params }) {
     localContentDir,
     mainBranch,
     navDataFile,
+    navDataFileHidden,
     params,
     product,
   })

website/pages/commands/[[...page]].jsx

@@ -1,14 +1,22 @@
 import { productName, productSlug } from 'data/metadata'
 import DocsPage from '@hashicorp/react-docs-page'
 // Imports below are only used server-side
+/**
+ * DEBT: short term patch for "hidden" docs-sidenav items.
+ * See components/_temp-enable-hidden-pages for details.
+ * Revert to importing from @hashicorp/react-docs-page/server
+ * once https://app.asana.com/0/1100423001970639/1200197752405255/f
+ * is complete.
+ **/
 import {
   generateStaticPaths,
   generateStaticProps,
-} from '@hashicorp/react-docs-page/server'
+} from 'components/_temp-enable-hidden-pages'
 
 //  Configure the docs path
 const baseRoute = 'commands'
 const navDataFile = `data/${baseRoute}-nav-data.json`
+const navDataFileHidden = `data/${baseRoute}-nav-data-hidden.json`
 const localContentDir = `content/${baseRoute}`
 const mainBranch = 'master'
 const product = { name: productName, slug: productSlug }
@@ -20,7 +28,11 @@ export default function CommandsLayout(props) {
 }
 
 export async function getStaticPaths() {
-  const paths = await generateStaticPaths({ localContentDir, navDataFile })
+  const paths = await generateStaticPaths({
+    localContentDir,
+    navDataFile,
+    navDataFileHidden,
+  })
   return { paths, fallback: false }
 }
 
@@ -29,6 +41,7 @@ export async function getStaticProps({ params }) {
     localContentDir,
     mainBranch,
     navDataFile,
+    navDataFileHidden,
     params,
     product,
   })

website/pages/docs/[[...page]].jsx

@@ -2,15 +2,23 @@ import { productName, productSlug } from 'data/metadata'
 import DocsPage from '@hashicorp/react-docs-page'
 import ConfigEntryReference from 'components/config-entry-reference'
 // Imports below are only used server-side
+/**
+ * DEBT: short term patch for "hidden" docs-sidenav items.
+ * See components/_temp-enable-hidden-pages for details.
+ * Revert to importing from @hashicorp/react-docs-page/server
+ * once https://app.asana.com/0/1100423001970639/1200197752405255/f
+ * is complete.
+ **/
 import {
   generateStaticPaths,
   generateStaticProps,
-} from '@hashicorp/react-docs-page/server'
+} from 'components/_temp-enable-hidden-pages'
 
 //  Configure the docs path
 const additionalComponents = { ConfigEntryReference }
 const baseRoute = 'docs'
 const navDataFile = `data/${baseRoute}-nav-data.json`
+const navDataFileHidden = `data/${baseRoute}-nav-data-hidden.json`
 const localContentDir = `content/${baseRoute}`
 const mainBranch = 'master'
 const product = { name: productName, slug: productSlug }
@@ -27,7 +35,11 @@ export default function DocsLayout(props) {
 }
 
 export async function getStaticPaths() {
-  const paths = await generateStaticPaths({ localContentDir, navDataFile })
+  const paths = await generateStaticPaths({
+    localContentDir,
+    navDataFile,
+    navDataFileHidden,
+  })
   return { paths, fallback: false }
 }
 
@@ -37,6 +49,7 @@ export async function getStaticProps({ params }) {
     localContentDir,
     mainBranch,
     navDataFile,
+    navDataFileHidden,
     params,
     product,
   })