From 6ba4ec7e5917d4a531eaa4446fbf12bdbc561dc9 Mon Sep 17 00:00:00 2001 From: Maya Raman Date: Mon, 20 Nov 2023 13:06:48 -0500 Subject: [PATCH 1/4] package.json --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 2252384d7..9ad5bd08c 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "name": "snooty", + "type": "module", "version": "0.15.1", "repository": "github:mongodb/snooty", "engines": { @@ -102,7 +103,7 @@ "clipboard": "^2.0.8", "dotenv": "^8.2.0", "eventsource": "^2.0.2", - "gatsby": "^5.0.0", + "gatsby": "^5.3.0", "gatsby-plugin-emotion": "^8.0.0", "gatsby-plugin-google-tagmanager": "^5.0.0", "gatsby-plugin-layout": "^4.7.0", From 8d31e56a92a0f18b0cf0bd5ca2dd56085a4e043e Mon Sep 17 00:00:00 2001 From: Maya Raman Date: Mon, 20 Nov 2023 13:08:34 -0500 Subject: [PATCH 2/4] package-lock.json --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 8b12c84a3..9691c61a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -55,7 +55,7 @@ "clipboard": "^2.0.8", "dotenv": "^8.2.0", "eventsource": "^2.0.2", - "gatsby": "^5.0.0", + "gatsby": "^5.3.0", "gatsby-plugin-emotion": "^8.0.0", "gatsby-plugin-google-tagmanager": "^5.0.0", "gatsby-plugin-layout": "^4.7.0", From ea5b6332ea4433b2580fb454fb6f2f18e568e89d Mon Sep 17 00:00:00 2001 From: Maya Raman Date: Mon, 20 Nov 2023 16:41:12 -0500 Subject: [PATCH 3/4] branch --- .eslintrc.js | 2 +- .lintstagedrc.js | 4 +- __mocks__/gatsby.js | 4 +- __mocks__/redoc.js | 2 +- gatsby-config.js | 39 --- jest-preprocess.js | 4 +- jest.config.js | 2 +- .../gatsby-node.js | 257 --------------- .../other-things-to-source.js | 14 - .../utils/data-consumer.js | 180 ----------- .../utils/kanopy-auth.js | 75 ----- .../utils/post-build.js | 55 ---- .../gatsby-source-snooty-prod/gatsby-node.js | 304 ------------------ plugins/utils/openapi.js | 117 ------- plugins/utils/products.js | 25 -- scripts/ensure-master.js | 2 +- src/build-constants.js | 12 +- src/components/Admonition.js | 2 +- src/components/Banner/SiteBanner.js | 2 +- src/components/Chapters/Chapter.js | 2 +- src/components/Chapters/index.js | 4 +- src/components/Code/code-context.js | 2 +- src/components/Cond.js | 2 +- .../DefinitionList/DefinitionListItem.js | 2 +- src/components/DocumentBody.js | 8 +- src/components/DocumentBodyPreview.js | 6 +- src/components/Emphasis.js | 2 +- src/components/Figure/Lightbox.js | 1 - src/components/Figure/index.js | 2 +- src/components/Footnote/FootnoteReference.js | 2 +- src/components/Footnote/index.js | 2 +- src/components/GuideNext/GuidesList.js | 2 +- src/components/HorizontalList.js | 2 +- src/components/Image.js | 2 +- src/components/Link.js | 2 +- src/components/Only.js | 2 +- src/components/OpenAPI/index.js | 2 +- .../utils/filterHiddenChanges.js | 2 +- .../utils/getResourceLinkUrl.js | 4 +- src/components/RefRole.js | 2 +- .../SearchResults/Facets/useFacets.js | 2 +- src/components/Sidenav/GuidesTOCTree.js | 2 +- src/components/Strong.js | 2 +- .../StructuredData/BreadcrumbSchema.js | 2 +- src/components/Tabs/TabSelectors.js | 2 +- src/components/Tabs/index.js | 2 +- src/components/Time.js | 2 +- src/components/Topic.js | 2 +- src/components/Twitter.js | 2 +- src/components/VersionDropdown/utils.js | 2 +- .../Widgets/QuizWidget/QuizChoice.js | 2 +- .../Widgets/QuizWidget/QuizWidget.js | 2 +- src/context/toc-context.js | 2 +- src/hooks/__mocks__/use-site-metadata.js | 2 +- src/hooks/use-canonical-url.js | 2 +- src/hooks/use-marian-manifests.js | 2 +- src/init/DocumentDatabase.js | 130 -------- src/templates/product-landing.js | 2 +- src/utils/assert-trailing-slash.js | 8 - src/utils/base-url.js | 4 +- src/utils/find-all-key-value-pairs.js | 2 +- src/utils/find-key-value-pair.js | 2 +- src/utils/generate-path-prefix.js | 5 +- src/utils/get-meta-from-directive.js | 4 +- src/utils/get-nested-value.js | 14 - src/utils/get-page-slug.js | 6 - src/utils/get-page-title.js | 2 +- src/utils/get-plaintext.js | 2 +- src/utils/is-gatsby-preview.js | 2 +- src/utils/normalize-path.js | 2 +- src/utils/remove-nested-value.js | 25 -- src/utils/search-params-to-url.js | 2 +- src/utils/setup/construct-build-filter.js | 4 +- src/utils/setup/construct-page-id-prefix.js | 4 - src/utils/setup/fetch-manifest-metadata.js | 6 +- src/utils/setup/init-realm.js | 6 +- src/utils/setup/save-asset-files.js | 40 --- src/utils/setup/transform-breadcrumbs.js | 16 - src/utils/setup/validate-env-variables.js | 103 ------ src/utils/site-metadata.js | 79 ----- src/utils/url-utils.js | 4 +- stubs/process.js | 17 +- tests/testSetup.js | 3 +- tests/unit/Chapters.test.js | 2 +- tests/unit/Meta.test.js | 2 +- tests/unit/__mockStyles.js | 2 +- tests/unit/__snapshots__/Figure.test.js.snap | 1 - .../unit/__snapshots__/Lightbox.test.js.snap | 1 - .../unit/utils/assert-trailing-slash.test.js | 2 +- tests/unit/utils/generate-path-prefix.test.js | 2 +- tests/unit/utils/join-class-names.test.js | 2 +- 91 files changed, 108 insertions(+), 1589 deletions(-) delete mode 100644 gatsby-config.js delete mode 100644 plugins/gatsby-source-snooty-preview/gatsby-node.js delete mode 100644 plugins/gatsby-source-snooty-preview/other-things-to-source.js delete mode 100644 plugins/gatsby-source-snooty-preview/utils/data-consumer.js delete mode 100644 plugins/gatsby-source-snooty-preview/utils/kanopy-auth.js delete mode 100644 plugins/gatsby-source-snooty-preview/utils/post-build.js delete mode 100644 plugins/gatsby-source-snooty-prod/gatsby-node.js delete mode 100644 plugins/utils/openapi.js delete mode 100644 plugins/utils/products.js delete mode 100644 src/init/DocumentDatabase.js delete mode 100644 src/utils/assert-trailing-slash.js delete mode 100644 src/utils/get-nested-value.js delete mode 100644 src/utils/get-page-slug.js delete mode 100644 src/utils/remove-nested-value.js delete mode 100644 src/utils/setup/construct-page-id-prefix.js delete mode 100644 src/utils/setup/save-asset-files.js delete mode 100644 src/utils/setup/transform-breadcrumbs.js delete mode 100644 src/utils/setup/validate-env-variables.js delete mode 100644 src/utils/site-metadata.js diff --git a/.eslintrc.js b/.eslintrc.js index cd311573b..3aecc590a 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,4 +1,4 @@ -module.exports = { +export default { globals: { __PATH_PREFIX__: true, browser: true, diff --git a/.lintstagedrc.js b/.lintstagedrc.js index 5b6e7c0f1..6722c1e86 100644 --- a/.lintstagedrc.js +++ b/.lintstagedrc.js @@ -1,4 +1,4 @@ -const { CLIEngine } = require('eslint'); +import CLIEngine from 'eslint'; // lint-staged seems to disregard eslint ignore rules if we have the eslint // "--max-warnings 0" option set. This will potentially break our linting process @@ -7,7 +7,7 @@ const { CLIEngine } = require('eslint'); // This was the recommended fix according to lint-staged's README: // https://github.com/okonet/lint-staged/blob/fa15d686deb90b7ffddfbcf644d56ed05fcd8a38/README.md#how-can-i-ignore-files-from-eslintignore -module.exports = { +export default { '*.js': (files) => { const cli = new CLIEngine({}); const filesToLint = files.filter((file) => !cli.isPathIgnored(file)).join(' '); diff --git a/__mocks__/gatsby.js b/__mocks__/gatsby.js index c7c4562ed..518af7019 100644 --- a/__mocks__/gatsby.js +++ b/__mocks__/gatsby.js @@ -1,8 +1,8 @@ -const React = require('react'); +import React from 'react'; const gatsby = jest.requireActual('gatsby'); -module.exports = { +export default { ...gatsby, graphql: jest.fn(), navigate: jest.fn(), diff --git a/__mocks__/redoc.js b/__mocks__/redoc.js index 2f16610aa..f51fa436a 100644 --- a/__mocks__/redoc.js +++ b/__mocks__/redoc.js @@ -1,5 +1,5 @@ // Mock redoc library by default to avoid nested import error in RedocStandalone component // import * as YAML from './dist/index.js' -module.exports = { +export default { RedocStandalone: jest.fn(), }; diff --git a/gatsby-config.js b/gatsby-config.js deleted file mode 100644 index a1ca68e57..000000000 --- a/gatsby-config.js +++ /dev/null @@ -1,39 +0,0 @@ -const { generatePathPrefix } = require('./src/utils/generate-path-prefix'); -const { siteMetadata } = require('./src/utils/site-metadata'); -const { isGatsbyPreview } = require('./src/utils/is-gatsby-preview'); - -const isPreview = isGatsbyPreview(); -const pathPrefix = !isPreview ? generatePathPrefix(siteMetadata) : undefined; - -console.log('PATH PREFIX', pathPrefix); - -// Specifies which plugins to use depending on build environment -const plugins = ['gatsby-plugin-emotion', isPreview ? 'gatsby-source-snooty-preview' : 'gatsby-source-snooty-prod']; - -// PRODUCTION DEPLOYMENTS -- -// If not a preview build, use the layout that includes the -// consistent navbar and footer and generate a sitemap. -if (!isPreview) { - plugins.push(`gatsby-plugin-sitemap`); - const layoutComponentRelativePath = `./src/layouts/index.js`; - plugins.push({ - resolve: 'gatsby-plugin-layout', - options: { - component: require.resolve(layoutComponentRelativePath), - }, - }); -} else { - const layoutComponentRelativePath = `./src/layouts/preview-layout-outer.js`; - plugins.push({ - resolve: 'gatsby-plugin-layout', - options: { - component: require.resolve(layoutComponentRelativePath), - }, - }); -} - -module.exports = { - plugins, - pathPrefix, - siteMetadata, -}; diff --git a/jest-preprocess.js b/jest-preprocess.js index c01b8c912..c7a987b4e 100644 --- a/jest-preprocess.js +++ b/jest-preprocess.js @@ -1,6 +1,8 @@ +import { createTransformer } from 'babel-jest'; + const babelOptions = { presets: ['babel-preset-gatsby', '@emotion/babel-preset-css-prop'], plugins: ['@emotion'], }; -module.exports = require('babel-jest').createTransformer(babelOptions); +export default createTransformer(babelOptions); diff --git a/jest.config.js b/jest.config.js index 32dd9dfcc..91e8090a1 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,4 +1,4 @@ -module.exports = { +export default { globals: { __PATH_PREFIX__: ``, }, diff --git a/plugins/gatsby-source-snooty-preview/gatsby-node.js b/plugins/gatsby-source-snooty-preview/gatsby-node.js deleted file mode 100644 index 9a7820842..000000000 --- a/plugins/gatsby-source-snooty-preview/gatsby-node.js +++ /dev/null @@ -1,257 +0,0 @@ -const { getDataStore } = require('gatsby/dist/datastore'); -const path = require('path'); -const stream = require('stream'); -const { promisify } = require('util'); -const pipeline = promisify(stream.pipeline); -const got = require(`got`); -const { parser } = require(`stream-json/jsonl/Parser`); -const { sourceNodes } = require(`./other-things-to-source`); -const { fetchClientAccessToken } = require('./utils/kanopy-auth.js'); -const { callPostBuildWebhook } = require('./utils/post-build.js'); -const { - consumeData, - createSnootyMetadataId, - KEY_LAST_FETCHED, - KEY_LAST_CLIENT_ACCESS_TOKEN, -} = require('./utils/data-consumer.js'); - -// Global variable to allow webhookBody from sourceNodes step to be passed down -// to other Gatsby build steps that might not pass webhookBody natively. -let currentWebhookBody = {}; - -exports.createSchemaCustomization = async ({ actions }) => { - const { createTypes } = actions; - const typeDefs = ` - type Page implements Node @dontInfer { - page_id: String - branch: String - pagePath: String - ast: JSON! - metadata: SnootyMetadata @link - } - - type PagePath implements Node @dontInfer { - page_id: String! - branch: String! - project: String! - pageNodeId: String! - } - - type SnootyMetadata implements Node @dontInfer { - metadata: JSON - branch: String - project: String - } - - type RemoteMetadata implements Node @dontInfer { - remoteMetadata: JSON - } - - type ChangelogData implements Node @dontInfer { - changelogData: JSON - } - `; - createTypes(typeDefs); -}; - -const APIBase = process.env.API_BASE || `https://snooty-data-api.mongodb.com`; -const GATSBY_CLOUD_SITE_USER = process.env.GATSBY_CLOUD_SITE_USER; - -let isFirstRun = true; -exports.sourceNodes = async ({ - actions, - createNodeId, - getNode, - getNodesByType, - reporter, - createContentDigest, - cache, - webhookBody, -}) => { - console.log({ webhookBody }); - currentWebhookBody = webhookBody; - let hasOpenAPIChangelog = false; - const { createNode, touchNode } = actions; - - const fileWritePromises = []; - const lastFetched = (await cache.get(KEY_LAST_FETCHED)) || 0; - const lastClientAccessToken = await cache.get(KEY_LAST_CLIENT_ACCESS_TOKEN); - console.log({ lastFetched }); - - if (isFirstRun && lastFetched) { - // nodes of following types are managed statefully: - // SnootyMetadata, Page, PagePath - // we need to touch on them on delta updates on first run of a process to prevent them from being garbage collected - const datastore = getDataStore(); - for (const nodeType of ['SnootyMetadata', 'Page', 'PagePath']) { - for (const node of datastore.iterateNodesByType(nodeType)) { - touchNode(node); - } - } - } - - try { - if (!GATSBY_CLOUD_SITE_USER) { - throw new Error('Missing GATSBY_CLOUD_SITE_USER'); - } - - // Generate client access token only if trying to access Snooty Data API's staging instance - const clientAccessToken = APIBase.includes('.staging') ? await fetchClientAccessToken(lastClientAccessToken) : ''; - let url; - if (lastFetched) { - url = `${APIBase}/user/${GATSBY_CLOUD_SITE_USER}/documents?updated=${lastFetched}`; - } else { - url = `${APIBase}/user/${GATSBY_CLOUD_SITE_USER}/documents`; - } - - const headers = {}; - if (clientAccessToken) { - headers['Authorization'] = `Bearer ${clientAccessToken}`; - } - const httpStream = got.stream(url, { headers }); - - let pageCount = 0; - // Callback function to be ran after a valid page has been found and handled. - // Tracks and updates information that spans across several data entries - const onHandlePage = (pageTemplate, pageId, pageNodeId) => { - if (pageTemplate === 'changelog') hasOpenAPIChangelog = true; - pageCount += 1; - if (pageCount % 1000 === 0) { - console.log({ pageCount, page_id: pageId, id: pageNodeId }); - } - }; - - // Since there's a lot of data incoming from the Snooty Data API, we stream - // the data in chunks and parse them as they come instead of fetching everything - // as a single JSON response - const decode = parser(); - decode.on('data', async (_entry) => { - // Un-nest data - const entry = _entry.value; - await consumeData(entry, { - actions, - cache, - clientAccessToken, - createContentDigest, - createNodeId, - fileWritePromises, - getNode, - onHandlePage, - }); - }); - - console.time(`source updates`); - // Wait for HTTP connection to close. - await pipeline(httpStream, decode); - console.timeEnd(`source updates`); - } catch (error) { - callPostBuildWebhook(webhookBody, 'failed'); - reporter.panic('There was an issue sourcing nodes', error); - } - - // Wait for all assets to be written. - await Promise.all(fileWritePromises); - - // Source old nodes. - console.time(`old source nodes`); - await sourceNodes({ - hasOpenAPIChangelog, - github_username: GATSBY_CLOUD_SITE_USER, - createNode, - createContentDigest, - createNodeId, - getNodesByType, - }); - console.timeEnd(`old source nodes`); - isFirstRun = false; -}; - -// Prevent errors when running gatsby build caused by browser packages run in a node environment. -exports.onCreateWebpackConfig = ({ plugins, actions }) => { - const providePlugins = { - Buffer: ['buffer', 'Buffer'], - process: require.resolve('../../stubs/process.js'), - }; - - const fallbacks = { stream: require.resolve('stream-browserify'), buffer: require.resolve('buffer/') }; - - actions.setWebpackConfig({ - plugins: [plugins.provide(providePlugins)], - resolve: { - fallback: fallbacks, - alias: { - process: 'process/browser', - }, - }, - }); -}; - -exports.createPages = async ({ actions, createNodeId, getNode, graphql, reporter }) => { - const { createPage } = actions; - const templatePath = path.join(__dirname, `../../src/components/DocumentBodyPreview.js`); - const result = await graphql(` - query { - allPagePath { - totalCount - nodes { - pageNodeId - branch - page_id - project - } - allProjects: distinct(field: { project: SELECT }) - } - } - `); - - if (result.errors) { - await callPostBuildWebhook(currentWebhookBody, 'failed'); - reporter.panic('There was an error in the graphql query', result.errors); - } - - try { - result.data.allPagePath.nodes.forEach((node) => { - const pagePath = path.join(node.project, node.branch, node.page_id); - let slug = node.page_id; - // Slices off leading slash to ensure slug matches an entry within the toctreeOrder and renders InternalPageNav components - if (slug !== '/' && slug[0] === '/') slug = slug.slice(1); - - const metadataNodeId = createSnootyMetadataId({ createNodeId, branch: node.branch, project: node.project }); - if (!getNode(metadataNodeId)) { - // Take into account the possibility of having new page data available through the API, - // but no metadata yet due to async uploads - console.warn( - `Skipping node creation for page "${node.page_id}", in project "${node.project}" on branch "${node.branch}". No metadata node "${metadataNodeId}" found.` - ); - return; - } - - createPage({ - path: pagePath, - component: templatePath, - context: { - id: node.pageNodeId, - slug, - // Hardcode static/safe values to prevent incremental builds from rebuilding versioned preview pages - repoBranches: {}, - associatedReposInfo: {}, - isAssociatedProduct: false, - project: node.project, - }, - }); - }); - } catch (err) { - await callPostBuildWebhook(currentWebhookBody, 'failed'); - reporter.panic('Could not build pages off of graphl query', err); - } -}; - -// `onPostBuild` is run by Gatsby Cloud after everything is built, but before the -// content is deployed to the preview site. This can result in a short delay between -// when the post-build webhook is called and when the content is updated. -// Ideally, we would use Gatsby Cloud's Outgoing Notifications feature once it can -// support passing through custom data from the preview webhook's body (to include the -// Autobuilder job ID associated with the GC build). -exports.onPostBuild = async () => { - await callPostBuildWebhook(currentWebhookBody, 'completed'); -}; diff --git a/plugins/gatsby-source-snooty-preview/other-things-to-source.js b/plugins/gatsby-source-snooty-preview/other-things-to-source.js deleted file mode 100644 index 4b88bcdf5..000000000 --- a/plugins/gatsby-source-snooty-preview/other-things-to-source.js +++ /dev/null @@ -1,14 +0,0 @@ -const { siteMetadata } = require('../../src/utils/site-metadata'); -const { realmDocumentDatabase } = require('../../src/init/DocumentDatabase.js'); -const { createOpenAPIChangelogNode } = require('../utils/openapi'); -const { createProductNodes } = require('../utils/products'); - -// Sources nodes for the preview plugin that are not directly related to data -// from the Snooty Data API -exports.sourceNodes = async ({ hasOpenAPIChangelog, createNode, createContentDigest, createNodeId }) => { - let db = realmDocumentDatabase; - await db.connect(); - await createProductNodes({ db, createNode, createNodeId, createContentDigest }); - if (hasOpenAPIChangelog) - await createOpenAPIChangelogNode({ createNode, createNodeId, createContentDigest, siteMetadata, db }); -}; diff --git a/plugins/gatsby-source-snooty-preview/utils/data-consumer.js b/plugins/gatsby-source-snooty-preview/utils/data-consumer.js deleted file mode 100644 index 8ead9cb77..000000000 --- a/plugins/gatsby-source-snooty-preview/utils/data-consumer.js +++ /dev/null @@ -1,180 +0,0 @@ -const { transformBreadcrumbs } = require('../../../src/utils/setup/transform-breadcrumbs.js'); -const { saveStaticFiles, saveFile } = require('../../../src/utils/setup/save-asset-files'); -const { getNestedValue } = require('../../../src/utils/get-nested-value'); - -const KEY_LAST_FETCHED = 'lastFetched'; -const KEY_LAST_CLIENT_ACCESS_TOKEN = 'lastClientAccessToken'; - -function createSnootyMetadataId({ branch, project, createNodeId }) { - return createNodeId(`metadata-${branch}-${project}`); -} - -// Syncs the plugin with timestamp of data being returned from the API. -const handleTimestamp = (data, { cache, clientAccessToken }) => { - cache.set(KEY_LAST_FETCHED, data); - cache.set(KEY_LAST_CLIENT_ACCESS_TOKEN, clientAccessToken); -}; - -const handleAsset = (data, { fileWritePromises }) => { - const { filenames, assetData } = data; - - // Incorrect asset format should not be acceptable - if (!filenames || !filenames.length) { - throw new Error('No filenames found for asset'); - } - if (!assetData) { - throw new Error('Missing asset data'); - } - - filenames.forEach((filePath) => { - // These promises will be resolved once all data is consumed - fileWritePromises.push(saveFile(filePath, Buffer.from(assetData, 'base64'))); - }); -}; - -const handleMetadata = async ( - data, - { createContentDigest, createNode, createNodeId, deleteNode, getNode, shouldDeleteContentNode } -) => { - const { _id, build_id, created_at, static_files: staticFiles, ...metadataMinusStatic } = data; - const { parentPaths, slugToTitle, branch, project } = metadataMinusStatic; - const nodeId = createSnootyMetadataId({ createNodeId, branch, project }); - - if (shouldDeleteContentNode) { - deleteNode(getNode(nodeId)); - return; - } - - if (parentPaths) { - transformBreadcrumbs(parentPaths, slugToTitle); - } - - // Save files in the static_files field of metadata document, including intersphinx inventories. - if (staticFiles) { - await saveStaticFiles(staticFiles); - } - - createNode({ - children: [], - id: nodeId, - internal: { - contentDigest: createContentDigest(metadataMinusStatic), - type: 'SnootyMetadata', - }, - branch, - project, - parent: null, - metadata: metadataMinusStatic, - }); -}; - -const handlePage = ( - data, - { createContentDigest, createNode, createNodeId, deleteNode, getNode, onHandlePage, shouldDeleteContentNode } -) => { - // Strip source string, in case it exists. We don't need the raw source of the AST - const { source, ...page } = data; - - const filename = getNestedValue(['filename'], page) || ''; - // There can be ASTs for included .rst files as well. We should skip these, in case - // we encounter them - if (!filename || !filename.endsWith('.txt')) { - console.warn(`Found an AST that is not for a page: ${filename}`); - return; - } - - const branch = page.page_id.split('/')[2]; - const raw_page_id = page.page_id.split('/').slice(3).join('/'); - const page_id = raw_page_id === 'index' ? '/' : `/${raw_page_id}`; - const project = page.page_id.split('/')[0]; - - const pageNodeId = createNodeId(page_id + project + branch); - const pagePathNodeId = pageNodeId + '/path'; - - if (shouldDeleteContentNode) { - deleteNode(getNode(pageNodeId)); - deleteNode(getNode(pagePathNodeId)); - return; - } - - page.page_id = page_id; - page.metadata = createSnootyMetadataId({ createNodeId, branch, project }); - page.id = pageNodeId; - page.internal = { - type: 'Page', - contentDigest: createContentDigest(page), - }; - - const pagePathNode = { - id: pagePathNodeId, - page_id, - branch, - project, - pageNodeId: page.id, - internal: { - type: 'PagePath', - contentDigest: page.internal.contentDigest, - }, - }; - - createNode(page); - createNode(pagePathNode); - - const pageTemplate = data.ast?.options?.template; - onHandlePage(pageTemplate, page_id, pageNodeId); -}; - -/** - * Handles incoming data accordingly based on its data type. Notably, this handles - * converting build data from the Snooty Data API into nodes and assets that the - * Gatsby site will need to render pages. - * @param {*} entry - A single data entry obtained from the Snooty Data API - * @param {*} options - Gatsby functions and other utilities to be used by handlers - */ -const consumeData = async ( - entry, - { actions, cache, createNodeId, createContentDigest, getNode, fileWritePromises, clientAccessToken, onHandlePage } -) => { - const { type, data } = entry; - - // Shape and format should be consistent across all data - if (!type) { - throw new Error('Data entry is missing data type'); - } - if (!data) { - throw new Error('Data entry is missing data'); - } - - const shouldDeleteContentNode = data.deleted; - const { createNode, deleteNode } = actions; - - if (type === 'timestamp') { - handleTimestamp(data, { cache, clientAccessToken }); - } else if (type === 'asset') { - handleAsset(data, { fileWritePromises }); - } else if (type === 'metadata') { - await handleMetadata(data, { - createContentDigest, - createNode, - createNodeId, - deleteNode, - getNode, - shouldDeleteContentNode, - }); - } else if (type === 'page') { - handlePage(data, { - createContentDigest, - createNode, - createNodeId, - deleteNode, - getNode, - onHandlePage, - shouldDeleteContentNode, - }); - } else { - // Shouldn't affect current builds - console.warn(`Unexpected data type: ${type}`); - } -}; - -module.exports = { consumeData, createSnootyMetadataId, KEY_LAST_FETCHED, KEY_LAST_CLIENT_ACCESS_TOKEN }; diff --git a/plugins/gatsby-source-snooty-preview/utils/kanopy-auth.js b/plugins/gatsby-source-snooty-preview/utils/kanopy-auth.js deleted file mode 100644 index 51fde316c..000000000 --- a/plugins/gatsby-source-snooty-preview/utils/kanopy-auth.js +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Checks if the configuration property exists, and throws an error otherwise. - * @param configProp - * @param configType - */ -const validateConfigType = (configProp, configType) => { - if (!configProp) { - throw new Error(`Missing ${configType} for Snooty Data API access`); - } -}; - -/** - * Generates the authZ token needed for requesting an access token to Kanopy. - */ -const getClientCredentialsHeader = () => { - const clientId = process.env.OAUTH_CLIENT_ID; - validateConfigType(clientId, 'client ID'); - const clientSecret = process.env.OAUTH_CLIENT_SECRET; - validateConfigType(clientSecret, 'client secret'); - return Buffer.from(`${clientId}:${clientSecret}`, 'utf-8').toString('base64'); -}; - -/** - * Generates a new access token to allow for authentication against Kanopy services. - */ -const generateNewAccessToken = async () => { - const grantType = process.env.OAUTH_GRANT_TYPE; - validateConfigType(grantType, 'grant type'); - const scope = process.env.OAUTH_SCOPE; - validateConfigType(scope, 'scope'); - const authUrl = process.env.OAUTH_TOKEN_AUTH_URL; - validateConfigType(authUrl, 'auth token url'); - - // Request a new access token from Kanopy's token authentication endpoint - const authRequestUrl = `${authUrl}?grant_type=${grantType}&scope=${scope}`; - const headers = { - authorization: `Basic ${getClientCredentialsHeader()}`, - accept: 'application/json', - 'Content-Type': 'application/x-www-form-urlencoded', - 'cache-control': 'no-cache', - }; - const res = await fetch(authRequestUrl, { method: 'POST', headers }); - if (!res.ok) { - throw new Error('Error trying to request new access token'); - } - - const data = await res.json(); - const token = data['access_token']; - if (!token) { - throw new Error('Could not find new access token'); - } - - return token; -}; - -/** - * Returns a valid client access token that can be used for machine-machine communication - * between a client and a service hosted on Kanopy. - * See: https://kanopy.corp.mongodb.com/docs/development/authentication_and_authorization/ - * @param prevToken - */ -const fetchClientAccessToken = async (prevToken) => { - if (!prevToken) { - return generateNewAccessToken(); - } - // Decoded value is the JSON object representation of the token string, with token's expiration date - const decodedValue = JSON.parse(Buffer.from(prevToken.split('.')[1], 'base64').toString('ascii')); - // Check if token is expired, or near expiration - if (decodedValue.exp < Date.now() / 1000) { - return generateNewAccessToken(); - } - return prevToken; -}; - -module.exports = { fetchClientAccessToken }; diff --git a/plugins/gatsby-source-snooty-preview/utils/post-build.js b/plugins/gatsby-source-snooty-preview/utils/post-build.js deleted file mode 100644 index 84f429290..000000000 --- a/plugins/gatsby-source-snooty-preview/utils/post-build.js +++ /dev/null @@ -1,55 +0,0 @@ -const crypto = require('crypto'); - -/** - * Constructs a signature using the payload and Snooty's secret. The signature - * can be used to help webhooks be more confident that the caller is Snooty. - * @param {string} payloadString - */ -const constructSnootyHeader = (payloadString) => - crypto.createHmac('sha256', process.env.SNOOTY_SECRET).update(payloadString).digest('hex'); - -/** - * Calls the post-build webhook to let the Autobuilder know that the Gatsby Cloud - * build is finished. - * @param {object} webhookBody - The webhook body passed to the source plugin to - * initiate the preview build. - * @param {string} status - The status of the build, typically "completed" or "failed". - * This value should coincide with the Autobuilder's job statuses. - */ -const callPostBuildWebhook = async (webhookBody, status) => { - // Webhook body could be empty if the Gatsby Cloud site is doing a fresh build - // that was not called by the preview webhook - if (!webhookBody || !Object.keys(webhookBody).length) { - console.log('No webhookBody found. This build will not call the post-build webhook.'); - return; - } - - const supportedStatuses = ['completed', 'failed']; - if (!supportedStatuses.includes(status)) { - console.log(`Post-build webhook call does not support status "${status}".`); - return; - } - - const payload = { - ...webhookBody, - status, - }; - const body = JSON.stringify(payload); - const headers = { - 'x-snooty-signature': constructSnootyHeader(body), - }; - - console.log(`Calling post-build webhook with status "${status}".`); - const res = await fetch(process.env.AUTOBUILDER_POST_BUILD_WEBHOOK, { method: 'POST', body, headers }); - // Calling the webhook from this function should assume we are fulfilling the requirements of the call. - // Any error thrown here is definitely unexpected. - if (!res.ok) { - const errMessage = await res.text(); - throw new Error( - `There was an issue calling the Autobuilder post-build webhook. Please have the DOP team check CloudWatch logs. ${errMessage}` - ); - } - console.log('Post-build webhook was successfully called!'); -}; - -module.exports = { callPostBuildWebhook }; diff --git a/plugins/gatsby-source-snooty-prod/gatsby-node.js b/plugins/gatsby-source-snooty-prod/gatsby-node.js deleted file mode 100644 index 0173a5439..000000000 --- a/plugins/gatsby-source-snooty-prod/gatsby-node.js +++ /dev/null @@ -1,304 +0,0 @@ -const path = require('path'); -const { transformBreadcrumbs } = require('../../src/utils/setup/transform-breadcrumbs.js'); -const { saveAssetFiles, saveStaticFiles } = require('../../src/utils/setup/save-asset-files'); -const { validateEnvVariables } = require('../../src/utils/setup/validate-env-variables'); -const { getNestedValue } = require('../../src/utils/get-nested-value'); -const { removeNestedValue } = require('../../src/utils/remove-nested-value.js'); -const { getPageSlug } = require('../../src/utils/get-page-slug'); -const { manifestMetadata, siteMetadata } = require('../../src/utils/site-metadata'); -const { assertTrailingSlash } = require('../../src/utils/assert-trailing-slash'); -const { constructPageIdPrefix } = require('../../src/utils/setup/construct-page-id-prefix'); -const { manifestDocumentDatabase, realmDocumentDatabase } = require('../../src/init/DocumentDatabase.js'); -const { createOpenAPIChangelogNode } = require('../utils/openapi.js'); -const { createProductNodes } = require('../utils/products.js'); - -// different types of references -const PAGES = []; - -// in-memory object with key/value = filename/document -let RESOLVED_REF_DOC_MAPPING = {}; - -const assets = new Map(); - -let db; - -let isAssociatedProduct = false; -const associatedReposInfo = {}; - -// Creates node for RemoteMetadata, mostly used for Embedded Versions. If no associated products -// or data are found, the node will be null -const createRemoteMetadataNode = async ({ createNode, createNodeId, createContentDigest }) => { - // fetch associated child products - const productList = manifestMetadata?.associated_products || []; - await Promise.all( - productList.map(async (product) => { - associatedReposInfo[product.name] = await db.realmInterface.fetchDocset({ project: product.name }); - }) - ); - // check if product is associated child product - try { - const umbrellaProduct = await db.realmInterface.getMetadata({ - 'associated_products.name': siteMetadata.project, - }); - isAssociatedProduct = !!umbrellaProduct; - } catch (e) { - console.log('No umbrella product found. Not an associated product.'); - isAssociatedProduct = false; - } - - // get remote metadata for updated ToC in Atlas - try { - const filter = { - project: manifestMetadata.project, - branch: manifestMetadata.branch, - }; - if (isAssociatedProduct || manifestMetadata?.associated_products?.length) { - filter['is_merged_toc'] = true; - } - const findOptions = { - sort: { build_id: -1 }, - }; - const remoteMetadata = await db.realmInterface.getMetadata(filter, findOptions); - - createNode({ - children: [], - id: createNodeId('remoteMetadata'), - internal: { - contentDigest: createContentDigest(remoteMetadata), - type: 'RemoteMetadata', - }, - parent: null, - remoteMetadata: remoteMetadata, - }); - } catch (e) { - console.error('Error while fetching metadata from Atlas, falling back to manifest metadata'); - console.error(e); - } -}; - -exports.sourceNodes = async ({ actions, createContentDigest, createNodeId }) => { - let hasOpenAPIChangelog = false; - const { createNode } = actions; - - // setup and validate env variables - const envResults = validateEnvVariables(manifestMetadata); - if (envResults.error) { - throw Error(envResults.message); - } - - // wait to connect to Realm - - if (siteMetadata.manifestPath) { - console.log('Loading documents from manifest'); - db = manifestDocumentDatabase; - } else { - console.log('Loading documents from realm'); - db = realmDocumentDatabase; - } - - await db.connect(); - - const documents = await db.getDocuments(); - - if (documents.length === 0) { - console.error( - 'Snooty could not find AST entries for the', - siteMetadata.parserBranch, - 'branch of', - siteMetadata.project, - 'within', - siteMetadata.database - ); - process.exit(1); - } - const pageIdPrefix = constructPageIdPrefix(siteMetadata); - documents.forEach((doc) => { - const { page_id, ...rest } = doc; - RESOLVED_REF_DOC_MAPPING[page_id.replace(`${pageIdPrefix}/`, '')] = rest; - }); - - // Identify page documents and parse each document for images - Object.entries(RESOLVED_REF_DOC_MAPPING).forEach(([key, val]) => { - const pageNode = getNestedValue(['ast', 'children'], val); - const filename = getNestedValue(['filename'], val) || ''; - - // Parse each document before pages are created via createPage - // to remove all positions fields as it is only used in the parser for logging - removeNestedValue('position', 'children', [val?.ast]); - - if (pageNode) { - val.static_assets.forEach((asset) => { - const checksum = asset.checksum; - if (assets.has(checksum)) { - assets.set(checksum, new Set([...assets.get(checksum), asset.key])); - } else { - assets.set(checksum, new Set([asset.key])); - } - }); - } - - if (filename.endsWith('.txt') && !manifestMetadata.openapi_pages?.[key]) { - PAGES.push(key); - } - if (val?.ast?.options?.template === 'changelog') hasOpenAPIChangelog = true; - }); - - await createProductNodes({ db, createNode, createNodeId, createContentDigest }); - - await createRemoteMetadataNode({ createNode, createNodeId, createContentDigest }); - if (siteMetadata.project === 'cloud-docs' && hasOpenAPIChangelog) - await createOpenAPIChangelogNode({ createNode, createNodeId, createContentDigest, siteMetadata, db }); - - await saveAssetFiles(assets, db); - const { static_files: staticFiles, ...metadataMinusStatic } = await db.getMetadata(); - - const { parentPaths, slugToTitle } = metadataMinusStatic; - if (parentPaths) { - transformBreadcrumbs(parentPaths, slugToTitle); - } - - //Save files in the static_files field of metadata document, including intersphinx inventories - if (staticFiles) { - await saveStaticFiles(staticFiles); - } - - createNode({ - children: [], - id: createNodeId('metadata'), - internal: { - contentDigest: createContentDigest(metadataMinusStatic), - type: 'SnootyMetadata', - }, - parent: null, - metadata: metadataMinusStatic, - }); -}; - -exports.createPages = async ({ actions }) => { - const { createPage } = actions; - - let repoBranches = null; - try { - const repoInfo = await db.realmInterface.fetchDocset(); - let errMsg; - - if (!repoInfo) { - errMsg = `Repo data for ${siteMetadata.project} could not be found.`; - } - - // We should expect the number of branches for a docs repo to be 1 or more. - if (!repoInfo.branches?.length) { - errMsg = `No version information found for ${siteMetadata.project}`; - } - - if (errMsg) { - throw errMsg; - } - - // Handle inconsistent env names. Default to 'dotcomprd' when possible since this is what we will most likely use. - // dotcom environments seem to be consistent. - let envKey = siteMetadata.snootyEnv; - if (!envKey || envKey === 'development') { - envKey = 'dotcomprd'; - } else if (envKey === 'production') { - envKey = 'prd'; - } else if (envKey === 'staging') { - envKey = 'stg'; - } - - // We're overfetching data here. We only need branches and prefix at the least - repoBranches = { - branches: repoInfo.branches, - siteBasePrefix: repoInfo.prefix[envKey], - }; - - if (repoInfo.groups?.length > 0) { - repoBranches.groups = repoInfo.groups; - } - } catch (err) { - console.error(err); - throw err; - } - - return new Promise((resolve, reject) => { - PAGES.forEach((page) => { - const pageNodes = RESOLVED_REF_DOC_MAPPING[page]?.ast; - const slug = getPageSlug(page); - - // TODO: Gatsby v4 will enable code splitting automatically. Delete duplicate component, add conditional for consistent-nav UnifiedFooter - const mainComponentRelativePath = `../../src/components/DocumentBody.js`; - - if (RESOLVED_REF_DOC_MAPPING[page] && Object.keys(RESOLVED_REF_DOC_MAPPING[page]).length > 0) { - createPage({ - path: assertTrailingSlash(slug), - component: path.resolve(__dirname, mainComponentRelativePath), - context: { - slug, - repoBranches, - associatedReposInfo, - isAssociatedProduct, - template: pageNodes?.options?.template, - page: pageNodes, - }, - }); - } - }); - - resolve(); - }); -}; - -// Prevent errors when running gatsby build caused by browser packages run in a node environment. -exports.onCreateWebpackConfig = ({ plugins, actions }) => { - const providePlugins = { - Buffer: ['buffer', 'Buffer'], - process: require.resolve('../../stubs/process.js'), - }; - - const fallbacks = { stream: require.resolve('stream-browserify'), buffer: require.resolve('buffer/') }; - - actions.setWebpackConfig({ - plugins: [plugins.provide(providePlugins)], - resolve: { - fallback: fallbacks, - alias: { - process: 'process/browser', - }, - }, - }); -}; - -// Remove type inference, as our schema is too ambiguous for this to be useful. -// https://www.gatsbyjs.com/docs/scaling-issues/#switch-off-type-inference-for-sitepagecontext -exports.createSchemaCustomization = ({ actions }) => { - actions.createTypes(` - type Page implements Node @dontInfer { - page_id: String - branch: String - pagePath: String - ast: JSON! - metadata: SnootyMetadata @link - } - - type SnootyMetadata implements Node @dontInfer { - metadata: JSON - branch: String - project: String - } - - type PagePath implements Node @dontInfer { - page_id: String! - branch: String! - project: String! - pageNodeId: String! - } - - type RemoteMetadata implements Node @dontInfer { - remoteMetadata: JSON - } - - type ChangelogData implements Node @dontInfer { - changelogData: JSON - } - `); -}; diff --git a/plugins/utils/openapi.js b/plugins/utils/openapi.js deleted file mode 100644 index 96700bd39..000000000 --- a/plugins/utils/openapi.js +++ /dev/null @@ -1,117 +0,0 @@ -const { hideChanges, hideDiffChanges } = require('../../src/components/OpenAPIChangelog/utils/filterHiddenChanges'); - -const atlasAdminProdChangelogS3Prefix = 'https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/changelog'; -const atlasAdminDevChangelogS3Prefix = 'https://mongodb-mms-build-server.s3.amazonaws.com/openapi/changelog'; - -const fetchChangelogData = async (runId, versions, s3Prefix) => { - try { - /* Using metadata runId, fetch OpenAPI Changelog full change list */ - const changelogResp = await fetch(`${s3Prefix}/${runId}/changelog.json`); - const unfilteredChangelog = await changelogResp.json(); - const changelog = hideChanges(unfilteredChangelog); - - /* Aggregate all Resources in changelog for frontend filter */ - const resourcesListSet = new Set(); - changelog.forEach((release) => - release.paths.forEach(({ httpMethod, path }) => resourcesListSet.add(`${httpMethod} ${path}`)) - ); - const changelogResourcesList = Array.from(resourcesListSet); - - /* Fetch most recent Resource Versions' diff */ - const mostRecentResourceVersions = versions.slice(-2); - const mostRecentDiffLabel = mostRecentResourceVersions.join('_'); - const mostRecentDiffResp = await fetch(`${s3Prefix}/${runId}/${mostRecentDiffLabel}.json`); - const mostRecentUnfilteredDiffData = await mostRecentDiffResp.json(); - const mostRecentDiffData = hideDiffChanges(mostRecentUnfilteredDiffData); - - return { - changelog, - changelogResourcesList, - mostRecentDiff: { - mostRecentDiffLabel, - mostRecentDiffData, - }, - }; - } catch (error) { - console.warn('Changelog error: Most recent runId not successful. Using last successful runId to build Changelog.'); - throw error; - } -}; - -/* Creates node for ChangelogData, cuyrrently only used for OpenAPI Changelog in cloud-docs. */ -const createOpenAPIChangelogNode = async ({ createNode, createNodeId, createContentDigest, siteMetadata, db }) => { - try { - const { snootyEnv } = siteMetadata; - let atlasAdminChangelogS3Prefix; - let indexLocation; - switch (snootyEnv) { - case 'staging': - case 'development': - atlasAdminChangelogS3Prefix = atlasAdminDevChangelogS3Prefix; - indexLocation = 'dev.json'; - break; - case 'production': - default: - atlasAdminChangelogS3Prefix = atlasAdminProdChangelogS3Prefix; - indexLocation = 'prod.json'; - } - /* Fetch OpenAPI Changelog metadata */ - const indexResp = await fetch(`${atlasAdminChangelogS3Prefix}/${indexLocation}`); - const index = await indexResp.json(); - - const { runId, versions } = index; - - if (!runId || typeof runId !== 'string') - throw new Error('OpenAPI Changelog Error: `runId` not available in S3 index.json!'); - - let changelogData = { - index, - }; - try { - const receivedChangelogData = await fetchChangelogData(runId, versions, atlasAdminChangelogS3Prefix); - changelogData = { ...changelogData, ...receivedChangelogData }; - if (snootyEnv === 'production') await db.realmInterface.updateOAChangelogMetadata(index); - } catch (error) { - /* If any error occurs, fetch last successful metadata and build changelog node */ - const lastSuccessfulIndex = await db.realmInterface.fetchDocument( - 'openapi_changelog', - 'atlas_admin_metadata', - {} - ); - const { runId: lastRunId, versions: lastVersions } = lastSuccessfulIndex; - const receivedChangelogData = fetchChangelogData(lastRunId, lastVersions, atlasAdminProdChangelogS3Prefix); - changelogData = { index: lastSuccessfulIndex, ...receivedChangelogData }; - } - - /* Create Node for useStaticQuery with all Changelog data */ - createNode({ - children: [], - id: createNodeId('changelogData'), - internal: { - contentDigest: createContentDigest(changelogData), - type: 'ChangelogData', - }, - parent: null, - changelogData: changelogData, - }); - } catch (e) { - console.error('Error while fetching OpenAPI Changelog data from S3'); - console.error(e); - - /* Create empty Node for useStaticQuery to ensure successful build */ - createNode({ - children: [], - id: createNodeId('changelogData'), - internal: { - contentDigest: createContentDigest({}), - type: 'ChangelogData', - }, - parent: null, - changelogData: {}, - }); - } -}; - -module.exports = { - createOpenAPIChangelogNode, -}; diff --git a/plugins/utils/products.js b/plugins/utils/products.js deleted file mode 100644 index 60f30bdea..000000000 --- a/plugins/utils/products.js +++ /dev/null @@ -1,25 +0,0 @@ -const { baseUrl } = require('../../src/utils/base-url'); - -const createProductNodes = async ({ db, createNode, createNodeId, createContentDigest }) => { - // Get all MongoDB products for the sidenav - const products = await db.fetchAllProducts(); - products.forEach((product) => { - const url = baseUrl(product.baseUrl + product.slug); - - createNode({ - children: [], - id: createNodeId(`Product-${product.title}`), - internal: { - contentDigest: createContentDigest(product), - type: 'Product', - }, - parent: null, - title: product.title, - url, - }); - }); -}; - -module.exports = { - createProductNodes, -}; diff --git a/scripts/ensure-master.js b/scripts/ensure-master.js index 8a71baea5..4a31a02f6 100644 --- a/scripts/ensure-master.js +++ b/scripts/ensure-master.js @@ -1,4 +1,4 @@ -const { execSync } = require('child_process'); +import { execSync } from 'child_process'; const getGitBranch = () => { return execSync('git rev-parse --abbrev-ref HEAD') diff --git a/src/build-constants.js b/src/build-constants.js index 8d62e4e83..c4172f91b 100644 --- a/src/build-constants.js +++ b/src/build-constants.js @@ -1,6 +1,6 @@ -module.exports = { - DOCUMENTS_COLLECTION: 'documents', - ASSETS_COLLECTION: 'assets', - METADATA_COLLECTION: 'metadata', - SNOOTY_REALM_APP_ID: 'snooty-koueq', -}; +const DOCUMENTS_COLLECTION = 'documents'; +const ASSETS_COLLECTION = 'assets'; +const METADATA_COLLECTION = 'metadata'; +const SNOOTY_REALM_APP_ID = 'snooty-koueq'; + +export { DOCUMENTS_COLLECTION, ASSETS_COLLECTION, METADATA_COLLECTION, SNOOTY_REALM_APP_ID }; diff --git a/src/components/Admonition.js b/src/components/Admonition.js index 009b4791a..e01d9e61c 100644 --- a/src/components/Admonition.js +++ b/src/components/Admonition.js @@ -2,7 +2,7 @@ import React from 'react'; import PropTypes from 'prop-types'; import Callout, { Variant } from '@leafygreen-ui/callout'; import { cx, css } from '@leafygreen-ui/emotion'; -import { getPlaintext } from '../utils/get-plaintext'; +import getPlaintext from '../utils/get-plaintext'; import { theme } from '../theme/docsTheme'; import ComponentFactory from './ComponentFactory'; diff --git a/src/components/Banner/SiteBanner.js b/src/components/Banner/SiteBanner.js index 9a31d9aa9..1d9d14575 100644 --- a/src/components/Banner/SiteBanner.js +++ b/src/components/Banner/SiteBanner.js @@ -5,7 +5,7 @@ import { SNOOTY_REALM_APP_ID } from '../../build-constants'; import { useSiteMetadata } from '../../hooks/use-site-metadata'; import { theme } from '../../theme/docsTheme'; import { isBrowser } from '../../utils/is-browser'; -import { normalizePath } from '../../utils/normalize-path'; +import normalizePath from '../../utils/normalize-path'; import { fetchBanner } from '../../utils/realm'; const getBannerSource = (src) => { diff --git a/src/components/Chapters/Chapter.js b/src/components/Chapters/Chapter.js index 55666c8da..6aa65a0fa 100644 --- a/src/components/Chapters/Chapter.js +++ b/src/components/Chapters/Chapter.js @@ -8,7 +8,7 @@ import { palette } from '@leafygreen-ui/palette'; import { Body } from '@leafygreen-ui/typography'; import Link from '../Link'; import { theme } from '../../theme/docsTheme'; -import { getPlaintext } from '../../utils/get-plaintext'; +import getPlaintext from '../../utils/get-plaintext'; import ChapterNumberLabel from './ChapterNumberLabel'; // Height and width of image diff --git a/src/components/Chapters/index.js b/src/components/Chapters/index.js index 84f4a1384..80fa8986f 100644 --- a/src/components/Chapters/index.js +++ b/src/components/Chapters/index.js @@ -8,8 +8,8 @@ import ComponentFactory from '../ComponentFactory'; import CardGroup from '../Card/CardGroup'; import BookIcon from '../icons/Book'; import { theme } from '../../theme/docsTheme'; -import { assertTrailingSlash } from '../../utils/assert-trailing-slash'; -import { getPlaintext } from '../../utils/get-plaintext'; +import assertTrailingSlash from '../../utils/assert-trailing-slash.mjs'; +import getPlaintext from '../../utils/get-plaintext'; import useStickyTopValues from '../../hooks/useStickyTopValues'; import RightColumn from './RightColumn'; diff --git a/src/components/Code/code-context.js b/src/components/Code/code-context.js index 76d77ed9b..a3eca1952 100644 --- a/src/components/Code/code-context.js +++ b/src/components/Code/code-context.js @@ -2,7 +2,7 @@ import React, { useContext, useMemo } from 'react'; import { Language } from '@leafygreen-ui/code'; import LeafyIcon from '@leafygreen-ui/icon'; import { TabContext } from '../Tabs/tab-context'; -import { getPlaintext } from '../../utils/get-plaintext'; +import getPlaintext from '../../utils/get-plaintext'; const defaultContextValue = { codeBlockLanguage: null, diff --git a/src/components/Cond.js b/src/components/Cond.js index 8bca538b6..c57f5ad05 100644 --- a/src/components/Cond.js +++ b/src/components/Cond.js @@ -1,6 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { getNestedValue } from '../utils/get-nested-value'; +import getNestedValue from '../utils/get-nested-value.mjs'; import ComponentFactory from './ComponentFactory'; // For now, explicitly define the arguments that should be accepted for Gatsby to build the node diff --git a/src/components/DefinitionList/DefinitionListItem.js b/src/components/DefinitionList/DefinitionListItem.js index 50102d240..cc6876c1b 100644 --- a/src/components/DefinitionList/DefinitionListItem.js +++ b/src/components/DefinitionList/DefinitionListItem.js @@ -2,7 +2,7 @@ import React from 'react'; import PropTypes from 'prop-types'; import { css } from '@emotion/react'; import ComponentFactory from '../ComponentFactory'; -import { findKeyValuePair } from '../../utils/find-key-value-pair'; +import findKeyValuePair from '../../utils/find-key-value-pair'; const DefinitionListItem = ({ nodeData: { children, term }, ...rest }) => { const termProps = {}; diff --git a/src/components/DocumentBody.js b/src/components/DocumentBody.js index 4d6ed8e68..d9da58b54 100644 --- a/src/components/DocumentBody.js +++ b/src/components/DocumentBody.js @@ -4,10 +4,10 @@ import { withPrefix } from 'gatsby'; import { UnifiedFooter } from '@mdb/consistent-nav'; import { usePresentationMode } from '../hooks/use-presentation-mode'; import { useCanonicalUrl } from '../hooks/use-canonical-url'; -import { findAllKeyValuePairs } from '../utils/find-all-key-value-pairs'; -import { getNestedValue } from '../utils/get-nested-value'; -import { getMetaFromDirective } from '../utils/get-meta-from-directive'; -import { getPlaintext } from '../utils/get-plaintext'; +import findAllKeyValuePairs from '../utils/find-all-key-value-pairs'; +import getNestedValue from '../utils/get-nested-value.mjs'; +import getMetaFromDirective from '../utils/get-meta-from-directive'; +import getPlaintext from '../utils/get-plaintext'; import { getTemplate } from '../utils/get-template'; import useSnootyMetadata from '../utils/use-snooty-metadata'; import { isBrowser } from '../utils/is-browser'; diff --git a/src/components/DocumentBodyPreview.js b/src/components/DocumentBodyPreview.js index c50594d95..3fdaf5792 100644 --- a/src/components/DocumentBodyPreview.js +++ b/src/components/DocumentBodyPreview.js @@ -1,9 +1,9 @@ import React, { useState } from 'react'; import { graphql } from 'gatsby'; import PropTypes from 'prop-types'; -import { findAllKeyValuePairs } from '../utils/find-all-key-value-pairs'; -import { getNestedValue } from '../utils/get-nested-value'; -import { getPlaintext } from '../utils/get-plaintext'; +import findAllKeyValuePairs from '../utils/find-all-key-value-pairs'; +import getNestedValue from '../utils/get-nested-value.mjs'; +import getPlaintext from '../utils/get-plaintext'; import { getTemplate } from '../utils/get-template'; import Layout from '../layouts/preview-layout'; import Widgets from './Widgets'; diff --git a/src/components/Emphasis.js b/src/components/Emphasis.js index b8629bca8..445ce472c 100644 --- a/src/components/Emphasis.js +++ b/src/components/Emphasis.js @@ -1,6 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { getNestedValue } from '../utils/get-nested-value'; +import getNestedValue from '../utils/get-nested-value.mjs'; const Emphasis = ({ nodeData }) => {getNestedValue(['children', 0, 'value'], nodeData)}; diff --git a/src/components/Figure/Lightbox.js b/src/components/Figure/Lightbox.js index e6eb93de4..771e8134b 100644 --- a/src/components/Figure/Lightbox.js +++ b/src/components/Figure/Lightbox.js @@ -46,7 +46,6 @@ const LightboxWrapper = styled('div')` margin-top: ${theme.size.medium}; margin-bottom: ${theme.size.medium}; display: block; - max-height: 620px; `; const Lightbox = ({ nodeData, ...rest }) => { diff --git a/src/components/Figure/index.js b/src/components/Figure/index.js index 311068ddc..f8a9988dc 100644 --- a/src/components/Figure/index.js +++ b/src/components/Figure/index.js @@ -2,7 +2,7 @@ import React, { Component } from 'react'; import PropTypes from 'prop-types'; import { css } from '@emotion/react'; import Image from '../Image'; -import { getNestedValue } from '../../utils/get-nested-value'; +import getNestedValue from '../../utils/get-nested-value.mjs'; import { theme } from '../../theme/docsTheme'; import Lightbox from './Lightbox'; import CaptionLegend from './CaptionLegend'; diff --git a/src/components/Footnote/FootnoteReference.js b/src/components/Footnote/FootnoteReference.js index a57a42c15..0cce8bd41 100644 --- a/src/components/Footnote/FootnoteReference.js +++ b/src/components/Footnote/FootnoteReference.js @@ -1,6 +1,6 @@ import React, { useContext } from 'react'; import PropTypes from 'prop-types'; -import { getNestedValue } from '../../utils/get-nested-value'; +import getNestedValue from '../../utils/get-nested-value.mjs'; import FootnoteContext from './footnote-context'; /** diff --git a/src/components/Footnote/index.js b/src/components/Footnote/index.js index e84da03be..6bc2c5507 100644 --- a/src/components/Footnote/index.js +++ b/src/components/Footnote/index.js @@ -2,7 +2,7 @@ import React, { useContext } from 'react'; import PropTypes from 'prop-types'; import { css } from '@emotion/react'; import ComponentFactory from '../ComponentFactory'; -import { getNestedValue } from '../../utils/get-nested-value'; +import getNestedValue from '../../utils/get-nested-value.mjs'; import { intersperse } from '../../utils/intersperse'; import FootnoteContext from './footnote-context'; diff --git a/src/components/GuideNext/GuidesList.js b/src/components/GuideNext/GuidesList.js index 6b9eddcd0..0f43e1e33 100644 --- a/src/components/GuideNext/GuidesList.js +++ b/src/components/GuideNext/GuidesList.js @@ -1,7 +1,7 @@ import React from 'react'; import PropTypes from 'prop-types'; import styled from '@emotion/styled'; -import { getPlaintext } from '../../utils/get-plaintext'; +import getPlaintext from '../../utils/get-plaintext'; import GuidesListItem from './GuidesListItem'; const List = styled('ul')` diff --git a/src/components/HorizontalList.js b/src/components/HorizontalList.js index f22149ada..2e5c40b76 100644 --- a/src/components/HorizontalList.js +++ b/src/components/HorizontalList.js @@ -1,6 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { getNestedValue } from '../utils/get-nested-value'; +import getNestedValue from '../utils/get-nested-value.mjs'; import ComponentFactory from './ComponentFactory'; const HorizontalList = ({ diff --git a/src/components/Image.js b/src/components/Image.js index 48856abf6..bd58d91d2 100644 --- a/src/components/Image.js +++ b/src/components/Image.js @@ -3,7 +3,7 @@ import PropTypes from 'prop-types'; import { withPrefix } from 'gatsby'; import { css } from '@emotion/react'; import { palette } from '@leafygreen-ui/palette'; -import { getNestedValue } from '../utils/get-nested-value'; +import getNestedValue from '../utils/get-nested-value.mjs'; const Image = ({ nodeData, handleImageLoaded, className }) => { const [height, setHeight] = useState(null); diff --git a/src/components/Link.js b/src/components/Link.js index 0aa17a13e..86c53a15c 100644 --- a/src/components/Link.js +++ b/src/components/Link.js @@ -8,7 +8,7 @@ import { palette } from '@leafygreen-ui/palette'; import ArrowRightIcon from '@leafygreen-ui/icon/dist/ArrowRight'; import { isRelativeUrl } from '../utils/is-relative-url'; import { joinClassNames } from '../utils/join-class-names'; -import { isGatsbyPreview } from '../utils/is-gatsby-preview'; +import isGatsbyPreview from '../utils/is-gatsby-preview'; /* * Note: This component is not suitable for internal page navigation: diff --git a/src/components/Only.js b/src/components/Only.js index 7864ea4bf..b1c2008ae 100644 --- a/src/components/Only.js +++ b/src/components/Only.js @@ -1,6 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { getNestedValue } from '../utils/get-nested-value'; +import getNestedValue from '../utils/get-nested-value'; import ComponentFactory from './ComponentFactory'; // For now, explicitly define the arguments that should be accepted for Gatsby to build the node diff --git a/src/components/OpenAPI/index.js b/src/components/OpenAPI/index.js index fb7f2cf1a..2880f992c 100644 --- a/src/components/OpenAPI/index.js +++ b/src/components/OpenAPI/index.js @@ -12,7 +12,7 @@ import { useSiteMetadata } from '../../hooks/use-site-metadata'; import useStickyTopValues from '../../hooks/useStickyTopValues'; import { isBrowser } from '../../utils/is-browser'; import { theme } from '../../theme/docsTheme'; -import { getPlaintext } from '../../utils/get-plaintext'; +import getPlaintext from '../../utils/get-plaintext'; import { fetchOASFile } from '../../utils/realm'; import { isLinkInWhitelist, WhitelistErrorCallout } from './whitelist'; import { diff --git a/src/components/OpenAPIChangelog/utils/filterHiddenChanges.js b/src/components/OpenAPIChangelog/utils/filterHiddenChanges.js index f4e1b4656..83c244754 100644 --- a/src/components/OpenAPIChangelog/utils/filterHiddenChanges.js +++ b/src/components/OpenAPIChangelog/utils/filterHiddenChanges.js @@ -43,4 +43,4 @@ const hideDiffChanges = (diffData) => { return updatedDiffData.filter((path) => path.changes?.length); }; -module.exports = { hideChanges, hideDiffChanges }; +export { hideChanges, hideDiffChanges }; diff --git a/src/components/OpenAPIChangelog/utils/getResourceLinkUrl.js b/src/components/OpenAPIChangelog/utils/getResourceLinkUrl.js index 9e028b0d0..20beaeba8 100644 --- a/src/components/OpenAPIChangelog/utils/getResourceLinkUrl.js +++ b/src/components/OpenAPIChangelog/utils/getResourceLinkUrl.js @@ -1,5 +1,5 @@ -import { generatePathPrefix } from '../../../utils/generate-path-prefix'; -import { normalizePath } from '../../../utils/normalize-path'; +import generatePathPrefix from '../../../utils/generate-path-prefix'; +import normalizePath from '../../../utils/normalize-path'; const getResourceLinkUrl = (metadata, project, tag, operationId, openapi_pages = {}) => { const pathPrefix = generatePathPrefix(metadata, project); diff --git a/src/components/RefRole.js b/src/components/RefRole.js index 7adca56de..d5a26e6a2 100644 --- a/src/components/RefRole.js +++ b/src/components/RefRole.js @@ -3,7 +3,7 @@ import PropTypes from 'prop-types'; import { css, cx } from '@leafygreen-ui/emotion'; import { palette } from '@leafygreen-ui/palette'; import { theme } from '../theme/docsTheme'; -import { normalizePath } from '../utils/normalize-path'; +import normalizePath from '../utils/normalize-path'; import ComponentFactory from './ComponentFactory'; import Link from './Link'; diff --git a/src/components/SearchResults/Facets/useFacets.js b/src/components/SearchResults/Facets/useFacets.js index 1fa5e73aa..484177c83 100644 --- a/src/components/SearchResults/Facets/useFacets.js +++ b/src/components/SearchResults/Facets/useFacets.js @@ -1,5 +1,5 @@ import { useEffect, useState } from 'react'; -import { assertTrailingSlash } from '../../../utils/assert-trailing-slash'; +import assertTrailingSlash from '../../../utils/assert-trailing-slash.mjs'; import { MARIAN_URL } from '../../../constants'; const useFacets = () => { diff --git a/src/components/Sidenav/GuidesTOCTree.js b/src/components/Sidenav/GuidesTOCTree.js index 8f86169e8..501e2a56a 100644 --- a/src/components/Sidenav/GuidesTOCTree.js +++ b/src/components/Sidenav/GuidesTOCTree.js @@ -5,7 +5,7 @@ import { SideNavItem } from '@leafygreen-ui/side-nav'; import Link from '../Link'; import { ContentsContext } from '../Contents/contents-context'; import { formatText } from '../../utils/format-text'; -import { getPlaintext } from '../../utils/get-plaintext'; +import getPlaintext from '../../utils/get-plaintext'; import { isCurrentPage } from '../../utils/is-current-page'; import { sideNavItemTOCStyling } from './styles/sideNavItem'; diff --git a/src/components/Strong.js b/src/components/Strong.js index 95da046a1..38a52f657 100644 --- a/src/components/Strong.js +++ b/src/components/Strong.js @@ -1,6 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { getNestedValue } from '../utils/get-nested-value'; +import getNestedValue from '../utils/get-nested-value.mjs'; const Strong = ({ nodeData }) => {getNestedValue(['children', 0, 'value'], nodeData)}; diff --git a/src/components/StructuredData/BreadcrumbSchema.js b/src/components/StructuredData/BreadcrumbSchema.js index 6e0610de7..d255c8bc7 100644 --- a/src/components/StructuredData/BreadcrumbSchema.js +++ b/src/components/StructuredData/BreadcrumbSchema.js @@ -3,7 +3,7 @@ import PropTypes from 'prop-types'; import { withPrefix } from 'gatsby'; import { useSiteMetadata } from '../../hooks/use-site-metadata'; -import { assertTrailingSlash } from '../../utils/assert-trailing-slash'; +import assertTrailingSlash from '../../utils/assert-trailing-slash.mjs'; import { baseUrl } from '../../utils/base-url'; import useSnootyMetadata from '../../utils/use-snooty-metadata'; diff --git a/src/components/Tabs/TabSelectors.js b/src/components/Tabs/TabSelectors.js index d310903b8..8c229e7e9 100644 --- a/src/components/Tabs/TabSelectors.js +++ b/src/components/Tabs/TabSelectors.js @@ -1,7 +1,7 @@ import React, { useContext, useMemo } from 'react'; import { useTheme, css } from '@emotion/react'; import Select from '../Select'; -import { getPlaintext } from '../../utils/get-plaintext'; +import getPlaintext from '../../utils/get-plaintext'; import { reportAnalytics } from '../../utils/report-analytics'; import { TabContext } from './tab-context'; diff --git a/src/components/Tabs/index.js b/src/components/Tabs/index.js index 96abf5c4d..27cc88213 100644 --- a/src/components/Tabs/index.js +++ b/src/components/Tabs/index.js @@ -6,7 +6,7 @@ import { CodeProvider } from '../Code/code-context'; import ComponentFactory from '../ComponentFactory'; import { theme } from '../../theme/docsTheme'; import { reportAnalytics } from '../../utils/report-analytics'; -import { getNestedValue } from '../../utils/get-nested-value'; +import getNestedValue from '../../utils/get-nested-value.mjs'; import { isBrowser } from '../../utils/is-browser'; import { TabContext } from './tab-context'; diff --git a/src/components/Time.js b/src/components/Time.js index cbb142ac1..6aaf64640 100644 --- a/src/components/Time.js +++ b/src/components/Time.js @@ -1,6 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { getPlaintext } from '../utils/get-plaintext'; +import getPlaintext from '../utils/get-plaintext'; const Time = ({ nodeData: { argument } }) => { const time = getPlaintext(argument); diff --git a/src/components/Topic.js b/src/components/Topic.js index e24ced1b8..70f0ac886 100644 --- a/src/components/Topic.js +++ b/src/components/Topic.js @@ -1,6 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { getNestedValue } from '../utils/get-nested-value'; +import getNestedValue from '../utils/get-nested-value.mjs'; import ComponentFactory from './ComponentFactory'; const Topic = ({ nodeData, ...rest }) => ( diff --git a/src/components/Twitter.js b/src/components/Twitter.js index 88b4217df..3b88410d6 100644 --- a/src/components/Twitter.js +++ b/src/components/Twitter.js @@ -2,7 +2,7 @@ import React from 'react'; import PropTypes from 'prop-types'; import { withPrefix } from 'gatsby'; import { useSiteMetadata } from '../hooks/use-site-metadata'; -import { getPlaintext } from '../utils/get-plaintext'; +import getPlaintext from '../utils/get-plaintext'; const isExternalUrl = /^http(s)?:\/\//; diff --git a/src/components/VersionDropdown/utils.js b/src/components/VersionDropdown/utils.js index af6c0e455..8dc35c28d 100644 --- a/src/components/VersionDropdown/utils.js +++ b/src/components/VersionDropdown/utils.js @@ -1,4 +1,4 @@ -import { generatePathPrefix } from '../../utils/generate-path-prefix'; +import generatePathPrefix from '../../utils/generate-path-prefix'; /** * Generates the prefix to be used for a version's URL. The prefix will typically consist of the docs repo's diff --git a/src/components/Widgets/QuizWidget/QuizChoice.js b/src/components/Widgets/QuizWidget/QuizChoice.js index 501d61e37..908a27178 100644 --- a/src/components/Widgets/QuizWidget/QuizChoice.js +++ b/src/components/Widgets/QuizWidget/QuizChoice.js @@ -7,7 +7,7 @@ import { palette } from '@leafygreen-ui/palette'; import Icon from '@leafygreen-ui/icon'; import ComponentFactory from '../../ComponentFactory'; import { theme } from '../../../theme/docsTheme'; -import { getPlaintext } from '../../../utils/get-plaintext'; +import getPlaintext from '../../../utils/get-plaintext'; const Dot = styled('span')` height: 10px; diff --git a/src/components/Widgets/QuizWidget/QuizWidget.js b/src/components/Widgets/QuizWidget/QuizWidget.js index 95fe2a7bd..8b966d6dd 100644 --- a/src/components/Widgets/QuizWidget/QuizWidget.js +++ b/src/components/Widgets/QuizWidget/QuizWidget.js @@ -8,7 +8,7 @@ import Icon from '@leafygreen-ui/icon'; import ComponentFactory from '../../ComponentFactory'; import { theme } from '../../../theme/docsTheme'; import { useSiteMetadata } from '../../../hooks/use-site-metadata'; -import { getPlaintext } from '../../../utils/get-plaintext'; +import getPlaintext from '../../../utils/get-plaintext'; import useSnootyMetadata from '../../../utils/use-snooty-metadata'; import { useRealmFuncs } from './RealmFuncs'; import { RealmAppProvider } from './RealmApp'; diff --git a/src/context/toc-context.js b/src/context/toc-context.js index 6cb900256..6829aad8a 100644 --- a/src/context/toc-context.js +++ b/src/context/toc-context.js @@ -4,7 +4,7 @@ import { METADATA_COLLECTION } from '../build-constants'; import { useSiteMetadata } from '../hooks/use-site-metadata'; import { fetchDocuments } from '../utils/realm'; import useSnootyMetadata from '../utils/use-snooty-metadata'; -import { isGatsbyPreview } from '../utils/is-gatsby-preview'; +import isGatsbyPreview from '../utils/is-gatsby-preview'; import { VersionContext } from './version-context'; const TocContext = createContext({ diff --git a/src/hooks/__mocks__/use-site-metadata.js b/src/hooks/__mocks__/use-site-metadata.js index 71580e437..38d6417a0 100644 --- a/src/hooks/__mocks__/use-site-metadata.js +++ b/src/hooks/__mocks__/use-site-metadata.js @@ -1,4 +1,4 @@ -module.exports = { +export default { useSiteMetadata: jest.fn().mockReturnValue({ branch: 'master', project: 'datalake', diff --git a/src/hooks/use-canonical-url.js b/src/hooks/use-canonical-url.js index 1f1018c52..4688fe7ba 100644 --- a/src/hooks/use-canonical-url.js +++ b/src/hooks/use-canonical-url.js @@ -1,4 +1,4 @@ -import { assertTrailingSlash } from '../utils/assert-trailing-slash'; +import assertTrailingSlash from '../utils/assert-trailing-slash.mjs'; import { useSiteMetadata } from './use-site-metadata'; import { usePathPrefix } from './use-path-prefix'; diff --git a/src/hooks/use-marian-manifests.js b/src/hooks/use-marian-manifests.js index 679eb9b54..b9a2346ae 100644 --- a/src/hooks/use-marian-manifests.js +++ b/src/hooks/use-marian-manifests.js @@ -1,5 +1,5 @@ import { useEffect, useState } from 'react'; -import { assertTrailingSlash } from '../utils/assert-trailing-slash'; +import assertTrailingSlash from '../utils/assert-trailing-slash.mjs'; import { parseMarianManifests } from '../utils/parse-marian-manifests'; import { MARIAN_URL } from '../constants'; import { fetchSearchPropertyMapping } from '../utils/realm'; diff --git a/src/init/DocumentDatabase.js b/src/init/DocumentDatabase.js deleted file mode 100644 index 67df17a30..000000000 --- a/src/init/DocumentDatabase.js +++ /dev/null @@ -1,130 +0,0 @@ -const AdmZip = require('adm-zip'); -const BSON = require('bson'); -const { initRealm } = require('../utils/setup/init-realm'); -const { DOCUMENTS_COLLECTION, METADATA_COLLECTION, ASSETS_COLLECTION } = require('../build-constants'); -const { manifestMetadata, siteMetadata } = require('../utils/site-metadata'); -const { constructBuildFilter } = require('../utils/setup/construct-build-filter'); - -const DB = siteMetadata.database; -const buildFilter = constructBuildFilter(siteMetadata); - -class RealmInterface { - constructor() { - this.realmClient = null; - } - - async connect() { - this.realmClient = await initRealm(); - } - - fetchAllProducts() { - return this.realmClient.callFunction('fetchAllProducts', siteMetadata.database); - } - - async fetchDocuments(collection, buildFilter) { - return this.realmClient.callFunction('fetchDocuments', DB, collection, buildFilter); - } - - async getMetadata(buildFilter, findOptions) { - return this.realmClient.callFunction('fetchDocument', DB, METADATA_COLLECTION, buildFilter, undefined, findOptions); - } - - async fetchDocument(database, collectionName, query) { - return this.realmClient.callFunction('fetchDocument', database, collectionName, query); - } - - async fetchDocset(matchConditions = { project: siteMetadata.project }) { - return this.realmClient.callFunction('fetchDocset', siteMetadata.reposDatabase, matchConditions); - } - - async updateOAChangelogMetadata(metadata) { - return this.realmClient.callFunction('updateOAChangelogMetadata', metadata); - } -} - -class ManifestDocumentDatabase { - constructor(path) { - this.zip = new AdmZip(path); - this.realmInterface = new RealmInterface(); - } - - async connect() { - await this.realmInterface.connect(); - } - - async getDocuments() { - const result = []; - const zipEntries = this.zip.getEntries(); - for (const entry of zipEntries) { - if (entry.entryName.startsWith('documents/')) { - const doc = BSON.deserialize(entry.getData()); - result.push(doc); - } - } - return result; - } - - async getMetadata() { - return manifestMetadata; - } - - async getAsset(checksum) { - const result = this.zip.getEntry(`assets/${checksum}`); - if (result) { - return result.getData(); - } - return null; - } - - async fetchAllProducts() { - return this.realmInterface.fetchAllProducts(); - } -} - -class RealmDocumentDatabase { - constructor() { - this.realmInterface = new RealmInterface(); - } - - async connect() { - await this.realmInterface.connect(); - } - - async getDocuments() { - return this.realmInterface.fetchDocuments(DOCUMENTS_COLLECTION, buildFilter); - } - - async getMetadata(queryFilters) { - const filter = queryFilters || buildFilter; - return this.realmInterface.getMetadata(filter); - } - - async getAsset(checksum) { - const assetQuery = { _id: checksum }; - const assetDataDocuments = await this.realmInterface.fetchDocuments(ASSETS_COLLECTION, assetQuery); - - for (const result of assetDataDocuments) { - return result.data.buffer; - } - return null; - } - - async fetchAllProducts() { - return this.realmInterface.fetchAllProducts(); - } - - async fetchDocument(database, collectionName, query) { - return this.realmInterface.fetchDocument(database, collectionName, query); - } - - async fetchDocset(matchConditions) { - return this.realmInterface.fetchDocset(matchConditions); - } - - async updateOAChangelogMetadata(metadata) { - return this.realmInterface.updateOAChangelogMetadata(metadata); - } -} - -exports.manifestDocumentDatabase = new ManifestDocumentDatabase(process.env.GATSBY_MANIFEST_PATH); -exports.realmDocumentDatabase = new RealmDocumentDatabase(); diff --git a/src/templates/product-landing.js b/src/templates/product-landing.js index 13c9a1f8f..136f7eb70 100644 --- a/src/templates/product-landing.js +++ b/src/templates/product-landing.js @@ -3,7 +3,7 @@ import PropTypes from 'prop-types'; import styled from '@emotion/styled'; import { palette } from '@leafygreen-ui/palette'; import { theme } from '../theme/docsTheme.js'; -import { findKeyValuePair } from '../utils/find-key-value-pair.js'; +import findKeyValuePair from '../utils/find-key-value-pair.js'; import useSnootyMetadata from '../utils/use-snooty-metadata.js'; const CONTENT_MAX_WIDTH = 1200; diff --git a/src/utils/assert-trailing-slash.js b/src/utils/assert-trailing-slash.js deleted file mode 100644 index b7739b6a4..000000000 --- a/src/utils/assert-trailing-slash.js +++ /dev/null @@ -1,8 +0,0 @@ -const assertTrailingSlash = (url) => { - if (url && url.match(/\/$/)) { - return url; - } - return `${url}/`; -}; - -module.exports.assertTrailingSlash = assertTrailingSlash; diff --git a/src/utils/base-url.js b/src/utils/base-url.js index 726e0b471..a6062ca62 100644 --- a/src/utils/base-url.js +++ b/src/utils/base-url.js @@ -1,4 +1,4 @@ -const { assertTrailingSlash } = require('./assert-trailing-slash'); +import assertTrailingSlash from './assert-trailing-slash.mjs'; const DOTCOM_BASE_URL = 'https://www.mongodb.com'; const DOTCOM_BASE_PREFIX = `docs`; @@ -59,4 +59,4 @@ const baseUrl = (url = DOTCOM_BASE_URL, options = {}) => { return assertTrailingSlash(needsProtocol ? baseUrl.toString() : baseUrl.toString().split('//')[1]); }; -module.exports = { baseUrl, DOTCOM_BASE_PREFIX, DOTCOM_BASE_URL }; +export { baseUrl, DOTCOM_BASE_PREFIX, DOTCOM_BASE_URL }; diff --git a/src/utils/find-all-key-value-pairs.js b/src/utils/find-all-key-value-pairs.js index 67986b223..329096d09 100644 --- a/src/utils/find-all-key-value-pairs.js +++ b/src/utils/find-all-key-value-pairs.js @@ -18,4 +18,4 @@ const findAllKeyValuePairs = (nodes, key, value) => { // TODO: switch to ES6 export syntax if Gatsby implements support for ES6 module imports // https://github.com/gatsbyjs/gatsby/issues/7810 -module.exports.findAllKeyValuePairs = findAllKeyValuePairs; +export default findAllKeyValuePairs; diff --git a/src/utils/find-key-value-pair.js b/src/utils/find-key-value-pair.js index 8a6817dbb..2d6b6c107 100644 --- a/src/utils/find-key-value-pair.js +++ b/src/utils/find-key-value-pair.js @@ -18,4 +18,4 @@ const findKeyValuePair = (nodes, key, value) => { // TODO: switch to ES6 export syntax if Gatsby implements support for ES6 module imports // https://github.com/gatsbyjs/gatsby/issues/7810 -module.exports.findKeyValuePair = findKeyValuePair; +export default findKeyValuePair; diff --git a/src/utils/generate-path-prefix.js b/src/utils/generate-path-prefix.js index 6eda621f4..74a5bc395 100644 --- a/src/utils/generate-path-prefix.js +++ b/src/utils/generate-path-prefix.js @@ -1,4 +1,5 @@ -const { normalizePath } = require('./normalize-path'); +import normalizePath from './normalize-path.js'; +// const { normalizePath } = require('./normalize-path'); const generatePathPrefix = ( { commitHash, parserBranch, patchId, pathPrefix, project: parserProject, snootyBranch, user }, @@ -34,4 +35,4 @@ const generatePathPrefix = ( // TODO: switch to ES6 export syntax if Gatsby implements support for ES6 module imports // https://github.com/gatsbyjs/gatsby/issues/7810 -module.exports.generatePathPrefix = generatePathPrefix; +export default generatePathPrefix; diff --git a/src/utils/get-meta-from-directive.js b/src/utils/get-meta-from-directive.js index dd218ba02..4509f841a 100644 --- a/src/utils/get-meta-from-directive.js +++ b/src/utils/get-meta-from-directive.js @@ -1,4 +1,4 @@ -const { getNestedValue } = require('./get-nested-value'); +import getNestedValue from './get-nested-value.mjs'; // Grabs the metadata values in question and returns them as an array // for the Meta & TwitterMeta tags @@ -33,4 +33,4 @@ const getMetaFromDirective = (type, nodes, target) => { return collectionOfMetadata; }; -module.exports.getMetaFromDirective = getMetaFromDirective; +export default getMetaFromDirective; diff --git a/src/utils/get-nested-value.js b/src/utils/get-nested-value.js deleted file mode 100644 index 9b703e7ee..000000000 --- a/src/utils/get-nested-value.js +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Safely return a deeply nested value from an object. If the property is not found, return undefined. - * Arguments: - * - p: an array containing the path to the desired return value - * - o: the object to be searched - */ -const getNestedValue = (p, o) => { - if (!o) return undefined; - return p.reduce((xs, x) => (xs && xs[x] ? xs[x] : undefined), o); -}; - -// TODO: switch to ES6 export syntax if Gatsby implements support for ES6 module imports -// https://github.com/gatsbyjs/gatsby/issues/7810 -module.exports.getNestedValue = getNestedValue; diff --git a/src/utils/get-page-slug.js b/src/utils/get-page-slug.js deleted file mode 100644 index ad2ca13bc..000000000 --- a/src/utils/get-page-slug.js +++ /dev/null @@ -1,6 +0,0 @@ -// Takes a look at the page name and returns the appropriate page url -const getPageSlug = (page) => { - return page === 'index' ? '/' : page; -}; - -module.exports.getPageSlug = getPageSlug; diff --git a/src/utils/get-page-title.js b/src/utils/get-page-title.js index f6ec0a99b..25b36b0f2 100644 --- a/src/utils/get-page-title.js +++ b/src/utils/get-page-title.js @@ -1,5 +1,5 @@ import { formatText } from './format-text'; -import { getNestedValue } from './get-nested-value'; +import getNestedValue from './get-nested-value.mjs'; /* * Given slug and a property's slug-title mapping, look up the title for a given page. diff --git a/src/utils/get-plaintext.js b/src/utils/get-plaintext.js index fd828fccc..0f3d44385 100644 --- a/src/utils/get-plaintext.js +++ b/src/utils/get-plaintext.js @@ -15,4 +15,4 @@ const getPlaintext = (nodeArray) => { return nodeArray && nodeArray.length > 0 ? nodeArray.reduce(extractText, '') : ''; }; -module.exports = { getPlaintext }; +export default getPlaintext; diff --git a/src/utils/is-gatsby-preview.js b/src/utils/is-gatsby-preview.js index f6e1c7b65..3ae51b5f7 100644 --- a/src/utils/is-gatsby-preview.js +++ b/src/utils/is-gatsby-preview.js @@ -3,4 +3,4 @@ */ const isGatsbyPreview = () => process.env.GATSBY_IS_PREVIEW === 'true'; -module.exports = { isGatsbyPreview }; +export default isGatsbyPreview; diff --git a/src/utils/normalize-path.js b/src/utils/normalize-path.js index f1a4307f1..3b56d939c 100644 --- a/src/utils/normalize-path.js +++ b/src/utils/normalize-path.js @@ -1,4 +1,4 @@ // Remove duplicate slashes in path string const normalizePath = (path) => path.replace(/\/+/g, `/`); -module.exports.normalizePath = normalizePath; +export default normalizePath; diff --git a/src/utils/remove-nested-value.js b/src/utils/remove-nested-value.js deleted file mode 100644 index 6ee71cced..000000000 --- a/src/utils/remove-nested-value.js +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Safely remove values from a deeply nested object. - * Arguments: - * - target: the target key to be deleted - * - traversalKey: the target key to traverse - * - arr: an array containing the objects to search for the target value - */ -const removeNestedValue = (target, traversalKey, arr) => { - if (!arr) return; - - arr.forEach((entry) => { - if (entry.hasOwnProperty(target)) { - delete entry[target]; - } - - if (entry.hasOwnProperty(traversalKey)) { - const children = entry[traversalKey]; - if (Array.isArray(children) && children.length) { - removeNestedValue(target, traversalKey, children); - } - } - }); -}; - -module.exports.removeNestedValue = removeNestedValue; diff --git a/src/utils/search-params-to-url.js b/src/utils/search-params-to-url.js index c59d5bc2b..6e8e01709 100644 --- a/src/utils/search-params-to-url.js +++ b/src/utils/search-params-to-url.js @@ -1,6 +1,6 @@ import { MARIAN_URL } from '../constants'; import { FACETS_KEY_PREFIX } from '../components/SearchResults/SearchContext'; -import { assertTrailingSlash } from './assert-trailing-slash'; +import assertTrailingSlash from './assert-trailing-slash.mjs'; const TERM_PARAM = 'q'; const PAGE_PARAM = 'page'; diff --git a/src/utils/setup/construct-build-filter.js b/src/utils/setup/construct-build-filter.js index 59c601c21..bd5a4926d 100644 --- a/src/utils/setup/construct-build-filter.js +++ b/src/utils/setup/construct-build-filter.js @@ -1,4 +1,4 @@ -const { constructPageIdPrefix } = require('./construct-page-id-prefix'); +import constructPageIdPrefix from './construct-page-id-prefix.mjs'; // Returns the query to be used by our Stitch/Realm function to fetch a site's documents const constructBuildFilter = ({ commitHash, patchId, ...rest }) => { @@ -10,4 +10,4 @@ const constructBuildFilter = ({ commitHash, patchId, ...rest }) => { }; }; -module.exports = { constructBuildFilter }; +export default constructBuildFilter; diff --git a/src/utils/setup/construct-page-id-prefix.js b/src/utils/setup/construct-page-id-prefix.js deleted file mode 100644 index 2537fe7c4..000000000 --- a/src/utils/setup/construct-page-id-prefix.js +++ /dev/null @@ -1,4 +0,0 @@ -// Concatenates a site prefix to return a page id -const constructPageIdPrefix = ({ project, parserUser, parserBranch }) => `${project}/${parserUser}/${parserBranch}`; - -module.exports = { constructPageIdPrefix }; diff --git a/src/utils/setup/fetch-manifest-metadata.js b/src/utils/setup/fetch-manifest-metadata.js index 2c5457bf8..657e53864 100644 --- a/src/utils/setup/fetch-manifest-metadata.js +++ b/src/utils/setup/fetch-manifest-metadata.js @@ -1,5 +1,5 @@ -const AdmZip = require('adm-zip'); -const BSON = require('bson'); +import AdmZip from 'adm-zip'; +import BSON from 'bson'; // Returns the metadata from the manifest file if provided const fetchManifestMetadata = () => { @@ -16,4 +16,4 @@ const fetchManifestMetadata = () => { return metadata; }; -module.exports = { fetchManifestMetadata }; +export default fetchManifestMetadata; diff --git a/src/utils/setup/init-realm.js b/src/utils/setup/init-realm.js index cd6de65d6..0a809decf 100644 --- a/src/utils/setup/init-realm.js +++ b/src/utils/setup/init-realm.js @@ -1,5 +1,5 @@ -const Realm = require('realm-web'); -const { SNOOTY_REALM_APP_ID } = require('../../build-constants'); +import Realm from 'realm-web'; +import { SNOOTY_REALM_APP_ID } from '../../build-constants.js'; const initRealm = async () => { // Returns an instance of an app. @@ -10,4 +10,4 @@ const initRealm = async () => { return anonymous; }; -module.exports = { initRealm }; +export default initRealm; diff --git a/src/utils/setup/save-asset-files.js b/src/utils/setup/save-asset-files.js deleted file mode 100644 index 17d2b956f..000000000 --- a/src/utils/setup/save-asset-files.js +++ /dev/null @@ -1,40 +0,0 @@ -const fs = require('fs').promises; -const path = require('path'); - -const saveFile = async (file, data) => { - await fs.mkdir(path.join('public', path.dirname(file)), { - recursive: true, - }); - await fs.writeFile(path.join('public', file), data, 'binary'); -}; - -// Write all assets to static directory -const saveAssetFiles = async (assets, db) => { - const imageWrites = []; - - for (const [id, filenames] of assets) { - if (filenames) { - const buffer = await db.getAsset(id); - if (!buffer) { - console.error( - `Failed to fetch asset with checksum ${id}. This should not be possible and indicates an internal problem with the document source.` - ); - process.exit(1); - } - filenames.forEach((filename) => imageWrites.push(saveFile(filename, buffer))); - } - } - await Promise.all(imageWrites); -}; - -const saveStaticFiles = async (staticFiles) => { - await Promise.all( - Object.entries(staticFiles).map(([file, data]) => { - // Certain files, like manpages, may not have buffers - const toSave = data.buffer ? data.buffer : data; - return saveFile(file, toSave); - }) - ); -}; - -module.exports = { saveAssetFiles, saveStaticFiles, saveFile }; diff --git a/src/utils/setup/transform-breadcrumbs.js b/src/utils/setup/transform-breadcrumbs.js deleted file mode 100644 index e7dee4b06..000000000 --- a/src/utils/setup/transform-breadcrumbs.js +++ /dev/null @@ -1,16 +0,0 @@ -const { getPlaintext } = require('../get-plaintext'); - -const transformBreadcrumbs = (breadcrumbs, slugToTitle) => { - Object.entries(breadcrumbs).forEach(([slug, breadcrumbList]) => { - breadcrumbs[slug] = breadcrumbList.map((path) => { - const title = slugToTitle?.[path] || []; - return { - path, - title, - plaintext: getPlaintext(title), - }; - }); - }); -}; - -module.exports = { transformBreadcrumbs }; diff --git a/src/utils/setup/validate-env-variables.js b/src/utils/setup/validate-env-variables.js deleted file mode 100644 index 56c11f3a1..000000000 --- a/src/utils/setup/validate-env-variables.js +++ /dev/null @@ -1,103 +0,0 @@ -const fs = require('fs'); - -// env variables for building site along with use in front-end -// https://www.gatsbyjs.org/docs/environment-variables/#defining-environment-variables -const validateEnvVariables = (manifestMetadata) => { - // only require env vars when no manifest path is specified - if ( - !process.env.GATSBY_MANIFEST_PATH && - (!process.env.GATSBY_SITE || !process.env.GATSBY_PARSER_USER || !process.env.GATSBY_PARSER_BRANCH) - ) { - return { - error: true, - message: `${process.env.NODE_ENV} requires the variables GATSBY_SITE, GATSBY_PARSER_USER, and GATSBY_PARSER_BRANCH, found: ${process.env.GATSBY_SITE}, ${process.env.GATSBY_PARSER_USER}, ${process.env.GATSBY_PARSER_BRANCH}`, - }; - } else if (process.env.GATSBY_MANIFEST_PATH) { - validateManifestEnvVars(manifestMetadata); - } - // create split prefix for use in stitch function - return { - error: false, - }; -}; - -// set env vars accordingly if none are provided -const validateManifestEnvVars = (manifestMetadata) => { - let envVars = ''; - const site = `\nGATSBY_SITE=${manifestMetadata['project']}`; - const branch = `\nGATSBY_PARSER_BRANCH=${manifestMetadata['branch']}`; - - if (!process.env.GATSBY_SITE) { - envVars += site; - } - if (!process.env.GATSBY_PARSER_BRANCH) { - envVars += branch; - } - - // write to .env files with updated env vars - try { - ['.env.production', '.env.development'].forEach((file) => { - fs.appendFile(file, envVars, function (err) { - if (err) { - console.error(err); - throw err; - } - }); - }); - } catch (err) { - console.error(err); - throw err; - } - - // if env variables are present, check that they are set to the correct values. Otherwise, override them. - let incorrectEnvVars = []; - let correctEnvVars = ''; - if (process.env.GATSBY_SITE && process.env.GATSBY_SITE !== manifestMetadata['project']) { - incorrectEnvVars.push('GATSBY_SITE'); - correctEnvVars += site; - } - if (process.env.GATSBY_PARSER_BRANCH && process.env.GATSBY_PARSER_BRANCH !== manifestMetadata['branch']) { - incorrectEnvVars.push('GATSBY_PARSER_BRANCH'); - correctEnvVars += branch; - } - if (incorrectEnvVars.length > 0) { - replaceIncorrectEnvVars(incorrectEnvVars, correctEnvVars); - } -}; - -const replaceIncorrectEnvVars = (incorrectEnvVars, envVars) => { - try { - ['.env.production', '.env.development'].forEach((file) => { - fs.readFile(file, 'utf8', (err, data) => { - if (err) { - console.error(err); - throw err; - } - - const dataArray = data.split('\n'); - - // remove incorrect variables from .env files - for (const dataEntry in dataArray) { - for (const incorrectVar in incorrectEnvVars) { - if (dataArray[dataEntry].includes(incorrectEnvVars[incorrectVar])) { - dataArray.splice(dataEntry, 1); - } - } - } - - // write to .env files with updated env vars - fs.writeFile(file, dataArray.join('\n').concat(envVars), (err) => { - if (err) { - console.error(err); - throw err; - } - }); - }); - }); - } catch (err) { - console.error(err); - throw err; - } -}; - -module.exports.validateEnvVariables = validateEnvVariables; diff --git a/src/utils/site-metadata.js b/src/utils/site-metadata.js deleted file mode 100644 index 6811620a5..000000000 --- a/src/utils/site-metadata.js +++ /dev/null @@ -1,79 +0,0 @@ -const { execSync } = require('child_process'); -const userInfo = require('os').userInfo; -const { fetchManifestMetadata } = require('../utils/setup/fetch-manifest-metadata'); -const { DOTCOM_BASE_URL } = require('./base-url'); - -// loads vars from the .env file into process.env object -const runningEnv = process.env.NODE_ENV || 'production'; - -require('dotenv').config({ - path: `.env.${runningEnv}`, -}); - -const manifestMetadata = fetchManifestMetadata(); - -const getDatabase = (env) => { - switch (env) { - case 'staging': - return 'snooty_stage'; - case 'production': - return 'snooty_prod'; - case 'dotcomstg': - return 'snooty_dotcomstg'; - case 'dotcomprd': - return 'snooty_dotcomprd'; - default: - return 'snooty_dev'; - } -}; - -const getReposDatabase = (env) => { - switch (env) { - case 'staging': - return 'pool_test'; - case 'production': - return 'pool'; - case 'dotcomstg': - return 'pool_test'; - case 'dotcomprd': - return 'pool'; - default: - return 'pool_test'; - } -}; - -const gitBranch = execSync('git rev-parse --abbrev-ref HEAD') - .toString('utf8') - .replace(/[\n\r\s]+$/, ''); - -const getPathPrefix = (pathPrefix) => { - if (!pathPrefix) { - return ''; - } - if (pathPrefix.startsWith('/')) { - return pathPrefix; - } - return `/${pathPrefix}`; -}; - -/** - * Get site metadata used to identify this build and query correct documents - */ -const siteMetadata = { - commitHash: process.env.COMMIT_HASH || '', - database: getDatabase(process.env.SNOOTY_ENV), - reposDatabase: getReposDatabase(process.env.SNOOTY_ENV), - parserBranch: manifestMetadata['branch'] || process.env.GATSBY_PARSER_BRANCH || `master`, - parserUser: process.env.GATSBY_PARSER_USER || userInfo().username, - patchId: process.env.PATCH_ID || '', - pathPrefix: getPathPrefix(process.env.PATH_PREFIX), - project: manifestMetadata['project'] || process.env.GATSBY_SITE, - siteUrl: DOTCOM_BASE_URL, - snootyBranch: gitBranch, - snootyEnv: process.env.SNOOTY_ENV || 'development', - user: userInfo().username, - manifestPath: process.env.GATSBY_MANIFEST_PATH, -}; - -module.exports.siteMetadata = siteMetadata; -module.exports.manifestMetadata = manifestMetadata; diff --git a/src/utils/url-utils.js b/src/utils/url-utils.js index b6a1b5a25..50f5b032f 100644 --- a/src/utils/url-utils.js +++ b/src/utils/url-utils.js @@ -1,7 +1,7 @@ import { generatePrefix } from '../components/VersionDropdown/utils'; import { baseUrl } from './base-url'; -import { assertTrailingSlash } from './assert-trailing-slash'; -import { normalizePath } from './normalize-path'; +import assertTrailingSlash from './assert-trailing-slash.mjs'; +import normalizePath from './normalize-path'; export const getUrl = (branchUrlSlug, project, siteMetadata, siteBasePrefix, slug) => { if (branchUrlSlug === 'legacy') { diff --git a/stubs/process.js b/stubs/process.js index 067b5eaf6..2a89bc0b3 100644 --- a/stubs/process.js +++ b/stubs/process.js @@ -1,7 +1,18 @@ -const process = require('process'); +import process from 'process'; //monkey patch process global.process = process; -module.exports = - typeof global !== 'undefined' && global.process ? global.process : require('../node_modules/process/browser.js'); +console.log('PROCESS MAYA:', process); + +// const exp = typeof global !== 'undefined' && global.process ? global.process : import {process} from '../node_modules/process/browser.js' +// MAYA TODO + +// module.exports = +// typeof global !== 'undefined' && global.process ? global.process : require('../node_modules/process/browser.js'); + +export default typeof global !== 'undefined' && global.process + ? global.process + : import('../node_modules/process/browser.js'); + +// require('../node_modules/process/browser.js'); diff --git a/tests/testSetup.js b/tests/testSetup.js index 14f177932..288131770 100644 --- a/tests/testSetup.js +++ b/tests/testSetup.js @@ -1,6 +1,7 @@ import '@testing-library/jest-dom'; import EventSource from 'eventsourcemock'; import { preloadAll } from 'react-loadable'; +import crypto from 'crypto'; global.navigator = { userAgent: 'node.js', @@ -35,8 +36,6 @@ afterAll(() => { process.removeListener('unhandledRejection', rejectionHandler); }); -const crypto = require('crypto'); - Object.defineProperty(global.self, 'crypto', { value: { getRandomValues: (arr) => crypto.randomBytes(arr.length), diff --git a/tests/unit/Chapters.test.js b/tests/unit/Chapters.test.js index 410a2f299..1b248433a 100644 --- a/tests/unit/Chapters.test.js +++ b/tests/unit/Chapters.test.js @@ -8,7 +8,7 @@ import { tick } from '../utils'; import { SidenavContext } from '../../src/components/Sidenav'; import { theme } from '../../src/theme/docsTheme'; import * as useActiveHeading from '../../src/hooks/useActiveHeading'; -import { getPlaintext } from '../../src/utils/get-plaintext'; +import getPlaintext from '../../src/utils/get-plaintext'; import mockData from './data/Chapters.test.json'; const mountChapters = () => { diff --git a/tests/unit/Meta.test.js b/tests/unit/Meta.test.js index dc0709814..a237abaf9 100644 --- a/tests/unit/Meta.test.js +++ b/tests/unit/Meta.test.js @@ -1,6 +1,6 @@ import React from 'react'; import { render, screen } from '@testing-library/react'; -import { getMetaFromDirective } from '../../src/utils/get-meta-from-directive'; +import getMetaFromDirective from '../../src/utils/get-meta-from-directive'; import Meta from '../../src/components/Meta'; // data for this component diff --git a/tests/unit/__mockStyles.js b/tests/unit/__mockStyles.js index f053ebf79..ff8b4c563 100644 --- a/tests/unit/__mockStyles.js +++ b/tests/unit/__mockStyles.js @@ -1 +1 @@ -module.exports = {}; +export default {}; diff --git a/tests/unit/__snapshots__/Figure.test.js.snap b/tests/unit/__snapshots__/Figure.test.js.snap index f9abc1d1a..add560759 100644 --- a/tests/unit/__snapshots__/Figure.test.js.snap +++ b/tests/unit/__snapshots__/Figure.test.js.snap @@ -61,7 +61,6 @@ exports[`renders lightbox correctly when specified as an option 1`] = ` margin-top: 24px; margin-bottom: 24px; display: block; - max-height: 620px; } .emotion-2 { diff --git a/tests/unit/__snapshots__/Lightbox.test.js.snap b/tests/unit/__snapshots__/Lightbox.test.js.snap index 40920de16..100a7b390 100644 --- a/tests/unit/__snapshots__/Lightbox.test.js.snap +++ b/tests/unit/__snapshots__/Lightbox.test.js.snap @@ -8,7 +8,6 @@ exports[`Lightbox renders correctly 1`] = ` margin-top: 24px; margin-bottom: 24px; display: block; - max-height: 620px; } .emotion-2 { diff --git a/tests/unit/utils/assert-trailing-slash.test.js b/tests/unit/utils/assert-trailing-slash.test.js index f17ffb354..2a826747a 100644 --- a/tests/unit/utils/assert-trailing-slash.test.js +++ b/tests/unit/utils/assert-trailing-slash.test.js @@ -1,4 +1,4 @@ -import { assertTrailingSlash } from '../../../src/utils/assert-trailing-slash'; +import assertTrailingSlash from '../../../src/utils/assert-trailing-slash'; it('should add trailing slashes to links if they are missing', () => { const linkWithoutSlash = 'foo.bar'; diff --git a/tests/unit/utils/generate-path-prefix.test.js b/tests/unit/utils/generate-path-prefix.test.js index b686d0422..4a05ce880 100644 --- a/tests/unit/utils/generate-path-prefix.test.js +++ b/tests/unit/utils/generate-path-prefix.test.js @@ -1,4 +1,4 @@ -import { generatePathPrefix } from '../../../src/utils/generate-path-prefix'; +import generatePathPrefix from '../../../src/utils/generate-path-prefix'; describe('path prefix testing', () => { const commitHash = 'COMMIT_HASH'; diff --git a/tests/unit/utils/join-class-names.test.js b/tests/unit/utils/join-class-names.test.js index 990d7754e..17677dec2 100644 --- a/tests/unit/utils/join-class-names.test.js +++ b/tests/unit/utils/join-class-names.test.js @@ -1,4 +1,4 @@ -const { joinClassNames } = require('../../../src/utils/join-class-names'); +import { joinClassNames } from '../../../src/utils/join-class-names.js'; describe('joinClassNames', () => { it('returns a string of all class names joined together', () => { From 6a661b4d37b44d209579cb42c28ad08c97b2476b Mon Sep 17 00:00:00 2001 From: Maya Raman Date: Mon, 20 Nov 2023 16:41:55 -0500 Subject: [PATCH 4/4] added untracked files --- gatsby-config.mjs | 41 +++ .../gatsby-node.mjs | 260 +++++++++++++++ .../other-things-to-source.mjs | 14 + .../utils/data-consumer.mjs | 180 ++++++++++ .../utils/kanopy-auth.mjs | 75 +++++ .../utils/post-build.mjs | 55 ++++ .../gatsby-source-snooty-prod/gatsby-node.mjs | 309 ++++++++++++++++++ plugins/utils/openapi.mjs | 115 +++++++ plugins/utils/products.mjs | 23 ++ src/init/DocumentDatabase.mjs | 132 ++++++++ src/utils/assert-trailing-slash.mjs | 8 + src/utils/get-nested-value.mjs | 14 + src/utils/get-page-slug.mjs | 6 + src/utils/remove-nested-value.mjs | 25 ++ src/utils/setup/construct-page-id-prefix.mjs | 4 + src/utils/setup/save-asset-files.mjs | 40 +++ src/utils/setup/transform-breadcrumbs.mjs | 16 + src/utils/setup/validate-env-variables.mjs | 103 ++++++ src/utils/site-metadata.mjs | 82 +++++ 19 files changed, 1502 insertions(+) create mode 100644 gatsby-config.mjs create mode 100644 plugins/gatsby-source-snooty-preview/gatsby-node.mjs create mode 100644 plugins/gatsby-source-snooty-preview/other-things-to-source.mjs create mode 100644 plugins/gatsby-source-snooty-preview/utils/data-consumer.mjs create mode 100644 plugins/gatsby-source-snooty-preview/utils/kanopy-auth.mjs create mode 100644 plugins/gatsby-source-snooty-preview/utils/post-build.mjs create mode 100644 plugins/gatsby-source-snooty-prod/gatsby-node.mjs create mode 100644 plugins/utils/openapi.mjs create mode 100644 plugins/utils/products.mjs create mode 100644 src/init/DocumentDatabase.mjs create mode 100644 src/utils/assert-trailing-slash.mjs create mode 100644 src/utils/get-nested-value.mjs create mode 100644 src/utils/get-page-slug.mjs create mode 100644 src/utils/remove-nested-value.mjs create mode 100644 src/utils/setup/construct-page-id-prefix.mjs create mode 100644 src/utils/setup/save-asset-files.mjs create mode 100644 src/utils/setup/transform-breadcrumbs.mjs create mode 100644 src/utils/setup/validate-env-variables.mjs create mode 100644 src/utils/site-metadata.mjs diff --git a/gatsby-config.mjs b/gatsby-config.mjs new file mode 100644 index 000000000..6920917d1 --- /dev/null +++ b/gatsby-config.mjs @@ -0,0 +1,41 @@ +import { createRequire } from 'module'; +import generatePathPrefix from './src/utils/generate-path-prefix.js'; +import { siteMetadata } from './src/utils/site-metadata.mjs'; +import isGatsbyPreview from './src/utils/is-gatsby-preview.js'; +// const { generatePathPrefix } = require('./src/utils/generate-path-prefix'); +// const { siteMetadata } = require('./src/utils/site-metadata'); +// const { isGatsbyPreview } = require('./src/utils/is-gatsby-preview'); + +const isPreview = isGatsbyPreview(); +const pathPrefix = !isPreview ? generatePathPrefix(siteMetadata) : undefined; + +const require = createRequire(import.meta.url); + +console.log('PATH PREFIX', pathPrefix); + +// Specifies which plugins to use depending on build environment +const plugins = ['gatsby-plugin-emotion', isPreview ? 'gatsby-source-snooty-preview' : 'gatsby-source-snooty-prod']; + +// PRODUCTION DEPLOYMENTS -- +// If not a preview build, use the layout that includes the +// consistent navbar and footer and generate a sitemap. +if (!isPreview) { + plugins.push(`gatsby-plugin-sitemap`); + const layoutComponentRelativePath = `./src/layouts/index.js`; + plugins.push({ + resolve: 'gatsby-plugin-layout', + options: { + component: require.resolve(layoutComponentRelativePath), + }, + }); +} else { + const layoutComponentRelativePath = `./src/layouts/preview-layout-outer.js`; + plugins.push({ + resolve: 'gatsby-plugin-layout', + options: { + component: require.resolve(layoutComponentRelativePath), + }, + }); +} + +export { plugins, pathPrefix, siteMetadata }; diff --git a/plugins/gatsby-source-snooty-preview/gatsby-node.mjs b/plugins/gatsby-source-snooty-preview/gatsby-node.mjs new file mode 100644 index 000000000..650bd9938 --- /dev/null +++ b/plugins/gatsby-source-snooty-preview/gatsby-node.mjs @@ -0,0 +1,260 @@ +import getDataStore from 'gatsby/dist/datastore'; +import path, { dirname } from 'path'; +import { fileURLToPath } from 'url'; +import { createRequire } from "module" +import stream from 'stream'; +import promisify from 'util'; +import got from `got`; +import parser from `stream-json/jsonl/Parser`; +import { sourceNodes } from `./other-things-to-source.mjs`; +import fetchClientAccessToken from './utils/kanopy-auth.mjs'; +import callPostBuildWebhook from './utils/post-build.mjs'; +import { consumeData, createSnootyMetadataId, KEY_LAST_FETCHED, KEY_LAST_CLIENT_ACCESS_TOKEN, } from './utils/data-consumer.mjs'; + +const pipeline = promisify(stream.pipeline); + +// Global variable to allow webhookBody from sourceNodes step to be passed down +// to other Gatsby build steps that might not pass webhookBody natively. +let currentWebhookBody = {}; + +export const createSchemaCustomization = async ({ actions }) => { + const { createTypes } = actions; + const typeDefs = ` + type Page implements Node @dontInfer { + page_id: String + branch: String + pagePath: String + ast: JSON! + metadata: SnootyMetadata @link + } + + type PagePath implements Node @dontInfer { + page_id: String! + branch: String! + project: String! + pageNodeId: String! + } + + type SnootyMetadata implements Node @dontInfer { + metadata: JSON + branch: String + project: String + } + + type RemoteMetadata implements Node @dontInfer { + remoteMetadata: JSON + } + + type ChangelogData implements Node @dontInfer { + changelogData: JSON + } + `; + createTypes(typeDefs); +}; + +const APIBase = process.env.API_BASE || `https://snooty-data-api.mongodb.com`; +const GATSBY_CLOUD_SITE_USER = process.env.GATSBY_CLOUD_SITE_USER; + +let isFirstRun = true; +export const sourceNodes = async ({ + actions, + createNodeId, + getNode, + getNodesByType, + reporter, + createContentDigest, + cache, + webhookBody, +}) => { + console.log({ webhookBody }); + currentWebhookBody = webhookBody; + let hasOpenAPIChangelog = false; + const { createNode, touchNode } = actions; + + const fileWritePromises = []; + const lastFetched = (await cache.get(KEY_LAST_FETCHED)) || 0; + const lastClientAccessToken = await cache.get(KEY_LAST_CLIENT_ACCESS_TOKEN); + console.log({ lastFetched }); + + if (isFirstRun && lastFetched) { + // nodes of following types are managed statefully: + // SnootyMetadata, Page, PagePath + // we need to touch on them on delta updates on first run of a process to prevent them from being garbage collected + const datastore = getDataStore(); + for (const nodeType of ['SnootyMetadata', 'Page', 'PagePath']) { + for (const node of datastore.iterateNodesByType(nodeType)) { + touchNode(node); + } + } + } + + try { + if (!GATSBY_CLOUD_SITE_USER) { + throw new Error('Missing GATSBY_CLOUD_SITE_USER'); + } + + // Generate client access token only if trying to access Snooty Data API's staging instance + const clientAccessToken = APIBase.includes('.staging') ? await fetchClientAccessToken(lastClientAccessToken) : ''; + let url; + if (lastFetched) { + url = `${APIBase}/user/${GATSBY_CLOUD_SITE_USER}/documents?updated=${lastFetched}`; + } else { + url = `${APIBase}/user/${GATSBY_CLOUD_SITE_USER}/documents`; + } + + const headers = {}; + if (clientAccessToken) { + headers['Authorization'] = `Bearer ${clientAccessToken}`; + } + const httpStream = got.stream(url, { headers }); + + let pageCount = 0; + // Callback function to be ran after a valid page has been found and handled. + // Tracks and updates information that spans across several data entries + const onHandlePage = (pageTemplate, pageId, pageNodeId) => { + if (pageTemplate === 'changelog') hasOpenAPIChangelog = true; + pageCount += 1; + if (pageCount % 1000 === 0) { + console.log({ pageCount, page_id: pageId, id: pageNodeId }); + } + }; + + // Since there's a lot of data incoming from the Snooty Data API, we stream + // the data in chunks and parse them as they come instead of fetching everything + // as a single JSON response + const decode = parser(); + decode.on('data', async (_entry) => { + // Un-nest data + const entry = _entry.value; + await consumeData(entry, { + actions, + cache, + clientAccessToken, + createContentDigest, + createNodeId, + fileWritePromises, + getNode, + onHandlePage, + }); + }); + + console.time(`source updates`); + // Wait for HTTP connection to close. + await pipeline(httpStream, decode); + console.timeEnd(`source updates`); + } catch (error) { + callPostBuildWebhook(webhookBody, 'failed'); + reporter.panic('There was an issue sourcing nodes', error); + } + + // Wait for all assets to be written. + await Promise.all(fileWritePromises); + + // Source old nodes. + console.time(`old source nodes`); + await sourceNodes({ + hasOpenAPIChangelog, + github_username: GATSBY_CLOUD_SITE_USER, + createNode, + createContentDigest, + createNodeId, + getNodesByType, + }); + console.timeEnd(`old source nodes`); + isFirstRun = false; +}; + + + +// Prevent errors when running gatsby build caused by browser packages run in a node environment. +export const onCreateWebpackConfig = ({ plugins, actions }) => { + const require = createRequire(import.meta.url); + const base = import.meta.url; + const providePlugins = { + Buffer: ['buffer', 'Buffer'], + process: require.resolve('../../stubs/process.js'), + }; + + const fallbacks = { stream: require.resolve('stream-browserify'), buffer: require.resolve('buffer/')}; + console.log("FALLBACKS", fallbacks) + actions.setWebpackConfig({ + plugins: [plugins.provide(providePlugins)], + resolve: { + fallback: fallbacks, + alias: { + process: 'process/browser', + }, + }, + }); +}; + +export const createPages = async ({ actions, createNodeId, getNode, graphql, reporter }) => { + const { createPage } = actions; + const __dirname = dirname(fileURLToPath(import.meta.url)); + const templatePath = path.join(__dirname, `../../src/components/DocumentBodyPreview.js`); + const result = await graphql(` + query { + allPagePath { + totalCount + nodes { + pageNodeId + branch + page_id + project + } + allProjects: distinct(field: { project: SELECT }) + } + } + `); + + if (result.errors) { + await callPostBuildWebhook(currentWebhookBody, 'failed'); + reporter.panic('There was an error in the graphql query', result.errors); + } + + try { + result.data.allPagePath.nodes.forEach((node) => { + const pagePath = path.join(node.project, node.branch, node.page_id); + let slug = node.page_id; + // Slices off leading slash to ensure slug matches an entry within the toctreeOrder and renders InternalPageNav components + if (slug !== '/' && slug[0] === '/') slug = slug.slice(1); + + const metadataNodeId = createSnootyMetadataId({ createNodeId, branch: node.branch, project: node.project }); + if (!getNode(metadataNodeId)) { + // Take into account the possibility of having new page data available through the API, + // but no metadata yet due to async uploads + console.warn( + `Skipping node creation for page "${node.page_id}", in project "${node.project}" on branch "${node.branch}". No metadata node "${metadataNodeId}" found.` + ); + return; + } + + createPage({ + path: pagePath, + component: templatePath, + context: { + id: node.pageNodeId, + slug, + // Hardcode static/safe values to prevent incremental builds from rebuilding versioned preview pages + repoBranches: {}, + associatedReposInfo: {}, + isAssociatedProduct: false, + project: node.project, + }, + }); + }); + } catch (err) { + await callPostBuildWebhook(currentWebhookBody, 'failed'); + reporter.panic('Could not build pages off of graphl query', err); + } +}; + +// `onPostBuild` is run by Gatsby Cloud after everything is built, but before the +// content is deployed to the preview site. This can result in a short delay between +// when the post-build webhook is called and when the content is updated. +// Ideally, we would use Gatsby Cloud's Outgoing Notifications feature once it can +// support passing through custom data from the preview webhook's body (to include the +// Autobuilder job ID associated with the GC build). +export const onPostBuild = async () => { + await callPostBuildWebhook(currentWebhookBody, 'completed'); +}; diff --git a/plugins/gatsby-source-snooty-preview/other-things-to-source.mjs b/plugins/gatsby-source-snooty-preview/other-things-to-source.mjs new file mode 100644 index 000000000..edaa5bf7b --- /dev/null +++ b/plugins/gatsby-source-snooty-preview/other-things-to-source.mjs @@ -0,0 +1,14 @@ +import { siteMetadata } from '../../src/utils/site-metadata.mjs'; +import { realmDocumentDatabase } from '../../src/init/DocumentDatabase.mjs'; +import createOpenAPIChangelogNode from '../utils/openapi.mjs'; +import createProductNodes from '../utils/products.mjs'; + +// Sources nodes for the preview plugin that are not directly related to data +// from the Snooty Data API +export const sourceNodes = async ({ hasOpenAPIChangelog, createNode, createContentDigest, createNodeId }) => { + let db = realmDocumentDatabase; + await db.connect(); + await createProductNodes({ db, createNode, createNodeId, createContentDigest }); + if (hasOpenAPIChangelog) + await createOpenAPIChangelogNode({ createNode, createNodeId, createContentDigest, siteMetadata, db }); +}; diff --git a/plugins/gatsby-source-snooty-preview/utils/data-consumer.mjs b/plugins/gatsby-source-snooty-preview/utils/data-consumer.mjs new file mode 100644 index 000000000..6dd419163 --- /dev/null +++ b/plugins/gatsby-source-snooty-preview/utils/data-consumer.mjs @@ -0,0 +1,180 @@ +import transformBreadcrumbs from '../../../src/utils/setup/transform-breadcrumbs.mjs'; +import { saveStaticFiles, saveFile } from '../../../src/utils/setup/save-asset-files.mjs'; +import getNestedValue from '../../../src/utils/get-nested-value.mjs'; + +const KEY_LAST_FETCHED = 'lastFetched'; +const KEY_LAST_CLIENT_ACCESS_TOKEN = 'lastClientAccessToken'; + +function createSnootyMetadataId({ branch, project, createNodeId }) { + return createNodeId(`metadata-${branch}-${project}`); +} + +// Syncs the plugin with timestamp of data being returned from the API. +const handleTimestamp = (data, { cache, clientAccessToken }) => { + cache.set(KEY_LAST_FETCHED, data); + cache.set(KEY_LAST_CLIENT_ACCESS_TOKEN, clientAccessToken); +}; + +const handleAsset = (data, { fileWritePromises }) => { + const { filenames, assetData } = data; + + // Incorrect asset format should not be acceptable + if (!filenames || !filenames.length) { + throw new Error('No filenames found for asset'); + } + if (!assetData) { + throw new Error('Missing asset data'); + } + + filenames.forEach((filePath) => { + // These promises will be resolved once all data is consumed + fileWritePromises.push(saveFile(filePath, Buffer.from(assetData, 'base64'))); + }); +}; + +const handleMetadata = async ( + data, + { createContentDigest, createNode, createNodeId, deleteNode, getNode, shouldDeleteContentNode } +) => { + const { _id, build_id, created_at, static_files: staticFiles, ...metadataMinusStatic } = data; + const { parentPaths, slugToTitle, branch, project } = metadataMinusStatic; + const nodeId = createSnootyMetadataId({ createNodeId, branch, project }); + + if (shouldDeleteContentNode) { + deleteNode(getNode(nodeId)); + return; + } + + if (parentPaths) { + transformBreadcrumbs(parentPaths, slugToTitle); + } + + // Save files in the static_files field of metadata document, including intersphinx inventories. + if (staticFiles) { + await saveStaticFiles(staticFiles); + } + + createNode({ + children: [], + id: nodeId, + internal: { + contentDigest: createContentDigest(metadataMinusStatic), + type: 'SnootyMetadata', + }, + branch, + project, + parent: null, + metadata: metadataMinusStatic, + }); +}; + +const handlePage = ( + data, + { createContentDigest, createNode, createNodeId, deleteNode, getNode, onHandlePage, shouldDeleteContentNode } +) => { + // Strip source string, in case it exists. We don't need the raw source of the AST + const { source, ...page } = data; + + const filename = getNestedValue(['filename'], page) || ''; + // There can be ASTs for included .rst files as well. We should skip these, in case + // we encounter them + if (!filename || !filename.endsWith('.txt')) { + console.warn(`Found an AST that is not for a page: ${filename}`); + return; + } + + const branch = page.page_id.split('/')[2]; + const raw_page_id = page.page_id.split('/').slice(3).join('/'); + const page_id = raw_page_id === 'index' ? '/' : `/${raw_page_id}`; + const project = page.page_id.split('/')[0]; + + const pageNodeId = createNodeId(page_id + project + branch); + const pagePathNodeId = pageNodeId + '/path'; + + if (shouldDeleteContentNode) { + deleteNode(getNode(pageNodeId)); + deleteNode(getNode(pagePathNodeId)); + return; + } + + page.page_id = page_id; + page.metadata = createSnootyMetadataId({ createNodeId, branch, project }); + page.id = pageNodeId; + page.internal = { + type: 'Page', + contentDigest: createContentDigest(page), + }; + + const pagePathNode = { + id: pagePathNodeId, + page_id, + branch, + project, + pageNodeId: page.id, + internal: { + type: 'PagePath', + contentDigest: page.internal.contentDigest, + }, + }; + + createNode(page); + createNode(pagePathNode); + + const pageTemplate = data.ast?.options?.template; + onHandlePage(pageTemplate, page_id, pageNodeId); +}; + +/** + * Handles incoming data accordingly based on its data type. Notably, this handles + * converting build data from the Snooty Data API into nodes and assets that the + * Gatsby site will need to render pages. + * @param {*} entry - A single data entry obtained from the Snooty Data API + * @param {*} options - Gatsby functions and other utilities to be used by handlers + */ +const consumeData = async ( + entry, + { actions, cache, createNodeId, createContentDigest, getNode, fileWritePromises, clientAccessToken, onHandlePage } +) => { + const { type, data } = entry; + + // Shape and format should be consistent across all data + if (!type) { + throw new Error('Data entry is missing data type'); + } + if (!data) { + throw new Error('Data entry is missing data'); + } + + const shouldDeleteContentNode = data.deleted; + const { createNode, deleteNode } = actions; + + if (type === 'timestamp') { + handleTimestamp(data, { cache, clientAccessToken }); + } else if (type === 'asset') { + handleAsset(data, { fileWritePromises }); + } else if (type === 'metadata') { + await handleMetadata(data, { + createContentDigest, + createNode, + createNodeId, + deleteNode, + getNode, + shouldDeleteContentNode, + }); + } else if (type === 'page') { + handlePage(data, { + createContentDigest, + createNode, + createNodeId, + deleteNode, + getNode, + onHandlePage, + shouldDeleteContentNode, + }); + } else { + // Shouldn't affect current builds + console.warn(`Unexpected data type: ${type}`); + } +}; + +export { consumeData, createSnootyMetadataId, KEY_LAST_FETCHED, KEY_LAST_CLIENT_ACCESS_TOKEN }; diff --git a/plugins/gatsby-source-snooty-preview/utils/kanopy-auth.mjs b/plugins/gatsby-source-snooty-preview/utils/kanopy-auth.mjs new file mode 100644 index 000000000..b95419e36 --- /dev/null +++ b/plugins/gatsby-source-snooty-preview/utils/kanopy-auth.mjs @@ -0,0 +1,75 @@ +/** + * Checks if the configuration property exists, and throws an error otherwise. + * @param configProp + * @param configType + */ +const validateConfigType = (configProp, configType) => { + if (!configProp) { + throw new Error(`Missing ${configType} for Snooty Data API access`); + } +}; + +/** + * Generates the authZ token needed for requesting an access token to Kanopy. + */ +const getClientCredentialsHeader = () => { + const clientId = process.env.OAUTH_CLIENT_ID; + validateConfigType(clientId, 'client ID'); + const clientSecret = process.env.OAUTH_CLIENT_SECRET; + validateConfigType(clientSecret, 'client secret'); + return Buffer.from(`${clientId}:${clientSecret}`, 'utf-8').toString('base64'); +}; + +/** + * Generates a new access token to allow for authentication against Kanopy services. + */ +const generateNewAccessToken = async () => { + const grantType = process.env.OAUTH_GRANT_TYPE; + validateConfigType(grantType, 'grant type'); + const scope = process.env.OAUTH_SCOPE; + validateConfigType(scope, 'scope'); + const authUrl = process.env.OAUTH_TOKEN_AUTH_URL; + validateConfigType(authUrl, 'auth token url'); + + // Request a new access token from Kanopy's token authentication endpoint + const authRequestUrl = `${authUrl}?grant_type=${grantType}&scope=${scope}`; + const headers = { + authorization: `Basic ${getClientCredentialsHeader()}`, + accept: 'application/json', + 'Content-Type': 'application/x-www-form-urlencoded', + 'cache-control': 'no-cache', + }; + const res = await fetch(authRequestUrl, { method: 'POST', headers }); + if (!res.ok) { + throw new Error('Error trying to request new access token'); + } + + const data = await res.json(); + const token = data['access_token']; + if (!token) { + throw new Error('Could not find new access token'); + } + + return token; +}; + +/** + * Returns a valid client access token that can be used for machine-machine communication + * between a client and a service hosted on Kanopy. + * See: https://kanopy.corp.mongodb.com/docs/development/authentication_and_authorization/ + * @param prevToken + */ +const fetchClientAccessToken = async (prevToken) => { + if (!prevToken) { + return generateNewAccessToken(); + } + // Decoded value is the JSON object representation of the token string, with token's expiration date + const decodedValue = JSON.parse(Buffer.from(prevToken.split('.')[1], 'base64').toString('ascii')); + // Check if token is expired, or near expiration + if (decodedValue.exp < Date.now() / 1000) { + return generateNewAccessToken(); + } + return prevToken; +}; + +export default fetchClientAccessToken; diff --git a/plugins/gatsby-source-snooty-preview/utils/post-build.mjs b/plugins/gatsby-source-snooty-preview/utils/post-build.mjs new file mode 100644 index 000000000..d3e08acd9 --- /dev/null +++ b/plugins/gatsby-source-snooty-preview/utils/post-build.mjs @@ -0,0 +1,55 @@ +import crypto from 'crypto'; + +/** + * Constructs a signature using the payload and Snooty's secret. The signature + * can be used to help webhooks be more confident that the caller is Snooty. + * @param {string} payloadString + */ +const constructSnootyHeader = (payloadString) => + crypto.createHmac('sha256', process.env.SNOOTY_SECRET).update(payloadString).digest('hex'); + +/** + * Calls the post-build webhook to let the Autobuilder know that the Gatsby Cloud + * build is finished. + * @param {object} webhookBody - The webhook body passed to the source plugin to + * initiate the preview build. + * @param {string} status - The status of the build, typically "completed" or "failed". + * This value should coincide with the Autobuilder's job statuses. + */ +const callPostBuildWebhook = async (webhookBody, status) => { + // Webhook body could be empty if the Gatsby Cloud site is doing a fresh build + // that was not called by the preview webhook + if (!webhookBody || !Object.keys(webhookBody).length) { + console.log('No webhookBody found. This build will not call the post-build webhook.'); + return; + } + + const supportedStatuses = ['completed', 'failed']; + if (!supportedStatuses.includes(status)) { + console.log(`Post-build webhook call does not support status "${status}".`); + return; + } + + const payload = { + ...webhookBody, + status, + }; + const body = JSON.stringify(payload); + const headers = { + 'x-snooty-signature': constructSnootyHeader(body), + }; + + console.log(`Calling post-build webhook with status "${status}".`); + const res = await fetch(process.env.AUTOBUILDER_POST_BUILD_WEBHOOK, { method: 'POST', body, headers }); + // Calling the webhook from this function should assume we are fulfilling the requirements of the call. + // Any error thrown here is definitely unexpected. + if (!res.ok) { + const errMessage = await res.text(); + throw new Error( + `There was an issue calling the Autobuilder post-build webhook. Please have the DOP team check CloudWatch logs. ${errMessage}` + ); + } + console.log('Post-build webhook was successfully called!'); +}; + +export default callPostBuildWebhook; diff --git a/plugins/gatsby-source-snooty-prod/gatsby-node.mjs b/plugins/gatsby-source-snooty-prod/gatsby-node.mjs new file mode 100644 index 000000000..04d52ea0c --- /dev/null +++ b/plugins/gatsby-source-snooty-prod/gatsby-node.mjs @@ -0,0 +1,309 @@ +import path, { dirname } from 'path'; +import { fileURLToPath } from 'url'; +import { createRequire } from 'module'; +import transformBreadcrumbs from '../../src/utils/setup/transform-breadcrumbs.mjs'; +import { saveAssetFiles, saveStaticFiles } from '../../src/utils/setup/save-asset-files.mjs'; +import validateEnvVariables from '../../src/utils/setup/validate-env-variables.mjs'; +import getNestedValue from '../../src/utils/get-nested-value.mjs'; +import removeNestedValue from '../../src/utils/remove-nested-value.mjs'; +import getPageSlug from '../../src/utils/get-page-slug.mjs'; +import { manifestMetadata, siteMetadata } from '../../src/utils/site-metadata.mjs'; +import assertTrailingSlash from '../../src/utils/assert-trailing-slash.mjs'; +import constructPageIdPrefix from '../../src/utils/setup/construct-page-id-prefix.mjs'; +import { manifestDocumentDatabase, realmDocumentDatabase } from '../../src/init/DocumentDatabase.mjs'; +import createOpenAPIChangelogNode from '../utils/openapi.mjs'; +import createProductNodes from '../utils/products.mjs'; + +// different types of references +const PAGES = []; + +// in-memory object with key/value = filename/document +let RESOLVED_REF_DOC_MAPPING = {}; + +const assets = new Map(); + +let db; + +let isAssociatedProduct = false; +const associatedReposInfo = {}; + +// Creates node for RemoteMetadata, mostly used for Embedded Versions. If no associated products +// or data are found, the node will be null +const createRemoteMetadataNode = async ({ createNode, createNodeId, createContentDigest }) => { + // fetch associated child products + const productList = manifestMetadata?.associated_products || []; + await Promise.all( + productList.map(async (product) => { + associatedReposInfo[product.name] = await db.realmInterface.fetchDocset({ project: product.name }); + }) + ); + // check if product is associated child product + try { + const umbrellaProduct = await db.realmInterface.getMetadata({ + 'associated_products.name': siteMetadata.project, + }); + isAssociatedProduct = !!umbrellaProduct; + } catch (e) { + console.log('No umbrella product found. Not an associated product.'); + isAssociatedProduct = false; + } + + // get remote metadata for updated ToC in Atlas + try { + const filter = { + project: manifestMetadata.project, + branch: manifestMetadata.branch, + }; + if (isAssociatedProduct || manifestMetadata?.associated_products?.length) { + filter['is_merged_toc'] = true; + } + const findOptions = { + sort: { build_id: -1 }, + }; + const remoteMetadata = await db.realmInterface.getMetadata(filter, findOptions); + + createNode({ + children: [], + id: createNodeId('remoteMetadata'), + internal: { + contentDigest: createContentDigest(remoteMetadata), + type: 'RemoteMetadata', + }, + parent: null, + remoteMetadata: remoteMetadata, + }); + } catch (e) { + console.error('Error while fetching metadata from Atlas, falling back to manifest metadata'); + console.error(e); + } +}; + +export const sourceNodes = async ({ actions, createContentDigest, createNodeId }) => { + let hasOpenAPIChangelog = false; + const { createNode } = actions; + + // setup and validate env variables + const envResults = validateEnvVariables(manifestMetadata); + if (envResults.error) { + throw Error(envResults.message); + } + + // wait to connect to Realm + + if (siteMetadata.manifestPath) { + console.log('Loading documents from manifest'); + db = manifestDocumentDatabase; + } else { + console.log('Loading documents from realm'); + db = realmDocumentDatabase; + } + + await db.connect(); + + const documents = await db.getDocuments(); + + if (documents.length === 0) { + console.error( + 'Snooty could not find AST entries for the', + siteMetadata.parserBranch, + 'branch of', + siteMetadata.project, + 'within', + siteMetadata.database + ); + process.exit(1); + } + const pageIdPrefix = constructPageIdPrefix(siteMetadata); + documents.forEach((doc) => { + const { page_id, ...rest } = doc; + RESOLVED_REF_DOC_MAPPING[page_id.replace(`${pageIdPrefix}/`, '')] = rest; + }); + + // Identify page documents and parse each document for images + Object.entries(RESOLVED_REF_DOC_MAPPING).forEach(([key, val]) => { + const pageNode = getNestedValue(['ast', 'children'], val); + const filename = getNestedValue(['filename'], val) || ''; + + // Parse each document before pages are created via createPage + // to remove all positions fields as it is only used in the parser for logging + removeNestedValue('position', 'children', [val?.ast]); + + if (pageNode) { + val.static_assets.forEach((asset) => { + const checksum = asset.checksum; + if (assets.has(checksum)) { + assets.set(checksum, new Set([...assets.get(checksum), asset.key])); + } else { + assets.set(checksum, new Set([asset.key])); + } + }); + } + + if (filename.endsWith('.txt') && !manifestMetadata.openapi_pages?.[key]) { + PAGES.push(key); + } + if (val?.ast?.options?.template === 'changelog') hasOpenAPIChangelog = true; + }); + + await createProductNodes({ db, createNode, createNodeId, createContentDigest }); + + await createRemoteMetadataNode({ createNode, createNodeId, createContentDigest }); + if (siteMetadata.project === 'cloud-docs' && hasOpenAPIChangelog) + await createOpenAPIChangelogNode({ createNode, createNodeId, createContentDigest, siteMetadata, db }); + + await saveAssetFiles(assets, db); + const { static_files: staticFiles, ...metadataMinusStatic } = await db.getMetadata(); + + const { parentPaths, slugToTitle } = metadataMinusStatic; + if (parentPaths) { + transformBreadcrumbs(parentPaths, slugToTitle); + } + + //Save files in the static_files field of metadata document, including intersphinx inventories + if (staticFiles) { + await saveStaticFiles(staticFiles); + } + + createNode({ + children: [], + id: createNodeId('metadata'), + internal: { + contentDigest: createContentDigest(metadataMinusStatic), + type: 'SnootyMetadata', + }, + parent: null, + metadata: metadataMinusStatic, + }); +}; + +export const createPages = async ({ actions }) => { + const { createPage } = actions; + + let repoBranches = null; + try { + const repoInfo = await db.realmInterface.fetchDocset(); + let errMsg; + + if (!repoInfo) { + errMsg = `Repo data for ${siteMetadata.project} could not be found.`; + } + + // We should expect the number of branches for a docs repo to be 1 or more. + if (!repoInfo.branches?.length) { + errMsg = `No version information found for ${siteMetadata.project}`; + } + + if (errMsg) { + throw errMsg; + } + + // Handle inconsistent env names. Default to 'dotcomprd' when possible since this is what we will most likely use. + // dotcom environments seem to be consistent. + let envKey = siteMetadata.snootyEnv; + if (!envKey || envKey === 'development') { + envKey = 'dotcomprd'; + } else if (envKey === 'production') { + envKey = 'prd'; + } else if (envKey === 'staging') { + envKey = 'stg'; + } + + // We're overfetching data here. We only need branches and prefix at the least + repoBranches = { + branches: repoInfo.branches, + siteBasePrefix: repoInfo.prefix[envKey], + }; + + if (repoInfo.groups?.length > 0) { + repoBranches.groups = repoInfo.groups; + } + } catch (err) { + console.error(err); + throw err; + } + + return new Promise((resolve, reject) => { + PAGES.forEach((page) => { + const pageNodes = RESOLVED_REF_DOC_MAPPING[page]?.ast; + const slug = getPageSlug(page); + + // TODO: Gatsby v4 will enable code splitting automatically. Delete duplicate component, add conditional for consistent-nav UnifiedFooter + const mainComponentRelativePath = `../../src/components/DocumentBody.js`; + + const __dirname = dirname(fileURLToPath(import.meta.url)); + + if (RESOLVED_REF_DOC_MAPPING[page] && Object.keys(RESOLVED_REF_DOC_MAPPING[page]).length > 0) { + createPage({ + path: assertTrailingSlash(slug), + component: path.resolve(__dirname, mainComponentRelativePath), + context: { + slug, + repoBranches, + associatedReposInfo, + isAssociatedProduct, + template: pageNodes?.options?.template, + page: pageNodes, + }, + }); + } + }); + + resolve(); + }); +}; + +// Prevent errors when running gatsby build caused by browser packages run in a node environment. +export const onCreateWebpackConfig = ({ plugins, actions }) => { + const require = createRequire(import.meta.url); + const providePlugins = { + Buffer: ['buffer', 'Buffer'], + process: require.resolve('../../stubs/process.js'), + }; + + const fallbacks = { stream: require.resolve('stream-browserify'), buffer: require.resolve('buffer/') }; + + actions.setWebpackConfig({ + plugins: [plugins.provide(providePlugins)], + resolve: { + fallback: fallbacks, + alias: { + process: 'process/browser', + }, + }, + }); +}; + +// Remove type inference, as our schema is too ambiguous for this to be useful. +// https://www.gatsbyjs.com/docs/scaling-issues/#switch-off-type-inference-for-sitepagecontext +export const createSchemaCustomization = ({ actions }) => { + actions.createTypes(` + type Page implements Node @dontInfer { + page_id: String + branch: String + pagePath: String + ast: JSON! + metadata: SnootyMetadata @link + } + + type SnootyMetadata implements Node @dontInfer { + metadata: JSON + branch: String + project: String + } + + type PagePath implements Node @dontInfer { + page_id: String! + branch: String! + project: String! + pageNodeId: String! + } + + type RemoteMetadata implements Node @dontInfer { + remoteMetadata: JSON + } + + type ChangelogData implements Node @dontInfer { + changelogData: JSON + } + `); +}; diff --git a/plugins/utils/openapi.mjs b/plugins/utils/openapi.mjs new file mode 100644 index 000000000..370def036 --- /dev/null +++ b/plugins/utils/openapi.mjs @@ -0,0 +1,115 @@ +import { hideChanges, hideDiffChanges } from '../../src/components/OpenAPIChangelog/utils/filterHiddenChanges.js'; + +const atlasAdminProdChangelogS3Prefix = 'https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/changelog'; +const atlasAdminDevChangelogS3Prefix = 'https://mongodb-mms-build-server.s3.amazonaws.com/openapi/changelog'; + +const fetchChangelogData = async (runId, versions, s3Prefix) => { + try { + /* Using metadata runId, fetch OpenAPI Changelog full change list */ + const changelogResp = await fetch(`${s3Prefix}/${runId}/changelog.json`); + const unfilteredChangelog = await changelogResp.json(); + const changelog = hideChanges(unfilteredChangelog); + + /* Aggregate all Resources in changelog for frontend filter */ + const resourcesListSet = new Set(); + changelog.forEach((release) => + release.paths.forEach(({ httpMethod, path }) => resourcesListSet.add(`${httpMethod} ${path}`)) + ); + const changelogResourcesList = Array.from(resourcesListSet); + + /* Fetch most recent Resource Versions' diff */ + const mostRecentResourceVersions = versions.slice(-2); + const mostRecentDiffLabel = mostRecentResourceVersions.join('_'); + const mostRecentDiffResp = await fetch(`${s3Prefix}/${runId}/${mostRecentDiffLabel}.json`); + const mostRecentUnfilteredDiffData = await mostRecentDiffResp.json(); + const mostRecentDiffData = hideDiffChanges(mostRecentUnfilteredDiffData); + + return { + changelog, + changelogResourcesList, + mostRecentDiff: { + mostRecentDiffLabel, + mostRecentDiffData, + }, + }; + } catch (error) { + console.warn('Changelog error: Most recent runId not successful. Using last successful runId to build Changelog.'); + throw error; + } +}; + +/* Creates node for ChangelogData, cuyrrently only used for OpenAPI Changelog in cloud-docs. */ +const createOpenAPIChangelogNode = async ({ createNode, createNodeId, createContentDigest, siteMetadata, db }) => { + try { + const { snootyEnv } = siteMetadata; + let atlasAdminChangelogS3Prefix; + let indexLocation; + switch (snootyEnv) { + case 'staging': + case 'development': + atlasAdminChangelogS3Prefix = atlasAdminDevChangelogS3Prefix; + indexLocation = 'dev.json'; + break; + case 'production': + default: + atlasAdminChangelogS3Prefix = atlasAdminProdChangelogS3Prefix; + indexLocation = 'prod.json'; + } + /* Fetch OpenAPI Changelog metadata */ + const indexResp = await fetch(`${atlasAdminChangelogS3Prefix}/${indexLocation}`); + const index = await indexResp.json(); + + const { runId, versions } = index; + + if (!runId || typeof runId !== 'string') + throw new Error('OpenAPI Changelog Error: `runId` not available in S3 index.json!'); + + let changelogData = { + index, + }; + try { + const receivedChangelogData = await fetchChangelogData(runId, versions, atlasAdminChangelogS3Prefix); + changelogData = { ...changelogData, ...receivedChangelogData }; + if (snootyEnv === 'production') await db.realmInterface.updateOAChangelogMetadata(index); + } catch (error) { + /* If any error occurs, fetch last successful metadata and build changelog node */ + const lastSuccessfulIndex = await db.realmInterface.fetchDocument( + 'openapi_changelog', + 'atlas_admin_metadata', + {} + ); + const { runId: lastRunId, versions: lastVersions } = lastSuccessfulIndex; + const receivedChangelogData = fetchChangelogData(lastRunId, lastVersions, atlasAdminProdChangelogS3Prefix); + changelogData = { index: lastSuccessfulIndex, ...receivedChangelogData }; + } + + /* Create Node for useStaticQuery with all Changelog data */ + createNode({ + children: [], + id: createNodeId('changelogData'), + internal: { + contentDigest: createContentDigest(changelogData), + type: 'ChangelogData', + }, + parent: null, + changelogData: changelogData, + }); + } catch (e) { + console.error('Error while fetching OpenAPI Changelog data from S3'); + console.error(e); + + /* Create empty Node for useStaticQuery to ensure successful build */ + createNode({ + children: [], + id: createNodeId('changelogData'), + internal: { + contentDigest: createContentDigest({}), + type: 'ChangelogData', + }, + parent: null, + changelogData: {}, + }); + } +}; + +export default createOpenAPIChangelogNode; diff --git a/plugins/utils/products.mjs b/plugins/utils/products.mjs new file mode 100644 index 000000000..2452a894a --- /dev/null +++ b/plugins/utils/products.mjs @@ -0,0 +1,23 @@ +import { baseUrl } from '../../src/utils/base-url.js'; + +const createProductNodes = async ({ db, createNode, createNodeId, createContentDigest }) => { + // Get all MongoDB products for the sidenav + const products = await db.fetchAllProducts(); + products.forEach((product) => { + const url = baseUrl(product.baseUrl + product.slug); + + createNode({ + children: [], + id: createNodeId(`Product-${product.title}`), + internal: { + contentDigest: createContentDigest(product), + type: 'Product', + }, + parent: null, + title: product.title, + url, + }); + }); +}; + +export default createProductNodes; diff --git a/src/init/DocumentDatabase.mjs b/src/init/DocumentDatabase.mjs new file mode 100644 index 000000000..d7d945988 --- /dev/null +++ b/src/init/DocumentDatabase.mjs @@ -0,0 +1,132 @@ +import AdmZip from 'adm-zip'; +import BSON from 'bson'; +import initRealm from '../utils/setup/init-realm.js'; +import { DOCUMENTS_COLLECTION, METADATA_COLLECTION, ASSETS_COLLECTION } from '../build-constants.js'; +import { manifestMetadata, siteMetadata } from '../utils/site-metadata.mjs'; +import constructBuildFilter from '../utils/setup/construct-build-filter.js'; + +const DB = siteMetadata.database; +const buildFilter = constructBuildFilter(siteMetadata); + +class RealmInterface { + constructor() { + this.realmClient = null; + } + + async connect() { + this.realmClient = await initRealm(); + } + + fetchAllProducts() { + return this.realmClient.callFunction('fetchAllProducts', siteMetadata.database); + } + + async fetchDocuments(collection, buildFilter) { + return this.realmClient.callFunction('fetchDocuments', DB, collection, buildFilter); + } + + async getMetadata(buildFilter, findOptions) { + return this.realmClient.callFunction('fetchDocument', DB, METADATA_COLLECTION, buildFilter, undefined, findOptions); + } + + async fetchDocument(database, collectionName, query) { + return this.realmClient.callFunction('fetchDocument', database, collectionName, query); + } + + async fetchDocset(matchConditions = { project: siteMetadata.project }) { + return this.realmClient.callFunction('fetchDocset', siteMetadata.reposDatabase, matchConditions); + } + + async updateOAChangelogMetadata(metadata) { + return this.realmClient.callFunction('updateOAChangelogMetadata', metadata); + } +} + +class ManifestDocumentDatabase { + constructor(path) { + this.zip = new AdmZip(path); + this.realmInterface = new RealmInterface(); + } + + async connect() { + await this.realmInterface.connect(); + } + + async getDocuments() { + const result = []; + const zipEntries = this.zip.getEntries(); + for (const entry of zipEntries) { + if (entry.entryName.startsWith('documents/')) { + const doc = BSON.deserialize(entry.getData()); + result.push(doc); + } + } + return result; + } + + async getMetadata() { + return manifestMetadata; + } + + async getAsset(checksum) { + const result = this.zip.getEntry(`assets/${checksum}`); + if (result) { + return result.getData(); + } + return null; + } + + async fetchAllProducts() { + return this.realmInterface.fetchAllProducts(); + } +} + +class RealmDocumentDatabase { + constructor() { + this.realmInterface = new RealmInterface(); + } + + async connect() { + await this.realmInterface.connect(); + } + + async getDocuments() { + return this.realmInterface.fetchDocuments(DOCUMENTS_COLLECTION, buildFilter); + } + + async getMetadata(queryFilters) { + const filter = queryFilters || buildFilter; + return this.realmInterface.getMetadata(filter); + } + + async getAsset(checksum) { + const assetQuery = { _id: checksum }; + const assetDataDocuments = await this.realmInterface.fetchDocuments(ASSETS_COLLECTION, assetQuery); + + for (const result of assetDataDocuments) { + return result.data.buffer; + } + return null; + } + + async fetchAllProducts() { + return this.realmInterface.fetchAllProducts(); + } + + async fetchDocument(database, collectionName, query) { + return this.realmInterface.fetchDocument(database, collectionName, query); + } + + async fetchDocset(matchConditions) { + return this.realmInterface.fetchDocset(matchConditions); + } + + async updateOAChangelogMetadata(metadata) { + return this.realmInterface.updateOAChangelogMetadata(metadata); + } +} + +const manifestDocumentDatabase = new ManifestDocumentDatabase(process.env.GATSBY_MANIFEST_PATH); +const realmDocumentDatabase = new RealmDocumentDatabase(); + +export { manifestDocumentDatabase, realmDocumentDatabase }; diff --git a/src/utils/assert-trailing-slash.mjs b/src/utils/assert-trailing-slash.mjs new file mode 100644 index 000000000..23509000c --- /dev/null +++ b/src/utils/assert-trailing-slash.mjs @@ -0,0 +1,8 @@ +const assertTrailingSlash = (url) => { + if (url && url.match(/\/$/)) { + return url; + } + return `${url}/`; +}; + +export default assertTrailingSlash; diff --git a/src/utils/get-nested-value.mjs b/src/utils/get-nested-value.mjs new file mode 100644 index 000000000..7ce4b12c9 --- /dev/null +++ b/src/utils/get-nested-value.mjs @@ -0,0 +1,14 @@ +/* + * Safely return a deeply nested value from an object. If the property is not found, return undefined. + * Arguments: + * - p: an array containing the path to the desired return value + * - o: the object to be searched + */ +const getNestedValue = (p, o) => { + if (!o) return undefined; + return p.reduce((xs, x) => (xs && xs[x] ? xs[x] : undefined), o); +}; + +// TODO: switch to ES6 export syntax if Gatsby implements support for ES6 module imports +// https://github.com/gatsbyjs/gatsby/issues/7810 +export default getNestedValue; diff --git a/src/utils/get-page-slug.mjs b/src/utils/get-page-slug.mjs new file mode 100644 index 000000000..eb5fd8712 --- /dev/null +++ b/src/utils/get-page-slug.mjs @@ -0,0 +1,6 @@ +// Takes a look at the page name and returns the appropriate page url +const getPageSlug = (page) => { + return page === 'index' ? '/' : page; +}; + +export default getPageSlug; diff --git a/src/utils/remove-nested-value.mjs b/src/utils/remove-nested-value.mjs new file mode 100644 index 000000000..2f3aed194 --- /dev/null +++ b/src/utils/remove-nested-value.mjs @@ -0,0 +1,25 @@ +/* + * Safely remove values from a deeply nested object. + * Arguments: + * - target: the target key to be deleted + * - traversalKey: the target key to traverse + * - arr: an array containing the objects to search for the target value + */ +const removeNestedValue = (target, traversalKey, arr) => { + if (!arr) return; + + arr.forEach((entry) => { + if (entry.hasOwnProperty(target)) { + delete entry[target]; + } + + if (entry.hasOwnProperty(traversalKey)) { + const children = entry[traversalKey]; + if (Array.isArray(children) && children.length) { + removeNestedValue(target, traversalKey, children); + } + } + }); +}; + +export default removeNestedValue; diff --git a/src/utils/setup/construct-page-id-prefix.mjs b/src/utils/setup/construct-page-id-prefix.mjs new file mode 100644 index 000000000..b1672e965 --- /dev/null +++ b/src/utils/setup/construct-page-id-prefix.mjs @@ -0,0 +1,4 @@ +// Concatenates a site prefix to return a page id +const constructPageIdPrefix = ({ project, parserUser, parserBranch }) => `${project}/${parserUser}/${parserBranch}`; + +export default constructPageIdPrefix; diff --git a/src/utils/setup/save-asset-files.mjs b/src/utils/setup/save-asset-files.mjs new file mode 100644 index 000000000..ba61042f7 --- /dev/null +++ b/src/utils/setup/save-asset-files.mjs @@ -0,0 +1,40 @@ +import fs from 'fs/promises'; +import path from 'path'; + +const saveFile = async (file, data) => { + await fs.mkdir(path.join('public', path.dirname(file)), { + recursive: true, + }); + await fs.writeFile(path.join('public', file), data, 'binary'); +}; + +// Write all assets to static directory +const saveAssetFiles = async (assets, db) => { + const imageWrites = []; + + for (const [id, filenames] of assets) { + if (filenames) { + const buffer = await db.getAsset(id); + if (!buffer) { + console.error( + `Failed to fetch asset with checksum ${id}. This should not be possible and indicates an internal problem with the document source.` + ); + process.exit(1); + } + filenames.forEach((filename) => imageWrites.push(saveFile(filename, buffer))); + } + } + await Promise.all(imageWrites); +}; + +const saveStaticFiles = async (staticFiles) => { + await Promise.all( + Object.entries(staticFiles).map(([file, data]) => { + // Certain files, like manpages, may not have buffers + const toSave = data.buffer ? data.buffer : data; + return saveFile(file, toSave); + }) + ); +}; + +export { saveAssetFiles, saveStaticFiles, saveFile }; diff --git a/src/utils/setup/transform-breadcrumbs.mjs b/src/utils/setup/transform-breadcrumbs.mjs new file mode 100644 index 000000000..6452080b1 --- /dev/null +++ b/src/utils/setup/transform-breadcrumbs.mjs @@ -0,0 +1,16 @@ +import getPlaintext from '../get-plaintext.js'; + +const transformBreadcrumbs = (breadcrumbs, slugToTitle) => { + Object.entries(breadcrumbs).forEach(([slug, breadcrumbList]) => { + breadcrumbs[slug] = breadcrumbList.map((path) => { + const title = slugToTitle?.[path] || []; + return { + path, + title, + plaintext: getPlaintext(title), + }; + }); + }); +}; + +export default transformBreadcrumbs; diff --git a/src/utils/setup/validate-env-variables.mjs b/src/utils/setup/validate-env-variables.mjs new file mode 100644 index 000000000..9553cb424 --- /dev/null +++ b/src/utils/setup/validate-env-variables.mjs @@ -0,0 +1,103 @@ +import fs from 'fs'; + +// env variables for building site along with use in front-end +// https://www.gatsbyjs.org/docs/environment-variables/#defining-environment-variables +const validateEnvVariables = (manifestMetadata) => { + // only require env vars when no manifest path is specified + if ( + !process.env.GATSBY_MANIFEST_PATH && + (!process.env.GATSBY_SITE || !process.env.GATSBY_PARSER_USER || !process.env.GATSBY_PARSER_BRANCH) + ) { + return { + error: true, + message: `${process.env.NODE_ENV} requires the variables GATSBY_SITE, GATSBY_PARSER_USER, and GATSBY_PARSER_BRANCH, found: ${process.env.GATSBY_SITE}, ${process.env.GATSBY_PARSER_USER}, ${process.env.GATSBY_PARSER_BRANCH}`, + }; + } else if (process.env.GATSBY_MANIFEST_PATH) { + validateManifestEnvVars(manifestMetadata); + } + // create split prefix for use in stitch function + return { + error: false, + }; +}; + +// set env vars accordingly if none are provided +const validateManifestEnvVars = (manifestMetadata) => { + let envVars = ''; + const site = `\nGATSBY_SITE=${manifestMetadata['project']}`; + const branch = `\nGATSBY_PARSER_BRANCH=${manifestMetadata['branch']}`; + + if (!process.env.GATSBY_SITE) { + envVars += site; + } + if (!process.env.GATSBY_PARSER_BRANCH) { + envVars += branch; + } + + // write to .env files with updated env vars + try { + ['.env.production', '.env.development'].forEach((file) => { + fs.appendFile(file, envVars, function (err) { + if (err) { + console.error(err); + throw err; + } + }); + }); + } catch (err) { + console.error(err); + throw err; + } + + // if env variables are present, check that they are set to the correct values. Otherwise, override them. + let incorrectEnvVars = []; + let correctEnvVars = ''; + if (process.env.GATSBY_SITE && process.env.GATSBY_SITE !== manifestMetadata['project']) { + incorrectEnvVars.push('GATSBY_SITE'); + correctEnvVars += site; + } + if (process.env.GATSBY_PARSER_BRANCH && process.env.GATSBY_PARSER_BRANCH !== manifestMetadata['branch']) { + incorrectEnvVars.push('GATSBY_PARSER_BRANCH'); + correctEnvVars += branch; + } + if (incorrectEnvVars.length > 0) { + replaceIncorrectEnvVars(incorrectEnvVars, correctEnvVars); + } +}; + +const replaceIncorrectEnvVars = (incorrectEnvVars, envVars) => { + try { + ['.env.production', '.env.development'].forEach((file) => { + fs.readFile(file, 'utf8', (err, data) => { + if (err) { + console.error(err); + throw err; + } + + const dataArray = data.split('\n'); + + // remove incorrect variables from .env files + for (const dataEntry in dataArray) { + for (const incorrectVar in incorrectEnvVars) { + if (dataArray[dataEntry].includes(incorrectEnvVars[incorrectVar])) { + dataArray.splice(dataEntry, 1); + } + } + } + + // write to .env files with updated env vars + fs.writeFile(file, dataArray.join('\n').concat(envVars), (err) => { + if (err) { + console.error(err); + throw err; + } + }); + }); + }); + } catch (err) { + console.error(err); + throw err; + } +}; + +export default validateEnvVariables; diff --git a/src/utils/site-metadata.mjs b/src/utils/site-metadata.mjs new file mode 100644 index 000000000..677790060 --- /dev/null +++ b/src/utils/site-metadata.mjs @@ -0,0 +1,82 @@ +import { execSync } from 'child_process'; +import os from 'os'; +import fetchManifestMetadata from './setup/fetch-manifest-metadata.js'; +import { DOTCOM_BASE_URL } from './base-url.js'; +import dotenv from 'dotenv'; + +// loads vars from the .env file into process.env object +const runningEnv = process.env.NODE_ENV || 'production'; + +dotenv.config({ + path: `.env.${runningEnv}`, +}); +// require('dotenv').config({ +// path: `.env.${runningEnv}`, +// }); + +const manifestMetadata = fetchManifestMetadata(); + +const getDatabase = (env) => { + switch (env) { + case 'staging': + return 'snooty_stage'; + case 'production': + return 'snooty_prod'; + case 'dotcomstg': + return 'snooty_dotcomstg'; + case 'dotcomprd': + return 'snooty_dotcomprd'; + default: + return 'snooty_dev'; + } +}; + +const getReposDatabase = (env) => { + switch (env) { + case 'staging': + return 'pool_test'; + case 'production': + return 'pool'; + case 'dotcomstg': + return 'pool_test'; + case 'dotcomprd': + return 'pool'; + default: + return 'pool_test'; + } +}; + +const gitBranch = execSync('git rev-parse --abbrev-ref HEAD') + .toString('utf8') + .replace(/[\n\r\s]+$/, ''); + +const getPathPrefix = (pathPrefix) => { + if (!pathPrefix) { + return ''; + } + if (pathPrefix.startsWith('/')) { + return pathPrefix; + } + return `/${pathPrefix}`; +}; + +/** + * Get site metadata used to identify this build and query correct documents + */ +const siteMetadata = { + commitHash: process.env.COMMIT_HASH || '', + database: getDatabase(process.env.SNOOTY_ENV), + reposDatabase: getReposDatabase(process.env.SNOOTY_ENV), + parserBranch: manifestMetadata['branch'] || process.env.GATSBY_PARSER_BRANCH || `master`, + parserUser: process.env.GATSBY_PARSER_USER || os.userInfo().username, + patchId: process.env.PATCH_ID || '', + pathPrefix: getPathPrefix(process.env.PATH_PREFIX), + project: manifestMetadata['project'] || process.env.GATSBY_SITE, + siteUrl: DOTCOM_BASE_URL, + snootyBranch: gitBranch, + snootyEnv: process.env.SNOOTY_ENV || 'development', + user: os.userInfo().username, + manifestPath: process.env.GATSBY_MANIFEST_PATH, +}; + +export { siteMetadata, manifestMetadata };