Bring in data-directory, let's go async file reads (github#16782)

* Bring in data-directory, let's go async file reads

* Lint fixes

* Update glossary.js

lib/data-directory.js

@@ -0,0 +1,68 @@
+const assert = require('assert')
+const fs = require('fs').promises
+const path = require('path')
+const walk = require('walk-sync')
+const yaml = require('js-yaml')
+const { isRegExp, set } = require('lodash')
+const filenameToKey = require('./filename-to-key')
+
+module.exports = async function dataDirectory (dir, opts = {}) {
+  const defaultOpts = {
+    preprocess: (content) => { return content },
+    ignorePatterns: [/README\.md$/i],
+    extensions: [
+      '.json',
+      '.md',
+      '.markdown',
+      '.yaml',
+      '.yml'
+    ]
+  }
+
+  opts = Object.assign({}, defaultOpts, opts)
+
+  // validate input
+  assert(Array.isArray(opts.ignorePatterns))
+  assert(opts.ignorePatterns.every(isRegExp))
+  assert(Array.isArray(opts.extensions))
+  assert(opts.extensions.length)
+
+  // start with an empty data object
+  const data = {}
+
+  // find YAML and Markdown files in the given directory, recursively
+  await Promise.all(walk(dir, { includeBasePath: true })
+    .filter(filename => {
+      // ignore files that match any of ignorePatterns regexes
+      if (opts.ignorePatterns.some(pattern => pattern.test(filename))) return false
+
+      // ignore files that don't have a whitelisted file extension
+      return opts.extensions.includes(path.extname(filename).toLowerCase())
+    })
+    .map(async filename => {
+      // derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename
+      const key = filenameToKey(path.relative(dir, filename))
+      const extension = path.extname(filename).toLowerCase()
+
+      let fileContent = await fs.readFile(filename, 'utf8')
+
+      if (opts.preprocess) fileContent = opts.preprocess(fileContent)
+
+      // add this file's data to the global data object
+      switch (extension) {
+        case '.json':
+          set(data, key, JSON.parse(fileContent))
+          break
+        case '.yaml':
+        case '.yml':
+          set(data, key, yaml.safeLoad(fileContent, { filename }))
+          break
+        case '.md':
+        case '.markdown':
+          set(data, key, fileContent)
+          break
+      }
+    }))
+
+  return data
+}

lib/filename-to-key.js

@@ -0,0 +1,28 @@
+/* eslint-disable prefer-regex-literals */
+const path = require('path')
+const { escapeRegExp } = require('lodash')
+
+// slash at the beginning of a filename
+const leadingPathSeparator = new RegExp(`^${escapeRegExp(path.sep)}`)
+const windowsLeadingPathSeparator = new RegExp('^/')
+
+// all slashes in the filename. path.sep is OS agnostic (windows, mac, etc)
+const pathSeparator = new RegExp(escapeRegExp(path.sep), 'g')
+const windowsPathSeparator = new RegExp('/', 'g')
+
+// handle MS Windows style double-backslashed filenames
+const windowsDoubleSlashSeparator = new RegExp('\\\\', 'g')
+
+// derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename
+module.exports = function filenameToKey (filename) {
+  const extension = new RegExp(`${path.extname(filename)}$`)
+  const key = filename
+    .replace(extension, '')
+    .replace(leadingPathSeparator, '')
+    .replace(windowsLeadingPathSeparator, '')
+    .replace(pathSeparator, '.')
+    .replace(windowsPathSeparator, '.')
+    .replace(windowsDoubleSlashSeparator, '.')
+
+  return key
+}

lib/site-data.js

@@ -2,12 +2,12 @@ const path = require('path')
 const flat = require('flat')
 const { get, set } = require('lodash')
 const languages = require('./languages')
-const dataDirectory = require('@github-docs/data-directory')
+const dataDirectory = require('./data-directory')
 const encodeBracketedParentheticals = require('./encode-bracketed-parentheticals')
 
-const loadSiteDataFromDir = dir => ({
+const loadSiteDataFromDir = async dir => ({
   site: {
-    data: dataDirectory(path.join(dir, 'data'), {
+    data: await dataDirectory(path.join(dir, 'data'), {
       preprocess: dataString =>
         encodeBracketedParentheticals(dataString.trimEnd()),
       ignorePatterns: [/README\.md$/]
@@ -18,15 +18,15 @@ const loadSiteDataFromDir = dir => ({
 module.exports = async function loadSiteData () {
   // load english site data
   const siteData = {
-    en: loadSiteDataFromDir(languages.en.dir)
+    en: await loadSiteDataFromDir(languages.en.dir)
   }
 
   // load and add other language data to siteData where keys match english keys,
   // filling holes with english site data
   const englishKeys = Object.keys(flat(siteData.en))
   for (const language of Object.values(languages)) {
     if (language.code === 'en') continue
-    const data = loadSiteDataFromDir(language.dir)
+    const data = await loadSiteDataFromDir(language.dir)
     for (const key of englishKeys) {
       set(
         siteData,

package.json

@@ -19,7 +19,6 @@
     "@babel/preset-env": "^7.12.7",
     "@babel/preset-react": "^7.12.7",
     "@babel/runtime": "^7.11.2",
-    "@github-docs/data-directory": "^1.2.0",
     "@github-docs/frontmatter": "^1.3.1",
     "@graphql-inspector/core": "^2.3.0",
     "@graphql-tools/load": "^6.2.5",

tests/unit/data-directory/filename-to-key.js

@@ -0,0 +1,15 @@
+const filenameToKey = require('../../../lib/filename-to-key')
+
+describe('filename-to-key', () => {
+  test('converts filenames to object keys', () => {
+    expect(filenameToKey('foo/bar/baz.txt')).toBe('foo.bar.baz')
+  })
+
+  test('ignores leading slash on filenames', () => {
+    expect(filenameToKey('/foo/bar/baz.txt')).toBe('foo.bar.baz')
+  })
+
+  test('supports MS Windows paths', () => {
+    expect(filenameToKey('path\\to\\file.txt')).toBe('path.to.file')
+  })
+})

tests/unit/data-directory/fixtures/README.md

@@ -0,0 +1 @@
+I am a README. I am ignored by default.

tests/unit/data-directory/fixtures/bar.yaml

@@ -0,0 +1 @@
+another_markup_language: 'yes'

tests/unit/data-directory/fixtures/foo.json

@@ -0,0 +1 @@
+{"meaningOfLife": 42}

tests/unit/data-directory/fixtures/nested/baz.md

@@ -0,0 +1 @@
+I am markdown!

tests/unit/data-directory/index.js

@@ -0,0 +1,40 @@
+const path = require('path')
+const dataDirectory = require('../../../lib/data-directory')
+const fixturesDir = path.join(__dirname, 'fixtures')
+
+describe('data-directory', () => {
+  test('works', async () => {
+    const data = await dataDirectory(fixturesDir)
+    const expected = {
+      bar: { another_markup_language: 'yes' },
+      foo: { meaningOfLife: 42 },
+      nested: { baz: 'I am markdown!' }
+    }
+    expect(data).toEqual(expected)
+  })
+
+  test('option: preprocess function', async () => {
+    const preprocess = function (content) {
+      return content.replace('markdown', 'MARKDOWN')
+    }
+    const data = await dataDirectory(fixturesDir, { preprocess })
+    expect(data.nested.baz).toBe('I am MARKDOWN!')
+  })
+
+  test('option: extensions array', async () => {
+    const extensions = ['.yaml', 'markdown']
+    const data = await dataDirectory(fixturesDir, { extensions })
+    expect('bar' in data).toBe(true)
+    expect('foo' in data).toBe(false) // JSON file should be ignored
+  })
+
+  test('option: ignorePatterns', async () => {
+    const ignorePatterns = []
+
+    // README is ignored by default
+    expect('README' in await dataDirectory(fixturesDir)).toBe(false)
+
+    // README can be included by setting empty ignorePatterns array
+    expect('README' in await dataDirectory(fixturesDir, { ignorePatterns })).toBe(true)
+  })
+})