Implemented wildcard selector (based on #238) (#488)

* Initial check-in of wildcard to get all secrets in path (Issue#234)
* Fix wildcard for K/V v2 and Cubbyhole.  Add more tests
* Refactored out selectAndAppendResults
* Use selectAndAppendResults for wildcard
* Use normalizeOutputKey in action.js
* Refactored wildcard

---------

Co-authored-by: Scott Lemme <68233981+slemme1@users.noreply.github.com>
Co-authored-by: Lemme <slemme@massmutual.com>

README.md

@@ -373,6 +373,13 @@ with:
         secret/data/ci/aws accessKey | AWS_ACCESS_KEY_ID ;
         secret/data/ci/aws secretKey | AWS_SECRET_ACCESS_KEY
 ```
+You can specify a wildcard * for the key name to get all keys in the path.  If you provide an output name with the wildcard, the name will be prepended to the key name:
+
+```yaml
+with:
+    secrets: |
+        secret/data/ci/aws * | MYAPP_ ;
+```
 
 ## Other Secret Engines
 

dist/index.js

@@ -18516,6 +18516,9 @@ const core = __nccwpck_require__(2186);
 const command = __nccwpck_require__(7351);
 const got = (__nccwpck_require__(3061)["default"]);
 const jsonata = __nccwpck_require__(4245);
+const { normalizeOutputKey } = __nccwpck_require__(1608);
+const { WILDCARD } = __nccwpck_require__(4438);
+
 const { auth: { retrieveToken }, secrets: { getSecrets } } = __nccwpck_require__(4351);
 
 const AUTH_METHODS = ['approle', 'token', 'github', 'jwt', 'kubernetes', 'ldap', 'userpass'];
@@ -18684,7 +18687,7 @@ function parseSecretsInput(secretsInput) {
         const selectorAst = jsonata(selectorQuoted).ast();
         const selector = selectorQuoted.replace(new RegExp('"', 'g'), '');
 
-        if ((selectorAst.type !== "path" || selectorAst.steps[0].stages) && selectorAst.type !== "string" && !outputVarName) {
+        if (selector !== WILDCARD && (selectorAst.type !== "path" || selectorAst.steps[0].stages) && selectorAst.type !== "string" && !outputVarName) {
             throw Error(`You must provide a name for the output key when using json selectors. Input: "${secret}"`);
         }
 
@@ -18704,20 +18707,6 @@ function parseSecretsInput(secretsInput) {
     return output;
 }
 
-/**
- * Replaces any dot chars to __ and removes non-ascii charts
- * @param {string} dataKey
- * @param {boolean=} isEnvVar
- */
-function normalizeOutputKey(dataKey, isEnvVar = false) {
-    let outputKey = dataKey
-        .replace('.', '__').replace(new RegExp('-', 'g'), '').replace(/[^\p{L}\p{N}_-]/gu, '');
-    if (isEnvVar) {
-        outputKey = outputKey.toUpperCase();
-    }
-    return outputKey;
-}
-
 /**
  * @param {string} inputKey
  * @param {any} inputOptions
@@ -18746,11 +18735,11 @@ function parseHeadersInput(inputKey, inputOptions) {
 module.exports = {
     exportSecrets,
     parseSecretsInput,
-    normalizeOutputKey,
-    parseHeadersInput
+    parseHeadersInput,
 };
 
 
+
 /***/ }),
 
 /***/ 4915:
@@ -18917,6 +18906,17 @@ module.exports = {
 };
 
 
+/***/ }),
+
+/***/ 4438:
+/***/ ((module) => {
+
+const WILDCARD = '*';
+
+module.exports = {
+    WILDCARD
+};
+
 /***/ }),
 
 /***/ 4351:
@@ -18936,8 +18936,8 @@ module.exports = {
 /***/ ((module, __unused_webpack_exports, __nccwpck_require__) => {
 
 const jsonata = __nccwpck_require__(4245);
-
-
+const { WILDCARD } = __nccwpck_require__(4438);
+const { normalizeOutputKey } = __nccwpck_require__(1608);
 /**
  * @typedef {Object} SecretRequest
  * @property {string} path
@@ -18960,7 +18960,8 @@ const jsonata = __nccwpck_require__(4245);
   */
 async function getSecrets(secretRequests, client) {
     const responseCache = new Map();
-    const results = [];
+    let results = [];
+
     for (const secretRequest of secretRequests) {
         let { path, selector } = secretRequest;
 
@@ -18983,22 +18984,53 @@ async function getSecrets(secretRequests, client) {
                 throw error
             }
         }
-        if (!selector.match(/.*[\.].*/)) {
-            selector = '"' + selector + '"'
-        }
-        selector = "data." + selector
-        body = JSON.parse(body)
+
+        body = JSON.parse(body);
+
+        if (selector == WILDCARD) {                     
+            let keys = body.data;
             if (body.data["data"] != undefined) {
-            selector = "data." + selector
+                keys = keys.data;
             }
 
-        const value = await selectData(body, selector);
-        results.push({
-            request: secretRequest,
-            value,
-            cachedResponse
-        });
+            for (let key in keys) {
+                let newRequest = Object.assign({},secretRequest);
+                newRequest.selector = key;                  
+                
+                if (secretRequest.selector === secretRequest.outputVarName) {
+                    newRequest.outputVarName = key;
+                    newRequest.envVarName = key;        		
                 }
+                else {
+                    newRequest.outputVarName = secretRequest.outputVarName+key;
+                    newRequest.envVarName = secretRequest.envVarName+key;        		
+                }
+
+                newRequest.outputVarName = normalizeOutputKey(newRequest.outputVarName);
+                newRequest.envVarName = normalizeOutputKey(newRequest.envVarName,true);   
+
+                selector = key;
+
+                results = await selectAndAppendResults(
+                  selector,
+                  body,
+                  cachedResponse,
+                  newRequest,
+                  results
+                );
+            }
+        }
+        else {
+          results = await selectAndAppendResults(
+            selector, 
+            body, 
+            cachedResponse, 
+            secretRequest, 
+            results
+          );
+        }   
+    }
+
     return results;
 }
 
@@ -19024,12 +19056,75 @@ async function selectData(data, selector) {
     return result;
 }
 
+/**
+ * Uses selectData with the selector to get the value and then appends it to the
+ * results. Returns a new array with all of the results.
+ * @param {string} selector
+ * @param {object} body
+ * @param {object} cachedResponse
+ * @param {TRequest} secretRequest
+ * @param {SecretResponse<TRequest>[]} results
+ * @return {Promise<SecretResponse<TRequest>[]>}
+ */
+const selectAndAppendResults = async (
+  selector,
+  body,
+  cachedResponse,
+  secretRequest,
+  results
+) => {
+  if (!selector.match(/.*[\.].*/)) {
+    selector = '"' + selector + '"';
+  }
+  selector = "data." + selector;
+
+  if (body.data["data"] != undefined) {
+    selector = "data." + selector;
+  }
+
+  const value = await selectData(body, selector);
+  return [
+    ...results,
+    {
+      request: secretRequest,
+      value,
+      cachedResponse,
+    },
+  ];
+};
+
 module.exports = {
     getSecrets,
     selectData
 }
 
 
+/***/ }),
+
+/***/ 1608:
+/***/ ((module) => {
+
+/**
+ * Replaces any dot chars to __ and removes non-ascii charts
+ * @param {string} dataKey
+ * @param {boolean=} isEnvVar
+ */
+function normalizeOutputKey(dataKey, isEnvVar = false) {
+  let outputKey = dataKey
+    .replace(".", "__")
+    .replace(new RegExp("-", "g"), "")
+    .replace(/[^\p{L}\p{N}_-]/gu, "");
+  if (isEnvVar) {
+    outputKey = outputKey.toUpperCase();
+  }
+  return outputKey;
+}
+
+module.exports = { 
+    normalizeOutputKey 
+};
+
+
 /***/ }),
 
 /***/ 9491:

integrationTests/basic/integration.test.js

@@ -171,6 +171,26 @@ describe('integration', () => {
         expect(core.exportVariable).toBeCalledWith('OTHERSECRETDASH', 'OTHERSUPERSECRET');
     });
 
+    it('get wildcard secrets', async () => {
+        mockInput(`secret/data/test * ;`);
+
+        await exportSecrets();
+
+        expect(core.exportVariable).toBeCalledTimes(1);
+
+        expect(core.exportVariable).toBeCalledWith('SECRET', 'SUPERSECRET');
+    });
+
+    it('get wildcard secrets with name prefix', async () => {
+        mockInput(`secret/data/test * | GROUP_ ;`);
+
+        await exportSecrets();
+
+        expect(core.exportVariable).toBeCalledTimes(1);
+
+        expect(core.exportVariable).toBeCalledWith('GROUP_SECRET', 'SUPERSECRET');
+    });
+
     it('leading slash kvv2', async () => {
         mockInput('/secret/data/foobar fookv2');
 
@@ -195,6 +215,34 @@ describe('integration', () => {
         expect(core.exportVariable).toBeCalledWith('OTHERSECRETDASH', 'OTHERCUSTOMSECRET');
     });
 
+    it('get K/V v1 wildcard secrets', async () => {
+        mockInput(`secret-kv1/test * ;`);
+
+        await exportSecrets();
+
+        expect(core.exportVariable).toBeCalledTimes(1);
+
+        expect(core.exportVariable).toBeCalledWith('SECRET', 'CUSTOMSECRET');
+    });
+
+    it('get K/V v1 wildcard secrets with name prefix', async () => {
+        mockInput(`secret-kv1/test * | GROUP_ ;`);
+
+        await exportSecrets();
+
+        expect(core.exportVariable).toBeCalledTimes(1);
+
+        expect(core.exportVariable).toBeCalledWith('GROUP_SECRET', 'CUSTOMSECRET');
+    });
+
+    it('get wildcard nested secret from K/V v1', async () => {
+        mockInput('secret-kv1/nested/test *');
+
+        await exportSecrets();
+
+        expect(core.exportVariable).toBeCalledWith('OTHERSECRETDASH', 'OTHERCUSTOMSECRET');
+    });
+
     it('leading slash kvv1', async () => {
         mockInput('/secret-kv1/foobar fookv1');
 
@@ -225,6 +273,17 @@ describe('integration', () => {
             expect(core.exportVariable).toBeCalledWith('FOO', 'bar');
         });
 
+        it('wildcard supports cubbyhole', async () => {            
+            mockInput('/cubbyhole/test *');
+
+            await exportSecrets();
+            
+            expect(core.exportVariable).toBeCalledTimes(2);
+
+            expect(core.exportVariable).toBeCalledWith('FOO', 'bar');
+            expect(core.exportVariable).toBeCalledWith('ZIP', 'zap');
+        });
+        
         it('caches responses', async () => {            
             mockInput(`
             /cubbyhole/test foo ;

integrationTests/enterprise/enterprise.test.js

@@ -72,6 +72,22 @@ describe('integration', () => {
         expect(core.exportVariable).toBeCalledWith('TEST_KEY', 'SUPERSECRET_IN_NAMESPACE');
     });
 
+    it('get wildcard secrets', async () => {
+        mockInput('secret/data/test *');
+
+        await exportSecrets();
+
+        expect(core.exportVariable).toBeCalledWith('SECRET', 'SUPERSECRET_IN_NAMESPACE');
+    });
+
+    it('get wildcard secrets with name prefix', async () => {
+        mockInput('secret/data/test * | GROUP_');
+
+        await exportSecrets();
+
+        expect(core.exportVariable).toBeCalledWith('GROUP_SECRET', 'SUPERSECRET_IN_NAMESPACE');
+    });
+
     it('get nested secret', async () => {
         mockInput('secret/data/nested/test otherSecret');
 
@@ -103,6 +119,22 @@ describe('integration', () => {
         expect(core.exportVariable).toBeCalledWith('SECRET', 'CUSTOMSECRET_IN_NAMESPACE');
     });
 
+    it('get wildcard secrets from K/V v1', async () => {
+        mockInput('my-secret/test *');
+
+        await exportSecrets();
+
+        expect(core.exportVariable).toBeCalledWith('SECRET', 'CUSTOMSECRET_IN_NAMESPACE');
+    });
+
+    it('get wildcard secrets from K/V v1 with name prefix', async () => {
+        mockInput('my-secret/test * | GROUP_');
+
+        await exportSecrets();
+
+        expect(core.exportVariable).toBeCalledWith('GROUP_SECRET', 'CUSTOMSECRET_IN_NAMESPACE');
+    });
+
     it('get nested secret from K/V v1', async () => {
         mockInput('my-secret/nested/test otherSecret');
 

src/action.js

@@ -3,6 +3,9 @@ const core = require('@actions/core');
 const command = require('@actions/core/lib/command');
 const got = require('got').default;
 const jsonata = require('jsonata');
+const { normalizeOutputKey } = require('./utils');
+const { WILDCARD } = require('./constants');
+
 const { auth: { retrieveToken }, secrets: { getSecrets } } = require('./index');
 
 const AUTH_METHODS = ['approle', 'token', 'github', 'jwt', 'kubernetes', 'ldap', 'userpass'];
@@ -171,7 +174,7 @@ function parseSecretsInput(secretsInput) {
         const selectorAst = jsonata(selectorQuoted).ast();
         const selector = selectorQuoted.replace(new RegExp('"', 'g'), '');
 
-        if ((selectorAst.type !== "path" || selectorAst.steps[0].stages) && selectorAst.type !== "string" && !outputVarName) {
+        if (selector !== WILDCARD && (selectorAst.type !== "path" || selectorAst.steps[0].stages) && selectorAst.type !== "string" && !outputVarName) {
             throw Error(`You must provide a name for the output key when using json selectors. Input: "${secret}"`);
         }
 
@@ -191,20 +194,6 @@ function parseSecretsInput(secretsInput) {
     return output;
 }
 
-/**
- * Replaces any dot chars to __ and removes non-ascii charts
- * @param {string} dataKey
- * @param {boolean=} isEnvVar
- */
-function normalizeOutputKey(dataKey, isEnvVar = false) {
-    let outputKey = dataKey
-        .replace('.', '__').replace(new RegExp('-', 'g'), '').replace(/[^\p{L}\p{N}_-]/gu, '');
-    if (isEnvVar) {
-        outputKey = outputKey.toUpperCase();
-    }
-    return outputKey;
-}
-
 /**
  * @param {string} inputKey
  * @param {any} inputOptions
@@ -233,6 +222,6 @@ function parseHeadersInput(inputKey, inputOptions) {
 module.exports = {
     exportSecrets,
     parseSecretsInput,
-    normalizeOutputKey,
-    parseHeadersInput
+    parseHeadersInput,
 };
+

src/constants.js

@@ -0,0 +1,5 @@
+const WILDCARD = '*';
+
+module.exports = {
+    WILDCARD
+};

src/secrets.js

@@ -1,6 +1,6 @@
 const jsonata = require("jsonata");
-
-
+const { WILDCARD } = require("./constants");
+const { normalizeOutputKey } = require("./utils");
 /**
  * @typedef {Object} SecretRequest
  * @property {string} path
@@ -23,7 +23,8 @@ const jsonata = require("jsonata");
   */
 async function getSecrets(secretRequests, client) {
     const responseCache = new Map();
-    const results = [];
+    let results = [];
+
     for (const secretRequest of secretRequests) {
         let { path, selector } = secretRequest;
 
@@ -46,22 +47,53 @@ async function getSecrets(secretRequests, client) {
                 throw error
             }
         }
-        if (!selector.match(/.*[\.].*/)) {
-            selector = '"' + selector + '"'
-        }
-        selector = "data." + selector
-        body = JSON.parse(body)
+
+        body = JSON.parse(body);
+
+        if (selector == WILDCARD) {                     
+            let keys = body.data;
             if (body.data["data"] != undefined) {
-            selector = "data." + selector
+                keys = keys.data;
             }
 
-        const value = await selectData(body, selector);
-        results.push({
-            request: secretRequest,
-            value,
-            cachedResponse
-        });
+            for (let key in keys) {
+                let newRequest = Object.assign({},secretRequest);
+                newRequest.selector = key;                  
+                
+                if (secretRequest.selector === secretRequest.outputVarName) {
+                    newRequest.outputVarName = key;
+                    newRequest.envVarName = key;        		
                 }
+                else {
+                    newRequest.outputVarName = secretRequest.outputVarName+key;
+                    newRequest.envVarName = secretRequest.envVarName+key;        		
+                }
+
+                newRequest.outputVarName = normalizeOutputKey(newRequest.outputVarName);
+                newRequest.envVarName = normalizeOutputKey(newRequest.envVarName,true);   
+
+                selector = key;
+
+                results = await selectAndAppendResults(
+                  selector,
+                  body,
+                  cachedResponse,
+                  newRequest,
+                  results
+                );
+            }
+        }
+        else {
+          results = await selectAndAppendResults(
+            selector, 
+            body, 
+            cachedResponse, 
+            secretRequest, 
+            results
+          );
+        }   
+    }
+
     return results;
 }
 
@@ -87,6 +119,43 @@ async function selectData(data, selector) {
     return result;
 }
 
+/**
+ * Uses selectData with the selector to get the value and then appends it to the
+ * results. Returns a new array with all of the results.
+ * @param {string} selector
+ * @param {object} body
+ * @param {object} cachedResponse
+ * @param {TRequest} secretRequest
+ * @param {SecretResponse<TRequest>[]} results
+ * @return {Promise<SecretResponse<TRequest>[]>}
+ */
+const selectAndAppendResults = async (
+  selector,
+  body,
+  cachedResponse,
+  secretRequest,
+  results
+) => {
+  if (!selector.match(/.*[\.].*/)) {
+    selector = '"' + selector + '"';
+  }
+  selector = "data." + selector;
+
+  if (body.data["data"] != undefined) {
+    selector = "data." + selector;
+  }
+
+  const value = await selectData(body, selector);
+  return [
+    ...results,
+    {
+      request: secretRequest,
+      value,
+      cachedResponse,
+    },
+  ];
+};
+
 module.exports = {
     getSecrets,
     selectData

src/utils.js

@@ -0,0 +1,19 @@
+/**
+ * Replaces any dot chars to __ and removes non-ascii charts
+ * @param {string} dataKey
+ * @param {boolean=} isEnvVar
+ */
+function normalizeOutputKey(dataKey, isEnvVar = false) {
+  let outputKey = dataKey
+    .replace(".", "__")
+    .replace(new RegExp("-", "g"), "")
+    .replace(/[^\p{L}\p{N}_-]/gu, "");
+  if (isEnvVar) {
+    outputKey = outputKey.toUpperCase();
+  }
+  return outputKey;
+}
+
+module.exports = { 
+    normalizeOutputKey 
+};