From 9f63033b54c20c0454042c6ca57621dd1ea7a984 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 24 Jul 2019 11:41:30 -0500 Subject: [PATCH 1/4] MQE-1647: read vault token from local file system --- .../Handlers/CredentialStore.php | 8 +- .../Handlers/SecretStorage/VaultStorage.php | 98 ++++++++++++++++++- 2 files changed, 96 insertions(+), 10 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 016360752..75707b941 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -59,13 +59,9 @@ private function __construct() // Initialize vault storage $csBaseUrl = getenv('CREDENTIAL_VAULT_BASE_URL'); - $csToken = getenv('CREDENTIAL_VAULT_TOKEN'); - if ($csBaseUrl !== false && $csToken !== false) { + if ($csBaseUrl !== false) { try { - $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( - rtrim($csBaseUrl, '/'), - $csToken - ); + $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage(rtrim($csBaseUrl, '/')); } catch (TestFrameworkException $e) { } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index de80b83e3..415627cda 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -21,6 +21,21 @@ class VaultStorage extends BaseStorage const BASE_PATH = '/dx_magento_qe'; const KV_DATA = 'data'; + /** + * Default vault token file + */ + const TOKEN_FILE = '.vault-token'; + /** + * Default vault config file + */ + const CONFIG_FILE = '.vault'; + /** + * Environment variable name for vault config path + */ + const CONFIG_PATH_ENV_VAR = 'VAULT_CONFIG_PATH'; + + const TOKEN_HELPER_REGEX = "~\s*token_helper\s*=(.+)$~"; + /** * Vault client * @@ -33,23 +48,22 @@ class VaultStorage extends BaseStorage * * @var string */ - private $token; + private $token = null; /** * CredentialVault constructor * * @param string $baseUrl - * @param string $token * @throws TestFrameworkException */ - public function __construct($baseUrl, $token) + public function __construct($baseUrl) { parent::__construct(); if (null === $this->client) { // Creating the client using Guzzle6 Transport and passing a custom url $this->client = new Client(new Guzzle6Transport(['base_uri' => $baseUrl])); } - $this->token = $token; + $this->readVaultTokenFromFileSystem(); if (!$this->authenticated()) { throw new TestFrameworkException("Credential vault is not used: cannot authenticate"); } @@ -120,4 +134,80 @@ private function authenticated() } return false; } + + /** + * Read vault token from file system + * + * @return void + * @throws TestFrameworkException + */ + private function readVaultTokenFromFileSystem() + { + // Find user home directory + $homeDir = getenv('HOME'); + if ($homeDir === false) { + // If HOME is not set, don't fail right away + $homeDir = '~/'; + } else { + $homeDir = rtrim($homeDir, '/') . '/'; + } + + $vaultTokenFile = $homeDir . self::TOKEN_FILE; + if (file_exists($vaultTokenFile)) { + // Found .vault-token file in default location, construct command + $cmd = 'cat ' . $vaultTokenFile; + } else { + // Otherwise search vault config file for custom token helper script + $vaultConfigPath = getenv(self::CONFIG_PATH_ENV_VAR); + if ($vaultConfigPath === false) { + $vaultConfigFile = $homeDir . self::CONFIG_FILE; + } else { + $vaultConfigFile = rtrim($vaultConfigPath, '/') . '/' . self::CONFIG_FILE; + } + // Found .vault config file, read custom token helper script and construct command + if (file_exists($vaultConfigFile) + && !empty($cmd = $this->getTokenHelperScript(file($vaultConfigFile, FILE_IGNORE_NEW_LINES)))) { + $cmd = $cmd . ' get'; + } else { + throw new TestFrameworkException( + 'Unable to read .vault-token file. Please authenticate to vault through vault CLI first.' + ); + } + } + $this->token = $this->execVaultTokenHelper($cmd); + } + + /** + * Get vault token helper script by parsing lines in vault config file + * + * @param array $lines + * @return array + */ + private function getTokenHelperScript($lines) + { + $tokenHelper = ''; + foreach ($lines as $line) { + preg_match(self::TOKEN_HELPER_REGEX, $line, $matches); + if (isset($matches[1])) { + $tokenHelper = trim(trim(trim($matches[1]), '"')); + } + } + return $tokenHelper; + } + + /** + * Execute vault token helper script and return the token it contains + * + * @param string $cmd + * @return string + */ + private function execVaultTokenHelper($cmd) + { + $output = ''; + exec($cmd, $out, $status); + if ($status === 0 && isset($out[0])) { + $output = $out[0]; + } + return $output; + } } From 2ddaa61feaa31e3b1b2a4595b290cf2841fa8018 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 24 Jul 2019 12:01:00 -0500 Subject: [PATCH 2/4] MQE-1647: read vault token from local file system --- .../Handlers/SecretStorage/VaultStorage.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 415627cda..bb88b18c5 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -181,7 +181,7 @@ private function readVaultTokenFromFileSystem() * Get vault token helper script by parsing lines in vault config file * * @param array $lines - * @return array + * @return string */ private function getTokenHelperScript($lines) { @@ -200,14 +200,16 @@ private function getTokenHelperScript($lines) * * @param string $cmd * @return string + * @throws TestFrameworkException */ private function execVaultTokenHelper($cmd) { - $output = ''; exec($cmd, $out, $status); - if ($status === 0 && isset($out[0])) { - $output = $out[0]; + if ($status === 0 && isset($out[0]) && !empty($out[0])) { + return $out[0]; } - return $output; + throw new TestFrameworkException( + 'Error running custom vault token helper script. Please make sure vault CLI works in your environment.' + ); } } From 599d8fded0c460e7ceb1292bebd18ad29c200d8d Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Thu, 25 Jul 2019 09:04:51 -0500 Subject: [PATCH 3/4] MQE-1647: read vault token from local file system --- etc/config/.env.example | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/config/.env.example b/etc/config/.env.example index cc82ae447..f091688f5 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -32,7 +32,6 @@ BROWSER=chrome #*** Uncomment and set vault base url and access token if you want to use vault to manage _CREDS secrets ***# #CREDENTIAL_VAULT_BASE_URL= -#CREDENTIAL_VAULT_TOKEN= #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= From 84d7a442be3d7eb29f6105a2307c2448e9762594 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Tue, 30 Jul 2019 16:51:32 -0500 Subject: [PATCH 4/4] =?UTF-8?q?MQE-1647:=C2=A0=20read=20vault=20token=20fr?= =?UTF-8?q?om=20local=20file=20system?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Handlers/SecretStorage/VaultStorage.php | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index bb88b18c5..785db036b 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -34,7 +34,11 @@ class VaultStorage extends BaseStorage */ const CONFIG_PATH_ENV_VAR = 'VAULT_CONFIG_PATH'; - const TOKEN_HELPER_REGEX = "~\s*token_helper\s*=(.+)$~"; + /** + * Regex to grab token helper script + */ + const TOKEN_HELPER_REGEX_GROUP_NAME = 'GROUP_NAME'; + const TOKEN_HELPER_REGEX = "~\s*token_helper\s*=(?<" . self::TOKEN_HELPER_REGEX_GROUP_NAME . ">.+)$~"; /** * Vault client @@ -146,35 +150,40 @@ private function readVaultTokenFromFileSystem() // Find user home directory $homeDir = getenv('HOME'); if ($homeDir === false) { - // If HOME is not set, don't fail right away - $homeDir = '~/'; - } else { - $homeDir = rtrim($homeDir, '/') . '/'; + throw new TestFrameworkException( + "HOME environment variable is not set. It's required when using vault." + ); } + $homeDir = realpath($homeDir) . DIRECTORY_SEPARATOR; + // Read .vault-token file if it is found in default location $vaultTokenFile = $homeDir . self::TOKEN_FILE; if (file_exists($vaultTokenFile)) { - // Found .vault-token file in default location, construct command - $cmd = 'cat ' . $vaultTokenFile; - } else { - // Otherwise search vault config file for custom token helper script - $vaultConfigPath = getenv(self::CONFIG_PATH_ENV_VAR); - if ($vaultConfigPath === false) { - $vaultConfigFile = $homeDir . self::CONFIG_FILE; - } else { - $vaultConfigFile = rtrim($vaultConfigPath, '/') . '/' . self::CONFIG_FILE; + $token = file_get_contents($vaultTokenFile); + if ($token !== false) { + $this->token = $token; + return; } - // Found .vault config file, read custom token helper script and construct command - if (file_exists($vaultConfigFile) - && !empty($cmd = $this->getTokenHelperScript(file($vaultConfigFile, FILE_IGNORE_NEW_LINES)))) { - $cmd = $cmd . ' get'; - } else { - throw new TestFrameworkException( - 'Unable to read .vault-token file. Please authenticate to vault through vault CLI first.' - ); + } + + // Otherwise search vault config file for custom token helper script + $vaultConfigPath = getenv(self::CONFIG_PATH_ENV_VAR); + if ($vaultConfigPath === false) { + $vaultConfigFile = $homeDir . self::CONFIG_FILE; + } else { + $vaultConfigFile = realpath($vaultConfigPath) . DIRECTORY_SEPARATOR . self::CONFIG_FILE; + } + // Get custom token helper script file from .vault config file + if (file_exists($vaultConfigFile)) { + $cmd = $this->getTokenHelperScript(file($vaultConfigFile, FILE_IGNORE_NEW_LINES)); + if (!empty($cmd)) { + $this->token = $this->execVaultTokenHelper($cmd . ' get'); + return; } } - $this->token = $this->execVaultTokenHelper($cmd); + throw new TestFrameworkException( + 'Unable to read .vault-token file. Please authenticate to vault through vault CLI first.' + ); } /** @@ -188,8 +197,8 @@ private function getTokenHelperScript($lines) $tokenHelper = ''; foreach ($lines as $line) { preg_match(self::TOKEN_HELPER_REGEX, $line, $matches); - if (isset($matches[1])) { - $tokenHelper = trim(trim(trim($matches[1]), '"')); + if (isset($matches[self::TOKEN_HELPER_REGEX_GROUP_NAME])) { + $tokenHelper = trim(trim(trim($matches[self::TOKEN_HELPER_REGEX_GROUP_NAME]), '"')); } } return $tokenHelper;